diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..6b948b10 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,47 @@ +# Path-based PR labels for actions/labeler v5+. +# See: https://github.com/actions/labeler#pull-request-labeler + +documentation: + - changed-files: + - any-glob-to-any-file: + - docs/** + - README.md + - .github/**/*.md + +ci: + - changed-files: + - any-glob-to-any-file: + - .github/** + +tests: + - changed-files: + - any-glob-to-any-file: + - test/** + +plugins-inband: + - changed-files: + - any-glob-to-any-file: + - nodescraper/plugins/inband/** + +plugins-ooband: + - changed-files: + - any-glob-to-any-file: + - nodescraper/plugins/ooband/** + +plugins-serviceability: + - changed-files: + - any-glob-to-any-file: + - nodescraper/plugins/serviceability/** + +plugins-generic: + - changed-files: + - any-glob-to-any-file: + - nodescraper/plugins/generic_collection/** + +framework: + - changed-files: + - any-glob-to-any-file: + - nodescraper/base/** + - nodescraper/interfaces/** + - nodescraper/cli/** + - nodescraper/models/** diff --git a/.github/scripts/plugin_convention_warnings.py b/.github/scripts/plugin_convention_warnings.py index caaf03f6..893e5797 100755 --- a/.github/scripts/plugin_convention_warnings.py +++ b/.github/scripts/plugin_convention_warnings.py @@ -14,6 +14,14 @@ ``pydantic.Field(...)`` with a non-empty ``description=`` (for help/CLI text). ``ClassVar`` fields, ``_``-prefixed names, and ``model_config`` are skipped. + +3. **Shell quoting** — In ``Collector`` / ``Analyzer`` methods, values from + ``args.*``, ``cmd_spec.*``, or sensitive function parameters (names ending + in ``_path`` / ``_file``, or ``url``, ``boot``, ``folder``, ``host``) that + are interpolated into shell commands (``_run_sut_cmd``, ``_run_amd_smi``, + ``_run_dell_command``) must be wrapped in ``shell_quote(...)`` or assigned + from a prior ``shell_quote(...)`` call in the same function. + ``GenericCollectionCollector`` is excluded (user commands are intentional). """ from __future__ import annotations @@ -40,6 +48,9 @@ } ) +_SHELL_RUN_METHODS = frozenset({"_run_sut_cmd", "_run_amd_smi", "_run_dell_command"}) +_SKIP_SHELL_QUOTE_CLASSES = frozenset({"GenericCollectionCollector"}) + def _is_stringish(expr: ast.expr) -> bool: if isinstance(expr, ast.Constant) and isinstance(expr.value, str): @@ -224,6 +235,179 @@ def _check_args_fields(path: Path, tree: ast.Module) -> list[str]: return msgs +def _is_shell_quote_call(expr: ast.expr) -> bool: + if not isinstance(expr, ast.Call): + return False + func = expr.func + if isinstance(func, ast.Name) and func.id == "shell_quote": + return True + return isinstance(func, ast.Attribute) and func.attr == "shell_quote" + + +def _func_param_names(func: ast.FunctionDef | ast.AsyncFunctionDef) -> set[str]: + names: set[str] = set() + for arg in func.args.args + func.args.kwonlyargs: + if arg.arg != "self": + names.add(arg.arg) + if func.args.vararg: + names.add(func.args.vararg.arg) + return names + + +def _collect_shell_quote_safe_names(func: ast.FunctionDef | ast.AsyncFunctionDef) -> set[str]: + safe: set[str] = set() + for node in ast.walk(func): + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + if isinstance(target, ast.Name) and _is_shell_quote_call(node.value): + safe.add(target.id) + return safe + + +def _expr_preview(expr: ast.expr) -> str: + try: + return ast.unparse(expr) + except Exception: + return "" + + +def _is_sensitive_param_name(name: str) -> bool: + if name in {"url", "boot", "folder", "hostname", "host"}: + return True + return name.endswith("_path") or name.endswith("_file") + + +def _is_user_controlled_leaf(expr: ast.expr, param_names: set[str]) -> bool: + if isinstance(expr, ast.Attribute) and isinstance(expr.value, ast.Name): + base = expr.value.id + if base in ("args", "cmd_spec"): + return True + if isinstance(expr, ast.Name) and expr.id in param_names: + return _is_sensitive_param_name(expr.id) + return False + + +def _contains_user_controlled(expr: ast.expr, param_names: set[str]) -> bool: + if _is_user_controlled_leaf(expr, param_names): + return True + for child in ast.iter_child_nodes(expr): + if isinstance(child, ast.expr) and _contains_user_controlled(child, param_names): + return True + return False + + +def _is_trivially_safe_expr(expr: ast.expr, safe_names: set[str]) -> bool: + if _is_shell_quote_call(expr): + return True + if isinstance(expr, ast.Constant): + return True + if isinstance(expr, ast.Name) and expr.id in safe_names: + return True + if isinstance(expr, ast.Attribute) and isinstance(expr.value, ast.Name): + base = expr.value.id + if base == "self" and ( + expr.attr == "CMD" or expr.attr.startswith("CMD_") or expr.attr in ("AMD_SMI_EXE",) + ): + return True + return False + + +def _command_insert_issues( + expr: ast.expr, + param_names: set[str], + safe_names: set[str], +) -> list[tuple[int, str]]: + issues: list[tuple[int, str]] = [] + + def note(value: ast.expr) -> None: + if _is_trivially_safe_expr(value, safe_names): + return + if _contains_user_controlled(value, param_names): + issues.append((getattr(value, "lineno", 0), _expr_preview(value))) + + if isinstance(expr, ast.JoinedStr): + for part in expr.values: + if isinstance(part, ast.FormattedValue): + note(part.value) + elif isinstance(expr, ast.Call) and isinstance(expr.func, ast.Attribute): + if expr.func.attr == "format": + for kw in expr.keywords: + if kw.arg is not None and kw.value is not None: + note(kw.value) + elif isinstance(expr, ast.BinOp): + note(expr.left) + note(expr.right) + + return issues + + +def _resolve_command_exprs( + func: ast.FunctionDef | ast.AsyncFunctionDef, expr: ast.expr +) -> list[ast.expr]: + if not isinstance(expr, ast.Name): + return [expr] + + matches: list[ast.expr] = [] + for node in ast.walk(func): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == expr.id: + matches.append(node.value) + elif ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == expr.id + and node.value is not None + ): + matches.append(node.value) + return matches or [expr] + + +def _check_function_shell_quoting( + path: Path, + class_name: str, + func: ast.FunctionDef | ast.AsyncFunctionDef, +) -> list[str]: + msgs: list[str] = [] + param_names = _func_param_names(func) + safe_names = _collect_shell_quote_safe_names(func) + + for node in ast.walk(func): + if not isinstance(node, ast.Call): + continue + if not isinstance(node.func, ast.Attribute): + continue + if node.func.attr not in _SHELL_RUN_METHODS: + continue + if not node.args: + continue + + command_expr = node.args[0] + for resolved in _resolve_command_exprs(func, command_expr): + for lineno, preview in _command_insert_issues(resolved, param_names, safe_names): + line = lineno or node.lineno + msgs.append( + f"{path}:{line}: [{class_name}.{func.name}] user-controlled value " + f"{preview} in shell command must use shell_quote(...)." + ) + return msgs + + +def _check_shell_quoting(path: Path, tree: ast.Module) -> list[str]: + """Rule #3: warn when config/param values reach shell commands without shell_quote.""" + msgs: list[str] = [] + for node in tree.body: + if not isinstance(node, ast.ClassDef) or not _is_collector_or_analyzer_class(node): + continue + if node.name in _SKIP_SHELL_QUOTE_CLASSES: + continue + for stmt in node.body: + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)): + msgs.extend(_check_function_shell_quoting(path, node.name, stmt)) + return msgs + + def main() -> None: if not PLUGIN_ROOT.is_dir(): sys.stderr.write(f"warning: plugins directory not found: {PLUGIN_ROOT}\n") @@ -242,8 +426,10 @@ def main() -> None: if "collector" in name and name.endswith(".py"): all_msgs.extend(_check_cmd_prefixes(rel, tree)) + all_msgs.extend(_check_shell_quoting(rel, tree)) if "analyzer" in name and name.endswith(".py"): all_msgs.extend(_check_cmd_prefixes(rel, tree)) + all_msgs.extend(_check_shell_quoting(rel, tree)) if name == "collector_args.py" or name == "analyzer_args.py": all_msgs.extend(_check_args_fields(rel, tree)) diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml new file mode 100644 index 00000000..6535a06c --- /dev/null +++ b/.github/workflows/label.yml @@ -0,0 +1,23 @@ +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. +# +# To use this workflow, you will need to set up a .github/labeler.yml +# file with configuration. For more information, see: +# https://github.com/actions/labeler + +name: Labeler +on: [pull_request_target] + +jobs: + label: + + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + + steps: + - uses: actions/labeler@v5 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + sync-labels: true diff --git a/docs/PLUGIN_DOC.md b/docs/PLUGIN_DOC.md index fe6612ae..4351a912 100644 --- a/docs/PLUGIN_DOC.md +++ b/docs/PLUGIN_DOC.md @@ -4,46 +4,46 @@ | Plugin | Collection | Analyzer Args | Collection Args | DataModel | Collector | Analyzer | | --- | --- | --- | --- | --- | --- | --- | -| GenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | +| GenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_\* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | | AmdSmiPlugin | bad-pages
firmware --json
list --json
metric -g all
partition --json
process --json
ras --cper --folder={folder}
ras --afid --cper-file {cper_file}
static -g all --json
static -g {gpu_id} --json
topology
version --json
xgmi -l
xgmi -m | **Analyzer Args:**
- `check_static_data`: bool — If True, run static data checks (e.g. driver version, partition mode).
- `expected_gpu_processes`: Optional[int] — Expected number of GPU processes.
- `expected_max_power`: Optional[int] — Expected maximum power value (e.g. watts).
- `expected_power_management`: Optional[str] — Expected amd-smi metric power_management value per GPU (e.g. DISABLED for active/full power, ENABLED for power-manage...
- `expected_driver_version`: Optional[str] — Expected AMD driver version string.
- `expected_memory_partition_mode`: Optional[str] — Expected memory partition mode (e.g. sp3, dp).
- `expected_compute_partition_mode`: Optional[str] — Expected compute partition mode.
- `expected_firmware_versions`: Optional[dict[str, str]] — Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).
- `l0_to_recovery_count_error_threshold`: Optional[int] — L0-to-recovery count above which an error is raised.
- `l0_to_recovery_count_warning_threshold`: Optional[int] — L0-to-recovery count above which a warning is raised.
- `vendorid_ep`: Optional[str] — Expected endpoint vendor ID (e.g. for PCIe).
- `vendorid_ep_vf`: Optional[str] — Expected endpoint VF vendor ID.
- `devid_ep`: Optional[str] — Expected endpoint device ID.
- `devid_ep_vf`: Optional[str] — Expected endpoint VF device ID.
- `sku_name`: Optional[str] — Expected SKU name string for GPU.
- `expected_xgmi_speed`: Optional[list[float]] — Expected xGMI speed value(s) (e.g. link rate).
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for time-windowed analysis.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for time-windowed analysis. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `analysis_firmware_ids`: Optional[list[str]] — amd-smi fw_id values to record in analysis_ref.firmware_versions
- `cper_file_path`: Optional[str] — Path to CPER folder or file for RAS AFID collection (ras --afid --cper-file). | [AmdSmiDataModel](#AmdSmiDataModel-Model) | [AmdSmiCollector](#Collector-Class-AmdSmiCollector) | [AmdSmiAnalyzer](#Data-Analyzer-Class-AmdSmiAnalyzer) | | BiosPlugin | sh -c 'cat /sys/devices/virtual/dmi/id/bios_version'
wmic bios get SMBIOSBIOSVersion /Value | **Analyzer Args:**
- `exp_bios_version`: list[str] — Expected BIOS version(s) to match against collected value (str or list).
- `regex_match`: bool — If True, match exp_bios_version as regex; otherwise exact match. | - | [BiosDataModel](#BiosDataModel-Model) | [BiosCollector](#Collector-Class-BiosCollector) | [BiosAnalyzer](#Data-Analyzer-Class-BiosAnalyzer) | | CmdlinePlugin | cat /proc/cmdline | **Analyzer Args:**
- `required_cmdline`: Union[str, List] — Command-line parameters that must be present (e.g. 'pci=bfsort').
- `banned_cmdline`: Union[str, List] — Command-line parameters that must not be present.
- `os_overrides`: Dict[str, nodescraper.plugins.inband.cmdline.cmdlineconfig.OverrideConfig] — Per-OS overrides for required_cmdline and banned_cmdline (keyed by OS identifier).
- `platform_overrides`: Dict[str, nodescraper.plugins.inband.cmdline.cmdlineconfig.OverrideConfig] — Per-platform overrides for required_cmdline and banned_cmdline (keyed by platform). | - | [CmdlineDataModel](#CmdlineDataModel-Model) | [CmdlineCollector](#Collector-Class-CmdlineCollector) | [CmdlineAnalyzer](#Data-Analyzer-Class-CmdlineAnalyzer) | | DeviceEnumerationPlugin | powershell -Command "(Get-WmiObject -Class Win32_Processor | Measure-Object).Count"
lspci -d {vendorid_ep}: | grep -iE 'VGA|Display|3D|Processing accelerators|Co-processor|Accelerator' | grep -vi 'Virtual Function' | wc -l
powershell -Command "(wmic path win32_VideoController get name | findstr AMD | Measure-Object).Count"
lscpu
lshw
lspci -d {vendorid_ep}: | grep -i 'Virtual Function' | wc -l
powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count" | **Analyzer Args:**
- `cpu_count`: Optional[list[int]] — Expected CPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `gpu_count`: Optional[list[int]] — Expected GPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `vf_count`: Optional[list[int]] — Expected virtual function count(s); pass as int or list of ints. Analysis passes if actual is in list. | - | [DeviceEnumerationDataModel](#DeviceEnumerationDataModel-Model) | [DeviceEnumerationCollector](#Collector-Class-DeviceEnumerationCollector) | [DeviceEnumerationAnalyzer](#Data-Analyzer-Class-DeviceEnumerationAnalyzer) | | DimmPlugin | sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\|None\|Module" | grep Size' 2>/dev/null
dmidecode
wmic memorychip get Capacity | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running dmidecode or wmic for memory info. | [DimmDataModel](#DimmDataModel-Model) | [DimmCollector](#Collector-Class-DimmCollector) | - | | DkmsPlugin | dkms status
dkms --version | **Analyzer Args:**
- `dkms_status`: Union[str, list] — Expected dkms status string(s) to match (e.g. 'amd/1.0.0'). At least one of dkms_status or dkms_version required.
- `dkms_version`: Union[str, list] — Expected dkms version string(s) to match. At least one of dkms_status or dkms_version required.
- `regex_match`: bool — If True, match dkms_status and dkms_version as regex; otherwise exact match. | - | [DkmsDataModel](#DkmsDataModel-Model) | [DkmsCollector](#Collector-Class-DkmsCollector) | [DkmsAnalyzer](#Data-Analyzer-Class-DkmsAnalyzer) | -| DmesgPlugin | dmesg --time-format iso -x
ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true | **Built-in Regexes:**
- Out of memory error: `(?:oom_kill_process.*)|(?:Out of memory.*)`
- I/O Page Fault: `IO_PAGE_FAULT`
- Kernel Panic: `\bkernel panic\b.*`
- SQ Interrupt: `sq_intr`
- SRAM ECC: `sram_ecc.*`
- Failed to load driver. IP hardware init error.: `\[amdgpu\]\] \*ERROR\* hw_init of IP block.*`
- Failed to load driver. IP software init error.: `\[amdgpu\]\] \*ERROR\* sw_init of IP block.*`
- Real Time throttling activated: `sched: RT throttling activated.*`
- RCU preempt detected stalls: `rcu_preempt detected stalls.*`
- RCU preempt self-detected stall: `rcu_preempt self-detected stall.*`
- QCM fence timeout: `qcm fence wait loop timeout.*`
- General protection fault: `(?:[\w-]+(?:\[[0-9.]+\])?\s+)?general protectio...`
- Segmentation fault: `(?:segfault.*in .*\[)|(?:[Ss]egmentation [Ff]au...`
- Failed to disallow cf state: `amdgpu: Failed to disallow cf state.*`
- Failed to terminate tmr: `\*ERROR\* Failed to terminate tmr.*`
- Suspend of IP block failed: `\*ERROR\* suspend of IP block <\w+> failed.*`
- amdgpu Page Fault: `(amdgpu \w{4}:\w{2}:\w{2}\.\w:\s+amdgpu:\s+\[\S...`
- Page Fault: `page fault for address.*`
- Fatal error during GPU init: `(?:amdgpu)(.*Fatal error during GPU init)|(Fata...`
- PCIe AER Error Status: `(pcieport [\w:.]+: AER: aer_status:[^\n]*(?:\n[...`
- PCIe AER Correctable Error Status: `(.*aer_cor_status: 0x[0-9a-fA-F]+, aer_cor_mask...`
- PCIe AER Uncorrectable Error Status: `(.*aer_uncor_status: 0x[0-9a-fA-F]+, aer_uncor_...`
- PCIe AER Uncorrectable Error Severity with TLP Header: `(.*aer_uncor_severity: 0x[0-9a-fA-F]+.*)(\n.*TL...`
- Failed to read journal file: `Failed to read journal file.*`
- Journal file corrupted or uncleanly shut down: `journal corrupted or uncleanly shut down.*`
- ACPI BIOS Error: `ACPI BIOS Error`
- ACPI Error: `ACPI Error`
- Filesystem corrupted!: `EXT4-fs error \(device .*\):`
- Error in buffered IO, check filesystem integrity: `(Buffer I\/O error on dev)(?:ice)? (\w+)`
- PCIe card no longer present: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- PCIe Link Down: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- Mismatched clock configuration between PCIe device and host: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(curren...`
- RAS Correctable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Uncorrectable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Deferred Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Corrected PCIe Error: `((?:\[Hardware Error\]:\s+)?event severity: cor...`
- GPU Reset: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- MCE Corrected Error: `\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|CE\|...`
- MCE Uncorrected Error: `\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|UC\|...`
- Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...`
- RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- SGX Error: `x86/cpu: SGX disabled by BIOS`
- MMP Error: `Failed to load MMP firmware qat_4xxx_mmp.bin`
- GPU Throttled: `amdgpu \w{4}:\w{2}:\w{2}.\w: amdgpu: WARN: GPU ...`
- RAS Poison Consumed: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- RAS Poison created: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- Bad page threshold exceeded: `(amdgpu: Saved bad pages (\d+) reaches threshol...`
- RAS Hardware Error: `Hardware error from APEI Generic Hardware Error...`
- Error Address: `Error Address.*(?:\s.*)`
- RAS EDR Event: `EDR: EDR event received`
- DPC Event: `DPC: .*`
- LNet: ko2iblnd has no matching interfaces: `(?:\[[^\]]+\]\s*)?LNetError:.*ko2iblnd:\s*No ma...`
- LNet: Error starting up LNI: `(?:\[[^\]]+\]\s*)?LNetError:\s*.*Error\s*-?\d+\...`
- Lustre: network initialisation failed: `LustreError:.*ptlrpc_init_portals\(\).*network ...` | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `collect_rotated_logs`: bool — If True, also collect rotated dmesg log files from /var/log/dmesg*.
- `skip_sudo`: bool — If True, do not use sudo when running dmesg or listing log files.
- `log_dmesg_data`: bool — If True, log the collected dmesg output in artifacts. | [DmesgData](#DmesgData-Model) | [DmesgCollector](#Collector-Class-DmesgCollector) | [DmesgAnalyzer](#Data-Analyzer-Class-DmesgAnalyzer) | -| FabricsPlugin | lspci | grep -i cassini
lsmod | grep cxi
cxi_stat
ibstat
ibv_devinfo
ls -l /sys/class/infiniband/*/device/net
fi_info -p cxi
mst start
mst status -v
ip link show
ofed_info -s | - | - | [FabricsDataModel](#FabricsDataModel-Model) | [FabricsCollector](#Collector-Class-FabricsCollector) | - | +| DmesgPlugin | dmesg --time-format iso -x
ls -1 /var/log/dmesg\* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true | **Built-in Regexes:**
- Out of memory error: `(?:oom_kill_process.*)|(?:Out of memory.*)`
- I/O Page Fault: `IO_PAGE_FAULT`
- Kernel Panic: `\bkernel panic\b.*`
- SQ Interrupt: `sq_intr`
- SRAM ECC: `sram_ecc.*`
- Failed to load driver. IP hardware init error.: `\[amdgpu\]\] \*ERROR\* hw_init of IP block.*`
- Failed to load driver. IP software init error.: `\[amdgpu\]\] \*ERROR\* sw_init of IP block.*`
- Real Time throttling activated: `sched: RT throttling activated.*`
- RCU preempt detected stalls: `rcu_preempt detected stalls.*`
- RCU preempt self-detected stall: `rcu_preempt self-detected stall.*`
- QCM fence timeout: `qcm fence wait loop timeout.*`
- General protection fault: `(?:[\w-]+(?:\[[0-9.]+\])?\s+)?general protectio...`
- Segmentation fault: `(?:segfault.*in .*\[)|(?:[Ss]egmentation [Ff]au...`
- Failed to disallow cf state: `amdgpu: Failed to disallow cf state.*`
- Failed to terminate tmr: `\*ERROR\* Failed to terminate tmr.*`
- Suspend of IP block failed: `\*ERROR\* suspend of IP block <\w+> failed.*`
- amdgpu Page Fault: `(amdgpu \w{4}:\w{2}:\w{2}\.\w:\s+amdgpu:\s+\[\S...`
- Page Fault: `page fault for address.*`
- Fatal error during GPU init: `(?:amdgpu)(.*Fatal error during GPU init)|(Fata...`
- PCIe AER Error Status: `(pcieport [\w:.]+: AER: aer_status:[^\n]*(?:\n[...`
- PCIe AER Correctable Error Status: `(.*aer_cor_status: 0x[0-9a-fA-F]+, aer_cor_mask...`
- PCIe AER Uncorrectable Error Status: `(.*aer_uncor_status: 0x[0-9a-fA-F]+, aer_uncor_...`
- PCIe AER Uncorrectable Error Severity with TLP Header: `(.*aer_uncor_severity: 0x[0-9a-fA-F]+.*)(\n.*TL...`
- Failed to read journal file: `Failed to read journal file.*`
- Journal file corrupted or uncleanly shut down: `journal corrupted or uncleanly shut down.*`
- ACPI BIOS Error: `ACPI BIOS Error`
- ACPI Error: `ACPI Error`
- Filesystem corrupted!: `EXT4-fs error \(device .*\):`
- Error in buffered IO, check filesystem integrity: `(Buffer I\/O error on dev)(?:ice)? (\w+)`
- PCIe card no longer present: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- PCIe Link Down: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- Mismatched clock configuration between PCIe device and host: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(curren...`
- RAS Correctable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Uncorrectable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Deferred Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Corrected PCIe Error: `((?:\[Hardware Error\]:\s+)?event severity: cor...`
- GPU Reset: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- MCE Corrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- MCE Uncorrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...`
- RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- SGX Error: `x86/cpu: SGX disabled by BIOS`
- MMP Error: `Failed to load MMP firmware qat_4xxx_mmp.bin`
- GPU Throttled: `amdgpu \w{4}:\w{2}:\w{2}.\w: amdgpu: WARN: GPU ...`
- RAS Poison Consumed: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- RAS Poison created: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- Bad page threshold exceeded: `(amdgpu: Saved bad pages (\d+) reaches threshol...`
- RAS Hardware Error: `Hardware error from APEI Generic Hardware Error...`
- Error Address: `Error Address.*(?:\s.*)`
- RAS EDR Event: `EDR: EDR event received`
- DPC Event: `DPC: .*`
- LNet: ko2iblnd has no matching interfaces: `(?:\[[^\]]+\]\s*)?LNetError:.*ko2iblnd:\s*No ma...`
- LNet: Error starting up LNI: `(?:\[[^\]]+\]\s*)?LNetError:\s*.*Error\s*-?\d+\...`
- Lustre: network initialisation failed: `LustreError:.*ptlrpc_init_portals\(\).*network ...`
**Analyzer Args:**
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for analysis (ISO format). Only events on or after this time are analyzed.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for analysis (ISO format). Only events before this time are analyzed.
- `check_unknown_dmesg_errors`: Optional[bool] — If True, treat unknown/unmatched dmesg error lines as failures.
- `exclude_category`: Optional[set[str]] — Set of error categories to exclude from analysis.
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one (for rate limiting).
- `num_timestamps`: int — Number of timestamps to include per event in output.
- `error_regex`: Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType] — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern.
- `priority_override_rules`: Optional[list[dict]] — Rules to override the priority of matched ErrorRegex objects. Each rule is a dict where all keys except 'new_priority...
- `mce_threshold`: Optional[int] — When set, raise ERROR if correctable MCE/RAS error count for any component (CPU, GPU BDF/block, etc.) reaches or exce...
- `ignore_match_rules`: Optional[list[dict[str, object]]] — Rules that skip regex matches during analysis. Each rule may use line_regex, match_regex, message, and/or mce_banks.... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `collect_rotated_logs`: bool — If True, also collect rotated dmesg log files from /var/log/dmesg\*.
- `skip_sudo`: bool — If True, do not use sudo when running dmesg or listing log files.
- `log_dmesg_data`: bool — If True, log the collected dmesg output in artifacts. | [DmesgData](#DmesgData-Model) | [DmesgCollector](#Collector-Class-DmesgCollector) | [DmesgAnalyzer](#Data-Analyzer-Class-DmesgAnalyzer) | +| FabricsPlugin | lspci | grep -i cassini
lsmod | grep cxi
cxi_stat
ibstat
ibv_devinfo
ls -l /sys/class/infiniband/\*/device/net
fi_info -p cxi
mst start
mst status -v
ip link show
ofed_info -s | - | - | [FabricsDataModel](#FabricsDataModel-Model) | [FabricsCollector](#Collector-Class-FabricsCollector) | - | | JournalPlugin | journalctl --no-pager --system --output=short-iso
journalctl --no-pager --system --output=json | **Analyzer Args:**
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for analysis (ISO format). Only events on or after this time are analyzed.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for analysis (ISO format). Only events before this time are analyzed.
- `check_priority`: Optional[int] — Check against journal log priority (0=emergency..7=debug). If an entry has priority <= check_priority, an ERROR event...
- `group`: bool — If True, group entries that have the same priority and message. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `boot`: Optional[int] — Optional boot ID to limit journal collection to a specific boot. | [JournalData](#JournalData-Model) | [JournalCollector](#Collector-Class-JournalCollector) | [JournalAnalyzer](#Data-Analyzer-Class-JournalAnalyzer) | | KernelPlugin | sh -c 'uname -a'
sh -c 'cat /proc/sys/kernel/numa_balancing'
wmic os get Version /Value | **Analyzer Args:**
- `exp_kernel`: Union[str, list] — Expected kernel version string(s) to match (e.g. from uname -a).
- `exp_numa`: Optional[int] — Expected value for kernel.numa_balancing (e.g. 0 or 1).
- `regex_match`: bool — If True, match exp_kernel as regex; otherwise exact match. | - | [KernelDataModel](#KernelDataModel-Model) | [KernelCollector](#Collector-Class-KernelCollector) | [KernelAnalyzer](#Data-Analyzer-Class-KernelAnalyzer) | | KernelModulePlugin | cat /proc/modules
modinfo amdgpu
wmic os get Version /Value | **Analyzer Args:**
- `kernel_modules`: dict[str, dict] — Expected kernel module name -> {version, etc.}. Analyzer checks collected modules match.
- `regex_filter`: list[str] — List of regex patterns to filter which collected modules are checked (default: amd). | - | [KernelModuleDataModel](#KernelModuleDataModel-Model) | [KernelModuleCollector](#Collector-Class-KernelModuleCollector) | [KernelModuleAnalyzer](#Data-Analyzer-Class-KernelModuleAnalyzer) | | MemoryPlugin | free -b
lsmem
numactl -H
wmic OS get FreePhysicalMemory /Value; wmic ComputerSystem get TotalPhysicalMemory /Value | **Analyzer Args:**
- `ratio`: float — Required free-memory ratio (0-1). Analysis fails if free/total < ratio.
- `memory_threshold`: str — Minimum free memory required (e.g. '30Gi', '1T'). Used when ratio is not sufficient. | - | [MemoryDataModel](#MemoryDataModel-Model) | [MemoryCollector](#Collector-Class-MemoryCollector) | [MemoryAnalyzer](#Data-Analyzer-Class-MemoryAnalyzer) | -| NetworkPlugin | ip addr show
curl
ethtool -S {interface}
ethtool {interface}
lldpcli show neighbor
lldpctl
ip neighbor show
ping
rdma link -j
ip route show
ip rule show
wget | **Analyzer Args:**
- `error_regex`: Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType] — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `url`: Optional[str] — Optional URL to probe for network connectivity (used with netprobe).
- `netprobe`: Optional[Literal['ping', 'wget', 'curl']] — Tool to use for network connectivity probe: ping, wget, or curl.
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it). | [NetworkDataModel](#NetworkDataModel-Model) | [NetworkCollector](#Collector-Class-NetworkCollector) | [NetworkAnalyzer](#Data-Analyzer-Class-NetworkAnalyzer) | -| NicPlugin | niccli --listdev
niccli --list
niccli --list_devices
niccli -dev {device_num} nvm -getoption pcie_relaxed_ordering
niccli --dev {device_num} nvm --getoption pcie_relaxed_ordering
niccli -dev {device_num} nvm -getoption performance_profile
niccli --dev {device_num} nvm --getoption performance_profile
niccli -dev {device_num} nvm -getoption support_rdma -scope 0
niccli -dev {device_num} getqos
niccli --dev {device_num} nvm --getoption support_rdma --scope 0
niccli --dev {device_num} qos --ets --show
niccli --version
nicctl show card
nicctl --version
nicctl show card flash partition --json
nicctl show card interrupts --json
nicctl show card logs --non-persistent
nicctl show card logs --boot-fault
nicctl show card logs --persistent
nicctl show card profile --json
nicctl show card time --json
nicctl show card statistics packet-buffer summary --json
nicctl show lif statistics --json
nicctl show lif internal queue-to-ud-pinning
nicctl show pipeline internal anomalies
nicctl show pipeline internal rsq-ring
nicctl show pipeline internal statistics memory
nicctl show port fsm
nicctl show port transceiver --json
nicctl show port statistics --json
nicctl show port internal mac
nicctl show qos headroom --json
nicctl show rdma queue --json
nicctl show rdma queue-pair --detail --json
nicctl show version firmware
nicctl show dcqcn
nicctl show environment
nicctl show lif
nicctl show pcie ats
nicctl show port
nicctl show qos
nicctl show rdma statistics
nicctl show version host-software
nicctl show dcqcn --card {card_id} --json
nicctl show card hardware-config --card {card_id} | **Analyzer Args:**
- `expected_values`: Optional[Dict[str, Dict[str, Any]]] — Per-command expected checks keyed by canonical key (see command_to_canonical_key).
- `performance_profile_expected`: str — Expected Broadcom performance_profile value (case-insensitive). Default RoCE.
- `support_rdma_disabled_values`: List[str] — Values that indicate RDMA is not supported (case-insensitive).
- `pcie_relaxed_ordering_expected`: str — Expected Broadcom pcie_relaxed_ordering value (e.g. 'Relaxed ordering = enabled'); checked case-insensitively. Defaul...
- `expected_qos_prio_map`: Optional[Dict[Any, Any]] — Expected priority-to-TC map (e.g. {0: 0, 1: 1}; keys may be int or str in config). Checked per device when set.
- `expected_qos_pfc_enabled`: Optional[int] — Expected PFC enabled value (0/1 or bitmask). Checked per device when set.
- `expected_qos_tsa_map`: Optional[Dict[Any, Any]] — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set.
- `expected_qos_tc_bandwidth`: Optional[List[int]] — Expected TC bandwidth percentages. Checked per device when set.
- `require_qos_consistent_across_adapters`: bool — When True and no expected_qos_* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map.
- `nicctl_log_error_regex`: Optional[List[Dict[str, Any]]] — Optional list of error patterns for nicctl show card logs. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: Optional[List[str]] — Optional list of niccli/nicctl commands to run. When None, default command set is used.
- `use_sudo_niccli`: bool — If True, run niccli commands with sudo when required.
- `use_sudo_nicctl`: bool — If True, run nicctl commands with sudo when required. | [NicDataModel](#NicDataModel-Model) | [NicCollector](#Collector-Class-NicCollector) | [NicAnalyzer](#Data-Analyzer-Class-NicAnalyzer) | +| NetworkPlugin | ip addr show
curl
ethtool -i {interface}
ethtool -S {interface}
ethtool {interface}
lldpcli show neighbor
lldpctl
ip neighbor show
ping
ip route show
ip rule show
wget | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `url`: Optional[str] — Optional URL to probe for network connectivity (used with netprobe).
- `netprobe`: Optional[Literal['ping', 'wget', 'curl']] — Tool to use for network connectivity probe: ping, wget, or curl.
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it). | [NetworkDataModel](#NetworkDataModel-Model) | [NetworkCollector](#Collector-Class-NetworkCollector) | [NetworkAnalyzer](#Data-Analyzer-Class-NetworkAnalyzer) | +| NicPlugin | niccli --listdev
niccli --list
niccli --list_devices
niccli -dev {device_num} nvm -getoption pcie_relaxed_ordering
niccli --dev {device_num} nvm --getoption pcie_relaxed_ordering
niccli -dev {device_num} nvm -getoption performance_profile
niccli --dev {device_num} nvm --getoption performance_profile
niccli -dev {device_num} nvm -getoption support_rdma -scope 0
niccli -dev {device_num} getqos
niccli --dev {device_num} nvm --getoption support_rdma --scope 0
niccli --dev {device_num} qos --ets --show
niccli --version
nicctl show card
nicctl --version
nicctl show card flash partition --json
nicctl show card interrupts --json
nicctl show card logs --non-persistent
nicctl show card logs --boot-fault
nicctl show card logs --persistent
nicctl show card profile --json
nicctl show card time --json
nicctl show card statistics packet-buffer summary --json
nicctl show lif statistics --json
nicctl show lif internal queue-to-ud-pinning
nicctl show pipeline internal anomalies
nicctl show pipeline internal rsq-ring
nicctl show pipeline internal statistics memory
nicctl show port fsm
nicctl show port transceiver --json
nicctl show port statistics --json
nicctl show port internal mac
nicctl show qos headroom --json
nicctl show rdma queue --json
nicctl show rdma queue-pair --detail --json
nicctl show version firmware
nicctl show dcqcn
nicctl show environment
nicctl show lif
nicctl show pcie ats
nicctl show port
nicctl show qos
nicctl show rdma statistics
nicctl show version host-software
nicctl show dcqcn --card {card_id} --json
nicctl show card hardware-config --card {card_id} | **Analyzer Args:**
- `expected_values`: Optional[Dict[str, Dict[str, Any]]] — Per-command expected checks keyed by canonical key (see command_to_canonical_key).
- `performance_profile_expected`: str — Expected Broadcom performance_profile value (case-insensitive). Default RoCE.
- `support_rdma_disabled_values`: List[str] — Values that indicate RDMA is not supported (case-insensitive).
- `pcie_relaxed_ordering_expected`: str — Expected Broadcom pcie_relaxed_ordering value (e.g. 'Relaxed ordering = enabled'); checked case-insensitively. Defaul...
- `expected_qos_prio_map`: Optional[Dict[Any, Any]] — Expected priority-to-TC map (e.g. {0: 0, 1: 1}; keys may be int or str in config). Checked per device when set.
- `expected_qos_pfc_enabled`: Optional[int] — Expected PFC enabled value (0/1 or bitmask). Checked per device when set.
- `expected_qos_tsa_map`: Optional[Dict[Any, Any]] — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set.
- `expected_qos_tc_bandwidth`: Optional[List[int]] — Expected TC bandwidth percentages. Checked per device when set.
- `require_qos_consistent_across_adapters`: bool — When True and no expected_qos_\* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map.
- `nicctl_log_error_regex`: Optional[List[Dict[str, Any]]] — Optional list of error patterns for nicctl show card logs. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: Optional[List[str]] — Optional list of niccli/nicctl commands to run. When None, default command set is used.
- `use_sudo_niccli`: bool — If True, run niccli commands with sudo when required.
- `use_sudo_nicctl`: bool — If True, run nicctl commands with sudo when required. | [NicDataModel](#NicDataModel-Model) | [NicCollector](#Collector-Class-NicCollector) | [NicAnalyzer](#Data-Analyzer-Class-NicAnalyzer) | | NvmePlugin | nvme smart-log {dev}
nvme error-log {dev} --log-entries=256
nvme id-ctrl {dev}
nvme id-ns {dev}{ns}
nvme fw-log {dev}
nvme self-test-log {dev}
nvme get-log {dev} --log-id=6 --log-len=512
nvme telemetry-log {dev} --output-file={dev}_{f_name}
nvme list -o json | - | - | [NvmeDataModel](#NvmeDataModel-Model) | [NvmeCollector](#Collector-Class-NvmeCollector) | - | -| OsPlugin | sh -c '( lsb_release -ds || (cat /etc/*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1'
cat /etc/*release | grep VERSION_ID
wmic os get Version /value
wmic os get Caption /Value | **Analyzer Args:**
- `exp_os`: Union[str, list] — Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release).
- `exact_match`: bool — If True, require exact match for exp_os; otherwise substring match. | - | [OsDataModel](#OsDataModel-Model) | [OsCollector](#Collector-Class-OsCollector) | [OsAnalyzer](#Data-Analyzer-Class-OsAnalyzer) | -| PackagePlugin | dnf list --installed
dpkg-query -W
pacman -Q
cat /etc/*release
wmic product get name,version | **Analyzer Args:**
- `exp_package_ver`: Dict[str, Optional[str]] — Map package name -> expected version (None = any version). Checked against installed packages.
- `regex_match`: bool — If True, match package versions with regex; otherwise exact or prefix match.
- `rocm_regex`: Optional[str] — Optional regex to identify ROCm package version (used when enable_rocm_regex is True).
- `enable_rocm_regex`: bool — If True, use rocm_regex (or default pattern) to extract ROCm version for checks. | - | [PackageDataModel](#PackageDataModel-Model) | [PackageCollector](#Collector-Class-PackageCollector) | [PackageAnalyzer](#Data-Analyzer-Class-PackageAnalyzer) | +| OsPlugin | sh -c '( lsb_release -ds || (cat /etc/\*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1'
cat /etc/\*release | grep VERSION_ID
wmic os get Version /value
wmic os get Caption /Value | **Analyzer Args:**
- `exp_os`: Union[str, list] — Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release).
- `exact_match`: bool — If True, require exact match for exp_os; otherwise substring match. | - | [OsDataModel](#OsDataModel-Model) | [OsCollector](#Collector-Class-OsCollector) | [OsAnalyzer](#Data-Analyzer-Class-OsAnalyzer) | +| PackagePlugin | dnf list --installed
dpkg-query -W
pacman -Q
cat /etc/\*release
wmic product get name,version | **Analyzer Args:**
- `exp_package_ver`: Dict[str, Optional[str]] — Map package name -> expected version (None = any version). Checked against installed packages.
- `regex_match`: bool — If True, match package versions with regex; otherwise exact or prefix match.
- `rocm_regex`: Optional[str] — Optional regex to identify ROCm package version (used when enable_rocm_regex is True).
- `enable_rocm_regex`: bool — If True, use rocm_regex (or default pattern) to extract ROCm version for checks. | - | [PackageDataModel](#PackageDataModel-Model) | [PackageCollector](#Collector-Class-PackageCollector) | [PackageAnalyzer](#Data-Analyzer-Class-PackageAnalyzer) | | PciePlugin | lspci -d {vendor_id}: -nn
lspci -x
lspci -xxxx
lspci -PP
lspci -PP -d {vendor_id}:{dev_id}
lspci -PP -D -d {vendor_id}:{dev_id}
lspci -PP -D
lspci -vvv
lspci -vvvt | **Analyzer Args:**
- `exp_speed`: int — Expected PCIe link speed (generation 1–5).
- `exp_width`: int — Expected PCIe link width in lanes (1–16).
- `exp_sriov_count`: int — Expected SR-IOV virtual function count.
- `exp_gpu_count_override`: Optional[int] — Override expected GPU count for validation.
- `exp_max_payload_size`: Union[Dict[int, int], int, NoneType] — Expected max payload size: int for all devices, or dict keyed by device ID.
- `exp_max_rd_req_size`: Union[Dict[int, int], int, NoneType] — Expected max read request size: int for all devices, or dict keyed by device ID.
- `exp_ten_bit_tag_req_en`: Union[Dict[int, int], int, NoneType] — Expected 10-bit tag request enable: int for all devices, or dict keyed by device ID. | - | [PcieDataModel](#PcieDataModel-Model) | [PcieCollector](#Collector-Class-PcieCollector) | [PcieAnalyzer](#Data-Analyzer-Class-PcieAnalyzer) | | ProcessPlugin | top -b -n 1
rocm-smi --showpids
top -b -n 1 -o %CPU | **Analyzer Args:**
- `max_kfd_processes`: int — Maximum allowed number of KFD (Kernel Fusion Driver) processes; 0 disables the check.
- `max_cpu_usage`: float — Maximum allowed CPU usage (percent) for process checks. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `top_n_process`: int — Number of top processes by CPU usage to collect (e.g. for top -b -n 1 -o %%CPU). | [ProcessDataModel](#ProcessDataModel-Model) | [ProcessCollector](#Collector-Class-ProcessCollector) | [ProcessAnalyzer](#Data-Analyzer-Class-ProcessAnalyzer) | -| RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | **Analyzer Args:**
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched against an interface netdev; matching interfaces are skipped. | - | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | -| RegexSearchPlugin | - | Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).
Emits regex match events with optional per-file source in the description when scanning directories.
**Analyzer Args:**
- `error_regex`: Optional[list[dict[str, Any]]] — Regex patterns to search for; each dict may include regex (str), message, event_category, event_priority (same as Dme...
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one.
- `num_timestamps`: int — Number of timestamps to include per event in output. | - | [RegexSearchData](#RegexSearchData-Model) | - | [RegexSearchAnalyzer](#Data-Analyzer-Class-RegexSearchAnalyzer) | -| RocmPlugin | {rocm_path}/opencl/bin/*/clinfo
env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen'
ls /sys/class/kfd/kfd/proc/
grep -i -E 'rocm' /etc/ld.so.conf.d/*
{rocm_path}/bin/rocminfo
ls -v -d {rocm_path}*
ls -v -d {rocm_path}-[3-7]* | tail -1
ldconfig -p | grep -i -E 'rocm'
grep . -H -r -i {rocm_path}/.info/* | **Analyzer Args:**
- `exp_rocm`: Union[str, list] — Expected ROCm version string(s) to match (e.g. from rocminfo).
- `exp_rocm_latest`: str — Expected 'latest' ROCm path or version string for versioned installs.
- `exp_rocm_sub_versions`: dict[str, Union[str, list]] — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `rocm_path`: str — Base path to ROCm installation (e.g. /opt/rocm). Used for rocminfo, clinfo, and version discovery. | [RocmDataModel](#RocmDataModel-Model) | [RocmCollector](#Collector-Class-RocmCollector) | [RocmAnalyzer](#Data-Analyzer-Class-RocmAnalyzer) | -| StoragePlugin | sh -c 'df -lH -B1 | grep -v 'boot''
wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running df and related storage commands. | [StorageDataModel](#StorageDataModel-Model) | [StorageCollector](#Collector-Class-StorageCollector) | [StorageAnalyzer](#Data-Analyzer-Class-StorageAnalyzer) | +| RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | **Analyzer Args:**
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched against an interface netdev; matching interfaces are skipped. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors... | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | +| RegexSearchPlugin | No collector step: reads local text from the CLI --data path (file or directory).
Directory scans load each file's contents into RegexSearchData for analysis. | Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).
Emits regex match events with optional per-file source in the description when scanning directories.
**Analyzer Args:**
- `error_regex`: Optional[list[dict[str, Any]]] — Regex patterns to search for; each dict may include regex (str), message, event_category, event_priority (same as Dme...
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one.
- `num_timestamps`: int — Number of timestamps to include per event in output. | - | [RegexSearchData](#RegexSearchData-Model) | - | [RegexSearchAnalyzer](#Data-Analyzer-Class-RegexSearchAnalyzer) | +| RocmPlugin | {rocm_path}/opencl/bin/\*/clinfo
env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen'
ls /sys/class/kfd/kfd/proc/
grep -i -E 'rocm' /etc/ld.so.conf.d/\*
{rocm_path}/bin/rocminfo
ls -v -d {rocm_path}\*
ls -v -d {rocm_path}-[3-7]\* | tail -1
ldconfig -p | grep -i -E 'rocm'
grep . -H -r -i {rocm_path}/.info/\* | **Analyzer Args:**
- `exp_rocm`: Union[str, list] — Expected ROCm version string(s) to match (e.g. from rocminfo).
- `exp_rocm_latest`: str — Expected 'latest' ROCm path or version string for versioned installs.
- `exp_rocm_sub_versions`: dict[str, Union[str, list]] — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `rocm_path`: str — Base path to ROCm installation (e.g. /opt/rocm). Used for rocminfo, clinfo, and version discovery. | [RocmDataModel](#RocmDataModel-Model) | [RocmCollector](#Collector-Class-RocmCollector) | [RocmAnalyzer](#Data-Analyzer-Class-RocmAnalyzer) | +| StoragePlugin | sh -c 'df -lH -B1 | grep -v 'boot''
wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace | **Analyzer Args:**
- `min_required_free_space_abs`: Optional[str] — Minimum required free space per mount (e.g. '10G', '1T').
- `min_required_free_space_prct`: Optional[int] — Minimum required free space as percentage of total (0–100).
- `ignore_devices`: Optional[list[str]] — Mount points or devices to exclude from free-space checks.
- `check_devices`: Optional[list[str]] — If non-empty, only these mount points or devices are checked.
- `regex_match`: bool — If True, match device/mount names with regex; otherwise exact match. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running df and related storage commands. | [StorageDataModel](#StorageDataModel-Model) | [StorageCollector](#Collector-Class-StorageCollector) | [StorageAnalyzer](#Data-Analyzer-Class-StorageAnalyzer) | | ScaleOutAristaPlugin | show interfaces counters bins | json | no-more
show interfaces counters queue | no-more
show interfaces counters queue drop-precedence | no-more
show qos interfaces ecn counters queue | json | no-more
show interfaces counters errors | json | no-more
show interfaces phy | no-more
show interfaces phy detail | no-more
show interfaces counters ip | json | no-more
show ip interface | no-more
show lldp | no-more
show lldp neighbors | json | no-more
show interfaces counters | json | no-more
show interfaces flow-control | json | no-more
show interfaces counters queue detail | no-more
show priority-flow-control counters | json | no-more
show priority-flow-control status | no-more
show interfaces status | json | no-more
show qos interfaces | no-more
show qos interfaces ecn | no-more
show qos interfaces trust | no-more
show qos maps | no-more
show qos profile | no-more
show qos profile summary | no-more
show interfaces counters rates | json | no-more
show running-config | no-more
show startup-config | no-more
show system environment cooling | json | no-more
show platform trident mmu queue status | no-more
show version | json | no-more | **Analyzer Args:**
- `analysis_ports`: Optional[List[str]] — Restrict per-port analysis to the given ports. Ports are S/P/[SP] where subport is optional (e.g. ['1/1', '1/31', '1/...
- `expected_port_bandwidth`: int — Expected interface bandwidth (bps) from show interfaces status (AristaPortStatus.bandwidth). Ports with a different b... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Re-runs successful... | [ScaleOutAristaDataModel](#ScaleOutAristaDataModel-Model) | [ScaleOutAristaCollector](#Collector-Class-ScaleOutAristaCollector) | [ScaleOutAristaAnalyzer](#Data-Analyzer-Class-ScaleOutAristaAnalyzer) | | ScaleOutDellPlugin | show alarm | no-more
show buffer pool | no-more
show buffer profile | no-more
show clock | no-more
show interface counters {port} | no-more
show event details | no-more
show interface fec status | no-more
show interface counters | no-more
show interface counters rate | no-more
show interface Eth | no-more
show interface phy counters | no-more
show interface status | no-more
show interface transceiver | no-more
show interface transceiver dom | no-more
show interface transceiver summary | no-more
show ip arp | no-more
show ip interfaces | no-more
show ip route | no-more
show lldp neighbor | no-more
show lldp table | no-more
show qos interface Ethall priority-flow-control statistics | no-more
show priority-flow-control watchdog | no-more
show qos interface Ethall queue all priority-flow-control watchdog-statistics | no-more
show platform environment | no-more
show platform firmware detail | no-more
show platform syseeprom | no-more
show qos interface Eth all | no-more
show qos interface Eth all queue all | no-more
show qos map dot1p-tc | no-more
show qos map dscp-tc | no-more
show qos map pfc-priority-pg | no-more
show qos map pfc-priority-queue | no-more
show qos map tc-dot1p | no-more
show qos map tc-dscp | no-more
show qos map tc-pg | no-more
show qos map tc-queue | no-more
show qos scheduler-policy | no-more
show qos wred-policy | no-more
show queue counters | no-more
show queue persistent-watermark multicast | no-more
show queue persistent-watermark unicast | no-more
show queue watermark multicast | no-more
show queue watermark unicast | no-more
show running-configuration | no-more
show version | no-more | **Analyzer Args:**
- `analysis_ports`: Optional[List[str]] — Restrict per-port analysis to the given ports. Accepts optional Eth prefix (e.g. ['1/1', '1/31', '1/1/1'] or ['Eth1/1...
- `expected_port_speed`: int — Expected interface speed (Mbps) from show interface status (DellInterfaceStatus.speed). Ports with a different speed... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `collection_ports`: Optional[List[str]] — Restrict detail counter collection to these ports. Accepts the same tokens as analysis_ports (e.g. ['1/1', '1/1/2'] o... | [ScaleOutDellDataModel](#ScaleOutDellDataModel-Model) | [ScaleOutDellCollector](#Collector-Class-ScaleOutDellCollector) | [ScaleOutDellAnalyzer](#Data-Analyzer-Class-ScaleOutDellAnalyzer) | -| SysSettingsPlugin | cat /sys/{}
ls -1 /sys/{}
ls -l /sys/{} | **Analyzer Args:**
- `checks`: Optional[list[nodescraper.plugins.inband.sys_settings.analyzer_args.SysfsCheck]] — List of sysfs checks (path, expected values or pattern, display name). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `paths`: list[str] — Sysfs paths to read (cat). Paths with '*' are collected with ls -l (e.g. class/net/*/device).
- `directory_paths`: list[str] — Sysfs paths to list (ls -1); used for checks that match entry names by regex. | [SysSettingsDataModel](#SysSettingsDataModel-Model) | [SysSettingsCollector](#Collector-Class-SysSettingsCollector) | [SysSettingsAnalyzer](#Data-Analyzer-Class-SysSettingsAnalyzer) | +| SysSettingsPlugin | cat /sys/{}
ls -1 /sys/{}
ls -l /sys/{} | **Analyzer Args:**
- `checks`: Optional[list[nodescraper.plugins.inband.sys_settings.analyzer_args.SysfsCheck]] — List of sysfs checks (path, expected values or pattern, display name). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `paths`: list[str] — Sysfs paths to read (cat). Paths with '\*' are collected with ls -l (e.g. class/net/\*/device).
- `directory_paths`: list[str] — Sysfs paths to list (ls -1); used for checks that match entry names by regex. | [SysSettingsDataModel](#SysSettingsDataModel-Model) | [SysSettingsCollector](#Collector-Class-SysSettingsCollector) | [SysSettingsAnalyzer](#Data-Analyzer-Class-SysSettingsAnalyzer) | | SysctlPlugin | sysctl -n | **Analyzer Args:**
- `exp_vm_swappiness`: Optional[int] — Expected vm.swappiness value.
- `exp_vm_numa_balancing`: Optional[int] — Expected vm.numa_balancing value.
- `exp_vm_oom_kill_allocating_task`: Optional[int] — Expected vm.oom_kill_allocating_task value.
- `exp_vm_compaction_proactiveness`: Optional[int] — Expected vm.compaction_proactiveness value.
- `exp_vm_compact_unevictable_allowed`: Optional[int] — Expected vm.compact_unevictable_allowed value.
- `exp_vm_extfrag_threshold`: Optional[int] — Expected vm.extfrag_threshold value.
- `exp_vm_zone_reclaim_mode`: Optional[int] — Expected vm.zone_reclaim_mode value.
- `exp_vm_dirty_background_ratio`: Optional[int] — Expected vm.dirty_background_ratio value.
- `exp_vm_dirty_ratio`: Optional[int] — Expected vm.dirty_ratio value.
- `exp_vm_dirty_writeback_centisecs`: Optional[int] — Expected vm.dirty_writeback_centisecs value.
- `exp_kernel_numa_balancing`: Optional[int] — Expected kernel.numa_balancing value. | - | [SysctlDataModel](#SysctlDataModel-Model) | [SysctlCollector](#Collector-Class-SysctlCollector) | [SysctlAnalyzer](#Data-Analyzer-Class-SysctlAnalyzer) | -| SyslogPlugin | ls -1 /var/log/syslog* 2>/dev/null | grep -E '^/var/log/syslog(\.[0-9]+(\.gz)?)?$' || true
ls -1 /var/log/messages* 2>/dev/null | grep -E '^/var/log/messages(\.[0-9]+(\.gz)?)?$' || true | - | - | [SyslogData](#SyslogData-Model) | [SyslogCollector](#Collector-Class-SyslogCollector) | - | +| SyslogPlugin | ls -1 /var/log/syslog\* 2>/dev/null | grep -E '^/var/log/syslog(\.[0-9]+(\.gz)?)?$' || true
ls -1 /var/log/messages\* 2>/dev/null | grep -E '^/var/log/messages(\.[0-9]+(\.gz)?)?$' || true | - | - | [SyslogData](#SyslogData-Model) | [SyslogCollector](#Collector-Class-SyslogCollector) | - | | UptimePlugin | uptime | - | - | [UptimeDataModel](#UptimeDataModel-Model) | [UptimeCollector](#Collector-Class-UptimeCollector) | - | # OOB plugins | Plugin | Collection | Analyzer Args | Collection Args | DataModel | Collector | Analyzer | | --- | --- | --- | --- | --- | --- | --- | -| OobGenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | +| OobGenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_\* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | | OobBmcArchivePlugin | SSH (BMC) shell: tar+gzip archives for each path in collection_args (see PathSpec entries).
Uses sudo on the BMC when collection_args paths require elevated access. | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `paths`: list[nodescraper.plugins.ooband.bmc_archive.collector_args.PathSpec] — Named BMC paths to archive with tar czf -. Configure in plugin config under plugins.OobBmcArchivePlugin.collection_ar...
- `sudo`: bool — Default sudo setting for paths that do not specify sudo.
- `timeout`: int — Default per-path tar timeout in seconds.
- `skip_if_missing`: bool — Skip paths that do not exist on the BMC instead of failing collection.
- `ignore_failed_read`: bool — When true, pass GNU tar's --ignore-failed-read when the remote tar supports it. | [BmcArchiveDataModel](#BmcArchiveDataModel-Model) | [BmcArchiveCollector](#Collector-Class-BmcArchiveCollector) | - | -| RedfishEndpointPlugin | Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).
Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.
Redfish GET tree: when discover_tree is true, walks from api_root using OData resource id links and Members navigation (depth and endpoint caps from collection_args). | For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.).
URI key "*" runs checks against every collected response body.
**Analyzer Args:**
- `checks`: dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]] — Map: URI or '*' -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match).... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uris`: list[str] — Redfish URIs to GET. Ignored when discover_tree is True.
- `discover_tree`: bool — If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.
- `tree_max_depth`: int — When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).
- `tree_max_endpoints`: int — When discover_tree is True: max endpoints to discover (0=no limit).
- `max_workers`: int — Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.
- `follow_next_link`: bool — If True, follow Redfish Members collection OData nextLink pagination for each URI and merge all pages into a single r...
- `max_pages`: int — When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200). | [RedfishEndpointDataModel](#RedfishEndpointDataModel-Model) | [RedfishEndpointCollector](#Collector-Class-RedfishEndpointCollector) | [RedfishEndpointAnalyzer](#Data-Analyzer-Class-RedfishEndpointAnalyzer) | +| RedfishEndpointPlugin | Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).
Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.
Redfish GET tree: when discover_tree is true, walks from api_root using OData resource id links and Members navigation (depth and endpoint caps from collection_args). | For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.).
URI key `*` runs checks against every collected response body.
**Analyzer Args:**
- `checks`: dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]] — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match).... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uris`: list[str] — Redfish URIs to GET. Ignored when discover_tree is True.
- `discover_tree`: bool — If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.
- `tree_max_depth`: int — When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).
- `tree_max_endpoints`: int — When discover_tree is True: max endpoints to discover (0=no limit).
- `max_workers`: int — Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.
- `follow_next_link`: bool — If True, follow Redfish Members collection OData nextLink pagination for each URI and merge all pages into a single r...
- `max_pages`: int — When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200). | [RedfishEndpointDataModel](#RedfishEndpointDataModel-Model) | [RedfishEndpointCollector](#Collector-Class-RedfishEndpointCollector) | [RedfishEndpointAnalyzer](#Data-Analyzer-Class-RedfishEndpointAnalyzer) | | RedfishOemDiagPlugin | Redfish LogService.CollectDiagnosticData for each entry in collection_args.oem_diagnostic_types (collection_args.log_service_path selects the LogService).
Optional binary archives under the plugin log path when log_path is set. | Summarizes success/failure per OEM diagnostic type from collected results.
When analysis_args.require_all_success is true, fails the run if any type failed collection.
**Analyzer Args:**
- `require_all_success`: bool — If True, analysis fails when any OEM type collection failed. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `log_service_path`: str — Redfish path to the LogService (e.g. DiagLogs).
- `oem_diagnostic_types_allowable`: Optional[list[str]] — Allowable OEM diagnostic types for this architecture/BMC. When set, used for validation and as default for oem_diagno...
- `oem_diagnostic_types`: list[str] — OEM diagnostic types to collect. When empty and oem_diagnostic_types_allowable is set, defaults to that list.
- `task_timeout_s`: int — Max seconds to wait for each BMC task. | [RedfishOemDiagDataModel](#RedfishOemDiagDataModel-Model) | [RedfishOemDiagCollector](#Collector-Class-RedfishOemDiagCollector) | [RedfishOemDiagAnalyzer](#Data-Analyzer-Class-RedfishOemDiagAnalyzer) | -| ServiceabilityPluginMI3XX | - | **Analyzer Args:**
- `hub_python_module`: Optional[str] — Import path for the hub module (class implements hub_analyze_method); hub_options forwards kwargs.
- `hub_display_name`: Optional[str] — Optional label for analyzer status messages.
- `afid_sag_path`: Optional[str] — Path to hub config (e.g. AFID_SAG.json); passed as hub_init_path_kwarg.
- `hub_init_path_kwarg`: str — Hub __init__ keyword that receives afid_sag_path.
- `hub_analyze_method`: str — Hub method called with rf_events first (default get_service_info).
- `skip_hub`: bool — If True, only build afid_events without running the service hub.
- `cper_decode_module`: Optional[str] — Module import path for CPER decoding when events include CPER attachments.
- `cper_decode_method`: str — Callable on cper_decode_module: file-like CPER in, (return_code, decode_dict) out.
- `hub_options`: Optional[dict[str, Any]] — Extra kwargs for hub __init__ and analyze; collected cper_data overrides cper_data key.
- `from_ac_cycle`: int — from_ac_cycle kwarg for the hub analyze call (merged after hub_options).
- `from_date`: Optional[str] — Optional from_date for the hub analyze call (merged after hub_options).
- `designation_serials`: Optional[dict[str, str]] — Optional designation_serials for the hub analyze call (merged after hub_options).
- `suppress_service_actions`: Optional[list[str]] — Optional suppress_service_actions for the hub analyze call (merged after hub_options). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uri`: Optional[str] — Optional alias for ``rf_event_log_uri``. When both ``uri`` and ``rf_event_log_uri`` are explicitly set to non-empty v...
- `rf_event_log_uri`: str — Redfish URI for the event log ``Entries`` collection.
- `rf_chassis_devices`: Optional[List[str]] — Chassis designations for Assembly GETs; required with ``rf_assembly_uri_template``.
- `rf_assembly_uri_template`: Optional[str] — Redfish URI template containing ``{device}`` for each chassis Assembly resource.
- `rf_firmware_bundle_uri`: Optional[str] — Redfish URI for firmware bundle inventory when subclasses extract component details.
- `follow_next_link`: bool — If True, follow Members@odata.nextLink up to max_pages; else single GET.
- `max_pages`: int — Safety cap on the number of pages when following event log pagination.
- `top`: Optional[int] — Most recent N entries via $skip after count probe; None collects full window.
- `reference_time`: Optional[str] — Optional ISO-8601 date or date-time used with time_operator (e.g. 2026-05-17 or 2026-05-17T13:01:00).
- `time_operator`: Optional[Literal['>', '>=', '<', '<=', '==']] — Comparison operator applied when reference_time is set. | [ServiceabilityDataModel](#ServiceabilityDataModel-Model) | [MI3XXCollector](#Collector-Class-MI3XXCollector) | [MI3XXAnalyzer](#Data-Analyzer-Class-MI3XXAnalyzer) | +| ServiceabilityPluginMI3XX | Redfish GET: BMC event log Entries (collection_args.rf_event_log_uri; optional uri alias).
Paginated Members collection and optional top, reference_time/time_operator filters.
Redfish GET: CPER AdditionalDataURI binaries for DiagnosticDataType=CPER events (base64 in data model).
Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices).
Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details. | Builds AFID events from collected Redfish event log members (and optional assembly metadata).
Optionally decodes CPER attachments via analysis_args.cper_decode_module before hub analysis.
Runs the configured Python service hub (hub_python_module) to produce service recommendations.
When analysis_args.skip_hub is true, only builds AFID events without running the hub.
**Analyzer Args:**
- `hub_python_module`: Optional[str] — Import path for the hub module (class implements hub_analyze_method); hub_options forwards kwargs.
- `hub_display_name`: Optional[str] — Optional label for analyzer status messages.
- `afid_sag_path`: Optional[str] — Path to hub config (e.g. AFID_SAG.json); passed as hub_init_path_kwarg.
- `hub_init_path_kwarg`: str — Hub __init__ keyword that receives afid_sag_path.
- `hub_analyze_method`: str — Hub method called with rf_events first (default get_service_info).
- `skip_hub`: bool — If True, only build afid_events without running the service hub.
- `cper_decode_module`: Optional[str] — Module import path for CPER decoding when events include CPER attachments.
- `cper_decode_method`: str — Callable on cper_decode_module: file-like CPER in, (return_code, decode_dict) out.
- `hub_options`: Optional[dict[str, Any]] — Extra kwargs for hub __init__ and analyze; collected cper_data overrides cper_data key.
- `from_ac_cycle`: int — from_ac_cycle kwarg for the hub analyze call (merged after hub_options).
- `from_date`: Optional[str] — Optional from_date for the hub analyze call (merged after hub_options).
- `designation_serials`: Optional[dict[str, str]] — Optional designation_serials for the hub analyze call (merged after hub_options).
- `suppress_service_actions`: Optional[list[str]] — Optional suppress_service_actions for the hub analyze call (merged after hub_options). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uri`: Optional[str] — Optional alias for ``rf_event_log_uri``. When both ``uri`` and ``rf_event_log_uri`` are explicitly set to non-empty v...
- `rf_event_log_uri`: str — Redfish URI for the event log ``Entries`` collection.
- `rf_chassis_devices`: Optional[List[str]] — Chassis designations for Assembly GETs; required with ``rf_assembly_uri_template``.
- `rf_assembly_uri_template`: Optional[str] — Redfish URI template containing ``{device}`` for each chassis Assembly resource.
- `rf_firmware_bundle_uri`: Optional[str] — Redfish URI for firmware bundle inventory when subclasses extract component details.
- `follow_next_link`: bool — If True, follow Members@odata.nextLink up to max_pages; else single GET.
- `max_pages`: int — Safety cap on the number of pages when following event log pagination.
- `top`: Optional[int] — Most recent N entries via $skip after count probe; None collects full window.
- `reference_time`: Optional[str] — Optional ISO-8601 date or date-time used with time_operator (e.g. 2026-05-17 or 2026-05-17T13:01:00).
- `time_operator`: Optional[Literal['>', '>=', '<', '<=', '==']] — Comparison operator applied when reference_time is set. | [ServiceabilityDataModel](#ServiceabilityDataModel-Model) | [MI3XXCollector](#Collector-Class-MI3XXCollector) | [MI3XXAnalyzer](#Data-Analyzer-Class-MI3XXAnalyzer) | # Collectors @@ -68,7 +68,7 @@ GenericCollectionDataModel ### Documented collection - Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH). -- Commands are user-configured; there are no fixed CMD_* class fields. +- Commands are user-configured; there are no fixed CMD_\* class fields. ## Collector Class AmdSmiCollector @@ -275,7 +275,7 @@ DmesgData ### Commands - dmesg --time-format iso -x -- ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true +- ls -1 /var/log/dmesg\* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true ## Collector Class FabricsCollector @@ -312,7 +312,7 @@ FabricsDataModel - cxi_stat - ibstat - ibv_devinfo -- ls -l /sys/class/infiniband/*/device/net +- ls -l /sys/class/infiniband/\*/device/net - fi_info -p cxi - mst start - mst status -v @@ -442,7 +442,7 @@ Collect network configuration details using ip command - **CMD_NEIGHBOR**: `ip neighbor show` - **CMD_ETHTOOL_TEMPLATE**: `ethtool {interface}` - **CMD_ETHTOOL_S_TEMPLATE**: `ethtool -S {interface}` -- **CMD_RDMA_LINK_JSON**: `rdma link -j` +- **CMD_ETHTOOL_I_TEMPLATE**: `ethtool -i {interface}` - **CMD_PING**: `ping` - **CMD_WGET**: `wget` - **CMD_CURL**: `curl` @@ -457,13 +457,13 @@ NetworkDataModel - ip addr show - curl +- ethtool -i {interface} - ethtool -S {interface} - ethtool {interface} - lldpcli show neighbor - lldpctl - ip neighbor show - ping -- rdma link -j - ip route show - ip rule show - wget @@ -673,8 +673,8 @@ OsDataModel ### Commands -- sh -c '( lsb_release -ds || (cat /etc/*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1' -- cat /etc/*release | grep VERSION_ID +- sh -c '( lsb_release -ds || (cat /etc/\*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1' +- cat /etc/\*release | grep VERSION_ID - wmic os get Version /value - wmic os get Caption /Value @@ -705,7 +705,7 @@ PackageDataModel - dnf list --installed - dpkg-query -W - pacman -Q -- cat /etc/*release +- cat /etc/\*release - wmic product get name,version ## Collector Class PcieCollector @@ -853,15 +853,15 @@ RocmDataModel ### Commands -- {rocm_path}/opencl/bin/*/clinfo +- {rocm_path}/opencl/bin/\*/clinfo - env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen' - ls /sys/class/kfd/kfd/proc/ -- grep -i -E 'rocm' /etc/ld.so.conf.d/* +- grep -i -E 'rocm' /etc/ld.so.conf.d/\* - {rocm_path}/bin/rocminfo -- ls -v -d {rocm_path}* -- ls -v -d {rocm_path}-[3-7]* | tail -1 +- ls -v -d {rocm_path}\* +- ls -v -d {rocm_path}-[3-7]\* | tail -1 - ldconfig -p | grep -i -E 'rocm' -- grep . -H -r -i {rocm_path}/.info/* +- grep . -H -r -i {rocm_path}/.info/\* ## Collector Class StorageCollector @@ -1208,8 +1208,8 @@ SyslogData ### Commands -- ls -1 /var/log/syslog* 2>/dev/null | grep -E '^/var/log/syslog(\.[0-9]+(\.gz)?)?$' || true -- ls -1 /var/log/messages* 2>/dev/null | grep -E '^/var/log/messages(\.[0-9]+(\.gz)?)?$' || true +- ls -1 /var/log/syslog\* 2>/dev/null | grep -E '^/var/log/syslog(\.[0-9]+(\.gz)?)?$' || true +- ls -1 /var/log/messages\* 2>/dev/null | grep -E '^/var/log/messages(\.[0-9]+(\.gz)?)?$' || true ## Collector Class UptimeCollector @@ -1313,6 +1313,14 @@ Collect MI3XX BMC Redfish data: event log members (with pagination), firmware in ServiceabilityDataModel +### Documented collection + +- Redfish GET: BMC event log Entries (collection_args.rf_event_log_uri; optional uri alias). +- Paginated Members collection and optional top, reference_time/time_operator filters. +- Redfish GET: CPER AdditionalDataURI binaries for DiagnosticDataType=CPER events (base64 in data model). +- Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices). +- Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details. + # Data Models ## GenericCollectionDataModel Model @@ -1525,8 +1533,8 @@ Complete network configuration data - **rules**: `List[nodescraper.plugins.inband.network.networkdata.RoutingRule]` - **neighbors**: `List[nodescraper.plugins.inband.network.networkdata.Neighbor]` - **ethtool_info**: `Dict[str, nodescraper.plugins.inband.network.networkdata.EthtoolInfo]` -- **rdma_ethtool_netdevs**: `List[str]` -- **rdma_ethtool_statistics**: `List[nodescraper.plugins.inband.network.ethtool_vendor.EthtoolStatistics]` +- **ethtool_netdevs**: `List[str]` +- **ethtool_statistics**: `List[nodescraper.plugins.inband.network.ethtool_vendor.EthtoolStatistics]` - **accessible**: `Optional[bool]` ## NicDataModel Model @@ -1993,8 +2001,8 @@ Check dmesg for errors regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)?(.*GPU reset(?:\\(\\d+\\))? failed.*)') message='GPU reset failed' event_category= event_priority=, regex=re.compile('(Accelerator Check Architecture[^\\n]*)(?:\\n[^\\n]*){0,10}?(amdgpu[ 0-9a-fA-F:.]+:? [^\\n]*entry\\[\\d+\\]\\.STATUS=0x[0-9a-fA-F]+)(?:\\n[^\\n]*){0,5}?(amdgpu[ 0-9a-fA-F:.]+:? [^\\n]*entry\\[\\d+\\], re.MULTILINE) message='ACA Error' event_category= event_priority=, regex=re.compile('(Accelerator Check Architecture[^\\n]*)(?:\\n[^\\n]*){0,10}?(amdgpu[ 0-9a-fA-F:.]+:? [^\\n]*CONTROL=0x[0-9a-fA-F]+)(?:\\n[^\\n]*){0,5}?(amdgpu[ 0-9a-fA-F:.]+:? [^\\n]*STATUS=0x[0-9a-fA-F]+)(?:\\n[^\\, re.MULTILINE) message='ACA Error' event_category= event_priority=, - regex=re.compile('\\[Hardware Error\\]:.+MC\\d+_STATUS\\[[^\\]]*\\|CE\\|[^\\]]*\\].*(?:\\n.*){0,5}') message='MCE Corrected Error' event_category= event_priority=, - regex=re.compile('\\[Hardware Error\\]:.+MC\\d+_STATUS\\[[^\\]]*\\|UC\\|[^\\]]*\\].*(?:\\n.*){0,5}') message='MCE Uncorrected Error' event_category= event_priority=, + regex=re.compile('\\[Hardware Error\\]:[^\\n]*MC\\d+_STATUS\\[[^\\]]*\\|CE\\|[^\\]]*\\][^\\n]*', re.IGNORECASE) message='MCE Corrected Error' event_category= event_priority=, + regex=re.compile('\\[Hardware Error\\]:[^\\n]*MC\\d+_STATUS\\[[^\\]]*\\|UC\\|[^\\]]*\\][^\\n]*', re.IGNORECASE) message='MCE Uncorrected Error' event_category= event_priority=, regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)? (.*Mode2 reset failed.*)') message='Mode 2 Reset Failed' event_category= event_priority=, regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)?(.*\\[Hardware Error\\]: Corrected error.*)') message='RAS Corrected Error' event_category= event_priority=, regex=re.compile('x86/cpu: SGX disabled by BIOS') message='SGX Error' event_category= event_priority=, @@ -2057,8 +2065,8 @@ Check dmesg for errors - - GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....` - - ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...` - - ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...` -- - MCE Corrected Error: `\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|CE\|...` -- - MCE Uncorrected Error: `\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|UC\|...` +- - MCE Corrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...` +- - MCE Uncorrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...` - - Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...` - - RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....` - - SGX Error: `x86/cpu: SGX disabled by BIOS` @@ -2125,10 +2133,6 @@ Check network statistics for errors. **Link to code**: [network_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/network/network_analyzer.py) -### Class Variables - -- **ERROR_REGEX**: `[]` - ## Data Analyzer Class NicAnalyzer ### Description @@ -2318,7 +2322,7 @@ Checks Redfish endpoint responses against configured thresholds and key/value ru ### Documented analysis - For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.). -- URI key "*" runs checks against every collected response body. +- URI key `*` runs checks against every collected response body. ## Data Analyzer Class RedfishOemDiagAnalyzer @@ -2345,6 +2349,13 @@ Build AFID events from collected data and run the configured service hub. **Link to code**: [mi3xx_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py) +### Documented analysis + +- Builds AFID events from collected Redfish event log members (and optional assembly metadata). +- Optionally decodes CPER attachments via analysis_args.cper_decode_module before hub analysis. +- Runs the configured Python service hub (hub_python_module) to produce service recommendations. +- When analysis_args.skip_hub is true, only builds AFID events without running the hub. + # Analyzer Args ## Analyzer Args Class GenericAnalyzerArgs @@ -2432,6 +2443,25 @@ Build AFID events from collected data and run the configured service hub. - **dkms_version**: `Union[str, list]` — Expected dkms version string(s) to match. At least one of dkms_status or dkms_version required. - **regex_match**: `bool` — If True, match dkms_status and dkms_version as regex; otherwise exact match. +## Analyzer Args Class DmesgAnalyzerArgs + +**Bases**: ['TimeRangeAnalysisArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/dmesg/analyzer_args.py) + +### Annotations / fields + +- **analysis_range_start**: `Optional[datetime.datetime]` — Start of time range for analysis (ISO format). Only events on or after this time are analyzed. +- **analysis_range_end**: `Optional[datetime.datetime]` — End of time range for analysis (ISO format). Only events before this time are analyzed. +- **check_unknown_dmesg_errors**: `Optional[bool]` — If True, treat unknown/unmatched dmesg error lines as failures. +- **exclude_category**: `Optional[set[str]]` — Set of error categories to exclude from analysis. +- **interval_to_collapse_event**: `int` — Seconds within which repeated events are collapsed into one (for rate limiting). +- **num_timestamps**: `int` — Number of timestamps to include per event in output. +- **error_regex**: `Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType]` — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern. +- **priority_override_rules**: `Optional[list[dict]]` — Rules to override the priority of matched ErrorRegex objects. Each rule is a dict where all keys except 'new_priority' and 'match_all' are filter fields matched against ErrorRegex attributes. 'new_priority' must be an EventPriority name (e.g. 'WARNING', 'ERROR') or 'NO_CHANGE' to leave the priority unchanged. +- **mce_threshold**: `Optional[int]` — When set, raise ERROR if correctable MCE/RAS error count for any component (CPU, GPU BDF/block, etc.) reaches or exceeds this value. +- **ignore_match_rules**: `Optional[list[dict[str, object]]]` — Rules that skip regex matches during analysis. Each rule may use line_regex, match_regex, message, and/or mce_banks. Within a rule all specified fields must match; any matching rule suppresses the hit. mce_banks accepts bank ids and inclusive ranges such as "60-63". When a rule matches an MCA bank, only [Hardware Error]: lines in that incident block are suppressed; interleaved non-MCE dmesg lines in the same block are not. + ## Analyzer Args Class JournalAnalyzerArgs ### Description @@ -2483,20 +2513,6 @@ Arguments for journal analyzer - **ratio**: `float` — Required free-memory ratio (0-1). Analysis fails if free/total < ratio. - **memory_threshold**: `str` — Minimum free memory required (e.g. '30Gi', '1T'). Used when ratio is not sufficient. -## Analyzer Args Class NetworkAnalyzerArgs - -### Description - -Arguments for the network analyzer plugin. - -**Bases**: ['AnalyzerArgs'] - -**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/network/analyzer_args.py) - -### Annotations / fields - -- **error_regex**: `Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType]` — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern. - ## Analyzer Args Class NicAnalyzerArgs ### Description @@ -2517,7 +2533,7 @@ Analyzer args for niccli/nicctl data, with expected_values keyed by canonical co - **expected_qos_pfc_enabled**: `Optional[int]` — Expected PFC enabled value (0/1 or bitmask). Checked per device when set. - **expected_qos_tsa_map**: `Optional[Dict[Any, Any]]` — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set. - **expected_qos_tc_bandwidth**: `Optional[List[int]]` — Expected TC bandwidth percentages. Checked per device when set. -- **require_qos_consistent_across_adapters**: `bool` — When True and no expected_qos_* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map. +- **require_qos_consistent_across_adapters**: `bool` — When True and no expected_qos_\* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map. - **nicctl_log_error_regex**: `Optional[List[Dict[str, Any]]]` — Optional list of error patterns for nicctl show card logs. ## Analyzer Args Class OsAnalyzerArgs @@ -2617,6 +2633,20 @@ Arguments for RegexSearchAnalyzer (dict items match Dmesg-style error_regex). - **exp_rocm_latest**: `str` — Expected 'latest' ROCm path or version string for versioned installs. - **exp_rocm_sub_versions**: `dict[str, Union[str, list]]` — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. +## Analyzer Args Class StorageAnalyzerArgs + +**Bases**: ['AnalyzerArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/storage/analyzer_args.py) + +### Annotations / fields + +- **min_required_free_space_abs**: `Optional[str]` — Minimum required free space per mount (e.g. '10G', '1T'). +- **min_required_free_space_prct**: `Optional[int]` — Minimum required free space as percentage of total (0–100). +- **ignore_devices**: `Optional[list[str]]` — Mount points or devices to exclude from free-space checks. +- **check_devices**: `Optional[list[str]]` — If non-empty, only these mount points or devices are checked. +- **regex_match**: `bool` — If True, match device/mount names with regex; otherwise exact match. + ## Analyzer Args Class ScaleOutAristaAnalyzerArgs ### Description @@ -2696,7 +2726,7 @@ Analyzer args for config-driven Redfish checks. ### Annotations / fields -- **checks**: `dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]]` — Map: URI or '*' -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match). Use '*' as the key to apply the inner constraints to every collected response body. Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. Constraints: 'eq' — value must equal the given literal (int, float, str, bool). 'min' — value must be numeric and >= the given number. 'max' — value must be numeric and <= the given number. 'anyOf' — value must be in the given list (OR; any match passes). Example: { "/redfish/v1/Systems/1": { "Status/Health": { "anyOf": ["OK", "Warning"] }, "PowerState": "On" }, "*": { "Status/Health": { "anyOf": ["OK"] } } }. +- **checks**: `dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]]` — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match). Use `*` as the key to apply the inner constraints to every collected response body. Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. Constraints: 'eq' — value must equal the given literal (int, float, str, bool). 'min' — value must be numeric and >= the given number. 'max' — value must be numeric and <= the given number. 'anyOf' — value must be in the given list (OR; any match passes). Example: { "/redfish/v1/Systems/1": { "Status/Health": { "anyOf": ["OK", "Warning"] }, "PowerState": "On" }, "\*": { "Status/Health": { "anyOf": ["OK"] } } }. ## Analyzer Args Class RedfishOemDiagAnalyzerArgs diff --git a/docs/generate_plugin_doc_bundle.py b/docs/generate_plugin_doc_bundle.py index 75c4d412..4e148d96 100644 --- a/docs/generate_plugin_doc_bundle.py +++ b/docs/generate_plugin_doc_bundle.py @@ -505,12 +505,25 @@ def extract_collection_args_from_collector_args(args_cls: Optional[type]) -> Lis return output +def escape_lone_markdown_asterisks(s: str) -> str: + """Escape single asterisks so GFM does not start emphasis spans; preserve **bold** and `code`.""" + parts = re.split(r"(`[^`]*`)", s) + out: list[str] = [] + for index, part in enumerate(parts): + if index % 2 == 1: + out.append(part) + else: + out.append(re.sub(r"(? str: """Escape content for a markdown table cell so pipes and newlines don't break columns. Use HTML entity for pipe so all markdown parsers treat it as content, not column separator. """ if not s: return s + s = escape_lone_markdown_asterisks(s) # Avoid @ in cells (e.g. OData property names) being turned into mail/mention links in Outlook/HTML viewers. return s.replace("|", "|").replace("@", "@").replace("\n", " ").replace("\r", " ") @@ -524,7 +537,9 @@ def md_kv(key: str, value: str) -> str: def md_list(items: List[str]) -> str: - return "".join(f"- {i}\n" for i in items) + ("\n" if items else "") + return "".join(f"- {escape_lone_markdown_asterisks(i)}\n" for i in items) + ( + "\n" if items else "" + ) def bases_list(cls: type) -> List[str]: diff --git a/nodescraper/connection/inband/inband.py b/nodescraper/connection/inband/inband.py index 9a52cb88..291491b4 100644 --- a/nodescraper/connection/inband/inband.py +++ b/nodescraper/connection/inband/inband.py @@ -151,12 +151,16 @@ class InBandConnection(abc.ABC): @abc.abstractmethod def run_command( - self, command: str, sudo: bool = False, timeout: int = 300, strip: bool = True + self, + command: str, + sudo: bool = False, + timeout: int = 300, + strip: bool = True, ) -> CommandArtifact: - """Run an in band shell command + """Run an in band shell command. Args: - command (str): command to run + command (str): shell command string. sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. diff --git a/nodescraper/connection/inband/inbandlocal.py b/nodescraper/connection/inband/inbandlocal.py index e36e460b..66e655f9 100644 --- a/nodescraper/connection/inband/inbandlocal.py +++ b/nodescraper/connection/inband/inbandlocal.py @@ -36,12 +36,16 @@ class LocalShell(InBandConnection): def run_command( - self, command: str, sudo: bool = False, timeout: int = 300, strip: bool = True + self, + command: str, + sudo: bool = False, + timeout: int = 300, + strip: bool = True, ) -> CommandArtifact: - """Run a local in band shell command + """Run a local in band shell command. Args: - command (str): command to run + command (str): shell command string. sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. diff --git a/nodescraper/connection/inband/inbandremote.py b/nodescraper/connection/inband/inbandremote.py index 9e2415ed..6b8dbf56 100644 --- a/nodescraper/connection/inband/inbandremote.py +++ b/nodescraper/connection/inband/inbandremote.py @@ -128,10 +128,10 @@ def run_command( timeout: int = 30, strip: bool = True, ) -> CommandArtifact: - """Run a shell command over ssh + """Run a shell command over ssh. Args: - command (str): command to run + command (str): shell command string. sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. @@ -140,13 +140,15 @@ def run_command( CommandArtifact: Command artifact with stdout, stderr, which have been decoded and stripped as well as exit code """ write_password = sudo and self.ssh_params.username != "root" and self.ssh_params.password + + cmd_str = command if write_password: - command = f"sudo -S -p '' {command}" + cmd_str = f"sudo -S -p '' {cmd_str}" elif sudo: - command = f"sudo {command}" + cmd_str = f"sudo {cmd_str}" try: - stdin, stdout, stderr = self.client.exec_command(command, timeout=timeout) + stdin, stdout, stderr = self.client.exec_command(cmd_str, timeout=timeout) if write_password: stdin.write( @@ -166,7 +168,7 @@ def run_command( exit_code = 124 return CommandArtifact( - command=command, + command=cmd_str, stdout=stdout_str.strip() if strip else stdout_str, stderr=stderr_str.strip() if strip else stderr_str, exit_code=exit_code, diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py b/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py index ef60995a..92d42252 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py @@ -76,7 +76,7 @@ normalize_static_limit_dict, ) from nodescraper.plugins.inband.amdsmi.collector_args import AmdSmiCollectorArgs -from nodescraper.utils import get_exception_traceback +from nodescraper.utils import get_exception_traceback, shell_quote class AmdSmiCollector(InBandDataCollector[AmdSmiDataModel, AmdSmiCollectorArgs]): @@ -1489,7 +1489,7 @@ def _get_cper_afid(self, cper_file_path: str) -> Optional[int]: Returns: Optional[int]: AFID value or None if command fails or no value found """ - cmd = self.CMD_RAS_AFID.format(cper_file=cper_file_path) + cmd = self.CMD_RAS_AFID.format(cper_file=shell_quote(cper_file_path)) result = self._run_amd_smi(cmd) if result is None: diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 9c7b11b2..5ae53f77 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -36,10 +36,13 @@ from .analyzer_args import DmesgAnalyzerArgs from .dmesgdata import DmesgData from .mce_utils import ( - ignored_mce_block_line_indices, - mce_block_all_line_indices, + compile_mce_ce_status_regex, + compile_mce_uc_status_regex, + mce_known_regex_skip_line_indices, + mce_unknown_suppress_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, + trim_mce_status_match_content, ) @@ -343,17 +346,13 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): event_category=EventCategory.RAS, ), ErrorRegex( - regex=re.compile( - r"\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\].*(?:\n.*){0,5}" - ), + regex=compile_mce_ce_status_regex(), message="MCE Corrected Error", event_category=EventCategory.RAS, event_priority=EventPriority.WARNING, ), ErrorRegex( - regex=re.compile( - r"\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|UC\|[^\]]*\].*(?:\n.*){0,5}" - ), + regex=compile_mce_uc_status_regex(), message="MCE Uncorrected Error", event_category=EventCategory.RAS, ), @@ -719,8 +718,8 @@ def analyze_data( dmesg_content = data.dmesg_content ignore_match_rules, ignore_mce_banks = parse_ignore_match_rules(args.ignore_match_rules) - ignored_mce_block_lines = ignored_mce_block_line_indices(dmesg_content, ignore_mce_banks) - mce_block_lines = mce_block_all_line_indices(dmesg_content) + known_skip_lines = mce_known_regex_skip_line_indices(dmesg_content, ignore_mce_banks) + unknown_skip_lines = mce_unknown_suppress_line_indices(dmesg_content) known_err_events = self.check_all_regexes( content=dmesg_content, @@ -729,8 +728,13 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, - skip_line_indices=ignored_mce_block_lines, + skip_line_indices=known_skip_lines, ) + for event in known_err_events: + if event.description in ("MCE Corrected Error", "MCE Uncorrected Error"): + event.data["match_content"] = trim_mce_status_match_content( + event.data["match_content"] + ) if args.exclude_category: known_err_events = [ event for event in known_err_events if event.category not in args.exclude_category @@ -760,7 +764,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, - skip_line_indices=mce_block_lines, + skip_line_indices=unknown_skip_lines, ) for err_event in err_events: diff --git a/nodescraper/plugins/inband/dmesg/dmesg_plugin.py b/nodescraper/plugins/inband/dmesg/dmesg_plugin.py index b1b42900..d4095ab9 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_plugin.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_plugin.py @@ -41,4 +41,6 @@ class DmesgPlugin(InBandDataPlugin[DmesgData, DmesgCollectorArgs, DmesgAnalyzerA ANALYZER = DmesgAnalyzer + ANALYZER_ARGS = DmesgAnalyzerArgs + COLLECTOR_ARGS = DmesgCollectorArgs diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index b08e3a83..9ec57246 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -24,7 +24,7 @@ # ############################################################################### import re -from typing import FrozenSet, Optional, Sequence +from typing import FrozenSet, Optional, Sequence, Union from nodescraper.base.match_ignore import extract_mce_banks_from_text @@ -35,38 +35,69 @@ _MCE_STATUS_START_RE = re.compile(r"\bMC\d+_STATUS\[", re.IGNORECASE) _MCE_DETAIL_LINE_RE = re.compile(r"\[Hardware Error\]:") -_CORRECTABLE_SUMMARY_RE = re.compile( - r"(?P\d+)\s+correctable hardware errors detected in total in (?P\w+) block" - r"(?:\s+on\s+(?PCPU:?\d+))?", +_MCE_CE_STATUS_RE = re.compile( + r"\[Hardware Error\]:.*?(?PCPU:?\d+).*?MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\]", re.IGNORECASE, ) -_UNCORRECTABLE_SUMMARY_RE = re.compile( - r"(?P\d+)\s+uncorrectable hardware errors detected in (?P\w+) block", +_MCE_UC_STATUS_RE = re.compile( + r"\[Hardware Error\]:.*?(?PCPU:?\d+).*?MC\d+_STATUS\[[^\]]*\|UC\|[^\]]*\]", re.IGNORECASE, ) -_GPU_CORRECTABLE_RE = re.compile( - r"amdgpu\s+(?P[\w:.]+):.*?(?P\d+)\s+correctable hardware errors detected in total in " - r"(?P\w+) block", +_MCE_CE_STATUS_LINE_RE = re.compile( + r"\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\][^\n]*", re.IGNORECASE, ) -_GPU_UNCORRECTABLE_RE = re.compile( - r"amdgpu\s+(?P[\w:.]+):.*?(?P\d+)\s+uncorrectable hardware errors detected in " - r"(?P\w+) block", +_MCE_UC_STATUS_LINE_RE = re.compile( + r"\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|UC\|[^\]]*\][^\n]*", re.IGNORECASE, ) -_MCE_CE_STATUS_RE = re.compile( - r"\[Hardware Error\]:.*?(?PCPU:?\d+).*?MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\]", +# MCE incident rows only +_MCE_INCIDENT_LINE_RE = re.compile( + r"\[Hardware Error\]:\s*(?:" + r"(?:Corrected error|Uncorrected error|Machine check events logged)|" + r"Machine Check:|" + r"CPU:?\d|" + r"MC\d+_STATUS|" + r"PPIN:|" + r"IPID:|" + r"Syndrome:|" + r"cache level:" + r")", re.IGNORECASE, ) -_MCE_UC_STATUS_RE = re.compile( - r"\[Hardware Error\]:.*?(?PCPU:?\d+).*?MC\d+_STATUS\[[^\]]*\|UC\|[^\]]*\]", - re.IGNORECASE, -) + +def compile_mce_ce_status_regex() -> re.Pattern[str]: + """Return a single-line regex for corrected MCn_STATUS hardware error rows.""" + return _MCE_CE_STATUS_LINE_RE + + +def compile_mce_uc_status_regex() -> re.Pattern[str]: + """Return a single-line regex for uncorrected MCn_STATUS hardware error rows.""" + return _MCE_UC_STATUS_LINE_RE + + +def trim_mce_status_match_content(match: Union[str, list[str]]) -> str: + """Keep only the MCn_STATUS [Hardware Error] row in match_content.""" + if isinstance(match, list): + for item in match: + trimmed = trim_mce_status_match_content(item) + if _MCE_STATUS_START_RE.search(trimmed): + return trimmed + return match[0] if match else "" + + for line in str(match).splitlines(): + ce_match = _MCE_CE_STATUS_LINE_RE.search(line) + if ce_match: + return ce_match.group(0) + uc_match = _MCE_UC_STATUS_LINE_RE.search(line) + if uc_match: + return uc_match.group(0) + return str(match) def _normalize_cpu_label(cpu: str) -> str: @@ -77,33 +108,6 @@ def _add_count(counts: dict[str, int], part: str, amount: int) -> None: counts[part] = counts.get(part, 0) + amount -def _part_label( - *, - cpu: Optional[str] = None, - block: Optional[str] = None, - bdf: Optional[str] = None, - gpu_index: Optional[int] = None, -) -> str: - if bdf is not None: - block_suffix = f"/{block}" if block else "" - if gpu_index is not None: - return f"GPU{gpu_index}{block_suffix}" - return f"GPU {bdf}{block_suffix}" - if cpu and block: - return f"{cpu}/{block}" - if cpu: - return cpu - if block: - return block - return "unknown" - - -def _gpu_index_for_bdf(bdf: str, bdf_order: list[str]) -> int: - if bdf not in bdf_order: - bdf_order.append(bdf) - return bdf_order.index(bdf) - - def _is_mce_primary_starter(line: str) -> bool: return _MCE_PRIMARY_START_RE.search(line) is not None @@ -121,6 +125,10 @@ def _is_mce_detail_line(line: str) -> bool: return _MCE_DETAIL_LINE_RE.search(line) is not None +def _is_mce_incident_line(line: str) -> bool: + return _MCE_INCIDENT_LINE_RE.search(line) is not None + + def _mce_detail_line_indices_in_range(lines: Sequence[str], start: int, end: int) -> set[int]: return {index for index in range(start, end) if _is_mce_detail_line(lines[index])} @@ -138,8 +146,8 @@ def _is_mce_status_only_starter(line: str) -> bool: return not _is_mce_primary_starter(line) and _MCE_STATUS_START_RE.search(line) is not None -def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: - """Return True when another [Hardware Error]: line appears before the next incident.""" +def _has_mce_incident_line_ahead(lines: Sequence[str], start_index: int) -> bool: + """Return True when another MCE incident [Hardware Error]: row appears before the next block.""" for idx in range(start_index + 1, len(lines)): line = lines[idx] if not line.strip(): @@ -148,20 +156,83 @@ def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: return False if _is_mce_status_only_starter(line): return False - if _is_mce_detail_line(line): + if _is_mce_incident_line(line): return True + if _is_mce_detail_line(line): + return False return False +def mce_defining_status_line_indices(content: str) -> frozenset[int]: + """Return line indices for MCn_STATUS rows that define a corrected or uncorrected MCE.""" + lines = content.splitlines() + indices: set[int] = set() + for index, line in enumerate(lines): + if _MCE_CE_STATUS_LINE_RE.search(line) or _MCE_UC_STATUS_LINE_RE.search(line): + indices.add(index) + return frozenset(indices) + + +def mce_hardware_error_line_indices(content: str) -> frozenset[int]: + """Return every line index containing [Hardware Error]:.""" + lines = content.splitlines() + return frozenset(index for index, line in enumerate(lines) if _is_mce_detail_line(line)) + + +def mce_non_status_hardware_error_line_indices(content: str) -> frozenset[int]: + """Return [Hardware Error]: detail lines that are not defining MCn_STATUS CE/UC rows.""" + defining = mce_defining_status_line_indices(content) + return frozenset( + index for index in mce_hardware_error_line_indices(content) if index not in defining + ) + + +def _primary_starters_in_mce_status_blocks( + lines: Sequence[str], defining: frozenset[int] +) -> frozenset[int]: + """Return primary MCE header lines that belong to blocks with MCn_STATUS CE/UC rows.""" + primary_starters = {index for index, line in enumerate(lines) if _is_mce_primary_starter(line)} + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + if any(index in defining for index in range(start, end)): + suppressed.update(index for index in range(start, end) if index in primary_starters) + return frozenset(suppressed) + + +def mce_known_regex_skip_line_indices( + content: str, + ignore_banks: Optional[FrozenSet[int]] = None, +) -> frozenset[int]: + """Skip non-defining MCE detail lines and ignored-bank incidents during known regex scan.""" + ignored = ignore_banks or frozenset() + lines = content.splitlines() + defining = mce_defining_status_line_indices(content) + primary_starters = frozenset( + index for index, line in enumerate(lines) if _is_mce_primary_starter(line) + ) + primary_in_status_blocks = _primary_starters_in_mce_status_blocks(lines, defining) + skipped = set(hardware_error_block_line_indices(content)) - defining - primary_starters + skipped.update(primary_in_status_blocks) + skipped.update(ignored_mce_block_line_indices(content, ignored)) + return frozenset(skipped) + + +def mce_unknown_suppress_line_indices(content: str) -> frozenset[int]: + """Suppress MCE block context and every [Hardware Error]: line from unknown scan.""" + suppressed = set(mce_block_all_line_indices(content)) + suppressed.update(mce_hardware_error_line_indices(content)) + return frozenset(suppressed) + + def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, int]]: """Return (start, end) line index ranges for MCE incident blocks. A block begins at a primary MCE header (Corrected/Uncorrected/Machine check logged) or at the first MCn_STATUS line when not already inside a block. The block then includes subsequent lines until the next primary header, a blank line before a bare - MCn_STATUS starter, trailing non-MCE lines with no further [Hardware Error]: detail, - or EOF. Blank lines, warn/err noise, and other non-MCE dmesg lines between detail - entries belong to the same block. + MCn_STATUS starter, a non-MCE [Hardware Error]: row, trailing lines with no further + MCE incident detail, or EOF. Blank lines, warn/err noise, and other non-MCE dmesg + lines between MCE incident entries belong to the same block. """ blocks: list[tuple[int, int]] = [] index = 0 @@ -177,9 +248,11 @@ def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, in next_line = _next_non_blank_line_index(lines, index) if next_line is not None and _is_mce_status_only_starter(lines[next_line]): break + elif _is_mce_detail_line(lines[index]) and not _is_mce_incident_line(lines[index]): + break elif _is_mce_block_starter(lines[index], in_block=True): break - elif not _is_mce_detail_line(lines[index]) and not _has_mce_detail_line_ahead( + elif not _is_mce_incident_line(lines[index]) and not _has_mce_incident_line_ahead( lines, index ): break @@ -225,39 +298,14 @@ def parse_correctable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, ) -> dict[str, int]: - """Count correctable MCE / RAS hardware errors per component from dmesg text. - - Handles summary lines (for example ``mce: 3 correctable ... on CPU1``), - amdgpu block summaries, and per-event ``MCn_STATUS[|CE|]`` hardware error lines. - """ + """Count correctable MCE hardware errors per CPU from MCn_STATUS[|CE|] rows.""" counts: dict[str, int] = {} - gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() ignored_block_lines = ignored_mce_block_line_indices(content, ignored) for line_no, line in enumerate(content.splitlines()): if line_no in ignored_block_lines: continue - gpu_match = _GPU_CORRECTABLE_RE.search(line) - if gpu_match: - bdf = gpu_match.group("bdf") - part = _part_label( - bdf=bdf, - block=gpu_match.group("block"), - gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order), - ) - _add_count(counts, part, int(gpu_match.group("count"))) - continue - - summary_match = _CORRECTABLE_SUMMARY_RE.search(line) - if summary_match: - cpu = summary_match.group("cpu") - part = _part_label( - cpu=_normalize_cpu_label(cpu) if cpu else None, - block=summary_match.group("block"), - ) - _add_count(counts, part, int(summary_match.group("count"))) - continue status_match = _MCE_CE_STATUS_RE.search(line) if status_match: @@ -275,31 +323,14 @@ def parse_uncorrectable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, ) -> dict[str, int]: - """Count uncorrectable MCE / RAS hardware errors per component from dmesg text.""" + """Count uncorrectable MCE hardware errors per CPU from MCn_STATUS[|UC|] rows.""" counts: dict[str, int] = {} - gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() ignored_block_lines = ignored_mce_block_line_indices(content, ignored) for line_no, line in enumerate(content.splitlines()): if line_no in ignored_block_lines: continue - gpu_match = _GPU_UNCORRECTABLE_RE.search(line) - if gpu_match: - bdf = gpu_match.group("bdf") - part = _part_label( - bdf=bdf, - block=gpu_match.group("block"), - gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order), - ) - _add_count(counts, part, int(gpu_match.group("count"))) - continue - - summary_match = _UNCORRECTABLE_SUMMARY_RE.search(line) - if summary_match: - part = _part_label(block=summary_match.group("block")) - _add_count(counts, part, int(summary_match.group("count"))) - continue status_match = _MCE_UC_STATUS_RE.search(line) if status_match: diff --git a/nodescraper/plugins/inband/journal/journal_collector.py b/nodescraper/plugins/inband/journal/journal_collector.py index d244eda3..e7a76c81 100644 --- a/nodescraper/plugins/inband/journal/journal_collector.py +++ b/nodescraper/plugins/inband/journal/journal_collector.py @@ -31,7 +31,7 @@ from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult -from nodescraper.utils import get_exception_details +from nodescraper.utils import get_exception_details, shell_quote from .collector_args import JournalCollectorArgs from .journaldata import JournalData, JournalJsonEntry @@ -56,7 +56,8 @@ def _read_with_journalctl(self, args: Optional[JournalCollectorArgs] = None): try: # safe check for args.boot if args is not None and getattr(args, "boot", None): - cmd = f"{self.CMD} -b {args.boot}" + boot_q = shell_quote(str(args.boot)) + cmd = f"{self.CMD} -b {boot_q}" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False, strip=False) @@ -99,7 +100,8 @@ def _read_with_journalctl_json( try: # safe check for args.boot if args is not None and getattr(args, "boot", None): - cmd = f"{self.CMD_JSON} -b {args.boot}" + boot_q = shell_quote(str(args.boot)) + cmd = f"{self.CMD_JSON} -b {boot_q}" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False, strip=False) diff --git a/nodescraper/plugins/inband/network/ethtool_vendor.py b/nodescraper/plugins/inband/network/ethtool_vendor.py index d43791b4..a631ba03 100644 --- a/nodescraper/plugins/inband/network/ethtool_vendor.py +++ b/nodescraper/plugins/inband/network/ethtool_vendor.py @@ -25,6 +25,7 @@ ############################################################################### """Vendor-specific ethtool -S statistics models (Pollara / Thor2 / ConnectX-7).""" +import re from typing import ClassVar, Optional, Union from pydantic import BaseModel, Field, model_validator @@ -32,7 +33,18 @@ class PollaraEthtoolStatistics(BaseModel): - """ifname ionic. Keeping only fields of interest. Skip queue-specific stats for now""" + """ionic (Pollara) ethtool -S counters.""" + + #: Regex matched against raw ``ethtool -S`` field names to identify per-queue + queue_counter_regex: ClassVar[str] = r"^(rx|tx)_\d+" + queue_error_regex: ClassVar[list[str]] = [ + r"dma_map_err", + r"hwstamp_invalid", + r"alloc_err", + r"csum_error", + r"dropped", + ] + queue_warning_regex: ClassVar[list[str]] = [] rx_csum_error: Optional[int] = None hw_tx_dropped: Optional[int] = None @@ -157,7 +169,27 @@ class PollaraEthtoolStatistics(BaseModel): class Thor2EthtoolStatistics(BaseModel): - """ifname bnxt. Keeping only fields of interest. Skip queue-specific stats for now""" + """bnxt (Thor2) ethtool -S counters.""" + + #: Regex matched against raw ``ethtool -S`` field names to identify per-queue + queue_counter_regex: ClassVar[str] = r"^\[\d+\]" + queue_error_regex: ClassVar[list[str]] = [ + r"rx_discards", + r"rx_errors", + r"tx_errors", + r"tx_discards", + r"rx_l4_csum_errors", + r"rx_buf_errors", + r"so_txtime_cmpl_errors", + r"missed_irqs", + r"xsk_rx_redirect_fail", + r"xsk_rx_alloc_fail", + r"xsk_rx_no_room", + r"xsk_tx_ring_full", + ] + queue_warning_regex: ClassVar[list[str]] = [ + r"rx_resets", + ] rx_total_l4_csum_errors: Optional[int] = None rx_total_resets: Optional[int] = None @@ -356,7 +388,22 @@ class Thor2EthtoolStatistics(BaseModel): class Cx7EthtoolStatistics(BaseModel): - """ifname mlx. Keeping only fields of interest. Skip queue-specific stats for now""" + """mlx (ConnectX-7) ethtool -S counters.""" + + #: Regex matched against raw ``ethtool -S`` field names to identify per-queue + queue_counter_regex: ClassVar[str] = r"^(rx|tx|ch)\d+" + queue_error_regex: ClassVar[list[str]] = [ + r"xdp_drop", + r"wqe_err", + r"oversize_pkts_sw_drop", + r"buff_alloc_err", + r"arfs_err", + r"tls_err", + r"xdp_tx_err", + r"dropped", + r"cqe_err", + ] + queue_warning_regex: ClassVar[list[str]] = [] rx_xdp_drop: Optional[int] = None rx_xdp_tx_err: Optional[int] = None @@ -628,16 +675,12 @@ class Cx7EthtoolStatistics(BaseModel): Cx7EthtoolStatistics, ] -VendorEthtoolStatisticsCls = Union[ - type[PollaraEthtoolStatistics], - type[Thor2EthtoolStatistics], - type[Cx7EthtoolStatistics], -] +VendorEthtoolStatisticsCls = type[VendorEthtoolStatisticsModel] -# Map ifname prefixes to vendor-specific statistic models -# If netdev is ens, use Cx7 -# If netdev is benic, check if it starts with ionic or bnxt to determine if it's Pollara or Thor2 + +# Map kernel driver name prefixes (from `ethtool -i`) to vendor-specific models +# ionic -> Pollara, bnxt -> Thor2, mlx -> ConnectX-7 VENDOR_PREFIX_MAP: dict[str, VendorEthtoolStatisticsCls] = { "ionic": PollaraEthtoolStatistics, "bnxt": Thor2EthtoolStatistics, @@ -645,15 +688,51 @@ class Cx7EthtoolStatistics(BaseModel): } +def extract_queue_counters(stats: dict[str, str], queue_counter_regex: str) -> dict[str, int]: + """Extract per-queue counters from a raw ``ethtool -S`` stats dict. + + Devices can report hundreds of per-queue counters. These are matched by the + vendor's ``queue_counter_regex`` and retained (rather than dropped like other + non-enumerated fields) so they remain available in the data model for error + detection. + + Args: + stats: Mapping of raw ``ethtool -S`` field name -> string value. + queue_counter_regex: Vendor regex matched against field names to identify + per-queue counters. + + Returns: + Mapping of matching field name -> integer value. The ``netdev`` marker key + and non-integer values are skipped. + """ + pattern = re.compile(queue_counter_regex) + queue_counters: dict[str, int] = {} + for name, value in stats.items(): + if name == "netdev" or not pattern.match(name): + continue + try: + queue_counters[name] = int(value) + except (TypeError, ValueError): + continue + return queue_counters + + class EthtoolStatistics(BaseModel): """Per-netdev ethtool -S row with optional vendor-parsed counters.""" netdev: Optional[str] = None - rdma_ifname: Optional[str] = Field( + driver: Optional[str] = Field( default=None, - description="RDMA interface name from 'rdma link -j' used for vendor prefix selection", + description="Kernel driver (from 'ethtool -i') used for vendor model selection", ) vendor_statistics: Optional[VendorEthtoolStatisticsModel] = None + queue_statistics: dict[str, int] = Field( + default_factory=dict, + description=( + "High-cardinality per-queue ethtool -S counters (raw field name -> value) " + "captured via the vendor queue_counter_regex for error detection" + ), + ) @model_validator(mode="after") def validate_atleast_one_field(self) -> Self: diff --git a/nodescraper/plugins/inband/network/network_analyzer.py b/nodescraper/plugins/inband/network/network_analyzer.py index 2967df59..90819e09 100644 --- a/nodescraper/plugins/inband/network/network_analyzer.py +++ b/nodescraper/plugins/inband/network/network_analyzer.py @@ -23,94 +23,42 @@ # SOFTWARE. # ############################################################################### -from typing import Optional +import re -from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer +from nodescraper.base.regexanalyzer import RegexAnalyzer from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.models import TaskResult -from .analyzer_args import NetworkAnalyzerArgs from .networkdata import NetworkDataModel -class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, NetworkAnalyzerArgs]): +class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, None]): """Check network statistics for errors.""" DATA_MODEL = NetworkDataModel - # No built-in regex patterns: RDMA-scoped counter classification lives in the vendor - # ethtool models (ethtool_vendor.py). This list is only extended by user-supplied - # NetworkAnalyzerArgs.error_regex patterns. - ERROR_REGEX: list[ErrorRegex] = [] - - def analyze_data( - self, data: NetworkDataModel, args: Optional[NetworkAnalyzerArgs] = None - ) -> TaskResult: - """Analyze ethtool -S statistics via RDMA-scoped vendor models and any user-supplied regex. + def analyze_data(self, data: NetworkDataModel, args=None) -> TaskResult: + """Analyze ethtool -S statistics via RDMA-scoped vendor models. Args: - data: Network data model with ethtool_info and/or rdma_ethtool_statistics. - args: Optional analyzer arguments with custom error regex support. + data: Network data model with ethtool_statistics. + args: Unused; retained for analyzer interface compatibility. Returns: TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR. """ - if not data.ethtool_info and not data.rdma_ethtool_statistics: + if not data.ethtool_info and not data.ethtool_statistics: self.result.message = "No network devices found" self.result.status = ExecutionStatus.WARNING return self.result - if not args: - args = NetworkAnalyzerArgs() - - final_error_regex = self._convert_and_extend_error_regex(args.error_regex, self.ERROR_REGEX) - - regex_error = False - regex_warning = False - for interface_name, ethtool_info in data.ethtool_info.items(): - matches_on_interface: list[tuple[str, int, EventPriority]] = [] - for stat_name, stat_value in ethtool_info.statistics.items(): - for error_regex_obj in final_error_regex: - if error_regex_obj.regex.match(stat_name): - try: - value = int(stat_value) - except (ValueError, TypeError): - break - - if value > 0: - matches_on_interface.append( - (stat_name, value, error_regex_obj.event_priority) - ) - break - - if matches_on_interface: - has_error = any( - priority == EventPriority.ERROR for _, _, priority in matches_on_interface - ) - if has_error: - regex_error = True - else: - regex_warning = True - priority = EventPriority.ERROR if has_error else EventPriority.WARNING - severity = "error" if has_error else "warning" - match_names = [match[0] for match in matches_on_interface] - matches_data = {name: value for name, value, _ in matches_on_interface} - self._log_event( - category=EventCategory.NETWORK, - description=f"Network {severity} detected on {interface_name}: [{', '.join(match_names)}]", - data={ - "interface": interface_name, - "errors": matches_data, - }, - priority=priority, - console_log=True, - ) - vendor_error = False vendor_warning = False vendor_error_fields: set[str] = set() vendor_warning_fields: set[str] = set() - for stat in data.rdma_ethtool_statistics: + vendor_queue_error_fields: set[str] = set() + vendor_queue_warning_fields: set[str] = set() + for stat in data.ethtool_statistics: if stat.vendor_statistics is None: continue @@ -129,24 +77,18 @@ def analyze_data( else: vendor_error = True vendor_error_fields.add(field_name) - # Use a single grouped description per severity so the run summary - # collapses every occurrence into one "Ethtool warning detected" / - # "Ethtool error detected" entry instead of one line per field. The - # specific field is still preserved in the event data below and in - # the consolidated console line emitted after the loop. + # Use a single grouped description per severity desc = ( "Ethtool warning detected" if is_warning_tier else "Ethtool error detected" ) # Per-field events are still recorded for the run summary and event - # log, but console logging is suppressed here to avoid repeating the - # same message once per device. A single consolidated line is emitted - # after the loop instead. + # log, but console logging is suppressed here self._log_event( category=EventCategory.NETWORK, description=desc, data={ "netdev": stat.netdev, - "rdma_ifname": stat.rdma_ifname, + "driver": stat.driver, "error_field": field_name, "error_count": error_value, }, @@ -154,17 +96,73 @@ def analyze_data( console_log=False, ) - if vendor_error: + # Per-queue counters matched against the vendor's adjustable regex + queue_error_patterns = [ + re.compile(pattern) for pattern in getattr(type(vs), "queue_error_regex", []) + ] + queue_warning_patterns = [ + re.compile(pattern) for pattern in getattr(type(vs), "queue_warning_regex", []) + ] + if stat.queue_statistics and (queue_error_patterns or queue_warning_patterns): + for counter_name, counter_value in stat.queue_statistics.items(): + if counter_value <= 0: + continue + if queue_error_patterns and any( + pattern.search(counter_name) for pattern in queue_error_patterns + ): + vendor_error = True + vendor_queue_error_fields.add(f"{stat.netdev} {counter_name}") + self._log_event( + category=EventCategory.NETWORK, + description="Ethtool queue error detected", + data={ + "netdev": stat.netdev, + "driver": stat.driver, + "error_field": counter_name, + "error_count": counter_value, + }, + priority=EventPriority.ERROR, + console_log=False, + ) + elif queue_warning_patterns and any( + pattern.search(counter_name) for pattern in queue_warning_patterns + ): + vendor_warning = True + vendor_queue_warning_fields.add(f"{stat.netdev} {counter_name}") + self._log_event( + category=EventCategory.NETWORK, + description="Ethtool queue warning detected", + data={ + "netdev": stat.netdev, + "driver": stat.driver, + "error_field": counter_name, + "error_count": counter_value, + }, + priority=EventPriority.WARNING, + console_log=False, + ) + + if vendor_error_fields: self.logger.error("Ethtool error detected: %s", ", ".join(sorted(vendor_error_fields))) - if vendor_warning: + if vendor_queue_error_fields: + self.logger.error( + "Ethtool queue error detected: %s", + ", ".join(sorted(vendor_queue_error_fields)), + ) + if vendor_warning_fields: self.logger.warning( "Ethtool warning detected: %s", ", ".join(sorted(vendor_warning_fields)) ) + if vendor_queue_warning_fields: + self.logger.warning( + "Ethtool queue warning detected: %s", + ", ".join(sorted(vendor_queue_warning_fields)), + ) - if regex_error or vendor_error: + if vendor_error: self.result.message = "Network errors detected in statistics" self.result.status = ExecutionStatus.ERROR - elif regex_warning or vendor_warning: + elif vendor_warning: self.result.message = "Network warning counters non-zero in statistics" self.result.status = ExecutionStatus.WARNING else: diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index b960e574..9f109f89 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -23,7 +23,6 @@ # SOFTWARE. # ############################################################################### -import json import re from typing import Dict, List, Optional, Tuple @@ -32,13 +31,14 @@ from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult -from nodescraper.utils import get_exception_traceback +from nodescraper.utils import get_exception_traceback, shell_quote from .collector_args import NetworkCollectorArgs from .ethtool_vendor import ( VENDOR_PREFIX_MAP, EthtoolStatistics, VendorEthtoolStatisticsModel, + extract_queue_counters, ) from .networkdata import ( EthtoolInfo, @@ -61,7 +61,7 @@ class NetworkCollector(InBandDataCollector[NetworkDataModel, NetworkCollectorArg CMD_NEIGHBOR = "ip neighbor show" CMD_ETHTOOL_TEMPLATE = "ethtool {interface}" CMD_ETHTOOL_S_TEMPLATE = "ethtool -S {interface}" - CMD_RDMA_LINK_JSON = "rdma link -j" + CMD_ETHTOOL_I_TEMPLATE = "ethtool -i {interface}" CMD_PING = "ping" CMD_WGET = "wget" CMD_CURL = "curl" @@ -453,6 +453,10 @@ def _parse_ethtool(self, interface: str, output: str) -> EthtoolInfo: def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, str]: """Parse 'ethtool -S ' output into a key-value dictionary. + Per-queue counters keep their full raw field name (e.g. "[0]: rx_ucast_packets") + so vendor ``queue_counter_regex`` patterns can match them; scalar counters are + stored under their plain name (e.g. "rx_total_l4_csum_errors"). + Args: output: Raw output from 'ethtool -S ' command interface: Name of the network interface (for netdev key) @@ -462,24 +466,18 @@ def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, st """ stats_dict: Dict[str, str] = {} for line in output.splitlines(): - if ":" not in line: + line = line.strip() + if not line: continue if "NIC statistics" in line: stats_dict["netdev"] = interface - elif "]: " in line and line.strip().startswith("["): - # Format: " [0]: rx_ucast_packets: 162" - bracket_part, rest = line.split("]: ", 1) - index = bracket_part.strip().lstrip("[") - if ": " in rest: - stat_key, stat_value = rest.split(": ", 1) - key = f"{index}_{stat_key.strip()}" - stats_dict[key] = stat_value.strip() - else: - key, value = line.split(":", 1) - stats_dict[key.strip()] = value.strip() - else: - key, value = line.split(":", 1) - stats_dict[key.strip()] = value.strip() + continue + if ": " not in line: + continue + # Split on the last ": " so per-queue lines like "[0]: rx_ucast_packets: 5" + # preserve their full "[0]: rx_ucast_packets" name for vendor queue matching. + name, value = line.rsplit(": ", 1) + stats_dict[name.strip()] = value.strip() return stats_dict def _collect_ethtool_info( @@ -526,47 +524,15 @@ def _collect_ethtool_info( return ethtool_data, skipped - def _collect_rdma_link_json(self) -> Optional[list[dict]]: - """Parse JSON from `rdma link -j`. Returns None on failure, [] when no links.""" - res = self._run_sut_cmd(self.CMD_RDMA_LINK_JSON) - if res.exit_code != 0: - self._log_event( - category=EventCategory.NETWORK, - description="rdma link -j failed (RDMA-scoped ethtool collection skipped)", - data={ - "command": self.CMD_RDMA_LINK_JSON, - "exit_code": res.exit_code, - "stderr": res.stderr, - }, - priority=EventPriority.WARNING, - ) - return None - if not res.stdout.strip(): - return [] - try: - parsed = json.loads(res.stdout) - except json.JSONDecodeError as e: - self._log_event( - category=EventCategory.NETWORK, - description="Failed to parse rdma link -j JSON", - data={"exception": get_exception_traceback(e)}, - priority=EventPriority.WARNING, - ) - return None - if not isinstance(parsed, list): - self._log_event( - category=EventCategory.NETWORK, - description="Unexpected rdma link -j JSON type", - data={"data_type": type(parsed).__name__}, - priority=EventPriority.WARNING, - ) - return None - return parsed + def _collect_ethtool_statistic(self, netdev: str, driver: str) -> Optional[EthtoolStatistics]: + """Run `ethtool -S` for netdev and attach vendor-parsed stats. + + The vendor model is selected by matching ``driver`` (from ``ethtool -i``) + against the known vendor prefixes in ``VENDOR_PREFIX_MAP``. - def _collect_rdma_scoped_ethtool_statistic( - self, netdev: str, ifname: str - ) -> Optional[EthtoolStatistics]: - """Run `ethtool -S` for netdev and attach vendor-parsed stats (prefix from RDMA ifname).""" + Returns: + EthtoolStatistics, or None on command failure. + """ cmd_s = self.CMD_ETHTOOL_S_TEMPLATE.format(interface=netdev) res = self._run_sut_cmd(cmd_s, sudo=True) if res.exit_code != 0: @@ -586,11 +552,16 @@ def _collect_rdma_scoped_ethtool_statistic( stats_dict = self._parse_ethtool_statistics(res.stdout, netdev) vendor_stats: Optional[VendorEthtoolStatisticsModel] = None + queue_statistics: Dict[str, int] = {} for prefix, vendor_cls in VENDOR_PREFIX_MAP.items(): - if ifname.startswith(prefix): + if driver.startswith(prefix): vendor_fields = set(vendor_cls.model_fields.keys()) stat_fields = set(stats_dict.keys()) - {"netdev"} + queue_statistics = extract_queue_counters( + stats_dict, vendor_cls.queue_counter_regex + ) + missing_fields = vendor_fields - stat_fields if missing_fields: sorted_missing = sorted(missing_fields) @@ -599,7 +570,7 @@ def _collect_rdma_scoped_ethtool_statistic( description=f"Missing fields in ethtool statistic for {netdev}", data={ "netdev": netdev, - "ifname": ifname, + "driver": driver, "missing_fields_count": len(sorted_missing), "missing_fields": sorted_missing[:50], }, @@ -620,18 +591,36 @@ def _collect_rdma_scoped_ethtool_statistic( return EthtoolStatistics( netdev=netdev, - rdma_ifname=ifname or None, + driver=driver or None, vendor_statistics=vendor_stats, + queue_statistics=queue_statistics, ) - def _collect_rdma_scoped_ethtool( - self, exclusions: Optional[List[re.Pattern]] = None + def _get_netdev_driver(self, netdev: str) -> str: + """Return the kernel driver for a netdev via `ethtool -i` ("" on failure).""" + cmd = self.CMD_ETHTOOL_I_TEMPLATE.format(interface=netdev) + res = self._run_sut_cmd(cmd, sudo=True) + if res.exit_code != 0: + return "" + for line in res.stdout.splitlines(): + line = line.strip() + if line.startswith("driver:"): + return line.split(":", 1)[1].strip() + return "" + + def _collect_ethtool_statistics( + self, + interfaces: List[NetworkInterface], + exclusions: Optional[List[re.Pattern]] = None, ) -> tuple[List[str], List[EthtoolStatistics], set[str]]: - """Collect ethtool -S for netdevs listed on RDMA links (error-scraper EthtoolCollector parity). + """Collect ethtool -S for vendor netdevs discovered from `ip addr`. + + Each interface's driver is resolved via `ethtool -i`; only interfaces whose + driver matches a known vendor prefix (see VENDOR_PREFIX_MAP) are collected. Args: - exclusions: Compiled regex patterns; netdevs whose name matches any pattern - are skipped (ethtool -S is not run against them). + interfaces: Interfaces discovered from `ip addr` to consider. + exclusions: Compiled regex patterns; matching netdevs are skipped. Returns: Tuple of (netdev list, statistics list, set of netdev names skipped due to @@ -641,38 +630,27 @@ def _collect_rdma_scoped_ethtool( statistics_list: List[EthtoolStatistics] = [] skipped: set[str] = set() - link_data = self._collect_rdma_link_json() - if link_data is None: - return netdev_list, statistics_list, skipped - - for link in link_data: - if not isinstance(link, dict): - self._log_event( - category=EventCategory.NETWORK, - description="Invalid data type for RDMA link entry", - data={"data_type": type(link).__name__}, - priority=EventPriority.WARNING, - ) + for iface in interfaces: + netdev = iface.name + if not netdev: continue - - netdev = link.get("netdev") or "" - ifname = link.get("ifname") or "" - - if netdev: - if exclusions and any(pattern.search(netdev) for pattern in exclusions): - skipped.add(netdev) - continue - netdev_list.append(netdev) - stat = self._collect_rdma_scoped_ethtool_statistic(netdev, ifname) - if stat is not None: - statistics_list.append(stat) + if exclusions and any(pattern.search(netdev) for pattern in exclusions): + skipped.add(netdev) + continue + driver = self._get_netdev_driver(netdev) + if not any(driver.startswith(prefix) for prefix in VENDOR_PREFIX_MAP): + continue + netdev_list.append(netdev) + stat = self._collect_ethtool_statistic(netdev, driver) + if stat is not None: + statistics_list.append(stat) if netdev_list: self._log_event( category=EventCategory.NETWORK, description=( - f"Collected RDMA-scoped ethtool -S for {len(statistics_list)}/" - f"{len(netdev_list)} netdev(s) from rdma link" + f"Collected ethtool -S for {len(statistics_list)}/" + f"{len(netdev_list)} vendor netdev(s)" ), priority=EventPriority.INFO, ) @@ -729,16 +707,16 @@ def _check_network_connectivity(self, cmd: str, url: str) -> bool: f"Valid options are: 'ping', 'wget', 'curl'" ) - # Determine ping options based on OS - ping_option = "-c 1" if self.system_info.os_family == OSFamily.LINUX else "-n 1" - - # Build command based on cmd parameter using class constants + url_q = shell_quote(url) if cmd == "ping": - result = self._run_sut_cmd(f"{self.CMD_PING} {url} {ping_option}") + if self.system_info.os_family == OSFamily.LINUX: + result = self._run_sut_cmd(f"{self.CMD_PING} {url_q} -c 1") + else: + result = self._run_sut_cmd(f"{self.CMD_PING} {url_q} -n 1") elif cmd == "wget": - result = self._run_sut_cmd(f"{self.CMD_WGET} {url}") + result = self._run_sut_cmd(f"{self.CMD_WGET} {url_q}") else: # curl - result = self._run_sut_cmd(f"{self.CMD_CURL} {url}") + result = self._run_sut_cmd(f"{self.CMD_CURL} {url_q}") if result.exit_code == 0: self._log_event( @@ -778,8 +756,8 @@ def collect_data( neighbors = [] ethtool_data: Dict[str, EthtoolInfo] = {} network_accessible: Optional[bool] = None - rdma_ethtool_netdevs: List[str] = [] - rdma_ethtool_statistics: List[EthtoolStatistics] = [] + ethtool_netdevs: List[str] = [] + ethtool_statistics: List[EthtoolStatistics] = [] skipped_devices: set[str] = set() # Compile device-exclusion regex patterns (netdevs matching any pattern are skipped) @@ -835,11 +813,11 @@ def collect_data( if self.system_info.os_family == OSFamily.LINUX: ( - rdma_ethtool_netdevs, - rdma_ethtool_statistics, - rdma_skipped, - ) = self._collect_rdma_scoped_ethtool(compiled_exclusions) - skipped_devices |= rdma_skipped + ethtool_netdevs, + ethtool_statistics, + ethtool_stat_skipped, + ) = self._collect_ethtool_statistics(interfaces, compiled_exclusions) + skipped_devices |= ethtool_stat_skipped # Collect routing table res_route = self._run_sut_cmd(self.CMD_ROUTE) @@ -905,8 +883,8 @@ def collect_data( rules=rules, neighbors=neighbors, ethtool_info=ethtool_data, - rdma_ethtool_netdevs=rdma_ethtool_netdevs, - rdma_ethtool_statistics=rdma_ethtool_statistics, + ethtool_netdevs=ethtool_netdevs, + ethtool_statistics=ethtool_statistics, accessible=network_accessible, ) self.result.status = ExecutionStatus.OK diff --git a/nodescraper/plugins/inband/network/network_plugin.py b/nodescraper/plugins/inband/network/network_plugin.py index 7e1fe518..69e7b671 100644 --- a/nodescraper/plugins/inband/network/network_plugin.py +++ b/nodescraper/plugins/inband/network/network_plugin.py @@ -25,14 +25,13 @@ ############################################################################### from nodescraper.base import InBandDataPlugin -from .analyzer_args import NetworkAnalyzerArgs from .collector_args import NetworkCollectorArgs from .network_analyzer import NetworkAnalyzer from .network_collector import NetworkCollector from .networkdata import NetworkDataModel -class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, NetworkAnalyzerArgs]): +class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, None]): """Plugin for collection of network configuration data""" DATA_MODEL = NetworkDataModel @@ -42,5 +41,3 @@ class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, Net COLLECTOR_ARGS = NetworkCollectorArgs ANALYZER = NetworkAnalyzer - - ANALYZER_ARGS = NetworkAnalyzerArgs diff --git a/nodescraper/plugins/inband/network/networkdata.py b/nodescraper/plugins/inband/network/networkdata.py index c90a1fc1..86e8cb6f 100644 --- a/nodescraper/plugins/inband/network/networkdata.py +++ b/nodescraper/plugins/inband/network/networkdata.py @@ -105,8 +105,6 @@ class EthtoolInfo(BaseModel): port: Optional[str] = None # Port type (e.g., "Twisted Pair") auto_negotiation: Optional[str] = None # Auto-negotiation status (e.g., "on", "off") link_detected: Optional[str] = None # Link detection status (e.g., "yes", "no") - # ethtool -S (statistics) output: parsed key-value for error/health analysis - statistics: Dict[str, str] = Field(default_factory=dict) class NetworkDataModel(DataModel): @@ -116,10 +114,8 @@ class NetworkDataModel(DataModel): routes: List[Route] = Field(default_factory=list) rules: List[RoutingRule] = Field(default_factory=list) neighbors: List[Neighbor] = Field(default_factory=list) - ethtool_info: Dict[str, EthtoolInfo] = Field( - default_factory=dict - ) # Interface name -> EthtoolInfo mapping - # RDMA-scoped ethtool -S: netdevs from `rdma link -j` with vendor-parsed counters - rdma_ethtool_netdevs: List[str] = Field(default_factory=list) - rdma_ethtool_statistics: List[EthtoolStatistics] = Field(default_factory=list) + ethtool_info: Dict[str, EthtoolInfo] = Field(default_factory=dict) + # ethtool -S vendor-parsed counters for netdevs whose driver matches a known vendor + ethtool_netdevs: List[str] = Field(default_factory=list) + ethtool_statistics: List[EthtoolStatistics] = Field(default_factory=list) accessible: Optional[bool] = None # Network accessibility check via ping diff --git a/nodescraper/plugins/inband/process/process_collector.py b/nodescraper/plugins/inband/process/process_collector.py index 9c92080d..72feb8c5 100644 --- a/nodescraper/plugins/inband/process/process_collector.py +++ b/nodescraper/plugins/inband/process/process_collector.py @@ -83,8 +83,9 @@ def collect_data( ) process_data.cpu_usage = 100 - float(cpu_idle) + last_line = args.top_n_process + 7 processes = self._run_sut_cmd( - f"self.CMD_PROCESS | sed -n '8,{args.top_n_process + 7}p'" + f"{self.CMD_PROCESS} | sed -n '8,{last_line}p'" ) # Remove system header if processes.exit_code == 0: for line in processes.stdout.splitlines(): diff --git a/nodescraper/plugins/inband/rdma/rdma_collector.py b/nodescraper/plugins/inband/rdma/rdma_collector.py index 740f5998..90d68aa4 100644 --- a/nodescraper/plugins/inband/rdma/rdma_collector.py +++ b/nodescraper/plugins/inband/rdma/rdma_collector.py @@ -31,7 +31,7 @@ from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily -from nodescraper.models import TaskResult +from nodescraper.models import CollectorArgs, TaskResult from nodescraper.utils import get_exception_traceback from .rdmadata import ( @@ -45,7 +45,7 @@ ) -class RdmaCollector(InBandDataCollector[RdmaDataModel, None]): +class RdmaCollector(InBandDataCollector[RdmaDataModel, CollectorArgs]): """Collect RDMA status and statistics via rdma link and rdma statistic commands.""" DATA_MODEL = RdmaDataModel @@ -291,7 +291,9 @@ def _get_rdma_link(self) -> Optional[list[RdmaLink]]: ) return None - def collect_data(self, args=None) -> tuple[TaskResult, Optional[RdmaDataModel]]: + def collect_data( + self, args: Optional[CollectorArgs] = None + ) -> tuple[TaskResult, Optional[RdmaDataModel]]: """Collect RDMA statistics, link data, and device/link text output. Returns: diff --git a/nodescraper/plugins/inband/rdma/rdma_plugin.py b/nodescraper/plugins/inband/rdma/rdma_plugin.py index 4bf5e8cc..f435ff7a 100644 --- a/nodescraper/plugins/inband/rdma/rdma_plugin.py +++ b/nodescraper/plugins/inband/rdma/rdma_plugin.py @@ -24,6 +24,7 @@ # ############################################################################### from nodescraper.base import InBandDataPlugin +from nodescraper.models import CollectorArgs from .analyzer_args import RdmaAnalyzerArgs from .rdma_analyzer import RdmaAnalyzer @@ -31,10 +32,11 @@ from .rdmadata import RdmaDataModel -class RdmaPlugin(InBandDataPlugin[RdmaDataModel, None, RdmaAnalyzerArgs]): +class RdmaPlugin(InBandDataPlugin[RdmaDataModel, CollectorArgs, RdmaAnalyzerArgs]): """Plugin for collection and analysis of RDMA statistics and link data.""" DATA_MODEL = RdmaDataModel COLLECTOR = RdmaCollector + COLLECTOR_ARGS = CollectorArgs ANALYZER = RdmaAnalyzer ANALYZER_ARGS = RdmaAnalyzerArgs diff --git a/nodescraper/plugins/inband/regex_search/regex_search_plugin.py b/nodescraper/plugins/inband/regex_search/regex_search_plugin.py index 2a101ff8..31a068ca 100644 --- a/nodescraper/plugins/inband/regex_search/regex_search_plugin.py +++ b/nodescraper/plugins/inband/regex_search/regex_search_plugin.py @@ -41,6 +41,11 @@ class RegexSearchPlugin(InBandDataPlugin[RegexSearchData, CollectorArgs, RegexSe ANALYZER = RegexSearchAnalyzer ANALYZER_ARGS = RegexSearchAnalyzerArgs + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "No collector step: reads local text from the CLI --data path (file or directory).", + "Directory scans load each file's contents into RegexSearchData for analysis.", + ) + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( "Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).", "Emits regex match events with optional per-file source in the description when scanning directories.", diff --git a/nodescraper/plugins/inband/rocm/rocm_collector.py b/nodescraper/plugins/inband/rocm/rocm_collector.py index 27a6c4f5..c586de9b 100644 --- a/nodescraper/plugins/inband/rocm/rocm_collector.py +++ b/nodescraper/plugins/inband/rocm/rocm_collector.py @@ -31,7 +31,7 @@ from nodescraper.connection.inband import TextFileArtifact from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult -from nodescraper.utils import strip_ansi_codes +from nodescraper.utils import shell_quote, strip_ansi_codes from .collector_args import RocmCollectorArgs from .rocmdata import RocmDataModel @@ -63,6 +63,7 @@ def collect_data( """ if args is None: args = RocmCollectorArgs() + rocm_path_q = shell_quote(args.rocm_path) version_paths = [ f"{args.rocm_path}/.info/version-rocm", f"{args.rocm_path}/.info/version", @@ -73,7 +74,7 @@ def collect_data( # First, try to collect all sub-versions sub_versions_res = self._run_sut_cmd( - self.CMD_ROCM_SUB_VERSIONS_TMPL.format(rocm_path=args.rocm_path) + self.CMD_ROCM_SUB_VERSIONS_TMPL.format(rocm_path=rocm_path_q) ) if sub_versions_res.exit_code == 0: for line in sub_versions_res.stdout.splitlines(): @@ -86,7 +87,7 @@ def collect_data( # Determine the main ROCm version for path in version_paths: - res = self._run_sut_cmd(f"grep . {path}") + res = self._run_sut_cmd(f"grep . {shell_quote(path)}") if res.exit_code == 0: try: rocm_data = RocmDataModel( @@ -124,15 +125,13 @@ def collect_data( if rocm_data: # Collect latest versioned ROCm path (rocm-[3-7]*) versioned_path_res = self._run_sut_cmd( - self.CMD_ROCM_LATEST_TMPL.format(rocm_path=args.rocm_path) + self.CMD_ROCM_LATEST_TMPL.format(rocm_path=rocm_path_q) ) if versioned_path_res.exit_code == 0: rocm_data.rocm_latest_versioned_path = versioned_path_res.stdout.strip() # Collect all ROCm paths as list - all_paths_res = self._run_sut_cmd( - self.CMD_ROCM_DIRS_TMPL.format(rocm_path=args.rocm_path) - ) + all_paths_res = self._run_sut_cmd(self.CMD_ROCM_DIRS_TMPL.format(rocm_path=rocm_path_q)) if all_paths_res.exit_code == 0: rocm_data.rocm_all_paths = [ path.strip() @@ -141,7 +140,7 @@ def collect_data( ] # Collect rocminfo output as list of lines with ANSI codes stripped - rocminfo_cmd = self.CMD_ROCMINFO_TMPL.format(rocm_path=args.rocm_path) + rocminfo_cmd = self.CMD_ROCMINFO_TMPL.format(rocm_path=rocm_path_q) rocminfo_res = self._run_sut_cmd(rocminfo_cmd) rocminfo_artifact_content = "" if rocminfo_res.exit_code == 0: @@ -178,7 +177,7 @@ def collect_data( ] # Collect clinfo output - clinfo_cmd = self.CMD_CLINFO_TMPL.format(rocm_path=args.rocm_path) + clinfo_cmd = self.CMD_CLINFO_TMPL.format(rocm_path=rocm_path_q) clinfo_res = self._run_sut_cmd(clinfo_cmd) # Always append clinfo section to artifact, even if empty or failed diff --git a/nodescraper/plugins/inband/storage/storage_plugin.py b/nodescraper/plugins/inband/storage/storage_plugin.py index 1687e20e..f0ca62fc 100644 --- a/nodescraper/plugins/inband/storage/storage_plugin.py +++ b/nodescraper/plugins/inband/storage/storage_plugin.py @@ -41,4 +41,6 @@ class StoragePlugin(InBandDataPlugin[StorageDataModel, StorageCollectorArgs, Sto ANALYZER = StorageAnalyzer + ANALYZER_ARGS = StorageAnalyzerArgs + COLLECTOR_ARGS = StorageCollectorArgs diff --git a/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py b/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py index 9162980e..5bb5acf9 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py @@ -53,9 +53,9 @@ class RedfishEndpointAnalyzerArgs(AnalyzerArgs): checks: dict[str, dict[str, RedfishConstraint]] = Field( default_factory=dict, description=( - "Map: URI or '*' -> { property_path: constraint }. " + "Map: URI or `*` -> { property_path: constraint }. " "URI keys must match a key in the collected responses (exact match). " - "Use '*' as the key to apply the inner constraints to every collected response body. " + "Use `*` as the key to apply the inner constraints to every collected response body. " "Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. " "Constraints: " "'eq' — value must equal the given literal (int, float, str, bool). " diff --git a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py index 1e43a71a..26412f94 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py @@ -92,7 +92,7 @@ class RedfishEndpointAnalyzer(DataAnalyzer[RedfishEndpointDataModel, RedfishEndp DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( "For each entry in analysis_args.checks, reads JSON paths in collected responses and " "compares values to constraints (eq, min/max, anyOf, regex, etc.).", - 'URI key "*" runs checks against every collected response body.', + "URI key `*` runs checks against every collected response body.", ) def analyze_data( diff --git a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py index b3e2644d..87bbaa3f 100644 --- a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py +++ b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py @@ -63,6 +63,13 @@ class MI3XXAnalyzer(DataAnalyzer[ServiceabilityDataModel, ServiceabilityAnalyzer DATA_MODEL = ServiceabilityDataModel + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( + "Builds AFID events from collected Redfish event log members (and optional assembly metadata).", + "Optionally decodes CPER attachments via analysis_args.cper_decode_module before hub analysis.", + "Runs the configured Python service hub (hub_python_module) to produce service recommendations.", + "When analysis_args.skip_hub is true, only builds AFID events without running the hub.", + ) + def analyze_data( self, data: ServiceabilityDataModel, diff --git a/nodescraper/plugins/serviceability/mi3xx/mi3xx_collector.py b/nodescraper/plugins/serviceability/mi3xx/mi3xx_collector.py index d155f14a..910b3c66 100644 --- a/nodescraper/plugins/serviceability/mi3xx/mi3xx_collector.py +++ b/nodescraper/plugins/serviceability/mi3xx/mi3xx_collector.py @@ -44,6 +44,14 @@ class MI3XXCollector(ServiceabilityCollectorBase[MI3XXCollectorArgs]): """Collect MI3XX BMC Redfish data: event log members (with pagination), firmware inventory, CPER attachment bytes for qualifying events, and optional assembly/chassis metadata.""" + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "Redfish GET: BMC event log Entries (collection_args.rf_event_log_uri; optional uri alias).", + "Paginated Members collection and optional top, reference_time/time_operator filters.", + "Redfish GET: CPER AdditionalDataURI binaries for DiagnosticDataType=CPER events (base64 in data model).", + "Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices).", + "Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details.", + ) + def satisfies_reference_time( self, candidate: str, diff --git a/nodescraper/plugins/inband/network/analyzer_args.py b/test/unit/connection/inband/test_shellcommand.py similarity index 63% rename from nodescraper/plugins/inband/network/analyzer_args.py rename to test/unit/connection/inband/test_shellcommand.py index f2e63047..0a1b3fc2 100644 --- a/nodescraper/plugins/inband/network/analyzer_args.py +++ b/test/unit/connection/inband/test_shellcommand.py @@ -23,18 +23,26 @@ # SOFTWARE. # ############################################################################### -from typing import Optional, Union +from unittest.mock import MagicMock, patch -from pydantic import Field +from nodescraper.connection.inband.inbandlocal import LocalShell -from nodescraper.base.regexanalyzer import ErrorRegex -from nodescraper.models import AnalyzerArgs +@patch("nodescraper.connection.inband.inbandlocal.subprocess.run") +def test_localshell_string_uses_shell_true(mock_run): + mock_run.return_value = MagicMock(stdout="ok\n", stderr="", returncode=0) -class NetworkAnalyzerArgs(AnalyzerArgs): - """Arguments for the network analyzer plugin.""" + shell = LocalShell() + shell.run_command("ip addr show") - error_regex: Optional[Union[list[ErrorRegex], list[dict]]] = Field( - default=None, - description="Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern.", - ) + assert mock_run.call_args.kwargs["shell"] is True + + +@patch("nodescraper.connection.inband.inbandlocal.subprocess.run") +def test_localshell_string_with_sudo(mock_run): + mock_run.return_value = MagicMock(stdout="", stderr="", returncode=0) + + shell = LocalShell() + shell.run_command("cat /etc/shadow", sudo=True) + + assert mock_run.call_args.args[0] == "sudo cat /etc/shadow" diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index 0de2ae21..784b5453 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -1022,11 +1022,11 @@ def test_priority_override_updates_unkown_dmesg_error(system_info): assert res.events[0].priority == EventPriority.ERROR -def test_mce_threshold_raises_error_for_gpu(system_info): +def test_mce_threshold_raises_error_for_status_lines(system_info): dmesg_content = ( - "kern :err : 2024-10-07T10:17:15,145363-04:00 " - "amdgpu 0000:c1:00.0: amdgpu: socket: 4, die: 0 " - "3 correctable hardware errors detected in total in gfx block\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x2\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x3\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1038,7 +1038,7 @@ def test_mce_threshold_raises_error_for_gpu(system_info): threshold_events = [e for e in res.events if e.data.get("mce_threshold") == 3] assert len(threshold_events) == 1 assert threshold_events[0].priority == EventPriority.ERROR - assert threshold_events[0].data["part"] == "GPU0/gfx" + assert threshold_events[0].data["part"] == "CPU0" assert threshold_events[0].data["correctable_mce_count"] == 3 assert res.status == ExecutionStatus.ERROR @@ -1065,8 +1065,8 @@ def test_mce_threshold_raises_error_for_cpu_colon_status(system_info): def test_mce_threshold_not_triggered_below_limit(system_info): dmesg_content = ( - "kern :warn : 2024-06-11T14:30:00,123456+00:00 " - "mce: 2 correctable hardware errors detected in total in mc0 block on CPU1\n" + "[Hardware Error]: CPU1 MC0_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU1 MC0_STATUS[0x0|CE|]: 0x2\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1260,7 +1260,11 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" - "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1270,12 +1274,18 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): ) descriptions = {event.description for event in res.events} + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + mce_events = [event for event in res.events if event.description == "MCE Corrected Error"] + assert unknown_events == [] + assert len(mce_events) == 1 + assert "MC60_STATUS" in str(mce_events[0].data["match_content"]) assert "Unknown dmesg error" not in descriptions - assert "MCE Corrected Error" in descriptions or "RAS Corrected Error" in descriptions + assert "MCE Corrected Error" in descriptions + assert "RAS Corrected Error" not in descriptions def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info): - """Dummy excerpt modeled on dmesg_mce_interleave.log: warn/blank lines inside MCE blocks.""" + """Dummy excerpt: warn/blank lines interleaved inside MCE hardware-error blocks.""" dmesg_content = ( "kern :info : 2038-01-19T00:00:00,000000+00:00 " "mce: [Hardware Error]: Machine check events logged\n" @@ -1326,3 +1336,102 @@ def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info) unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] assert len(unknown_events) == 1 assert unknown_events[0].data["match_content"] == "dummy harness fault outside mce blocks" + + +def test_orphan_mce_detail_lines_not_reported_as_unknown(system_info): + """When analysis window omits MCn_STATUS, trailing detail lines must not become unknowns.""" + dmesg_content = ( + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 unrelated plugin failure\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs(check_unknown_dmesg_errors=True), + ) + + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + assert len(unknown_events) == 1 + assert unknown_events[0].data["match_content"] == "unrelated plugin failure" + + +def test_filtered_window_orphan_mce_tail_not_unknown(system_info): + """Analysis window can start after MCn_STATUS but before IPID/cache tail lines.""" + dmesg_content = ( + "kern :info : 2038-01-19T00:00:10,254124+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:10,266984+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:10,280615+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:10,303837+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:10,315164+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:10,335516+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + ) + analysis_range_start = datetime.datetime.fromisoformat( + "2038-01-19T00:00:10.314000+00:00" + ).astimezone(datetime.timezone.utc) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=True, + analysis_range_start=analysis_range_start, + ), + ) + + filtered_artifact = next( + artifact for artifact in res.artifacts if artifact.filename == "filtered_dmesg.log" + ) + assert "MC60_STATUS" not in filtered_artifact.contents + assert "IPID:" in filtered_artifact.contents + + descriptions = {event.description for event in res.events} + assert "Unknown dmesg error" not in descriptions + assert "MCE Corrected Error" not in descriptions + assert "RAS Corrected Error" not in descriptions + + +def test_mce_match_content_is_single_status_line(system_info): + """Adjacent incidents: events.json match_content must be one MCn_STATUS row.""" + dmesg_content = ( + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 " + "[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: CPU:8 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xccc\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=False, + mce_threshold=1, + ignore_match_rules=[{"mce_banks": ["60-63"]}], + ), + ) + + mce_events = [event for event in res.events if event.description == "MCE Corrected Error"] + assert len(mce_events) == 1 + match_content = str(mce_events[0].data["match_content"]) + assert match_content.startswith("[Hardware Error]:") + assert "MC49_STATUS" in match_content + assert "MC60_STATUS" not in match_content + assert "CPU:29" in match_content + assert "CPU:8" not in match_content + assert "\n" not in match_content diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index 53a58f00..4a3034e6 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -27,8 +27,13 @@ hardware_error_block_line_indices, ignored_mce_block_line_indices, iter_hardware_error_block_ranges, + mce_defining_status_line_indices, + mce_hardware_error_line_indices, + mce_non_status_hardware_error_line_indices, + mce_unknown_suppress_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, + trim_mce_status_match_content, ) @@ -37,12 +42,12 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): "kern :warn : 2038-01-19T00:00:00,000000+00:00 " "mce: 3 correctable hardware errors detected in total in mc0 block on CPU1\n" "kern :warn : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xabc\n" + "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xaaa\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU1/mc0": 3, "CPU0": 1} + assert counts == {"CPU0": 1} def test_parse_correctable_mce_counts_gpu_summary(): @@ -54,19 +59,19 @@ def test_parse_correctable_mce_counts_gpu_summary(): counts = parse_correctable_mce_counts(content) - assert counts == {"GPU0/gfx": 3} + assert counts == {} def test_parse_uncorrectable_mce_counts(): content = ( "kern :err : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: Machine Check: CPU1 MC1_STATUS[0xfeed|UC|AddrV]: 0x0\n" + "[Hardware Error]: Machine Check: CPU1 MC1_STATUS[0xbbb|UC|AddrV]: 0x0\n" "amdgpu 0000:de:ad.0: amdgpu: socket: 0 2 uncorrectable hardware errors detected in gfx block\n" ) counts = parse_uncorrectable_mce_counts(content) - assert counts == {"CPU1": 1, "GPU0/gfx": 2} + assert counts == {"CPU1": 1} def test_parse_correctable_mce_counts_skips_ignored_banks(): @@ -96,10 +101,12 @@ def test_parse_correctable_mce_counts_skips_ignored_banks_in_separate_block(): def test_hardware_error_block_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" - "kern :info : ts unrelated line\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :info : 2038-01-19T00:00:03,000000+00:00 dummy: unrelated line\n" ) assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2}) @@ -113,10 +120,10 @@ def test_hardware_error_block_splits_info_preamble_from_emerg_details(): "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" - "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " - "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" - "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) lines = content.splitlines() @@ -128,11 +135,15 @@ def test_hardware_error_block_splits_info_preamble_from_emerg_details(): def test_hardware_error_block_includes_interleaved_non_mce_lines(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts amdgpu 0000:de:ad.0: amdgpu: unrelated reset notice\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" - "kern :emerg : ts [Hardware Error]: cache level: L3/GEN\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "amdgpu 0000:de:ad.0: amdgpu: dummy unrelated reset notice\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN\n" ) assert hardware_error_block_line_indices(content) == frozenset({0, 1, 3, 4}) @@ -141,12 +152,16 @@ def test_hardware_error_block_includes_interleaved_non_mce_lines(): def test_ignored_mce_block_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" "\n" - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :err : ts [Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0x3\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :err : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0xccc\n" ) ignored = ignored_mce_block_line_indices(content, frozenset({60})) @@ -168,7 +183,7 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): "workqueue: dummy_mce_worker hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND\n" "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " - "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" "\n" "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" @@ -177,7 +192,7 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): "kern :emerg : 2038-01-19T00:00:08,000000+00:00 " "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:09,000000+00:00 " - "[Hardware Error]: CPU:24 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "[Hardware Error]: CPU:99 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" "kern :emerg : 2038-01-19T00:00:10,000000+00:00 [Hardware Error]: PPIN: 0xcccccccccccccccc\n" ) lines = content.splitlines() @@ -191,37 +206,102 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): assert 6 not in ignored_mce_block_line_indices(content, frozenset({60})) +def test_mce_non_status_hardware_error_line_indices(): + content = ( + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + ) + + assert mce_defining_status_line_indices(content) == frozenset({1}) + assert mce_non_status_hardware_error_line_indices(content) == frozenset({0, 2}) + + +def test_mce_unknown_suppress_orphan_detail_lines(): + """Tail of an MCE block without the defining MCn_STATUS row still suppresses unknowns.""" + content = ( + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 dummy: unrelated plugin failure\n" + ) + + suppressed = mce_unknown_suppress_line_indices(content) + + assert 0 in suppressed + assert 1 in suppressed + assert 2 not in suppressed + + +def test_mce_unknown_suppress_status_and_tail_lines(): + """Status row plus PPIN/IPID/cache tail must never reach unknown dmesg matching.""" + content = ( + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 dummy: unrelated plugin failure\n" + ) + + suppressed = mce_unknown_suppress_line_indices(content) + + assert suppressed == frozenset({0, 1, 2, 3, 4}) + assert mce_hardware_error_line_indices(content) == frozenset({0, 1, 2, 3, 4}) + + +def test_trim_mce_status_match_content_keeps_status_row_only(): + multiline = ( + "[Hardware Error]: CPU:42 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" + "[Hardware Error]: Corrected error, no action required.\n" + "[Hardware Error]: CPU:99 (00:00:0) MC60_STATUS[Over|CE|MiscV]: 0xccc\n" + ) + + trimmed = trim_mce_status_match_content(multiline) + + assert trimmed == ("[Hardware Error]: CPU:42 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb") + assert "MC60_STATUS" not in trimmed + assert "\n" not in trimmed + + def test_parse_correctable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:00,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xaaa\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU72": 1} + assert counts == {"CPU12": 1} def test_parse_correctable_mce_counts_both_cpu_formats(): content = ( "[Hardware Error]: CPU0 MC1_STATUS[0x0|CE|]: 0x1\n" "kern :err : 2038-01-19T00:00:00,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xaaa\n" "kern :warn : 2038-01-19T00:00:02,000000+00:00 " "mce: 2 correctable hardware errors detected in total in mc0 block on CPU:1\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU0": 1, "CPU72": 1, "CPU1/mc0": 2} + assert counts == {"CPU0": 1, "CPU12": 1} def test_parse_uncorrectable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|UC|Misc]: 0xdef\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|UC|Misc]: 0xbbb\n" ) counts = parse_uncorrectable_mce_counts(content) - assert counts == {"CPU72": 1} + assert counts == {"CPU12": 1} diff --git a/test/unit/plugin/test_mi3xx_collector.py b/test/unit/plugin/test_mi3xx_collector.py index 96a9d556..d27b80c1 100644 --- a/test/unit/plugin/test_mi3xx_collector.py +++ b/test/unit/plugin/test_mi3xx_collector.py @@ -140,6 +140,8 @@ def test_serviceability_plugin_mi3xx_wiring(): assert ServiceabilityPluginMI3XX.COLLECTOR is MI3XXCollector assert ServiceabilityPluginMI3XX.COLLECTOR_ARGS is MI3XXCollectorArgs assert ServiceabilityPluginMI3XX.ANALYZER is MI3XXAnalyzer + assert MI3XXCollector.DOCUMENTATION_COLLECTION_ITEMS + assert MI3XXAnalyzer.DOCUMENTATION_ANALYSIS_ITEMS def test_mi3xx_collector_no_args(mi3xx_collector): diff --git a/test/unit/plugin/test_network_analyzer.py b/test/unit/plugin/test_network_analyzer.py index 832125e5..9796a173 100644 --- a/test/unit/plugin/test_network_analyzer.py +++ b/test/unit/plugin/test_network_analyzer.py @@ -26,7 +26,6 @@ import pytest from nodescraper.enums import EventPriority, ExecutionStatus -from nodescraper.plugins.inband.network.analyzer_args import NetworkAnalyzerArgs from nodescraper.plugins.inband.network.ethtool_vendor import ( EthtoolStatistics, Thor2EthtoolStatistics, @@ -37,16 +36,6 @@ NetworkDataModel, ) -# Built-in regex defaults were removed; the vendor ethtool models own RDMA counter -# classification. These custom patterns exercise the user-supplied regex path. -CUSTOM_REGEX = [ - { - "regex": r"^custom_err_\d+$", - "message": "custom err counter non-zero", - "event_category": "NETWORK", - } -] - @pytest.fixture def network_analyzer(system_info): @@ -55,31 +44,10 @@ def network_analyzer(system_info): @pytest.fixture def clean_ethtool_info(): - """EthtoolInfo with no errors (all counters zero).""" + """EthtoolInfo with no vendor statistics.""" return EthtoolInfo( interface="eth0", raw_output="dummy output", - statistics={ - "tx_pfc_frames": "0", - "tx_pfc_ena_frames_pri0": "0", - "tx_pfc_ena_frames_pri1": "0", - "tx_pfc_ena_frames_pri2": "0", - "tx_pfc_ena_frames_pri3": "0", - "tx_pfc_ena_frames_pri4": "0", - "tx_pfc_ena_frames_pri5": "0", - "tx_pfc_ena_frames_pri6": "0", - "tx_pfc_ena_frames_pri7": "0", - "pfc_pri0_tx_transitions": "0", - "pfc_pri1_tx_transitions": "0", - "pfc_pri2_tx_transitions": "0", - "pfc_pri3_tx_transitions": "0", - "pfc_pri4_tx_transitions": "0", - "pfc_pri5_tx_transitions": "0", - "pfc_pri6_tx_transitions": "0", - "pfc_pri7_tx_transitions": "0", - "some_other_stat": "100", # Should be ignored - "rx_bytes": "1234567", # Should be ignored - }, ) @@ -101,88 +69,6 @@ def test_no_errors_detected(network_analyzer, clean_network_model): assert len(result.events) == 0 -def test_single_custom_match_detected(network_analyzer): - """A single non-zero counter matched by a user-supplied custom regex reports as ERROR.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={"custom_err_0": "5"}, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - assert "errors detected" in result.message - assert len(result.events) == 1 - assert result.events[0].description == "Network error detected on eth0: [custom_err_0]" - assert result.events[0].priority == EventPriority.ERROR - assert result.events[0].data["errors"] == {"custom_err_0": 5} - assert result.events[0].data["interface"] == "eth0" - - -def test_multiple_matches_same_interface(network_analyzer): - """Multiple non-zero counters on one interface produce a single grouped event.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={"custom_err_0": "10", "custom_err_1": "3", "custom_err_2": "7"}, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - assert "errors detected" in result.message - assert len(result.events) == 1 # one event per interface - assert result.events[0].priority == EventPriority.ERROR - # Check all 3 counters are present - assert len(result.events[0].data["errors"]) == 3 - assert result.events[0].data["errors"]["custom_err_0"] == 10 - assert result.events[0].data["errors"]["custom_err_1"] == 3 - assert result.events[0].data["errors"]["custom_err_2"] == 7 - - -def test_multiple_interfaces_with_matches(network_analyzer): - """Test with custom-regex matches across multiple interfaces.""" - eth0 = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "15", - "custom_err_1": "0", - }, - ) - eth1 = EthtoolInfo( - interface="eth1", - raw_output="dummy", - statistics={ - "custom_err_3": "8", - }, - ) - eth2 = EthtoolInfo( - interface="eth2", - raw_output="dummy", - statistics={ - "custom_err_7": "100", - }, - ) - model = NetworkDataModel( - ethtool_info={ - "eth0": eth0, - "eth1": eth1, - "eth2": eth2, - } - ) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 3 - interfaces = {event.data["interface"] for event in result.events} - assert interfaces == {"eth0", "eth1", "eth2"} - - def test_empty_ethtool_info(network_analyzer): """Test with empty ethtool_info and no RDMA ethtool: WARNING and message logged.""" model = NetworkDataModel(ethtool_info={}) @@ -192,13 +78,13 @@ def test_empty_ethtool_info(network_analyzer): def test_rdma_ethtool_vendor_error_only(network_analyzer): - """RDMA-scoped vendor ethtool: error-tier counter raises ERROR.""" + """Vendor ethtool: error-tier counter raises ERROR.""" stat = EthtoolStatistics( netdev="eth0", - rdma_ifname="bnxt0", + driver="bnxt_en", vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=4), ) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR assert "Network errors detected" in result.message @@ -209,13 +95,13 @@ def test_rdma_ethtool_vendor_error_only(network_analyzer): def test_rdma_ethtool_vendor_warning_only(network_analyzer): - """RDMA-scoped vendor ethtool: only warning-tier counters -> WARNING status.""" + """Vendor ethtool: only warning-tier counters -> WARNING status.""" stat = EthtoolStatistics( netdev="eth0", - rdma_ifname="bnxt0", + driver="bnxt_en", vendor_statistics=Thor2EthtoolStatistics(rx_pause_frames=2), ) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.WARNING assert "warning counters" in result.message @@ -234,195 +120,210 @@ def test_thor2_tx_pause_counters_are_warning_tier(): def test_rdma_ethtool_no_vendor_model_ok(network_analyzer): - """RDMA ethtool row without parsed vendor statistics is ignored by vendor path.""" - stat = EthtoolStatistics(netdev="eth0", rdma_ifname="unknown0", vendor_statistics=None) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + """Ethtool row without parsed vendor statistics is ignored by vendor path.""" + stat = EthtoolStatistics(netdev="eth0", driver="unknown", vendor_statistics=None) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 -def test_custom_regex_matches_multi_digit_suffixes(network_analyzer): - """Test that a user-supplied \\d+ pattern matches single- and double-digit suffixes.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "1", - "custom_err_3": "2", - "custom_err_7": "3", - "custom_err_10": "4", # Test double-digit - "custom_err_15": "5", # Test double-digit +def test_queue_counter_error_detected(network_analyzer): + """A non-zero per-queue error counter (e.g. rx_discards) is flagged as ERROR.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_ucast_packets": 100, # benign, ignored + "[0]: rx_discards": 5, # error + "[3]: tx_errors": 2, # error }, ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - # All 5 counters should be detected - assert len(result.events[0].data["errors"]) == 5 + assert "Network errors detected" in result.message + queue_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + assert len(queue_events) == 2 + flagged = {e.data["error_field"] for e in queue_events} + assert flagged == {"[0]: rx_discards", "[3]: tx_errors"} + assert all(e.priority == EventPriority.ERROR for e in queue_events) -def test_non_numeric_values_ignored(network_analyzer): - """Test that non-numeric values in statistics are gracefully ignored.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "N/A", # Non-numeric - "custom_err_1": "invalid", # Non-numeric - "custom_err_2": "5", # Valid + +def test_queue_counter_thor2_specific_error_fields(network_analyzer): + """Each curated Thor2 per-queue error pattern flags its non-zero counter as ERROR.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_discards": 1, + "[1]: rx_errors": 2, + "[2]: tx_errors": 3, + "[3]: rx_l4_csum_errors": 4, + "[4]: rx_buf_errors": 5, + "[5]: so_txtime_cmpl_errors": 6, + "[6]: missed_irqs": 7, + "[7]: xsk_rx_redirect_fail": 8, + "[8]: xsk_rx_alloc_fail": 9, + "[9]: xsk_rx_no_room": 10, + "[10]: xsk_tx_ring_full": 11, + # benign counters must NOT be flagged + "[0]: rx_ucast_packets": 12345, + "[0]: tx_ucast_bytes": 67890, }, ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - # Only the valid numeric counter should be reported - assert len(result.events[0].data["errors"]) == 1 - assert result.events[0].data["errors"]["custom_err_2"] == 5 + + error_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + flagged = {e.data["error_field"] for e in error_events} + assert flagged == { + "[0]: rx_discards", + "[1]: rx_errors", + "[2]: tx_errors", + "[3]: rx_l4_csum_errors", + "[4]: rx_buf_errors", + "[5]: so_txtime_cmpl_errors", + "[6]: missed_irqs", + "[7]: xsk_rx_redirect_fail", + "[8]: xsk_rx_alloc_fail", + "[9]: xsk_rx_no_room", + "[10]: xsk_tx_ring_full", + } + assert "[0]: rx_ucast_packets" not in flagged + assert "[0]: tx_ucast_bytes" not in flagged + assert all(e.priority == EventPriority.ERROR for e in error_events) -def test_zero_values_not_reported(network_analyzer): - """Test that zero values are not reported as errors.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "0", - "custom_err_1": "0", - "custom_err_2": "0", +def test_queue_counter_benign_not_reported(network_analyzer): + """Non-zero benign per-queue counters (packets/bytes) do not raise errors.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_ucast_packets": 12345, + "[0]: tx_ucast_bytes": 67890, }, ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 -def test_non_matching_fields_ignored(network_analyzer): - """Test that statistics not matching error patterns are ignored.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "rx_bytes": "999999999", # High value but not a matched field - "tx_bytes": "888888888", # High value but not a matched field - "some_random_counter": "12345", # Not a matched field - "custom_err_0": "5", # This SHOULD be detected - }, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) +def test_queue_counter_zero_not_reported(network_analyzer): + """A per-queue error counter that is zero does not raise an error.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={"[0]: rx_discards": 0, "[1]: tx_errors": 0}, ) - assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - # Only custom_err_0 should be reported - assert len(result.events[0].data["errors"]) == 1 - assert "custom_err_0" in result.events[0].data["errors"] + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 -def test_mixed_interfaces_with_and_without_errors(network_analyzer): - """Test with some interfaces having matches and others clean.""" - eth0_error = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "10", - }, - ) - eth1_clean = EthtoolInfo( - interface="eth1", - raw_output="dummy", - statistics={ - "custom_err_0": "0", - "custom_err_1": "0", +def test_queue_counter_warning_detected(network_analyzer): + """A non-zero warning-tier per-queue counter (e.g. rx_resets) -> WARNING.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_ucast_packets": 100, # benign, ignored + "[0]: rx_resets": 3, # warning-tier }, ) - eth2_error = EthtoolInfo( - interface="eth2", - raw_output="dummy", - statistics={ - "custom_err_5": "20", + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.WARNING + assert "warning counters" in result.message + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert len(warn_events) == 1 + assert warn_events[0].data["error_field"] == "[0]: rx_resets" + assert warn_events[0].priority == EventPriority.WARNING + + +def test_queue_counter_error_precedence_over_warning(network_analyzer): + """When both error and warning per-queue counters are non-zero, status is ERROR.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_discards": 5, # error-tier + "[0]: rx_resets": 3, # warning-tier }, ) - model = NetworkDataModel( - ethtool_info={ - "eth0": eth0_error, - "eth1": eth1_clean, - "eth2": eth2_error, - } - ) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR - # Only 2 events (eth0 and eth2), eth1 should not generate an event - assert len(result.events) == 2 - interfaces_with_errors = {event.data["interface"] for event in result.events} - assert interfaces_with_errors == {"eth0", "eth2"} + err_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert {e.data["error_field"] for e in err_events} == {"[0]: rx_discards"} + assert {e.data["error_field"] for e in warn_events} == {"[0]: rx_resets"} -def test_custom_error_regex_detected(network_analyzer): - """Test that user-supplied custom regex patterns are applied (no built-in defaults).""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_tx_drops": "9", # Matched via custom regex only - "tx_pfc_frames": "0", # No built-in pattern; ignored unless RDMA vendor-scoped +def test_queue_counter_empty_warning_list_skips_warning_check(network_analyzer, monkeypatch): + """An empty queue_warning_regex skips warning checking; error checking still runs.""" + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_warning_regex", []) + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_resets": 3, # would be a warning, but warning list is empty + "[0]: rx_discards": 5, # still flagged as an error }, ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - args = NetworkAnalyzerArgs( - error_regex=[ - { - "regex": r"^custom_tx_drops$", - "message": "Custom tx drops", - "event_category": "NETWORK", - } - ] - ) - - result = network_analyzer.analyze_data(model, args=args) - + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - assert result.events[0].data["interface"] == "eth0" - assert result.events[0].data["errors"] == {"custom_tx_drops": 9} + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert warn_events == [] + err_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + assert {e.data["error_field"] for e in err_events} == {"[0]: rx_discards"} -def test_custom_warning_priority_regex(network_analyzer): - """A user-supplied WARNING-priority custom pattern yields WARNING status via the regex path.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={"custom_warn": "3"}, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - args = NetworkAnalyzerArgs( - error_regex=[ - { - "regex": r"^custom_warn$", - "message": "custom warn counter", - "event_category": "NETWORK", - "event_priority": "WARNING", - } - ] +def test_queue_counter_empty_error_list_skips_error_check(network_analyzer, monkeypatch): + """An empty queue_error_regex skips error checking; warning checking still runs.""" + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_error_regex", []) + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_discards": 5, # would be an error, but error list is empty + "[0]: rx_resets": 3, # still flagged as a warning + }, ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.WARNING + err_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + assert err_events == [] + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert {e.data["error_field"] for e in warn_events} == {"[0]: rx_resets"} - result = network_analyzer.analyze_data(model, args=args) - assert result.status == ExecutionStatus.WARNING - assert "warning counters" in result.message - assert len(result.events) == 1 - assert result.events[0].priority == EventPriority.WARNING - assert result.events[0].description == "Network warning detected on eth0: [custom_warn]" - assert result.events[0].data["errors"] == {"custom_warn": 3} +def test_queue_counter_both_lists_empty_skips_all_checks(network_analyzer, monkeypatch): + """Empty error and warning lists skip per-queue checking entirely (status OK).""" + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_error_regex", []) + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_warning_regex", []) + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={"[0]: rx_discards": 5, "[0]: rx_resets": 3}, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 diff --git a/test/unit/plugin/test_network_collector.py b/test/unit/plugin/test_network_collector.py index a7b1faae..909e1319 100644 --- a/test/unit/plugin/test_network_collector.py +++ b/test/unit/plugin/test_network_collector.py @@ -30,6 +30,12 @@ from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.models.systeminfo import OSFamily +from nodescraper.plugins.inband.network.ethtool_vendor import ( + Cx7EthtoolStatistics, + PollaraEthtoolStatistics, + Thor2EthtoolStatistics, + extract_queue_counters, +) from nodescraper.plugins.inband.network.network_collector import NetworkCollector from nodescraper.plugins.inband.network.networkdata import ( EthtoolInfo, @@ -510,7 +516,10 @@ def test_parse_ethtool_empty_output(collector): def test_parse_ethtool_statistics(collector): - """Test parsing ethtool -S output (statistics) for error/health analysis.""" + """Test parsing ethtool -S output (statistics) for error/health analysis. + + Per-queue bracket lines keep their full raw name so vendor queue patterns match. + """ output = """NIC statistics: [0]: rx_ucast_packets: 162692536538787551 [0]: rx_errors: 0 @@ -519,13 +528,51 @@ def test_parse_ethtool_statistics(collector): rx_total_buf_errors: 0""" stats = collector._parse_ethtool_statistics(output, "abc1p1") assert stats.get("netdev") == "abc1p1" - assert stats.get("0_rx_ucast_packets") == "162692536538787551" - assert stats.get("0_rx_errors") == "0" - assert stats.get("1_rx_ucast_packets") == "79657418409137764" + assert stats.get("[0]: rx_ucast_packets") == "162692536538787551" + assert stats.get("[0]: rx_errors") == "0" + assert stats.get("[1]: rx_ucast_packets") == "79657418409137764" assert stats.get("rx_total_l4_csum_errors") == "0" assert stats.get("rx_total_buf_errors") == "0" +def test_extract_queue_counters_thor2(): + """Thor2 per-queue counters are the bracket-indexed "[]" lines.""" + stats = { + "netdev": "benic1p1", + "[0]: rx_ucast_packets": "5", + "[15]: tx_errors": "0", + "rx_total_l4_csum_errors": "0", + } + queue = extract_queue_counters(stats, Thor2EthtoolStatistics.queue_counter_regex) + assert queue == {"[0]: rx_ucast_packets": 5, "[15]: tx_errors": 0} + assert "rx_total_l4_csum_errors" not in queue + assert "netdev" not in queue + + +def test_extract_queue_counters_pollara(): + """Pollara per-queue counters start with "rx_" / "tx_".""" + stats = { + "rx_0_pkts": "3", + "tx_12_bytes": "9", + "rx_csum_error": "0", + "frames_tx_pri_0": "1", + } + queue = extract_queue_counters(stats, PollaraEthtoolStatistics.queue_counter_regex) + assert queue == {"rx_0_pkts": 3, "tx_12_bytes": 9} + + +def test_extract_queue_counters_cx7(): + """CX7 per-queue counters start with "rx" / "tx".""" + stats = { + "rx0_packets": "7", + "tx3_bytes": "8", + "rx_out_of_buffer": "0", + "rx_prio0_pause": "2", + } + queue = extract_queue_counters(stats, Cx7EthtoolStatistics.queue_counter_regex) + assert queue == {"rx0_packets": 7, "tx3_bytes": 8} + + def test_network_data_model_creation(collector): """Test creating NetworkDataModel with all components""" interface = NetworkInterface( @@ -650,13 +697,10 @@ def run_sut_cmd_side_effect(cmd, **kwargs): assert accessible is False -def test_collect_data_includes_rdma_ethtool(collector, conn_mock): - """RDMA-scoped ethtool -S is stored on NetworkDataModel when rdma link succeeds.""" - import json - +def test_collect_data_includes_ethtool_statistics(collector, conn_mock): + """ethtool -S is collected for vendor NICs via driver detection (no RDMA).""" collector.system_info.os_family = OSFamily.LINUX - rdma_link = [{"netdev": "eth0", "ifname": "bnxt0"}] ethtool_s_bnxt = "NIC statistics:\n tx_pfc_frames: 0\n rx_pause_frames: 0\n" def run_sut_cmd_side_effect(cmd, **kwargs): @@ -668,8 +712,8 @@ def run_sut_cmd_side_effect(cmd, **kwargs): return MagicMock(exit_code=0, stdout=IP_RULE_OUTPUT, command=cmd) elif "neighbor show" in cmd: return MagicMock(exit_code=0, stdout=IP_NEIGHBOR_OUTPUT, command=cmd) - elif "rdma link -j" in cmd: - return MagicMock(exit_code=0, stdout=json.dumps(rdma_link), command=cmd) + elif "ethtool -i" in cmd and "eth0" in cmd: + return MagicMock(exit_code=0, stdout="driver: bnxt_en\nversion: 1.0\n", command=cmd) elif "ethtool -S" in cmd and "eth0" in cmd: return MagicMock(exit_code=0, stdout=ethtool_s_bnxt, command=cmd) elif "ethtool" in cmd: @@ -684,8 +728,46 @@ def run_sut_cmd_side_effect(cmd, **kwargs): assert result.status == ExecutionStatus.OK assert data is not None - assert "eth0" in data.rdma_ethtool_netdevs - assert len(data.rdma_ethtool_statistics) == 1 - assert data.rdma_ethtool_statistics[0].netdev == "eth0" - assert data.rdma_ethtool_statistics[0].rdma_ifname == "bnxt0" - assert data.rdma_ethtool_statistics[0].vendor_statistics is not None + # lo has no vendor driver and is skipped; eth0 (bnxt_en) is collected + assert data.ethtool_netdevs == ["eth0"] + assert len(data.ethtool_statistics) == 1 + assert data.ethtool_statistics[0].netdev == "eth0" + assert data.ethtool_statistics[0].driver == "bnxt_en" + assert data.ethtool_statistics[0].vendor_statistics is not None + + +def test_collect_data_skips_non_vendor_netdev(collector, conn_mock): + """ethtool -S is not run for netdevs whose driver is not a known vendor.""" + collector.system_info.os_family = OSFamily.LINUX + + ethtool_s_called = {"value": False} + + def run_sut_cmd_side_effect(cmd, **kwargs): + if "addr show" in cmd: + return MagicMock(exit_code=0, stdout=IP_ADDR_OUTPUT, command=cmd) + elif "route show" in cmd: + return MagicMock(exit_code=0, stdout=IP_ROUTE_OUTPUT, command=cmd) + elif "rule show" in cmd: + return MagicMock(exit_code=0, stdout=IP_RULE_OUTPUT, command=cmd) + elif "neighbor show" in cmd: + return MagicMock(exit_code=0, stdout=IP_NEIGHBOR_OUTPUT, command=cmd) + elif "ethtool -i" in cmd and "eth0" in cmd: + return MagicMock(exit_code=0, stdout="driver: e1000e\n", command=cmd) + elif "ethtool -S" in cmd: + ethtool_s_called["value"] = True + return MagicMock(exit_code=0, stdout="NIC statistics:\n", command=cmd) + elif "ethtool" in cmd: + return MagicMock(exit_code=1, stdout="", command=cmd) + elif "lldpcli" in cmd or "lldpctl" in cmd: + return MagicMock(exit_code=1, stdout="", command=cmd) + return MagicMock(exit_code=1, stdout="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) + + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.OK + assert data is not None + assert data.ethtool_netdevs == [] + assert data.ethtool_statistics == [] + assert ethtool_s_called["value"] is False diff --git a/test/unit/plugin/test_rocm_collector.py b/test/unit/plugin/test_rocm_collector.py index 0cb8523c..63cef21d 100644 --- a/test/unit/plugin/test_rocm_collector.py +++ b/test/unit/plugin/test_rocm_collector.py @@ -32,7 +32,7 @@ from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.plugins.inband.rocm.rocm_collector import RocmCollector -ROCM_SUB_VERSIONS_GREP_CMD = "grep . -H -r -i /opt/rocm/.info/*" +ROCM_SUB_VERSIONS_GREP_CMD = "grep . -H -r -i '/opt/rocm'/.info/*" # gfx942 (CDNA3 / MI300) and gfx950 (CDNA4 / MI350) — released ROCm 7.13 LLVM targets ROCM_7_13_GFX_VERSION = "7.13.0-123-gfx942;gfx950"