From c46ece65da13595f7b4e8b112447bdb38cc68afa Mon Sep 17 00:00:00 2001 From: wallentx Date: Wed, 24 Jun 2026 16:39:55 -0500 Subject: [PATCH] style: Format code and logging calls for readability Signed-off-by: wallentx --- netbox_agent/arp_reporter.py | 37 +- netbox_agent/cli.py | 11 +- netbox_agent/dependencies.py | 16 +- netbox_agent/ethtool.py | 2 +- netbox_agent/ipmi.py | 12 +- netbox_agent/lshw.py | 39 +- netbox_agent/misc.py | 9 +- netbox_agent/modules.py | 400 +++++++++----- netbox_agent/network.py | 86 +-- netbox_agent/server.py | 144 +++-- netbox_agent/state.py | 4 +- netbox_agent/vendors/supermicro.py | 18 +- scripts/arp_discover.py | 90 ++-- scripts/fixtures/collect_hardware_fixture.py | 25 +- scripts/infra/compute/sync_asset_tags.py | 141 +++-- scripts/infra/credentials/push_to_vaults.py | 43 +- .../infra/credentials/sync_to_1password.py | 73 ++- .../credentials/validate_machines_csv.py | 118 +++-- scripts/infra/pdus/delete_garbage_entries.py | 13 +- scripts/infra/pdus/sync_asset_tags.py | 131 +++-- scripts/migration/assign_asset_tags.py | 4 +- scripts/migration/cleanup_bad_modules.py | 8 +- scripts/migration/cleanup_legacy_bays.py | 8 +- scripts/migration/migrate_ipam.py | 63 ++- .../migration/verify_modules_vs_inventory.py | 4 +- scripts/rollback/delete_all_modules.py | 4 +- .../02_create_custom_field_choice_sets.py | 4 +- scripts/schema/03_create_custom_fields.py | 28 +- .../schema/04_create_module_type_profiles.py | 26 +- scripts/schema/05_create_module_types.py | 171 ++++-- ...pdate_device_types_module_bay_templates.py | 31 +- .../07_create_spare_inventory_device.py | 74 +-- scripts/schema/08_run_all_schema_setup.py | 5 +- .../validate_record_completeness.py | 28 +- tests/test_dependencies.py | 3 +- tests/test_fixture_integration.py | 197 ++++--- tests/test_ipmi.py | 32 +- tests/test_modules.py | 493 +++++++++++++----- tests/test_network_ip.py | 19 +- tests/test_state.py | 1 - 40 files changed, 1725 insertions(+), 890 deletions(-) diff --git a/netbox_agent/arp_reporter.py b/netbox_agent/arp_reporter.py index e2029802..64b577ec 100644 --- a/netbox_agent/arp_reporter.py +++ b/netbox_agent/arp_reporter.py @@ -45,7 +45,9 @@ def _scan_arp_scan(interface: str, timeout: int) -> list[tuple[str, str]]: try: result = subprocess.run( ["arp-scan", "--localnet", f"--interface={interface}", "--plain"], - capture_output=True, text=True, timeout=timeout, + capture_output=True, + text=True, + timeout=timeout, ) for line in result.stdout.strip().splitlines(): # arp-scan --plain output: IP\tMAC\tVendor (or IP\tMAC) @@ -115,7 +117,9 @@ def _scan_nmap(interface: str, timeout: int) -> list[tuple[str, str]]: try: result = subprocess.run( ["nmap", "-sn", "-oX", "-", "-e", interface, cidr], - capture_output=True, text=True, timeout=timeout, + capture_output=True, + text=True, + timeout=timeout, ) root = ET.fromstring(result.stdout) for host in root.findall("host"): @@ -149,7 +153,9 @@ def _scan_ip_neigh() -> list[tuple[str, str]]: try: result = subprocess.run( ["ip", "-j", "neigh", "show"], - capture_output=True, text=True, timeout=10, + capture_output=True, + text=True, + timeout=10, ) entries = json.loads(result.stdout) if result.stdout.strip() else [] for entry in entries: @@ -195,17 +201,17 @@ def _get_scan_interfaces(config) -> list[str]: # These waste time (arp-scan times out on bridges) and produce # noise (container-internal MACs are not useful for inventory). _SKIP_PREFIXES = ( - "br-", # Docker custom network bridges - "veth", # Docker/container veth pairs - "vnet", # libvirt/KVM virtual NICs - "virbr", # libvirt default bridge - "cni", # Kubernetes CNI + "br-", # Docker custom network bridges + "veth", # Docker/container veth pairs + "vnet", # libvirt/KVM virtual NICs + "virbr", # libvirt default bridge + "cni", # Kubernetes CNI "flannel", # Kubernetes flannel overlay - "calico", # Kubernetes Calico - "tunl", # tunnel interfaces - "podman", # Podman container bridge - "wg", # WireGuard tunnels - "tailscale", # Tailscale tunnel + "calico", # Kubernetes Calico + "tunl", # tunnel interfaces + "podman", # Podman container bridge + "wg", # WireGuard tunnels + "tailscale", # Tailscale tunnel ) for iface in sorted(os.listdir("/sys/class/net/")): @@ -230,6 +236,7 @@ def _get_scan_interfaces(config) -> list[str]: # Check for IPv4 address (skip interfaces with no IP) try: import netifaces + addrs = netifaces.ifaddresses(iface).get(netifaces.AF_INET, []) if not addrs: continue @@ -319,7 +326,9 @@ def scan_and_report(config) -> dict: timeout=post_timeout, ) result["pairs_submitted"] = len(pairs_list) - result["response"] = resp.json() if resp.ok else {"status_code": resp.status_code, "text": resp.text} + result["response"] = ( + resp.json() if resp.ok else {"status_code": resp.status_code, "text": resp.text} + ) if not resp.ok: logging.warning("ARP report POST failed: HTTP %d — %s", resp.status_code, resp.text) else: diff --git a/netbox_agent/cli.py b/netbox_agent/cli.py index a057316e..5e8b25c6 100644 --- a/netbox_agent/cli.py +++ b/netbox_agent/cli.py @@ -71,19 +71,22 @@ def run(config): or network_only ): server.netbox_create_or_update( - config, deps=deps, network_only=network_only, state=state, + config, + deps=deps, + network_only=network_only, + state=state, ) if config.debug: server.print_debug() # ARP neighbor reporting (after main sync) - arp_enabled = ( - getattr(config, "arp_report", None) - and getattr(config.arp_report, "enabled", False) + arp_enabled = getattr(config, "arp_report", None) and getattr( + config.arp_report, "enabled", False ) arp_flag = getattr(config, "arp_report_flag", False) if arp_enabled or arp_flag: from netbox_agent.arp_reporter import scan_and_report + try: arp_result = scan_and_report(config) logging.info( diff --git a/netbox_agent/dependencies.py b/netbox_agent/dependencies.py index 067b10dd..37b32ddb 100644 --- a/netbox_agent/dependencies.py +++ b/netbox_agent/dependencies.py @@ -13,14 +13,14 @@ # tool_name -> (description, required) TOOLS = { - "dmidecode": ("DMI/SMBIOS data (serial, chassis, PSUs, DIMMs)", True), - "lshw": ("Hardware tree (GPUs, NICs, storage fallback)", True), - "lsblk": ("Block device enumeration (primary storage)", False), - "lscpu": ("CPU socket detection (primary CPU path)", False), - "ipmitool": ("IPMI/BMC data (OOB IP, MAC, asset tag)", False), - "ethtool": ("NIC speed/duplex detection", False), - "lldpctl": ("LLDP neighbor discovery (auto-cabling)", False), - "nvme": ("NVMe device enrichment (vendor, firmware)", False), + "dmidecode": ("DMI/SMBIOS data (serial, chassis, PSUs, DIMMs)", True), + "lshw": ("Hardware tree (GPUs, NICs, storage fallback)", True), + "lsblk": ("Block device enumeration (primary storage)", False), + "lscpu": ("CPU socket detection (primary CPU path)", False), + "ipmitool": ("IPMI/BMC data (OOB IP, MAC, asset tag)", False), + "ethtool": ("NIC speed/duplex detection", False), + "lldpctl": ("LLDP neighbor discovery (auto-cabling)", False), + "nvme": ("NVMe device enrichment (vendor, firmware)", False), "nvidia-smi": ("NVIDIA GPU serial numbers", False), } diff --git a/netbox_agent/ethtool.py b/netbox_agent/ethtool.py index 4ccb158a..67b1fc73 100644 --- a/netbox_agent/ethtool.py +++ b/netbox_agent/ethtool.py @@ -129,7 +129,7 @@ def _parse_ethtool_module_output(self): if colon <= 0: continue key = line[:colon].strip() - value = line[colon + 1:].strip() + value = line[colon + 1 :].strip() if key in module_field_map: field_key = module_field_map[key] diff --git a/netbox_agent/ipmi.py b/netbox_agent/ipmi.py index f72a39d6..54542a9e 100644 --- a/netbox_agent/ipmi.py +++ b/netbox_agent/ipmi.py @@ -29,9 +29,7 @@ def __init__(self): return for ch in _CHANNELS: - ret, output = subprocess.getstatusoutput( - f"ipmitool lan print {ch}" - ) + ret, output = subprocess.getstatusoutput(f"ipmitool lan print {ch}") if ret != 0: continue @@ -41,9 +39,7 @@ def __init__(self): self.output = output self.channel = ch ip = self._extract_field(output, "IP Address") or "0.0.0.0" - logger.debug( - "IPMI: valid response on channel %d (MAC=%s, IP=%s)", ch, mac, ip - ) + logger.debug("IPMI: valid response on channel %d (MAC=%s, IP=%s)", ch, mac, ip) break else: logger.warning("IPMI: no valid response on channels %s", _CHANNELS) @@ -81,7 +77,9 @@ def parse(self): if ip and ip != "0.0.0.0": ip_list = [f"{ip}/32"] else: - logger.info("IPMI: MAC=%s but IP unassigned (0.0.0.0) — interface created without IP", mac) + logger.info( + "IPMI: MAC=%s but IP unassigned (0.0.0.0) — interface created without IP", mac + ) return { "name": "IPMI", diff --git a/netbox_agent/lshw.py b/netbox_agent/lshw.py index 0ce20bce..d68d9673 100644 --- a/netbox_agent/lshw.py +++ b/netbox_agent/lshw.py @@ -193,13 +193,22 @@ def find_gpus(self, obj): # These are IOMMU, host bridges, system peripherals, etc. that lshw # classifies as "generic" but are not compute accelerators. _INFRA_DESCRIPTIONS = { - "iommu", "system peripheral", "generic system peripheral", - "non-essential instrumentation", "encryption controller", - "host bridge", "pci bridge", "isa bridge", "smi bridge", - "signal processing controller", "communication controller", - "pic", "dma controller", "timer", - "performance counters", # Intel CPU uncore PMU counters - "scsi enclosure", # Dell storage enclosure managers (Fryer U.2 etc.) + "iommu", + "system peripheral", + "generic system peripheral", + "non-essential instrumentation", + "encryption controller", + "host bridge", + "pci bridge", + "isa bridge", + "smi bridge", + "signal processing controller", + "communication controller", + "pic", + "dma controller", + "timer", + "performance counters", # Intel CPU uncore PMU counters + "scsi enclosure", # Dell storage enclosure managers (Fryer U.2 etc.) } def find_accelerators(self, obj): @@ -227,13 +236,15 @@ def find_accelerators(self, obj): return # Everything else is a true accelerator (Pliops, FPGA, custom hardware) - self.accelerators.append({ - "product": obj.get("product", "Unknown Accelerator"), - "vendor": vendor, - "description": obj.get("description", ""), - "businfo": obj.get("businfo", ""), - "class": obj.get("class", ""), - }) + self.accelerators.append( + { + "product": obj.get("product", "Unknown Accelerator"), + "vendor": vendor, + "description": obj.get("description", ""), + "businfo": obj.get("businfo", ""), + "class": obj.get("class", ""), + } + ) # PCI device classes that indicate compute accelerators (not CPUs, not GPUs) _ACCELERATOR_CLASSES = {"coprocessor", "generic", "processing"} diff --git a/netbox_agent/misc.py b/netbox_agent/misc.py index 2a7a40dd..e5f24cda 100644 --- a/netbox_agent/misc.py +++ b/netbox_agent/misc.py @@ -46,9 +46,7 @@ def get_device_type(model, manufacturer=None): device_type = nb.dcim.device_types.get(model=model) if device_type is None and manufacturer: mfr = get_or_create_manufacturer(manufacturer) - logging.info( - 'Auto-creating DeviceType "%s" (manufacturer: %s)', model, manufacturer - ) + logging.info('Auto-creating DeviceType "%s" (manufacturer: %s)', model, manufacturer) device_type = nb.dcim.device_types.create( model=model, slug=slugify(model), @@ -56,8 +54,9 @@ def get_device_type(model, manufacturer=None): ) if device_type is None: raise Exception( - 'DeviceType "{}" does not exist and no manufacturer provided' - " for auto-creation".format(model) + 'DeviceType "{}" does not exist and no manufacturer provided for auto-creation'.format( + model + ) ) return device_type diff --git a/netbox_agent/modules.py b/netbox_agent/modules.py index 7e37eac2..4a6874bb 100644 --- a/netbox_agent/modules.py +++ b/netbox_agent/modules.py @@ -42,7 +42,7 @@ def _api_retry(func, *args, **kwargs): return func(*args, **kwargs) except Exception as e: if attempt < MAX_RETRIES - 1: - wait = RETRY_BACKOFF * (2 ** attempt) + wait = RETRY_BACKOFF * (2**attempt) logger.warning("API call failed (%s), retrying in %ds...", e, wait) time.sleep(wait) else: @@ -80,8 +80,19 @@ def __init__(self, server, config): # ------------------------------------------------------------------ # # Vendor ID normalization (from SILO cpu.py) - _INTEL_ALIASES = {"genuineintel", "intel", "intel(r) corporation", "intel corporation", "intel corp."} - _AMD_ALIASES = {"authenticamd", "advanced micro devices", "amd", "advanced micro devices [amd]"} + _INTEL_ALIASES = { + "genuineintel", + "intel", + "intel(r) corporation", + "intel corporation", + "intel corp.", + } + _AMD_ALIASES = { + "authenticamd", + "advanced micro devices", + "amd", + "advanced micro devices [amd]", + } def _normalize_cpu_vendor(self, vendor_id): """Normalize CPU vendor string. Informed by SILO's normalize_cpu_make().""" @@ -148,12 +159,14 @@ def _parse_lscpu(self, lscpu_data): items = [] for i in range(sockets): - items.append({ - "product": model.strip(), - "vendor": vendor, - "serial": None, # CPUs don't report serials - "slot": f"CPU{i}", - }) + items.append( + { + "product": model.strip(), + "vendor": vendor, + "serial": None, # CPUs don't report serials + "slot": f"CPU{i}", + } + ) return items def _get_local_cpus_lshw_fallback(self): @@ -166,20 +179,31 @@ def _get_local_cpus_lshw_fallback(self): # Basic filtering for lshw fallback — skip obvious accelerators combined = f"{product} {description}".lower() - skip_keywords = {"qat", "quickassist", "dlb", "iaa", "dsa", - "co-processor", "coprocessor", "accelerator", "4xxx"} + skip_keywords = { + "qat", + "quickassist", + "dlb", + "iaa", + "dsa", + "co-processor", + "coprocessor", + "accelerator", + "4xxx", + } if any(kw in combined for kw in skip_keywords): continue # Skip entries where product is just a vendor name if product.lower().strip() in self._INTEL_ALIASES | self._AMD_ALIASES: continue - items.append({ - "product": product, - "vendor": self._normalize_cpu_vendor(vendor), - "serial": None, - "slot": cpu.get("location", ""), - }) + items.append( + { + "product": product, + "vendor": self._normalize_cpu_vendor(vendor), + "serial": None, + "slot": cpu.get("location", ""), + } + ) return items # BMC/onboard VGA controllers that should NOT be tracked as GPU modules @@ -201,8 +225,12 @@ def _get_local_gpus(self): amd_serials = self._get_amd_gpu_serials() if amd_driver else {} amd_gpu_idx = 0 # Counter for AMD GPUs (separate from NVIDIA index) gaudi_driver, gaudi_devices = self._get_intel_gaudi_info() - gaudi_serials = {d.get("businfo", ""): d.get("serial") for d in gaudi_devices if d.get("serial")} - gaudi_names = {d.get("businfo", ""): d.get("product") for d in gaudi_devices if d.get("product")} + gaudi_serials = { + d.get("businfo", ""): d.get("serial") for d in gaudi_devices if d.get("serial") + } + gaudi_names = { + d.get("businfo", ""): d.get("product") for d in gaudi_devices if d.get("product") + } items = [] real_idx = 0 # index into nvidia-smi GPU info (only real GPUs) @@ -223,7 +251,11 @@ def _get_local_gpus(self): # Skip onboard VGA — but NOT discrete NVIDIA/AMD GPUs which may also # report as "VGA compatible" when lshw doesn't have the PCI ID mapping. is_known_gpu_vendor = any(v in vendor_lower for v in ("nvidia", "amd", "ati")) - if "VGA compatible" in description and "3D" not in description and not is_known_gpu_vendor: + if ( + "VGA compatible" in description + and "3D" not in description + and not is_known_gpu_vendor + ): logger.debug("Skipping VGA-only device: %s %s", vendor, product) continue @@ -273,12 +305,14 @@ def _get_local_gpus(self): desc_parts.append(f"SynapseAI: habanalabs {gaudi_driver}") full_desc = " | ".join(p for p in desc_parts if p) - items.append({ - "product": product, - "vendor": vendor, - "serial": serial, - "description": full_desc, - }) + items.append( + { + "product": product, + "vendor": vendor, + "serial": serial, + "description": full_desc, + } + ) # Intel Gaudi: lshw classifies these as "network" (not "display"), # so they don't appear in lshw.gpus. Add them from hl-smi if not @@ -291,12 +325,14 @@ def _get_local_gpus(self): if gaudi_driver: desc_parts.append(f"driver: habanalabs {gaudi_driver}") desc_parts.append(f"SynapseAI: habanalabs {gaudi_driver}") - items.append({ - "product": gdev.get("product", "Gaudi GPU"), - "vendor": "Habana Labs (Intel)", - "serial": gdev.get("serial"), - "description": " | ".join(desc_parts), - }) + items.append( + { + "product": gdev.get("product", "Gaudi GPU"), + "vendor": "Habana Labs (Intel)", + "serial": gdev.get("serial"), + "description": " | ".join(desc_parts), + } + ) return items @@ -326,13 +362,15 @@ def _get_local_accelerators(self): if driver: desc_parts.append(f"driver: {driver}") - items.append({ - "product": product, - "vendor": vendor, - "serial": None, - "description": " | ".join(p for p in desc_parts if p), - "businfo": businfo, - }) + items.append( + { + "product": product, + "vendor": vendor, + "serial": None, + "description": " | ".join(p for p in desc_parts if p), + "businfo": businfo, + } + ) # Note: Intel Gaudi (Habana Labs) is now routed to GPUs via lshw.py, # not to accelerators. Gaudi enrichment (hl-smi serials, driver) happens @@ -340,9 +378,11 @@ def _get_local_accelerators(self): # (Pliops, FPGAs, QAT, custom hardware). if items: - logger.info("Detected %d accelerator(s): %s", - len(items), - ", ".join(f'{a["vendor"]} {a["product"]}' for a in items)) + logger.info( + "Detected %d accelerator(s): %s", + len(items), + ", ".join(f"{a['vendor']} {a['product']}" for a in items), + ) return items @@ -359,8 +399,11 @@ def _get_nvidia_gpu_info(self): return driver, cuda, gpu_info try: output = subprocess.check_output( - ["nvidia-smi", "--query-gpu=index,name,serial,driver_version", - "--format=csv,noheader,nounits"], + [ + "nvidia-smi", + "--query-gpu=index,name,serial,driver_version", + "--format=csv,noheader,nounits", + ], encoding="utf-8", timeout=30, ).strip() @@ -383,8 +426,11 @@ def _get_nvidia_gpu_info(self): # CUDA version from nvidia-smi header (not available via --query-gpu) try: import re + header = subprocess.check_output( - ["nvidia-smi"], encoding="utf-8", timeout=10, + ["nvidia-smi"], + encoding="utf-8", + timeout=10, ) m = re.search(r"CUDA Version:\s*([0-9.]+)", header) if m: @@ -401,7 +447,8 @@ def _get_amd_rocm_version(self): try: output = subprocess.check_output( ["rocm-smi", "--showdriverversion"], - encoding="utf-8", timeout=10, + encoding="utf-8", + timeout=10, ).strip() for line in output.splitlines(): if "Driver version" in line: @@ -412,7 +459,9 @@ def _get_amd_rocm_version(self): if is_tool("rocminfo"): try: output = subprocess.check_output( - ["rocminfo"], encoding="utf-8", timeout=10, + ["rocminfo"], + encoding="utf-8", + timeout=10, ).strip() for line in output.splitlines(): if "Runtime Version" in line: @@ -439,7 +488,9 @@ def _get_amd_gpu_driver(self): try: output = subprocess.check_output( ["modinfo", "amdgpu", "-F", "version"], - encoding="utf-8", timeout=10, stderr=subprocess.DEVNULL, + encoding="utf-8", + timeout=10, + stderr=subprocess.DEVNULL, ).strip() if output: return output.splitlines()[0] @@ -459,9 +510,12 @@ def _get_amd_gpu_serials(self): if is_tool("rocm-smi"): try: import re + output = subprocess.check_output( ["rocm-smi", "--showserial"], - encoding="utf-8", timeout=15, stderr=subprocess.DEVNULL, + encoding="utf-8", + timeout=15, + stderr=subprocess.DEVNULL, ).strip() # Parse "GPU[0] : Serial Number: XXXX" format for m in re.finditer(r"GPU\[(\d+)\]\s*:\s*Serial Number:\s*(\S+)", output): @@ -477,6 +531,7 @@ def _get_amd_gpu_serials(self): try: import os import glob + for card_dir in sorted(glob.glob("/sys/class/drm/card[0-9]*/device/")): serial_path = os.path.join(card_dir, "serial_number") vendor_path = os.path.join(card_dir, "vendor") @@ -517,14 +572,21 @@ def _get_driver_for_pci_device(self, businfo: str) -> str: driver_link = f"/sys/bus/pci/devices/{pci_addr}/driver" try: import os + driver_path = os.readlink(driver_link) driver_name = os.path.basename(driver_path) # Get driver version from modinfo try: - version = subprocess.check_output( - ["modinfo", driver_name, "-F", "version"], - encoding="utf-8", timeout=10, stderr=subprocess.DEVNULL, - ).strip().splitlines()[0] + version = ( + subprocess.check_output( + ["modinfo", driver_name, "-F", "version"], + encoding="utf-8", + timeout=10, + stderr=subprocess.DEVNULL, + ) + .strip() + .splitlines()[0] + ) return f"{driver_name} {version}" except Exception: return driver_name @@ -552,7 +614,9 @@ def _get_intel_gaudi_info(self): try: output = subprocess.check_output( ["modinfo", "habanalabs", "-F", "version"], - encoding="utf-8", timeout=10, stderr=subprocess.DEVNULL, + encoding="utf-8", + timeout=10, + stderr=subprocess.DEVNULL, ).strip() if output: driver = output.splitlines()[0] @@ -564,17 +628,20 @@ def _get_intel_gaudi_info(self): try: output = subprocess.check_output( ["hl-smi", "-Q", "index,name,serial,bus_id", "-f", "csv,noheader"], - encoding="utf-8", timeout=30, + encoding="utf-8", + timeout=30, ).strip() for line in output.splitlines(): parts = [p.strip() for p in line.split(",")] if len(parts) >= 4: - devices.append({ - "index": int(parts[0]), - "product": parts[1], - "serial": parts[2] if parts[2] not in ("N/A", "") else None, - "businfo": parts[3], - }) + devices.append( + { + "index": int(parts[0]), + "product": parts[1], + "serial": parts[2] if parts[2] not in ("N/A", "") else None, + "businfo": parts[3], + } + ) except Exception as e: logger.debug("hl-smi query failed: %s", e) @@ -591,14 +658,16 @@ def _get_local_dimms(self): if isinstance(size_gb, (int, float)): size_gb = int(size_gb) product = dimm.get("product", "Unknown") - items.append({ - "product": f"{product} {size_gb}GB" if size_gb else product, - "vendor": dimm.get("vendor", "Unknown"), - "serial": serial, - "slot": dimm.get("slot", ""), - "size_gb": size_gb, - "description": dimm.get("description", ""), - }) + items.append( + { + "product": f"{product} {size_gb}GB" if size_gb else product, + "vendor": dimm.get("vendor", "Unknown"), + "serial": serial, + "slot": dimm.get("slot", ""), + "size_gb": size_gb, + "description": dimm.get("description", ""), + } + ) return items # Device names that are never physical storage @@ -721,7 +790,9 @@ def _parse_lsblk_storage(self, lsblk_data): if not serial: serial = (nvme_info.get("SerialNumber") or "").strip() or None if not vendor: - vendor = (nvme_info.get("Vendor") or nvme_info.get("Manufacturer") or "").strip() or None + vendor = ( + nvme_info.get("Vendor") or nvme_info.get("Manufacturer") or "" + ).strip() or None if not size_bytes: size_bytes = nvme_info.get("PhysicalSize") or nvme_info.get("UsedBytes") if not rev: @@ -737,16 +808,18 @@ def _parse_lsblk_storage(self, lsblk_data): # Build description from interface + rotational status description = self._build_storage_description(interface, rota) - items.append({ - "product": model or f"Unknown ({name})", - "vendor": vendor or "Unknown", - "serial": serial, - "description": description, - "interface": interface, - "size_bytes": int(size_bytes) if size_bytes else None, - "firmware": rev, - "name": name, - }) + items.append( + { + "product": model or f"Unknown ({name})", + "vendor": vendor or "Unknown", + "serial": serial, + "description": description, + "interface": interface, + "size_bytes": int(size_bytes) if size_bytes else None, + "firmware": rev, + "name": name, + } + ) return items @@ -814,12 +887,14 @@ def _get_local_ssds_lshw_fallback(self): if not vendor and product: vendor = self._guess_vendor(product) - items.append({ - "product": product, - "vendor": vendor or "Unknown", - "serial": serial, - "description": disk.get("description", ""), - }) + items.append( + { + "product": product, + "vendor": vendor or "Unknown", + "serial": serial, + "description": disk.get("description", ""), + } + ) return items @@ -853,18 +928,22 @@ def _get_local_nics(self): if any(gv in vendor_lower for gv in self._GPU_AS_NIC_VENDORS): logger.debug( "Skipping GPU-as-NIC: %s %s (%s)", - vendor, product, iface.get("name", ""), + vendor, + product, + iface.get("name", ""), ) continue seen_macs.add(mac) - items.append({ - "product": product, - "vendor": vendor, - "serial": mac, # MAC as serial proxy - "description": iface.get("description", ""), - "name": iface.get("name", ""), - }) + items.append( + { + "product": product, + "vendor": vendor, + "serial": mac, # MAC as serial proxy + "description": iface.get("description", ""), + "name": iface.get("name", ""), + } + ) return items @@ -873,6 +952,7 @@ def _get_local_psus(self): items = [] try: from netbox_agent import dmidecode + dmi = self.server.dmi # Use numeric type ID 39 because _str2type has " Power Supply" # (with leading space), causing string lookup to fail. @@ -885,12 +965,14 @@ def _get_local_psus(self): serial = None if name in ("Not Specified", "To Be Filled By O.E.M."): name = "Unknown PSU" - items.append({ - "product": f"{manufacturer} {name}".strip(), - "vendor": manufacturer, - "serial": serial, - "description": "Power Supply", - }) + items.append( + { + "product": f"{manufacturer} {name}".strip(), + "vendor": manufacturer, + "serial": serial, + "description": "Power Supply", + } + ) except Exception as e: logger.warning("PSU detection failed: %s", e) @@ -918,10 +1000,10 @@ def _get_local_psus(self): # Model number prefixes → vendor (for models that don't contain the vendor name) _VENDOR_PREFIXES = ( - ("st", "Seagate"), # ST4000NM000A, ST8000NM000A + ("st", "Seagate"), # ST4000NM000A, ST8000NM000A ("wdc ", "Western Digital"), # WDC WD4003FFBX ("wdc_", "Western Digital"), - ("wd", "Western Digital"), # WD4003FFBX + ("wd", "Western Digital"), # WD4003FFBX ) def _guess_vendor(self, product): @@ -1006,7 +1088,9 @@ def _resolve_module_type(self, category, item): create_params["profile"] = profile.id mt = _api_retry(nb.dcim.module_types.create, create_params) - logger.info("Auto-created module type '%s / %s' (profile=%s)", vendor, product, profile_name) + logger.info( + "Auto-created module type '%s / %s' (profile=%s)", vendor, product, profile_name + ) self._module_type_cache[cache_key] = mt return mt @@ -1037,11 +1121,14 @@ def _ensure_module_bays(self, device, category, count): for i in range(count): bay_name = f"{prefix}-{i}" if bay_name not in existing_names: - _api_retry(nb.dcim.module_bays.create, { - "device": device.id, - "name": bay_name, - "position": bay_name, - }) + _api_retry( + nb.dcim.module_bays.create, + { + "device": device.id, + "name": bay_name, + "position": bay_name, + }, + ) logger.info("Created module bay '%s' on device '%s'", bay_name, device.name) # Re-fetch to get the complete list @@ -1091,14 +1178,21 @@ def _find_module_by_serial(self, serial): return None results = list(_api_retry(nb.dcim.modules.filter, serial=serial)) if len(results) > 1: - logger.warning("Duplicate serial '%s' found on %d modules — using first match", serial, len(results)) + logger.warning( + "Duplicate serial '%s' found on %d modules — using first match", + serial, + len(results), + ) return results[0] if results else None def _reparent_module(self, module, target_device, target_bay): """Move a module to a different device and bay.""" logger.info( "Re-parenting module '%s' (serial=%s) → device '%s' bay '%s'", - module, module.serial, target_device.name, target_bay.name, + module, + module.serial, + target_device.name, + target_bay.name, ) module.device = target_device.id module.module_bay = target_bay.id @@ -1133,9 +1227,7 @@ def _move_to_spare(self, module, category): break if not target_bay: - logger.error( - "No free %s bay on spare device — admin must expand spare bays", prefix - ) + logger.error("No free %s bay on spare device — admin must expand spare bays", prefix) return False self._reparent_module(module, spare, target_bay) @@ -1145,7 +1237,9 @@ def _vacate_bay(self, bay, category): """If a bay is occupied, move its occupant to spare.""" modules_in_bay = list(_api_retry(nb.dcim.modules.filter, module_bay_id=bay.id)) for mod in modules_in_bay: - logger.info("Bay '%s' occupied by module (serial=%s) — moving to spare", bay.name, mod.serial) + logger.info( + "Bay '%s' occupied by module (serial=%s) — moving to spare", bay.name, mod.serial + ) self._move_to_spare(mod, category) # ------------------------------------------------------------------ # @@ -1195,7 +1289,9 @@ def _sync_category(self, category, local_items): if not bay: logger.warning( "No bay available at index %d for %s on %s", - idx, prefix, self.device.name, + idx, + prefix, + self.device.name, ) continue @@ -1212,7 +1308,9 @@ def _sync_category(self, category, local_items): # Check bay assignment mod_bay_name = None if mod.module_bay: - mod_bay_name = getattr(mod.module_bay, "name", None) or getattr(mod.module_bay, "display", None) + mod_bay_name = getattr(mod.module_bay, "name", None) or getattr( + mod.module_bay, "display", None + ) if mod_bay_name != bay.name: self._vacate_bay(bay, category) mod.module_bay = bay.id @@ -1221,7 +1319,11 @@ def _sync_category(self, category, local_items): # Check module type mod_mt_id = None if mod.module_type: - mod_mt_id = mod.module_type.id if hasattr(mod.module_type, "id") else mod.module_type + mod_mt_id = ( + mod.module_type.id + if hasattr(mod.module_type, "id") + else mod.module_type + ) if mod_mt_id != module_type.id: mod.module_type = module_type.id updated = True @@ -1241,7 +1343,11 @@ def _sync_category(self, category, local_items): # Update module type if changed mod_mt_id = None if remote_mod.module_type: - mod_mt_id = remote_mod.module_type.id if hasattr(remote_mod.module_type, "id") else remote_mod.module_type + mod_mt_id = ( + remote_mod.module_type.id + if hasattr(remote_mod.module_type, "id") + else remote_mod.module_type + ) if mod_mt_id != module_type.id: remote_mod.module_type = module_type.id _api_retry(remote_mod.save) @@ -1249,18 +1355,24 @@ def _sync_category(self, category, local_items): # Not found anywhere — create new self._vacate_bay(bay, category) - new_mod = _api_retry(nb.dcim.modules.create, { - "device": self.device.id, - "module_bay": bay.id, - "module_type": module_type.id, - "serial": serial, - "status": "active", - "custom_fields": self._default_module_custom_fields(), - }) + new_mod = _api_retry( + nb.dcim.modules.create, + { + "device": self.device.id, + "module_bay": bay.id, + "module_type": module_type.id, + "serial": serial, + "status": "active", + "custom_fields": self._default_module_custom_fields(), + }, + ) matched_module_ids.add(new_mod.id) logger.info( "Created module %s serial=%s on %s bay=%s", - item["product"], serial, self.device.name, bay.name, + item["product"], + serial, + self.device.name, + bay.name, ) else: @@ -1273,23 +1385,32 @@ def _sync_category(self, category, local_items): # Update module type if changed mod_mt_id = None if mod.module_type: - mod_mt_id = mod.module_type.id if hasattr(mod.module_type, "id") else mod.module_type + mod_mt_id = ( + mod.module_type.id + if hasattr(mod.module_type, "id") + else mod.module_type + ) if mod_mt_id != module_type.id: mod.module_type = module_type.id _api_retry(mod.save) logger.info("Updated module type at %s on %s", bay.name, self.device.name) else: - new_mod = _api_retry(nb.dcim.modules.create, { - "device": self.device.id, - "module_bay": bay.id, - "module_type": module_type.id, - "status": "active", - "custom_fields": self._default_module_custom_fields(), - }) + new_mod = _api_retry( + nb.dcim.modules.create, + { + "device": self.device.id, + "module_bay": bay.id, + "module_type": module_type.id, + "status": "active", + "custom_fields": self._default_module_custom_fields(), + }, + ) matched_module_ids.add(new_mod.id) logger.info( "Created module %s (no serial) on %s bay=%s", - item["product"], self.device.name, bay.name, + item["product"], + self.device.name, + bay.name, ) # Step 3: Move unmatched existing modules to spare @@ -1298,7 +1419,8 @@ def _sync_category(self, category, local_items): if mod.id not in matched_module_ids: logger.info( "Module serial=%s no longer detected on %s — moving to spare", - mod.serial, self.device.name, + mod.serial, + self.device.name, ) self._move_to_spare(mod, category) @@ -1320,7 +1442,9 @@ def create_or_update(self, deps=None, state=None): logger.error("Device not found in NetBox — cannot sync modules") return False - logger.info("Starting module sync for device '%s' (id=%d)", self.device.name, self.device.id) + logger.info( + "Starting module sync for device '%s' (id=%d)", self.device.name, self.device.id + ) # Skip PSU detection when dmidecode is unavailable skip_psu = deps is not None and not deps.get("dmidecode", True) diff --git a/netbox_agent/network.py b/netbox_agent/network.py index ba39473b..f77f6338 100644 --- a/netbox_agent/network.py +++ b/netbox_agent/network.py @@ -101,8 +101,9 @@ def _sync_transceiver_module(device_id, interface, ethtool_data): vendor = (ethtool_data.get("transceiver_vendor") or "").strip() part_number = (ethtool_data.get("transceiver_part_number") or "").strip() serial = (ethtool_data.get("transceiver_serial") or "").strip() - form_factor = (ethtool_data.get("transceiver_type") or - ethtool_data.get("form_factor") or "").strip() + form_factor = ( + ethtool_data.get("transceiver_type") or ethtool_data.get("form_factor") or "" + ).strip() # Need at least vendor or part number to create a module type if not vendor and not part_number: @@ -130,13 +131,13 @@ def _sync_transceiver_module(device_id, interface, ethtool_data): # --- Transceiver ModuleType --- module_type = None if part_number: - existing = list(nb.dcim.module_types.filter( - part_number=part_number, manufacturer_id=mfr.id)) + existing = list( + nb.dcim.module_types.filter(part_number=part_number, manufacturer_id=mfr.id) + ) if existing: module_type = existing[0] if not module_type: - existing = list(nb.dcim.module_types.filter( - model=model, manufacturer_id=mfr.id)) + existing = list(nb.dcim.module_types.filter(model=model, manufacturer_id=mfr.id)) if existing: module_type = existing[0] if not module_type: @@ -151,24 +152,27 @@ def _sync_transceiver_module(device_id, interface, ethtool_data): if nic_module: # Each per-port NIC module gets one XCVR child bay xcvr_bay_name = "XCVR-0" - xcvr_bays = list(nb.dcim.module_bays.filter( - module_id=nic_module.id, name=xcvr_bay_name)) + xcvr_bays = list( + nb.dcim.module_bays.filter(module_id=nic_module.id, name=xcvr_bay_name) + ) if xcvr_bays: bay = xcvr_bays[0] else: # NetBox requires device even for module-level bays bay = nb.dcim.module_bays.create( - device=device_id, module=nic_module.id, - name=xcvr_bay_name) - logging.info("Created XCVR bay: %s on NIC module %s (id=%s)", - xcvr_bay_name, nic_module.module_type, nic_module.id) + device=device_id, module=nic_module.id, name=xcvr_bay_name + ) + logging.info( + "Created XCVR bay: %s on NIC module %s (id=%s)", + xcvr_bay_name, + nic_module.module_type, + nic_module.id, + ) else: # Legacy fallback: device-level bay bay_name = "%s-xcvr" % interface.name - bays = list(nb.dcim.module_bays.filter( - device_id=device_id, name=bay_name)) - bay = bays[0] if bays else nb.dcim.module_bays.create( - device=device_id, name=bay_name) + bays = list(nb.dcim.module_bays.filter(device_id=device_id, name=bay_name)) + bay = bays[0] if bays else nb.dcim.module_bays.create(device=device_id, name=bay_name) # --- Transceiver Module --- existing_modules = list(nb.dcim.modules.filter(module_bay_id=bay.id)) @@ -183,14 +187,18 @@ def _sync_transceiver_module(device_id, interface, ethtool_data): dirty = True if dirty: module.save() - logging.info("Updated transceiver: %s %s (SN:%s) on %s", - vendor, model, serial, interface.name) + logging.info( + "Updated transceiver: %s %s (SN:%s) on %s", + vendor, + model, + serial, + interface.name, + ) return # Check by serial — optic may have moved bays if serial: - by_sn = list(nb.dcim.modules.filter( - serial=serial, device_id=device_id)) + by_sn = list(nb.dcim.modules.filter(serial=serial, device_id=device_id)) if by_sn: module = by_sn[0] module.module_bay = bay.id @@ -209,12 +217,16 @@ def _sync_transceiver_module(device_id, interface, ethtool_data): ) logging.info( "Created transceiver: %s %s (SN:%s) on %s", - vendor, model, serial, interface.name, + vendor, + model, + serial, + interface.name, ) except Exception: logging.debug( - "Failed to sync transceiver for %s", interface.name, + "Failed to sync transceiver for %s", + interface.name, exc_info=True, ) @@ -246,8 +258,12 @@ def _build_transceiver_description(ethtool_data): parts.append(vendor_str) # Cable length - for length_key in ("transceiver_length_copper", "transceiver_length_om3", - "transceiver_length_om4", "transceiver_length_smf"): + for length_key in ( + "transceiver_length_copper", + "transceiver_length_om3", + "transceiver_length_om4", + "transceiver_length_smf", + ): length = ethtool_data.get(length_key, "").strip() if length and length != "0m" and length != "0": connector = ethtool_data.get("transceiver_connector", "").strip() @@ -376,7 +392,9 @@ def scan(self): if len(mac) != 17: logging.debug( "Skipping non-Ethernet MAC on %s: %s (%d chars)", - interface, mac, len(mac), + interface, + mac, + len(mac), ) mac = None @@ -784,8 +802,9 @@ def _enrich_existing_ip(self, netbox_ip): netbox_ip.tenant = self.tenant.id dirty = True if dirty: - logging.info("Enriching IP %s: dns_name=%s tenant=%s", - netbox_ip.address, dns, self.tenant) + logging.info( + "Enriching IP %s: dns_name=%s tenant=%s", netbox_ip.address, dns, self.tenant + ) netbox_ip.save() def _enrich_ip(self, netbox_ip, interface): @@ -836,7 +855,8 @@ def create_or_update_netbox_network_cards(self): if managed_by and managed_by != "netbox-agent": logging.debug( "Skipping deletion of '%s' (managed_by=%s)", - nic.name, managed_by, + nic.name, + managed_by, ) continue logging.info( @@ -886,7 +906,8 @@ def batched(it, n): if "oob_ip" in err_str: logging.warning( "oob_ip validation failed during primary_ip4 clear — " - "also clearing oob_ip: %s", e, + "also clearing oob_ip: %s", + e, ) fresh_device.oob_ip = None fresh_device.save() @@ -957,8 +978,11 @@ def batched(it, n): mac_objs = self.update_interface_macs(interface, [nic["mac"]]) # Find the MAC object matching nic["mac"] and set as primary primary_mac_id = None - for mac_obj in (mac_objs or []): - if mac_obj.mac_address and mac_obj.mac_address.upper() == nic["mac"].upper(): + for mac_obj in mac_objs or []: + if ( + mac_obj.mac_address + and mac_obj.mac_address.upper() == nic["mac"].upper() + ): primary_mac_id = mac_obj.id break current_primary = getattr(interface, "primary_mac_address", None) diff --git a/netbox_agent/server.py b/netbox_agent/server.py index 362d12ad..a43412a6 100644 --- a/netbox_agent/server.py +++ b/netbox_agent/server.py @@ -28,8 +28,14 @@ # Base-36 asset tag validation: 4-char alphanumeric _ASSET_TAG_RE = re.compile(r"^[0-9A-Z]{4}$", re.IGNORECASE) _ASSET_TAG_PLACEHOLDERS = { - "Not Specified", "None", "N/A", "To Be Filled By O.E.M.", "", - "Chassis Asset Tag", "Default string", "No Asset Tag", + "Not Specified", + "None", + "N/A", + "To Be Filled By O.E.M.", + "", + "Chassis Asset Tag", + "Default string", + "No Asset Tag", } @@ -235,7 +241,9 @@ def _netbox_create_chassis(self, datacenter, tenant, rack): def _netbox_create_blade(self, chassis, datacenter, tenant, rack): device_role = get_device_role(config.device.blade_role) - device_type = get_device_type(self.get_product_name(), manufacturer=self.get_manufacturer()) + device_type = get_device_type( + self.get_product_name(), manufacturer=self.get_manufacturer() + ) serial = self.get_service_tag() hostname = self.get_hostname() logging.info( @@ -260,7 +268,9 @@ def _netbox_create_blade(self, chassis, datacenter, tenant, rack): def _netbox_create_blade_expansion(self, chassis, datacenter, tenant, rack): device_role = get_device_role(config.device.blade_role) - device_type = get_device_type(self.get_expansion_product(), manufacturer=self.get_manufacturer()) + device_type = get_device_type( + self.get_expansion_product(), manufacturer=self.get_manufacturer() + ) serial = self.get_expansion_service_tag() hostname = self.get_hostname() + " expansion" logging.info( @@ -335,7 +345,7 @@ def _refine_role(self, server): if not current_role: return - role_name = current_role.name if hasattr(current_role, 'name') else str(current_role) + role_name = current_role.name if hasattr(current_role, "name") else str(current_role) if role_name not in self._AUTO_ASSIGNABLE_ROLES: return # Manually set role — don't touch @@ -354,12 +364,15 @@ def _refine_role(self, server): server.save() logging.info( "Refined role for '%s': %s → %s", - server.name, old_role, new_role_name, + server.name, + old_role, + new_role_name, ) else: logging.warning( "Role '%s' not found in NetBox — skipping refinement for '%s'", - new_role_name, server.name, + new_role_name, + server.name, ) # Vendors/keywords for filtering onboard VGA from real GPUs @@ -397,6 +410,7 @@ def _detect_server_type(self) -> str: gpu_count = 0 try: from netbox_agent.lshw import LSHW + lshw = LSHW() gpus = lshw.get_hw_linux("gpu") @@ -429,14 +443,16 @@ def _detect_server_type(self) -> str: try: output = subprocess.check_output( ["lsblk", "-J", "-b", "-d", "-o", "NAME,TYPE,SIZE"], - encoding="utf-8", timeout=10, + encoding="utf-8", + timeout=10, ) data = json.loads(output) - disks = [d for d in data.get("blockdevices", []) - if d.get("type") == "disk" - and not d.get("name", "").startswith( - ("loop", "ram", "zram", "dm-", "md") - )] + disks = [ + d + for d in data.get("blockdevices", []) + if d.get("type") == "disk" + and not d.get("name", "").startswith(("loop", "ram", "zram", "dm-", "md")) + ] disk_count = len(disks) except Exception as e: logging.warning("Storage detection failed during role refinement: %s", e) @@ -453,8 +469,11 @@ def _detect_server_type(self) -> str: _RUNPOD_SERVICES = ("runpod", "safe_runpod", "runpod-worker") # MooseFS services indicate dedicated storage for RunPod _MOOSEFS_SERVICES = ( - "moosefs-chunkserver", "moosefs-master", "moosefs-metalogger", - "mfschunkserver", "mfsmaster", + "moosefs-chunkserver", + "moosefs-master", + "moosefs-metalogger", + "mfschunkserver", + "mfsmaster", ) def _detect_tenant(self) -> str: @@ -468,7 +487,9 @@ def _detect_tenant(self) -> str: try: result = subprocess.run( ["systemctl", "is-active", svc], - capture_output=True, encoding="utf-8", timeout=5, + capture_output=True, + encoding="utf-8", + timeout=5, ) if result.stdout.strip() == "active": logging.debug("Tenant detection: service '%s' is active → runpod", svc) @@ -488,7 +509,8 @@ def _sync_tenant(self, server): if not nb_tenant: logging.warning( "Tenant '%s' not found in NetBox — skipping tenant sync for '%s'", - tenant_slug, server.name, + tenant_slug, + server.name, ) return @@ -499,12 +521,16 @@ def _sync_tenant(self, server): server.save() logging.info( "Tenant for '%s': %s → %s", - server.name, old_name, nb_tenant.name, + server.name, + old_name, + nb_tenant.name, ) def _netbox_create_server(self, datacenter, tenant, rack): device_role = get_device_role(config.device.server_role) - device_type = get_device_type(self.get_product_name(), manufacturer=self.get_manufacturer()) + device_type = get_device_type( + self.get_product_name(), manufacturer=self.get_manufacturer() + ) if not device_type: raise Exception('Chassis "{}" doesn\'t exist'.format(self.get_chassis())) serial = self.get_service_tag() @@ -604,7 +630,9 @@ def get_asset_tag(self): try: output = subprocess.check_output( ["ipmitool", "fru", "print", "0"], - encoding="utf-8", timeout=10, stderr=subprocess.DEVNULL, + encoding="utf-8", + timeout=10, + stderr=subprocess.DEVNULL, ) for line in output.splitlines(): if "Product Asset Tag" in line and ":" in line: @@ -661,7 +689,9 @@ def get_netbox_server(self, expansion=False): if devices: logging.info( "Matched device by BMC MAC %s → %s (id=%s)", - bmc_mac, devices[0].name, devices[0].id, + bmc_mac, + devices[0].name, + devices[0].id, ) return devices[0] logging.debug("No device found with bmc_mac=%s", bmc_mac) @@ -858,6 +888,7 @@ def netbox_create_or_update(self, config, deps=None, network_only=False, state=N ) if update_modules: from netbox_agent.modules import ModuleManager + self.module_manager = ModuleManager(server=self, config=config) self.module_manager.create_or_update(deps=deps, state=state) # update psu @@ -874,12 +905,13 @@ def netbox_create_or_update(self, config, deps=None, network_only=False, state=N cluster = nb.virtualization.clusters.get(name=cluster_name) if cluster: nb_server = self.get_netbox_server() - if nb_server and getattr(nb_server, 'cluster', None) != cluster: + if nb_server and getattr(nb_server, "cluster", None) != cluster: nb_server.cluster = cluster.id nb_server.save() logging.info( "Auto-assigned Proxmox host '%s' to cluster '%s'", - nb_server.name, cluster_name, + nb_server.name, + cluster_name, ) else: logging.warning( @@ -923,7 +955,9 @@ def netbox_create_or_update(self, config, deps=None, network_only=False, state=N if server.serial != local_serial: logging.info( "Updating serial on '%s': %s -> %s", - server.name, server.serial, local_serial or "(empty)", + server.name, + server.serial, + local_serial or "(empty)", ) server.serial = local_serial update += 1 @@ -967,7 +1001,8 @@ def netbox_create_or_update(self, config, deps=None, network_only=False, state=N if current_status_value in _ACTIVATABLE_STATUSES: logging.info( "Transitioning device '%s' status: %s → active", - server.name, current_status_value, + server.name, + current_status_value, ) server.status = "active" update += 1 @@ -1023,7 +1058,11 @@ def netbox_create_or_update(self, config, deps=None, network_only=False, state=N # Set oob_ip to the IPMI interface IP if not oob_update: for ip in myips: - if ip.assigned_object and ip.assigned_object.display == "IPMI" and ip != server.oob_ip: + if ( + ip.assigned_object + and ip.assigned_object.display == "IPMI" + and ip != server.oob_ip + ): server.oob_ip = ip.id oob_update = True break @@ -1033,12 +1072,15 @@ def netbox_create_or_update(self, config, deps=None, network_only=False, state=N server.save() logging.info( "Saved oob_ip for device %s (id=%s)", - server.name, server.id, + server.name, + server.id, ) except Exception as e: logging.error( "Failed to save oob_ip for device %s (id=%s): %s", - server.name, server.id, e, + server.name, + server.id, + e, ) # --- Primary IPv4 assignment --- saved separately to avoid atomic failure --- @@ -1075,24 +1117,43 @@ def netbox_create_or_update(self, config, deps=None, network_only=False, state=N server.save() logging.info( "Saved primary_ip4 for device %s (id=%s)", - server.name, server.id, + server.name, + server.id, ) except Exception as e: logging.error( "Failed to save primary_ip4 for device %s (id=%s): %s", - server.name, server.id, e, + server.name, + server.id, + e, ) logging.debug("Finished updating Server!") # DMI placeholder values that should be treated as "no serial" _DMI_PLACEHOLDERS = { - "", "none", "n/a", "na", "not specified", "not available", - "not applicable", "to be filled by o.e.m.", "default string", - "0123456789", "..................", "system serial number", - "chassis serial number", "base board serial number", - "default", "unknown", "unspecified", "no asset information", - "empty", "xxxxxxxxxxxx", "0000000000", "____________", + "", + "none", + "n/a", + "na", + "not specified", + "not available", + "not applicable", + "to be filled by o.e.m.", + "default string", + "0123456789", + "..................", + "system serial number", + "chassis serial number", + "base board serial number", + "default", + "unknown", + "unspecified", + "no asset information", + "empty", + "xxxxxxxxxxxx", + "0000000000", + "____________", } def _is_valid_serial(self, value): @@ -1133,8 +1194,14 @@ def _get_chassis_serial(self): Distinct from the system serial (service tag) on many servers. """ _PLACEHOLDERS = { - "", "none", "n/a", "not specified", "not available", - "to be filled by o.e.m.", "default string", "0123456789", + "", + "none", + "n/a", + "not specified", + "not available", + "to be filled by o.e.m.", + "default string", + "0123456789", "..................", } try: @@ -1150,6 +1217,7 @@ def _get_bmc_mac(self): """Return the BMC MAC address from IPMI, if available.""" try: from netbox_agent.ipmi import IPMI + ipmi_data = IPMI().parse() if ipmi_data and ipmi_data.get("mac"): return ipmi_data["mac"].upper() diff --git a/netbox_agent/state.py b/netbox_agent/state.py index 45e5b799..1edf166f 100644 --- a/netbox_agent/state.py +++ b/netbox_agent/state.py @@ -63,9 +63,7 @@ def save(self, hostname, hardware, network=None, dependencies=None): os.makedirs(self.state_dir, exist_ok=True) # Atomic write: write to temp file, then rename - fd, tmp_path = tempfile.mkstemp( - dir=self.state_dir, prefix=".state_", suffix=".tmp" - ) + fd, tmp_path = tempfile.mkstemp(dir=self.state_dir, prefix=".state_", suffix=".tmp") try: with os.fdopen(fd, "w") as f: json.dump(state, f, indent=2, sort_keys=True) diff --git a/netbox_agent/vendors/supermicro.py b/netbox_agent/vendors/supermicro.py index ecccc20b..bf2b3e61 100644 --- a/netbox_agent/vendors/supermicro.py +++ b/netbox_agent/vendors/supermicro.py @@ -128,9 +128,7 @@ def get_power_consumption(self): psu_amps[psu_num] = value if psu_amps: - logging.debug( - "SuperMicro power: found current sensors for %d PSU(s)", len(psu_amps) - ) + logging.debug("SuperMicro power: found current sensors for %d PSU(s)", len(psu_amps)) return [str(psu_amps[k]) for k in sorted(psu_amps)] # Strategy 2: Compute amps from per-PSU power (Watts) / voltage (Volts) @@ -153,9 +151,7 @@ def get_power_consumption(self): volts = psu_volts.get(psu_num, self._DEFAULT_VOLTAGE) amps = watts / volts if volts > 0 else 0 result.append(str(round(amps, 3))) - logging.debug( - "SuperMicro power: computed amps from watts for %d PSU(s)", len(result) - ) + logging.debug("SuperMicro power: computed amps from watts for %d PSU(s)", len(result)) return result # Strategy 3: DCMI total system power, split across present PSUs @@ -193,7 +189,10 @@ def _get_power_from_dcmi(self): logging.debug( "SuperMicro power: DCMI total %dW / %d PSU(s) = %.1fW each (%.3fA @ %dV)", - total_watts, psu_count, per_psu_watts, per_psu_amps, + total_watts, + psu_count, + per_psu_watts, + per_psu_amps, int(self._DEFAULT_VOLTAGE), ) return [str(round(per_psu_amps, 3))] * psu_count @@ -202,10 +201,7 @@ def _count_present_psus(self): """Count PSUs reporting 'Presence detected' in SDR.""" try: sdr_out = subprocess.getoutput('ipmitool sdr type "Power Supply"') - count = sum( - 1 for line in sdr_out.splitlines() - if "Presence detected" in line - ) + count = sum(1 for line in sdr_out.splitlines() if "Presence detected" in line) return count if count > 0 else 1 except Exception: return 1 diff --git a/scripts/arp_discover.py b/scripts/arp_discover.py index d04509f5..ae9c24f9 100644 --- a/scripts/arp_discover.py +++ b/scripts/arp_discover.py @@ -46,19 +46,19 @@ # --------------------------------------------------------------------------- SUBNETS = { "10.100.200.0/24": { - "pivot": "10.100.200.44", # hickory09 + "pivot": "10.100.200.44", # hickory09 "name": "mgmt-200 (storage/compute)", }, "10.100.208.0/24": { - "pivot": "10.100.208.40", # anaheim14 + "pivot": "10.100.208.40", # anaheim14 "name": "mgmt-208 (GPU servers)", }, "10.100.10.0/24": { - "pivot": "10.100.10.56", # tyan-milan-1 + "pivot": "10.100.10.56", # tyan-milan-1 "name": "mgmt-10 (infrastructure)", }, "192.168.211.0/24": { - "pivot": "192.168.211.36", # bell01 (via jump host) + "pivot": "192.168.211.36", # bell01 (via jump host) "name": "mgmt-211 (legacy GPU)", "jump": "fgpu@10.100.191.46", }, @@ -71,11 +71,16 @@ ] SSH_USER = "fgpu" SSH_OPTS = [ - "-o", "BatchMode=yes", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "ConnectTimeout=10", - "-o", "LogLevel=ERROR", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "ConnectTimeout=10", + "-o", + "LogLevel=ERROR", ] # NetBox @@ -140,9 +145,7 @@ def arp_scan_subnet(subnet_cidr, pivot_host, jump=None): cmd = _ssh_cmd(pivot_host, remote_script, jump=jump) try: - result = subprocess.run( - cmd, capture_output=True, text=True, timeout=120 - ) + result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) if result.returncode != 0: logger.error("SSH to %s failed: %s", pivot_host, result.stderr.strip()) return [] @@ -176,6 +179,7 @@ def load_netbox_data(): try: import pynetbox import urllib3 + urllib3.disable_warnings() except ImportError: logger.warning("pynetbox not installed — skipping NetBox cross-reference") @@ -184,14 +188,12 @@ def load_netbox_data(): token = NETBOX_TOKEN if not token: # Try to read from deploy script - deploy_script = os.path.join( - os.path.dirname(__file__), "deploy_and_run.sh" - ) + deploy_script = os.path.join(os.path.dirname(__file__), "deploy_and_run.sh") if os.path.exists(deploy_script): with open(deploy_script) as f: for line in f: if line.startswith("NETBOX_TOKEN="): - token = line.split("=", 1)[1].strip().strip('"\'') + token = line.split("=", 1)[1].strip().strip("\"'") break if not token: @@ -205,7 +207,7 @@ def load_netbox_data(): # Build MAC → device mapping from interfaces mac_to_device = {} # MAC → {device_name, device_id, iface_name, primary_ip} - device_info = {} # device_id → {name, serial, primary_ip, bmc_mac} + device_info = {} # device_id → {name, serial, primary_ip, bmc_mac} devices = list(nb.dcim.devices.filter(status="active", role_id=2)) for dev in devices: @@ -236,7 +238,11 @@ def load_netbox_data(): def identify_host(ip, jump=None): """Try to SSH into a discovered IP and get hostname + serial.""" - cmd = _ssh_cmd(ip, "hostname -f 2>/dev/null; sudo dmidecode -s system-serial-number 2>/dev/null || echo UNKNOWN", jump=jump) + cmd = _ssh_cmd( + ip, + "hostname -f 2>/dev/null; sudo dmidecode -s system-serial-number 2>/dev/null || echo UNKNOWN", + jump=jump, + ) try: result = subprocess.run(cmd, capture_output=True, text=True, timeout=15) if result.returncode == 0: @@ -250,23 +256,25 @@ def identify_host(ip, jump=None): def main(): - parser = argparse.ArgumentParser( - description="ARP-based host discovery for NetBox population" - ) + parser = argparse.ArgumentParser(description="ARP-based host discovery for NetBox population") parser.add_argument( - "--subnet", "-s", + "--subnet", + "-s", help="Specific subnet to scan (e.g., 10.100.200.0/24). Default: all configured.", ) parser.add_argument( - "--csv", action="store_true", + "--csv", + action="store_true", help="Output results as CSV", ) parser.add_argument( - "--identify", action="store_true", + "--identify", + action="store_true", help="Try SSH into unknown hosts to get hostname/serial", ) parser.add_argument( - "--json", action="store_true", + "--json", + action="store_true", help="Output results as JSON", ) args = parser.parse_args() @@ -345,8 +353,16 @@ def main(): if args.csv: writer = csv.DictWriter( sys.stdout, - fieldnames=["subnet", "ip", "mac", "nb_match", "nb_device", "nb_device_ip", - "ssh_hostname", "ssh_serial"], + fieldnames=[ + "subnet", + "ip", + "mac", + "nb_match", + "nb_device", + "nb_device_ip", + "ssh_hostname", + "ssh_serial", + ], extrasaction="ignore", ) writer.writeheader() @@ -358,9 +374,11 @@ def main(): known = [r for r in all_results if r.get("nb_match")] unknown = [r for r in all_results if not r.get("nb_match")] - print(f"\n{'='*110}") - print(f"ARP Discovery Results — {len(all_results)} hosts found across {len(targets)} subnet(s)") - print(f"{'='*110}") + print(f"\n{'=' * 110}") + print( + f"ARP Discovery Results — {len(all_results)} hosts found across {len(targets)} subnet(s)" + ) + print(f"{'=' * 110}") if known: print(f"\n--- KNOWN DEVICES ({len(known)}) — matched in NetBox ---") @@ -373,8 +391,10 @@ def main(): ip_status = " ← MISSING IN NB" elif nb_ip != r["ip"]: ip_status = f" ← MISMATCH (ARP={r['ip']})" - print(f" {r['ip']:<16} {r['mac']:<19} {r.get('nb_match',''):<10} " - f"{r.get('nb_device',''):<45} {nb_ip}{ip_status}") + print( + f" {r['ip']:<16} {r['mac']:<19} {r.get('nb_match', ''):<10} " + f"{r.get('nb_device', ''):<45} {nb_ip}{ip_status}" + ) if unknown: print(f"\n--- UNKNOWN DEVICES ({len(unknown)}) — NOT in NetBox ---") @@ -387,14 +407,14 @@ def main(): # Summary missing_ip = [r for r in known if not r.get("nb_device_ip")] - print(f"\n{'='*110}") - print(f"Summary:") + print(f"\n{'=' * 110}") + print("Summary:") print(f" Total discovered: {len(all_results)}") print(f" Matched in NetBox: {len(known)}") print(f" Unknown (not in NetBox): {len(unknown)}") if missing_ip: print(f" Known but MISSING mgmt IP: {len(missing_ip)} ← can be populated now") - print(f"{'='*110}") + print(f"{'=' * 110}") if __name__ == "__main__": diff --git a/scripts/fixtures/collect_hardware_fixture.py b/scripts/fixtures/collect_hardware_fixture.py index 37a73def..695fbe2e 100644 --- a/scripts/fixtures/collect_hardware_fixture.py +++ b/scripts/fixtures/collect_hardware_fixture.py @@ -85,7 +85,9 @@ def collect_lsblk(): if err: # Some kernels don't support all columns — try minimal set logger.warning("lsblk full columns failed (%s), trying minimal...", err) - data, raw, err = _run_json(["lsblk", "-J", "-b", "-o", "NAME,TYPE,SIZE,MODEL,SERIAL,VENDOR,TRAN,ROTA"]) + data, raw, err = _run_json( + ["lsblk", "-J", "-b", "-o", "NAME,TYPE,SIZE,MODEL,SERIAL,VENDOR,TRAN,ROTA"] + ) if err: return {"available": True, "error": err, "raw": raw[:5000]} return {"available": True, "data": data} @@ -153,11 +155,13 @@ def collect_nvidia_smi(): # Query GPU details query_fields = "index,name,serial,uuid,pci.bus_id,memory.total,driver_version,power.limit" - stdout, stderr, rc = _run([ - "nvidia-smi", - f"--query-gpu={query_fields}", - "--format=csv,noheader", - ]) + stdout, stderr, rc = _run( + [ + "nvidia-smi", + f"--query-gpu={query_fields}", + "--format=csv,noheader", + ] + ) if rc == 0: results["query_csv"] = stdout else: @@ -264,7 +268,8 @@ def main(): description="Collect hardware fixture data from a real server" ) parser.add_argument( - "--output", "-o", + "--output", + "-o", default=None, help="Output file path (default: _fixture.json)", ) @@ -319,9 +324,9 @@ def main(): logger.info("Transfer this file to tests/fixtures/ in the netbox-agent repo") # Print summary - print(f"\n{'='*60}") + print(f"\n{'=' * 60}") print(f"Fixture Summary: {hostname}") - print(f"{'='*60}") + print(f"{'=' * 60}") for name, data in fixture.items(): if name == "system": continue @@ -341,7 +346,7 @@ def main(): elif "lscpu" in d: detail = f" — {len(d['lscpu'])} fields" print(f" {status} {name}{detail}") - print(f"{'='*60}") + print(f"{'=' * 60}") return 0 diff --git a/scripts/infra/compute/sync_asset_tags.py b/scripts/infra/compute/sync_asset_tags.py index c7ba81e4..035f5fbc 100755 --- a/scripts/infra/compute/sync_asset_tags.py +++ b/scripts/infra/compute/sync_asset_tags.py @@ -44,6 +44,7 @@ # ── Helpers ────────────────────────────────────────────────────────────────── + def normalize_mac(mac: str) -> str: """Strip separators and lowercase: '3C:EC:EF:33:8C:EC' → '3cecef338cec'.""" return re.sub(r"[:.\-]", "", mac).lower().strip() @@ -51,6 +52,7 @@ def normalize_mac(mac: str) -> str: # ── Phase 1: Load CSV ──────────────────────────────────────────────────────── + def load_hostnames_csv(path): """ Read hostnames.csv and return (entries, skip_count). @@ -106,21 +108,24 @@ def load_hostnames_csv(path): skipped += 1 continue - entries.append({ - "row": row_num, - "hostname": hostname, - "full_hostname": row[2].strip(), - "asset_tag": asset_tag, - "serial": serial, - "macs": macs, - "hostname_variants": hostname_variants, - }) + entries.append( + { + "row": row_num, + "hostname": hostname, + "full_hostname": row[2].strip(), + "asset_tag": asset_tag, + "serial": serial, + "macs": macs, + "hostname_variants": hostname_variants, + } + ) return entries, skipped # ── Phase 2: Match to NetBox ───────────────────────────────────────────────── + def load_netbox_devices(nb): """Load all devices from target roles with their MACs, serial, and name.""" devices = [] @@ -136,14 +141,16 @@ def load_netbox_devices(nb): if iface.mac_address: dev_macs.add(normalize_mac(str(iface.mac_address))) - devices.append({ - "id": dev.id, - "name": dev.name or "", - "serial": dev.serial or "", - "asset_tag": dev.asset_tag or "", - "owner": (dev.custom_fields or {}).get("owner", ""), - "macs": dev_macs, - }) + devices.append( + { + "id": dev.id, + "name": dev.name or "", + "serial": dev.serial or "", + "asset_tag": dev.asset_tag or "", + "owner": (dev.custom_fields or {}).get("owner", ""), + "macs": dev_macs, + } + ) return devices @@ -221,9 +228,12 @@ def match_csv_to_netbox(csv_entries, mac_index, serial_index, name_index): logger.warning( "CONFLICT: CSV '%s' (tag=%s) matches NB device '%s' (id=%d) " "already claimed by CSV '%s' (tag=%s) — skipping", - label, entry["asset_tag"], - matched_dev["name"], dev_id, - prior_label, prior["asset_tag"], + label, + entry["asset_tag"], + matched_dev["name"], + dev_id, + prior_label, + prior["asset_tag"], ) unmatched.append(entry) continue @@ -241,6 +251,7 @@ def match_csv_to_netbox(csv_entries, mac_index, serial_index, name_index): # ── Phase 3: NetBox update ─────────────────────────────────────────────────── + def _get_api_session(): """ Build a requests.Session for direct NetBox API calls. @@ -260,11 +271,13 @@ def _get_api_session(): session = requests.Session() session.verify = ssl_verify - session.headers.update({ - "Authorization": f"Token {token}", - "Content-Type": "application/json", - "Accept": "application/json", - }) + session.headers.update( + { + "Authorization": f"Token {token}", + "Content-Type": "application/json", + "Accept": "application/json", + } + ) return session, url @@ -299,9 +312,9 @@ def update_device_asset_tag(device_id, device_name, asset_tag, owner, dry_run): def run_netbox_updates(matches, dry_run): """Phase 3: Update asset_tags in NetBox for matched entries.""" - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("Phase 3: NetBox asset tag updates") - print(f"{'='*80}\n") + print(f"{'=' * 80}\n") successes = 0 failures = 0 @@ -316,12 +329,15 @@ def run_netbox_updates(matches, dry_run): if current_tag == csv_tag: already_set += 1 - print(f" [SKIP] {label:<45} tag={csv_tag:<6} " - f"NB='{nb_name}' — Already set") + print(f" [SKIP] {label:<45} tag={csv_tag:<6} NB='{nb_name}' — Already set") continue ok, msg = update_device_asset_tag( - nb_dev["id"], nb_name, csv_tag, nb_dev["owner"], dry_run, + nb_dev["id"], + nb_name, + csv_tag, + nb_dev["owner"], + dry_run, ) if ok: successes += 1 @@ -331,22 +347,26 @@ def run_netbox_updates(matches, dry_run): status = "FAIL" current_str = f"'{current_tag}'" if current_tag else "none" - print(f" [{status}] {label:<45} tag={csv_tag:<6} " - f"NB='{nb_name}' (was {current_str}) {msg}") + print( + f" [{status}] {label:<45} tag={csv_tag:<6} NB='{nb_name}' (was {current_str}) {msg}" + ) total = successes + failures + already_set - print(f"\nResults: {successes} updated, {already_set} already set, " - f"{failures} failed (of {total} matched)") + print( + f"\nResults: {successes} updated, {already_set} already set, " + f"{failures} failed (of {total} matched)" + ) return failures == 0 # ── Phase 4: Post-update verification ──────────────────────────────────────── + def run_post_verification(nb, matches): """Re-read each updated device by ID and confirm asset_tag matches.""" - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("Phase 4: Post-update verification") - print(f"{'='*80}\n") + print(f"{'=' * 80}\n") ok_count = 0 fail_count = 0 @@ -370,13 +390,13 @@ def run_post_verification(nb, matches): print(f" FAIL {label:<45} expected='{expected_tag}' got='{actual_tag}'") fail_count += 1 - print(f"\nVerification: {ok_count} OK, {fail_count} failed " - f"(of {len(matches)} total)") + print(f"\nVerification: {ok_count} OK, {fail_count} failed (of {len(matches)} total)") return fail_count == 0 # ── Main ───────────────────────────────────────────────────────────────────── + def main(): parser = argparse.ArgumentParser( description="Sync compute asset tags from CSV into NetBox.", @@ -388,23 +408,30 @@ def main(): %(prog)s --filter potato --dry-run # preview potato nodes only %(prog)s --filter potato # program potato nodes only %(prog)s --hostnames-csv /custom.csv # custom CSV path -""") - parser.add_argument("--dry-run", action="store_true", - help="show what would change, don't write to NetBox") - parser.add_argument("--filter", - help="only process entries whose friendly name, hostname, " - "or full hostname contains this substring (case-insensitive)") - parser.add_argument("--hostnames-csv", default=HOSTNAMES_CSV, - help=f"path to hostnames.csv (default: {HOSTNAMES_CSV})") +""", + ) + parser.add_argument( + "--dry-run", action="store_true", help="show what would change, don't write to NetBox" + ) + parser.add_argument( + "--filter", + help="only process entries whose friendly name, hostname, " + "or full hostname contains this substring (case-insensitive)", + ) + parser.add_argument( + "--hostnames-csv", + default=HOSTNAMES_CSV, + help=f"path to hostnames.csv (default: {HOSTNAMES_CSV})", + ) args = parser.parse_args() if args.dry_run: print("=== DRY RUN — no changes will be made ===\n") # ── Phase 1 ────────────────────────────────────────────────────────── - print(f"{'='*80}") + print(f"{'=' * 80}") print("Phase 1: Loading CSV data") - print(f"{'='*80}") + print(f"{'=' * 80}") print(f" hostnames.csv: {args.hostnames_csv}") csv_entries, skip_count = load_hostnames_csv(args.hostnames_csv) @@ -416,7 +443,8 @@ def main(): if args.filter: pattern = args.filter.lower() csv_entries = [ - e for e in csv_entries + e + for e in csv_entries if pattern in (e["hostname"] or "").lower() or pattern in (e["full_hostname"] or "").lower() or any(pattern in v for v in e["hostname_variants"]) @@ -428,9 +456,9 @@ def main(): return # ── Phase 2 ────────────────────────────────────────────────────────── - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("Phase 2: Matching CSV entries to NetBox devices") - print(f"{'='*80}") + print(f"{'=' * 80}") nb = get_api() print(f"\n Loading devices from roles: {', '.join(DEVICE_ROLES)}...") @@ -438,11 +466,11 @@ def main(): print(f" Loaded {len(nb_devices)} NetBox devices") mac_index, serial_index, name_index = build_match_indexes(nb_devices) - print(f" Indexes: {len(mac_index)} MACs, {len(serial_index)} serials, " - f"{len(name_index)} names") + print( + f" Indexes: {len(mac_index)} MACs, {len(serial_index)} serials, {len(name_index)} names" + ) - matches, unmatched = match_csv_to_netbox(csv_entries, mac_index, - serial_index, name_index) + matches, unmatched = match_csv_to_netbox(csv_entries, mac_index, serial_index, name_index) # Report match methods method_counts = {} @@ -460,8 +488,7 @@ def main(): for entry in unmatched: label = entry["hostname"] or entry["full_hostname"] macs_str = ", ".join(entry["macs"][:1]) if entry["macs"] else "none" - print(f" {label:<45} tag={entry['asset_tag']:<6} " - f"mac={macs_str}") + print(f" {label:<45} tag={entry['asset_tag']:<6} mac={macs_str}") if not matches: print("\nNo matches found. Nothing to update.") diff --git a/scripts/infra/credentials/push_to_vaults.py b/scripts/infra/credentials/push_to_vaults.py index a0b2ad23..8b5df0cb 100644 --- a/scripts/infra/credentials/push_to_vaults.py +++ b/scripts/infra/credentials/push_to_vaults.py @@ -78,7 +78,8 @@ def create_login_item(vault, title, username, password, dry_run=False, update=Fa # Check if item exists and delete it result = subprocess.run( ["op", "item", "get", title, f"--vault={vault}", "--format=json"], - capture_output=True, text=True, + capture_output=True, + text=True, ) if result.returncode == 0: try: @@ -93,14 +94,17 @@ def create_login_item(vault, title, username, password, dry_run=False, update=Fa result = subprocess.run( [ - "op", "item", "create", + "op", + "item", + "create", "--category=Login", f"--vault={vault}", f"--title={title}", f"username={username}", f"password={password}", ], - capture_output=True, text=True, + capture_output=True, + text=True, ) if result.returncode == 0: @@ -118,10 +122,8 @@ def main(): parser = argparse.ArgumentParser( description="Push credentials to 1Password vaults (individual items per device)", ) - parser.add_argument("--dry-run", action="store_true", - help="Preview without creating items") - parser.add_argument("--update", action="store_true", - help="Delete and recreate existing items") + parser.add_argument("--dry-run", action="store_true", help="Preview without creating items") + parser.add_argument("--update", action="store_true", help="Delete and recreate existing items") parser.add_argument("--fgpu-vault", default=VAULT_FGPU) parser.add_argument("--oem-vault", default=VAULT_OEM) args = parser.parse_args() @@ -130,7 +132,8 @@ def main(): # Verify signed in and vaults exist result = subprocess.run( ["op", "vault", "list", "--format=json"], - capture_output=True, text=True, + capture_output=True, + text=True, ) if result.returncode != 0: print("ERROR: Not signed in to 1Password. Run: eval $(op signin)") @@ -148,12 +151,14 @@ def main(): switches = load_csv_credentials(SWITCHES_CSV) all_creds = pdu + compute + machines + switches - print(f"Loaded: {len(pdu)} PDU + {len(compute)} compute + {len(machines)} machines OEM + {len(switches)} switch = {len(all_creds)} total") + print( + f"Loaded: {len(pdu)} PDU + {len(compute)} compute + {len(machines)} machines OEM + {len(switches)} switch = {len(all_creds)} total" + ) # Classify and deduplicate: one entry per (tag, vault) # If same tag has multiple creds for same vault, keep first fgpu_items = {} # tag -> (user, pw) - oem_items = {} # tag -> (user, pw) + oem_items = {} # tag -> (user, pw) for tag, user, pw in all_creds: if is_default(user): @@ -163,12 +168,12 @@ def main(): if tag not in fgpu_items: fgpu_items[tag] = (user, pw) - print(f"\nItems to create:") + print("\nItems to create:") print(f" {args.fgpu_vault}: {len(fgpu_items)} devices") print(f" {args.oem_vault}: {len(oem_items)} devices") if args.dry_run: - print(f"\n=== DRY RUN ===\n") + print("\n=== DRY RUN ===\n") # Create OEM items print(f"\n── {args.oem_vault} ({len(oem_items)} items) ──") @@ -176,8 +181,9 @@ def main(): oem_fail = 0 for tag in sorted(oem_items): user, pw = oem_items[tag] - if create_login_item(args.oem_vault, tag, user, pw, - dry_run=args.dry_run, update=args.update): + if create_login_item( + args.oem_vault, tag, user, pw, dry_run=args.dry_run, update=args.update + ): oem_ok += 1 else: oem_fail += 1 @@ -188,15 +194,16 @@ def main(): fgpu_fail = 0 for tag in sorted(fgpu_items): user, pw = fgpu_items[tag] - if create_login_item(args.fgpu_vault, tag, user, pw, - dry_run=args.dry_run, update=args.update): + if create_login_item( + args.fgpu_vault, tag, user, pw, dry_run=args.dry_run, update=args.update + ): fgpu_ok += 1 else: fgpu_fail += 1 - print(f"\n{'='*50}") + print(f"\n{'=' * 50}") if args.dry_run: - print(f"DRY RUN complete.") + print("DRY RUN complete.") else: print(f"{args.oem_vault}: {oem_ok} created, {oem_fail} failed") print(f"{args.fgpu_vault}: {fgpu_ok} created, {fgpu_fail} failed") diff --git a/scripts/infra/credentials/sync_to_1password.py b/scripts/infra/credentials/sync_to_1password.py index e7b37a2d..b0c5810e 100644 --- a/scripts/infra/credentials/sync_to_1password.py +++ b/scripts/infra/credentials/sync_to_1password.py @@ -57,12 +57,12 @@ # Item title template TITLES = { - ("pdu", "default"): "PDU — Default Credentials", - ("pdu", "farmgpu"): "PDU — FarmGPU Credentials", - ("compute-bmc", "default"): "Compute BMC — Default Credentials", - ("compute-bmc", "farmgpu"): "Compute BMC — FarmGPU Credentials", - ("switches", "default"): "Switches — Default Credentials", - ("switches", "farmgpu"): "Switches — FarmGPU Credentials", + ("pdu", "default"): "PDU — Default Credentials", + ("pdu", "farmgpu"): "PDU — FarmGPU Credentials", + ("compute-bmc", "default"): "Compute BMC — Default Credentials", + ("compute-bmc", "farmgpu"): "Compute BMC — FarmGPU Credentials", + ("switches", "default"): "Switches — Default Credentials", + ("switches", "farmgpu"): "Switches — FarmGPU Credentials", } @@ -70,6 +70,7 @@ # Each loader returns list of (asset_tag, user, password) tuples. # Only entries WITH an asset_tag are included. + def load_pdu_credentials(path): """Load PDU credentials for entries with asset tags.""" entries = [] @@ -169,6 +170,7 @@ def classify_credentials(entries): # ── 1Password item generation ──────────────────────────────────────────────── + def escape_op_field(s): """Escape periods, equals signs, and backslashes for op field names.""" s = s.replace("\\", "\\\\") @@ -248,7 +250,9 @@ def create_or_update_item(title, vault, domain, cred_type, entries, dry_run=Fals # Build op command cmd = [ - "op", "item", "create", + "op", + "item", + "create", "--category=Secure Note", f"--vault={vault}", f"--title={title}", @@ -262,8 +266,9 @@ def create_or_update_item(title, vault, domain, cred_type, entries, dry_run=Fals if result.returncode == 0: item_id = json.loads(result.stdout).get("id", "") if item_id: - subprocess.run(["op", "item", "delete", item_id, f"--vault={vault}"], - capture_output=True) + subprocess.run( + ["op", "item", "delete", item_id, f"--vault={vault}"], capture_output=True + ) print(f" Deleted existing item {item_id} for recreation.") result = subprocess.run(cmd, capture_output=True, text=True) @@ -279,6 +284,7 @@ def create_or_update_item(title, vault, domain, cred_type, entries, dry_run=Fals # ── Main ───────────────────────────────────────────────────────────────────── + def main(): parser = argparse.ArgumentParser( description="Sync infrastructure credentials to 1Password", @@ -288,17 +294,22 @@ def main(): %(prog)s --dry-run # preview (no op signin needed) %(prog)s --vault=Employee # create items %(prog)s --vault=Employee --update # replace existing items -""") - parser.add_argument("--dry-run", action="store_true", - help="preview items without creating them") - parser.add_argument("--vault", default=VAULT_DEFAULT, - help=f"1Password vault name (default: {VAULT_DEFAULT})") - parser.add_argument("--update", action="store_true", - help="delete and recreate existing items") +""", + ) + parser.add_argument( + "--dry-run", action="store_true", help="preview items without creating them" + ) + parser.add_argument( + "--vault", default=VAULT_DEFAULT, help=f"1Password vault name (default: {VAULT_DEFAULT})" + ) + parser.add_argument("--update", action="store_true", help="delete and recreate existing items") parser.add_argument("--pdu-csv", default=PDU_CSV) parser.add_argument("--compute-csv", default=COMPUTE_CSV) - parser.add_argument("--machines-csv", default=MACHINES_CSV, - help="OEM BMC credentials from machines_validated.csv") + parser.add_argument( + "--machines-csv", + default=MACHINES_CSV, + help="OEM BMC credentials from machines_validated.csv", + ) parser.add_argument("--switches-csv", default=SWITCHES_CSV) args = parser.parse_args() @@ -306,8 +317,9 @@ def main(): print("=== DRY RUN — no 1Password changes ===\n") else: # Verify op is signed in - result = subprocess.run(["op", "vault", "list", "--format=json"], - capture_output=True, text=True) + result = subprocess.run( + ["op", "vault", "list", "--format=json"], capture_output=True, text=True + ) if result.returncode != 0: print("ERROR: Not signed in to 1Password. Run: op signin") sys.exit(1) @@ -329,8 +341,10 @@ def main(): machines_all = load_machines_credentials(args.machines_csv) switch_all = load_switch_credentials(args.switches_csv) - print(f"\nLoaded: {len(pdu_all)} PDU, {len(compute_all)} compute, " - f"{len(machines_all)} machines OEM, {len(switch_all)} switch credentials (with asset tags)") + print( + f"\nLoaded: {len(pdu_all)} PDU, {len(compute_all)} compute, " + f"{len(machines_all)} machines OEM, {len(switch_all)} switch credentials (with asset tags)" + ) # Merge machines OEM creds into compute — these are all ADMIN/default. # Deduplicate by asset_tag: if the same tag exists in both compute_all @@ -351,22 +365,27 @@ def main(): ("switches", "farmgpu", switch_farmgpu), ] - print(f"\n{'='*60}") + print(f"\n{'=' * 60}") print("Items to create:") - print(f"{'='*60}") + print(f"{'=' * 60}") all_ok = True for domain, cred_type, entries in items: title = TITLES[(domain, cred_type)] ok = create_or_update_item( - title, args.vault, domain, cred_type, entries, - dry_run=args.dry_run, update=args.update, + title, + args.vault, + domain, + cred_type, + entries, + dry_run=args.dry_run, + update=args.update, ) if not ok: all_ok = False if args.dry_run: - print(f"\n=== DRY RUN complete ===") + print("\n=== DRY RUN complete ===") else: status = "All items created." if all_ok else "Some items failed." print(f"\n{status}") diff --git a/scripts/infra/credentials/validate_machines_csv.py b/scripts/infra/credentials/validate_machines_csv.py index e9ba0324..c0296b4a 100644 --- a/scripts/infra/credentials/validate_machines_csv.py +++ b/scripts/infra/credentials/validate_machines_csv.py @@ -85,17 +85,19 @@ def load_machines_csv(path, mac_index): asset_tag = mac_index.get(mac_norm, "") - entries.append({ - "friendly": friendly, - "hostname": hostname, - "serial": serial, - "mac": mac_raw, - "mac_norm": mac_norm, - "bmc_ip": bmc_ip, - "password": pw, - "model": model, - "asset_tag": asset_tag, - }) + entries.append( + { + "friendly": friendly, + "hostname": hostname, + "serial": serial, + "mac": mac_raw, + "mac_norm": mac_norm, + "bmc_ip": bmc_ip, + "password": pw, + "model": model, + "asset_tag": asset_tag, + } + ) return entries @@ -109,13 +111,23 @@ def try_redfish(ip, username, password): try: result = subprocess.run( [ - "curl", "-sk", "--connect-timeout", str(TIMEOUT_SECS), - "--max-time", str(TIMEOUT_SECS), - "-o", "/dev/null", "-w", "%{http_code}", - "-u", f"{username}:{password}", + "curl", + "-sk", + "--connect-timeout", + str(TIMEOUT_SECS), + "--max-time", + str(TIMEOUT_SECS), + "-o", + "/dev/null", + "-w", + "%{http_code}", + "-u", + f"{username}:{password}", f"https://{ip}/redfish/v1/Systems/", ], - capture_output=True, text=True, timeout=TIMEOUT_SECS + 5, + capture_output=True, + text=True, + timeout=TIMEOUT_SECS + 5, ) code = result.stdout.strip() return code == "200" @@ -128,11 +140,21 @@ def try_ipmi(ip, username, password): try: result = subprocess.run( [ - "ipmitool", "-I", "lanplus", - "-H", ip, "-U", username, "-P", password, - "chassis", "status", + "ipmitool", + "-I", + "lanplus", + "-H", + ip, + "-U", + username, + "-P", + password, + "chassis", + "status", ], - capture_output=True, text=True, timeout=TIMEOUT_SECS + 5, + capture_output=True, + text=True, + timeout=TIMEOUT_SECS + 5, ) return result.returncode == 0 and "Power" in result.stdout except Exception: @@ -184,7 +206,10 @@ def main(): "--output", default=os.path.join( os.path.expanduser("~"), - "asset-tag-testing", "csvs", "compute", "machines_validated.csv", + "asset-tag-testing", + "csvs", + "compute", + "machines_validated.csv", ), ) args = parser.parse_args() @@ -225,38 +250,53 @@ def main(): print(f" - {entry['friendly']:<14} {tag_str:<10} (no BMC IP)") else: failed += 1 - print( - f" ✗ {entry['friendly']:<14} {tag_str:<10} " - f"FAILED @ {entry['bmc_ip']}" - ) + print(f" ✗ {entry['friendly']:<14} {tag_str:<10} FAILED @ {entry['bmc_ip']}") # Sort results by friendly name for consistent output results.sort(key=lambda e: e["friendly"]) - print(f"\n{'='*60}") + print(f"\n{'=' * 60}") print(f"Results: {ok} OK, {failed} FAILED, {no_ip} no IP") - print(f"{'='*60}") + print(f"{'=' * 60}") # Write validated CSV os.makedirs(os.path.dirname(args.output), exist_ok=True) with open(args.output, "w", newline="") as f: writer = csv.writer(f) - writer.writerow([ - "hostname", "friendly", "model", "serial", "bmc_mac", "bmc_ip", - "asset_tag", "validated_user", "validated_password", - "validated_protocol", "status", - ]) + writer.writerow( + [ + "hostname", + "friendly", + "model", + "serial", + "bmc_mac", + "bmc_ip", + "asset_tag", + "validated_user", + "validated_password", + "validated_protocol", + "status", + ] + ) for e in results: # Always write the password from machines.csv — even for # FAILED/NO_IP entries. The password came from the CSV and # is likely correct; failure just means the BMC was unreachable. - writer.writerow([ - e["hostname"], e["friendly"], e["model"], e["serial"], - e["mac"], e["bmc_ip"], e["asset_tag"], - e["validated_user"] or "ADMIN", - e["password"], - e.get("validated_protocol", ""), e["status"], - ]) + writer.writerow( + [ + e["hostname"], + e["friendly"], + e["model"], + e["serial"], + e["mac"], + e["bmc_ip"], + e["asset_tag"], + e["validated_user"] or "ADMIN", + e["password"], + e.get("validated_protocol", ""), + e["status"], + ] + ) print(f"\nWrote: {args.output}") print(f" {ok} entries with validated credentials") diff --git a/scripts/infra/pdus/delete_garbage_entries.py b/scripts/infra/pdus/delete_garbage_entries.py index 56cfdb81..73cd7514 100644 --- a/scripts/infra/pdus/delete_garbage_entries.py +++ b/scripts/infra/pdus/delete_garbage_entries.py @@ -21,7 +21,7 @@ # Hard-coded targets — each tuple: (device_id, expected_name, reason) GARBAGE_DEVICES = [ - (202, "|PDU19-7-14", "Duplicate with pipe char prefix"), + (202, "|PDU19-7-14", "Duplicate with pipe char prefix"), (232, "smf010301-pdu01", "Empty OEM placeholder"), (233, "smf010301-pdu02", "Empty OEM placeholder"), ] @@ -50,16 +50,19 @@ def main(): continue if device.name != expected_name: - msg = (f" ID {device_id}: NAME MISMATCH — expected '{expected_name}', " - f"got '{device.name}'. SKIPPING for safety.") + msg = ( + f" ID {device_id}: NAME MISMATCH — expected '{expected_name}', " + f"got '{device.name}'. SKIPPING for safety." + ) print(msg) errors.append(msg) continue serial = device.serial or "(none)" dtype = device.device_type.model if device.device_type else "(none)" - print(f" ID {device_id}: name='{device.name}' serial={serial} " - f"type={dtype} reason={reason}") + print( + f" ID {device_id}: name='{device.name}' serial={serial} type={dtype} reason={reason}" + ) if errors: print(f"\n{len(errors)} problem(s) found. Review above before proceeding.") diff --git a/scripts/infra/pdus/sync_asset_tags.py b/scripts/infra/pdus/sync_asset_tags.py index 8e961bfc..3921b58d 100644 --- a/scripts/infra/pdus/sync_asset_tags.py +++ b/scripts/infra/pdus/sync_asset_tags.py @@ -68,6 +68,7 @@ # ── Phase 1: Load and merge CSV data ───────────────────────────────────────── + def load_hostnames_csv(path): """ Read hostnames.csv and return {serial: record} plus a list of @@ -105,13 +106,15 @@ def load_hostnames_csv(path): "csv_ip": csv_ip, } else: - unverifiable.append({ - "hostname": hostname, - "asset_tag": asset_tag, - "serial": "", - "live_ip": "", - "reason": "No serial in CSV", - }) + unverifiable.append( + { + "hostname": hostname, + "asset_tag": asset_tag, + "serial": "", + "live_ip": "", + "reason": "No serial in CSV", + } + ) return by_serial, unverifiable @@ -160,26 +163,31 @@ def merge_csv_data(hostnames_path, validated_path): live_ip = hn_rec.get("csv_ip", "") if live_ip: - verifiable.append({ - "hostname": hn_rec["hostname"], - "asset_tag": hn_rec["asset_tag"], - "serial": serial, - "live_ip": live_ip, - }) + verifiable.append( + { + "hostname": hn_rec["hostname"], + "asset_tag": hn_rec["asset_tag"], + "serial": serial, + "live_ip": live_ip, + } + ) else: - unverifiable.append({ - "hostname": hn_rec["hostname"], - "asset_tag": hn_rec["asset_tag"], - "serial": serial, - "live_ip": "", - "reason": "No reachable IP", - }) + unverifiable.append( + { + "hostname": hn_rec["hostname"], + "asset_tag": hn_rec["asset_tag"], + "serial": serial, + "live_ip": "", + "reason": "No reachable IP", + } + ) return verifiable, unverifiable # ── Phase 2: Pre-flight serial verification ────────────────────────────────── + def verify_one_serial(entry): """Fetch serial from live PDU via HTTP and compare to CSV serial.""" ip = entry["live_ip"] @@ -203,9 +211,9 @@ def run_live_verification(verifiable): Verify serials in parallel via HTTP. Returns (results, had_failures). """ - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("Phase 2: Live serial verification") - print(f"{'='*80}") + print(f"{'=' * 80}") print(f"Verifying {len(verifiable)} PDUs (workers={LIVE_VERIFY_WORKERS})...\n") with ThreadPoolExecutor(max_workers=LIVE_VERIFY_WORKERS) as pool: @@ -227,14 +235,17 @@ def run_live_verification(verifiable): else: icon = "\u274c " + e["status"] had_failures = True - print(f"{e['hostname']:<30} {e['serial']:<12} " - f"{e.get('live_serial', ''):<12} {e['live_ip']:<18} {icon}") + print( + f"{e['hostname']:<30} {e['serial']:<12} " + f"{e.get('live_serial', ''):<12} {e['live_ip']:<18} {icon}" + ) return results, had_failures # ── Phase 3: NetBox update ─────────────────────────────────────────────────── + def _get_api_session(): """ Build a requests.Session for direct NetBox API calls. @@ -254,11 +265,13 @@ def _get_api_session(): session = requests.Session() session.verify = ssl_verify - session.headers.update({ - "Authorization": f"Token {token}", - "Content-Type": "application/json", - "Accept": "application/json", - }) + session.headers.update( + { + "Authorization": f"Token {token}", + "Content-Type": "application/json", + "Accept": "application/json", + } + ) return session, url @@ -284,8 +297,9 @@ def update_device_asset_tag(nb, hostname, expected_serial, asset_tag, dry_run): # Belt-and-suspenders: if we have an expected serial, confirm NetBox agrees if expected_serial and device.serial and device.serial != expected_serial: - return False, (f"Serial mismatch in NetBox: expected '{expected_serial}', " - f"got '{device.serial}'") + return False, ( + f"Serial mismatch in NetBox: expected '{expected_serial}', got '{device.serial}'" + ) if device.asset_tag == asset_tag: return True, f"Already set to '{asset_tag}'" @@ -320,9 +334,9 @@ def run_netbox_updates(nb, entries, dry_run): """ Phase 3: Update asset_tags in NetBox for the given entries. """ - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("Phase 3: NetBox asset tag updates") - print(f"{'='*80}\n") + print(f"{'=' * 80}\n") successes = 0 failures = 0 @@ -338,7 +352,9 @@ def run_netbox_updates(nb, entries, dry_run): successes += 1 else: failures += 1 - print(f" [{status}] {hostname:<30} tag={asset_tag:<6} serial={serial or '(none)':<10} {msg}") + print( + f" [{status}] {hostname:<30} tag={asset_tag:<6} serial={serial or '(none)':<10} {msg}" + ) print(f"\nResults: {successes} succeeded, {failures} failed (of {len(entries)} total)") return failures == 0 @@ -346,13 +362,14 @@ def run_netbox_updates(nb, entries, dry_run): # ── Phase 4: Post-update verification ──────────────────────────────────────── + def run_post_verification(nb, entries): """ Re-read all updated devices from NetBox and confirm asset_tags match. """ - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print("Phase 4: Post-update verification") - print(f"{'='*80}\n") + print(f"{'=' * 80}\n") ok_count = 0 fail_count = 0 @@ -381,6 +398,7 @@ def run_post_verification(nb, entries): # ── Main ───────────────────────────────────────────────────────────────────── + def main(): parser = argparse.ArgumentParser( description="Sync PDU asset tags from CSV into NetBox with live serial verification.", @@ -391,27 +409,40 @@ def main(): %(prog)s --verified-only # program verified PDUs (safest) %(prog)s --dry-run # preview all PDUs %(prog)s # program all PDUs -""") - parser.add_argument("--dry-run", action="store_true", - help="show what would change, don't write to NetBox") - parser.add_argument("--verified-only", action="store_true", - help="only program PDUs whose serials were live-verified " - "(skips unverifiable entries)") - parser.add_argument("--skip-live-verify", action="store_true", - help="skip HTTP serial verification (trust CSV data)") - parser.add_argument("--hostnames-csv", default=HOSTNAMES_CSV, - help=f"path to hostnames.csv (default: {HOSTNAMES_CSV})") - parser.add_argument("--validated-csv", default=VALIDATED_CSV, - help=f"path to validated_hosts.csv (default: {VALIDATED_CSV})") +""", + ) + parser.add_argument( + "--dry-run", action="store_true", help="show what would change, don't write to NetBox" + ) + parser.add_argument( + "--verified-only", + action="store_true", + help="only program PDUs whose serials were live-verified (skips unverifiable entries)", + ) + parser.add_argument( + "--skip-live-verify", + action="store_true", + help="skip HTTP serial verification (trust CSV data)", + ) + parser.add_argument( + "--hostnames-csv", + default=HOSTNAMES_CSV, + help=f"path to hostnames.csv (default: {HOSTNAMES_CSV})", + ) + parser.add_argument( + "--validated-csv", + default=VALIDATED_CSV, + help=f"path to validated_hosts.csv (default: {VALIDATED_CSV})", + ) args = parser.parse_args() if args.dry_run: print("=== DRY RUN — no changes will be made ===\n") # ── Phase 1 ────────────────────────────────────────────────────────── - print(f"{'='*80}") + print(f"{'=' * 80}") print("Phase 1: Loading CSV data") - print(f"{'='*80}") + print(f"{'=' * 80}") print(f" hostnames.csv: {args.hostnames_csv}") print(f" validated.csv: {args.validated_csv}") diff --git a/scripts/migration/assign_asset_tags.py b/scripts/migration/assign_asset_tags.py index 726a7b07..b733cd7b 100644 --- a/scripts/migration/assign_asset_tags.py +++ b/scripts/migration/assign_asset_tags.py @@ -96,7 +96,9 @@ def run(nb, dry_run=True): def main(): parser = argparse.ArgumentParser(description="Assign Base-36 asset tags to devices") - parser.add_argument("--dry-run", action="store_true", help="Preview assignments without applying") + parser.add_argument( + "--dry-run", action="store_true", help="Preview assignments without applying" + ) args = parser.parse_args() nb = get_api() diff --git a/scripts/migration/cleanup_bad_modules.py b/scripts/migration/cleanup_bad_modules.py index 5fff1036..18123299 100644 --- a/scripts/migration/cleanup_bad_modules.py +++ b/scripts/migration/cleanup_bad_modules.py @@ -42,7 +42,9 @@ def main(): for mod in all_modules: mt_model = mod.module_type.model if mod.module_type else "" if mt_model in BAD_MODULE_TYPE_MODELS: - print(f"DELETE module id={mod.id} bay={mod.module_bay.name} type={mt_model} device={mod.device.name}") + print( + f"DELETE module id={mod.id} bay={mod.module_bay.name} type={mt_model} device={mod.device.name}" + ) if not dry: mod.delete() deleted_modules += 1 @@ -76,7 +78,9 @@ def main(): idx = bay.name.split("-")[1] try: if int(idx) >= 2: - print(f"DELETE module_bay id={bay.id} name={bay.name} device={device.name}") + print( + f"DELETE module_bay id={bay.id} name={bay.name} device={device.name}" + ) if not dry: bay.delete() deleted_bays += 1 diff --git a/scripts/migration/cleanup_legacy_bays.py b/scripts/migration/cleanup_legacy_bays.py index c5fa1892..e15f8b76 100644 --- a/scripts/migration/cleanup_legacy_bays.py +++ b/scripts/migration/cleanup_legacy_bays.py @@ -123,12 +123,8 @@ def cleanup_bays(nb, dry): def main(): - parser = argparse.ArgumentParser( - description="Clean up legacy module bay templates and bays" - ) - parser.add_argument( - "--dry-run", action="store_true", help="Print what would be deleted" - ) + parser = argparse.ArgumentParser(description="Clean up legacy module bay templates and bays") + parser.add_argument("--dry-run", action="store_true", help="Print what would be deleted") args = parser.parse_args() dry = args.dry_run diff --git a/scripts/migration/migrate_ipam.py b/scripts/migration/migrate_ipam.py index 1a44ebbd..fd462471 100644 --- a/scripts/migration/migrate_ipam.py +++ b/scripts/migration/migrate_ipam.py @@ -36,6 +36,7 @@ # Helpers # --------------------------------------------------------------------------- + class IdMapper: """Track old_id → new_id mappings across migration phases.""" @@ -121,6 +122,7 @@ def connect(): # Phase 1: Tags # --------------------------------------------------------------------------- + def migrate_tags(src, dst, mapper, dry_run): stats = PhaseStats("Tags") print("\n" + "=" * 60) @@ -165,6 +167,7 @@ def migrate_tags(src, dst, mapper, dry_run): # Phase 2: Tenant Groups # --------------------------------------------------------------------------- + def migrate_tenant_groups(src, dst, mapper, dry_run): stats = PhaseStats("Tenant Groups") print("\n" + "=" * 60) @@ -208,6 +211,7 @@ def migrate_tenant_groups(src, dst, mapper, dry_run): # Phase 3: Tenants # --------------------------------------------------------------------------- + def migrate_tenants(src, dst, mapper, dry_run): stats = PhaseStats("Tenants") print("\n" + "=" * 60) @@ -257,6 +261,7 @@ def migrate_tenants(src, dst, mapper, dry_run): # Phase 4: RIRs # --------------------------------------------------------------------------- + def migrate_rirs(src, dst, mapper, dry_run): stats = PhaseStats("RIRs") print("\n" + "=" * 60) @@ -301,6 +306,7 @@ def migrate_rirs(src, dst, mapper, dry_run): # Phase 5: Aggregates # --------------------------------------------------------------------------- + def migrate_aggregates(src, dst, mapper, dry_run): stats = PhaseStats("Aggregates") print("\n" + "=" * 60) @@ -347,6 +353,7 @@ def migrate_aggregates(src, dst, mapper, dry_run): # Phase 6: IPAM Roles # --------------------------------------------------------------------------- + def migrate_roles(src, dst, mapper, dry_run): stats = PhaseStats("IPAM Roles") print("\n" + "=" * 60) @@ -391,6 +398,7 @@ def migrate_roles(src, dst, mapper, dry_run): # Phase 7: VLANs (101 source → merge with 4 existing bare VLANs) # --------------------------------------------------------------------------- + def migrate_vlans(src, dst, mapper, dry_run): stats = PhaseStats("VLANs") print("\n" + "=" * 60) @@ -422,7 +430,8 @@ def migrate_vlans(src, dst, mapper, dry_run): if dry_run: logger.info( "[DRY RUN] Would update VLAN %d '%s' with source data", - vlan.vid, vlan.name, + vlan.vid, + vlan.name, ) else: match.name = vlan.name @@ -476,6 +485,7 @@ def migrate_vlans(src, dst, mapper, dry_run): # Phase 8: VRFs + Route Targets # --------------------------------------------------------------------------- + def migrate_vrfs(src, dst, mapper, dry_run): stats = PhaseStats("VRFs") print("\n" + "=" * 60) @@ -496,9 +506,7 @@ def migrate_vrfs(src, dst, mapper, dry_run): if dry_run: logger.info("[DRY RUN] Would create route target '%s'", rt.name) continue - new = dst.ipam.route_targets.create( - name=rt.name, description=rt.description or "" - ) + new = dst.ipam.route_targets.create(name=rt.name, description=rt.description or "") mapper.set("route_targets", rt.id, new.id) logger.info("Created route target '%s'", rt.name) except Exception as e: @@ -549,6 +557,7 @@ def migrate_vrfs(src, dst, mapper, dry_run): # Phase 9: Prefixes (sorted by prefix length — containers before children) # --------------------------------------------------------------------------- + def migrate_prefixes(src, dst, mapper, dry_run): stats = PhaseStats("Prefixes") print("\n" + "=" * 60) @@ -556,9 +565,7 @@ def migrate_prefixes(src, dst, mapper, dry_run): print("=" * 60) src_prefixes = list(src.ipam.prefixes.all()) - src_prefixes.sort( - key=lambda p: ipaddress.ip_network(p.prefix, strict=False).prefixlen - ) + src_prefixes.sort(key=lambda p: ipaddress.ip_network(p.prefix, strict=False).prefixlen) print(f" Source: {len(src_prefixes)} (sorted by prefix length)") for pfx in src_prefixes: @@ -585,7 +592,8 @@ def migrate_prefixes(src, dst, mapper, dry_run): if dry_run: logger.info( "[DRY RUN] Would create prefix '%s' (status=%s)", - pfx.prefix, _status_val(pfx.status), + pfx.prefix, + _status_val(pfx.status), ) stats.created += 1 continue @@ -619,7 +627,9 @@ def migrate_prefixes(src, dst, mapper, dry_run): logger.info(" ... %d / %d prefixes created", stats.created, len(src_prefixes)) logger.info( "Created prefix '%s' (id=%d, status=%s)", - pfx.prefix, new.id, data["status"], + pfx.prefix, + new.id, + data["status"], ) except Exception as e: stats.failed += 1 @@ -633,6 +643,7 @@ def migrate_prefixes(src, dst, mapper, dry_run): # Phase 10: IP Ranges # --------------------------------------------------------------------------- + def migrate_ip_ranges(src, dst, mapper, dry_run): stats = PhaseStats("IP Ranges") print("\n" + "=" * 60) @@ -653,14 +664,16 @@ def migrate_ip_ranges(src, dst, mapper, dry_run): stats.skipped += 1 logger.info( "IP range %s–%s exists, skipped", - ipr.start_address, ipr.end_address, + ipr.start_address, + ipr.end_address, ) continue if dry_run: logger.info( "[DRY RUN] Would create IP range %s–%s", - ipr.start_address, ipr.end_address, + ipr.start_address, + ipr.end_address, ) stats.created += 1 continue @@ -691,13 +704,17 @@ def migrate_ip_ranges(src, dst, mapper, dry_run): stats.created += 1 logger.info( "Created IP range %s–%s (id=%d)", - ipr.start_address, ipr.end_address, new.id, + ipr.start_address, + ipr.end_address, + new.id, ) except Exception as e: stats.failed += 1 logger.error( "Failed IP range %s–%s: %s", - ipr.start_address, ipr.end_address, e, + ipr.start_address, + ipr.end_address, + e, ) stats.print_summary() @@ -708,6 +725,7 @@ def migrate_ip_ranges(src, dst, mapper, dry_run): # Verify-only mode # --------------------------------------------------------------------------- + def verify(src, dst): print("\n" + "=" * 60) print("VERIFICATION REPORT") @@ -753,6 +771,7 @@ def verify(src, dst): # Pre-populate mapper (for --phase N single-phase runs) # --------------------------------------------------------------------------- + def _prepopulate_mapper(src, dst, mapper, up_to_phase): """Match existing dest objects to source IDs for phases before up_to_phase.""" logger.info("Pre-populating ID mapper for phases 1–%d ...", up_to_phase - 1) @@ -825,19 +844,22 @@ def _prepopulate_mapper(src, dst, mapper, up_to_phase): def main(): - parser = argparse.ArgumentParser( - description="Migrate IPAM data from old NetBox to new NetBox" - ) + parser = argparse.ArgumentParser(description="Migrate IPAM data from old NetBox to new NetBox") parser.add_argument( - "--dry-run", action="store_true", + "--dry-run", + action="store_true", help="Preview migration without writing to destination", ) parser.add_argument( - "--verify-only", action="store_true", + "--verify-only", + action="store_true", help="Compare source vs destination counts and report gaps", ) parser.add_argument( - "--phase", type=int, choices=range(1, 11), metavar="N", + "--phase", + type=int, + choices=range(1, 11), + metavar="N", help="Run only phase N (1–10)", ) args = parser.parse_args() @@ -879,8 +901,7 @@ def main(): total_u = sum(s.updated for s in all_stats) total_s = sum(s.skipped for s in all_stats) total_f = sum(s.failed for s in all_stats) - print(f"\n TOTAL: {total_c} created, {total_u} updated," - f" {total_s} skipped, {total_f} failed") + print(f"\n TOTAL: {total_c} created, {total_u} updated, {total_s} skipped, {total_f} failed") if total_f: print(f"\n WARNING: {total_f} failures — review log output above") diff --git a/scripts/migration/verify_modules_vs_inventory.py b/scripts/migration/verify_modules_vs_inventory.py index dc4bf7a2..fbe0c5a7 100644 --- a/scripts/migration/verify_modules_vs_inventory.py +++ b/scripts/migration/verify_modules_vs_inventory.py @@ -47,7 +47,9 @@ def _get_modules_by_device(nb, device_id): for mod in all_modules: bay_name = "" if mod.module_bay: - bay_name = getattr(mod.module_bay, "name", "") or getattr(mod.module_bay, "display", "") + bay_name = getattr(mod.module_bay, "name", "") or getattr( + mod.module_bay, "display", "" + ) for category in TAG_TO_CATEGORY.values(): if bay_name.startswith(f"{category}-"): result[category]["count"] += 1 diff --git a/scripts/rollback/delete_all_modules.py b/scripts/rollback/delete_all_modules.py index fdba07c5..1f206d25 100644 --- a/scripts/rollback/delete_all_modules.py +++ b/scripts/rollback/delete_all_modules.py @@ -31,7 +31,9 @@ def run(nb, dry_run=True): for mod in modules: device_name = "Unknown" if mod.device: - device_name = getattr(mod.device, "name", None) or getattr(mod.device, "display", str(mod.device)) + device_name = getattr(mod.device, "name", None) or getattr( + mod.device, "display", str(mod.device) + ) by_device.setdefault(device_name, []).append(mod) print("\nModules by device:") diff --git a/scripts/schema/02_create_custom_field_choice_sets.py b/scripts/schema/02_create_custom_field_choice_sets.py index 850457c2..583b2a67 100644 --- a/scripts/schema/02_create_custom_field_choice_sets.py +++ b/scripts/schema/02_create_custom_field_choice_sets.py @@ -45,7 +45,9 @@ def run(nb): for cs_def in CHOICE_SETS: existing = nb.extras.custom_field_choice_sets.get(name=cs_def["name"]) if existing: - logger.info("Choice set '%s' already exists (id=%d) — skipping", cs_def["name"], existing.id) + logger.info( + "Choice set '%s' already exists (id=%d) — skipping", cs_def["name"], existing.id + ) continue result = nb.extras.custom_field_choice_sets.create(cs_def) diff --git a/scripts/schema/03_create_custom_fields.py b/scripts/schema/03_create_custom_fields.py index 40ce0025..16e1f5fc 100644 --- a/scripts/schema/03_create_custom_fields.py +++ b/scripts/schema/03_create_custom_fields.py @@ -101,7 +101,11 @@ def run(nb): for field_def in fields: existing = nb.extras.custom_fields.get(name=field_def["name"]) if existing: - logger.info("Custom field '%s' already exists (id=%d) — skipping", field_def["name"], existing.id) + logger.info( + "Custom field '%s' already exists (id=%d) — skipping", + field_def["name"], + existing.id, + ) continue result = nb.extras.custom_fields.create(field_def) @@ -125,16 +129,18 @@ def run(nb): logger.info("bmc_mac_address already configured correctly — skipping") else: logger.warning("bmc_mac_address custom field not found — creating it") - nb.extras.custom_fields.create({ - "name": "bmc_mac_address", - "label": "BMC MAC Address", - "type": "text", - "object_types": ["dcim.device"], - "required": False, - "validation_regex": r"^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$", - "description": "BMC/IPMI MAC address", - "weight": 600, - }) + nb.extras.custom_fields.create( + { + "name": "bmc_mac_address", + "label": "BMC MAC Address", + "type": "text", + "object_types": ["dcim.device"], + "required": False, + "validation_regex": r"^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$", + "description": "BMC/IPMI MAC address", + "weight": 600, + } + ) logger.info("Created bmc_mac_address custom field") diff --git a/scripts/schema/04_create_module_type_profiles.py b/scripts/schema/04_create_module_type_profiles.py index 0a84e615..a2e5c57a 100644 --- a/scripts/schema/04_create_module_type_profiles.py +++ b/scripts/schema/04_create_module_type_profiles.py @@ -68,8 +68,16 @@ "Memory": { "required": ["class", "size"], "properties": { - "ecc": {"type": "boolean", "title": "ECC", "description": "Error-correcting code is enabled"}, - "size": {"type": "integer", "title": "Size (GB)", "description": "Raw capacity of the module"}, + "ecc": { + "type": "boolean", + "title": "ECC", + "description": "Error-correcting code is enabled", + }, + "size": { + "type": "integer", + "title": "Size (GB)", + "description": "Raw capacity of the module", + }, "class": { "enum": ["DDR3", "DDR4", "DDR5"], "type": "string", @@ -125,14 +133,18 @@ def run(nb): if existing.schema != schema: existing.schema = schema existing.save() - logger.info("Updated profile '%s' (id=%d) with enriched schema", profile_name, existing.id) + logger.info( + "Updated profile '%s' (id=%d) with enriched schema", profile_name, existing.id + ) else: logger.info("Profile '%s' already has correct schema — skipping", profile_name) else: - result = nb.dcim.module_type_profiles.create({ - "name": profile_name, - "schema": schema, - }) + result = nb.dcim.module_type_profiles.create( + { + "name": profile_name, + "schema": schema, + } + ) logger.info("Created profile '%s' (id=%d)", profile_name, result.id) diff --git a/scripts/schema/05_create_module_types.py b/scripts/schema/05_create_module_types.py index 1a4367d8..e6773128 100644 --- a/scripts/schema/05_create_module_types.py +++ b/scripts/schema/05_create_module_types.py @@ -26,7 +26,9 @@ def _get_or_create_manufacturer(nb, name): slug = re.sub(r"[^A-Za-z0-9]+", "-", name).lower().strip("-") mfr = nb.dcim.manufacturers.get(slug=slug) if mfr: - logger.info("Manufacturer '%s' found by slug '%s' (actual name='%s')", name, slug, mfr.name) + logger.info( + "Manufacturer '%s' found by slug '%s' (actual name='%s')", name, slug, mfr.name + ) else: mfr = nb.dcim.manufacturers.create(name=name, slug=slug) logger.info("Created manufacturer '%s'", name) @@ -50,7 +52,9 @@ def _ensure_module_type(nb, manufacturer_name, model, profile, attribute_data=No update_needed = False existing_profile_id = None if hasattr(existing, "profile") and existing.profile: - existing_profile_id = existing.profile.id if hasattr(existing.profile, "id") else existing.profile + existing_profile_id = ( + existing.profile.id if hasattr(existing.profile, "id") else existing.profile + ) if existing_profile_id != profile.id: existing.profile = profile.id update_needed = True @@ -61,7 +65,9 @@ def _ensure_module_type(nb, manufacturer_name, model, profile, attribute_data=No existing.save() logger.info("Updated module type '%s / %s'", manufacturer_name, model) else: - logger.info("Module type '%s / %s' already correct — skipping", manufacturer_name, model) + logger.info( + "Module type '%s / %s' already correct — skipping", manufacturer_name, model + ) return existing create_params = { @@ -79,42 +85,137 @@ def _ensure_module_type(nb, manufacturer_name, model, profile, attribute_data=No # Existing GPU module types to update (these 6 already exist) GPU_TYPES = [ - {"manufacturer": "NVIDIA", "model": "RTX 4000 Ada", "attributes": {"memory_gb": 20, "form_factor": "PCIe"}}, - {"manufacturer": "NVIDIA", "model": "RTX A5000", "attributes": {"memory_gb": 24, "form_factor": "PCIe"}}, - {"manufacturer": "NVIDIA", "model": "RTX 6000 Ada", "attributes": {"memory_gb": 48, "form_factor": "PCIe"}}, - {"manufacturer": "NVIDIA", "model": "RTX 4090", "attributes": {"memory_gb": 24, "form_factor": "PCIe"}}, - {"manufacturer": "NVIDIA", "model": "RTX A6000", "attributes": {"memory_gb": 48, "form_factor": "PCIe"}}, - {"manufacturer": "NVIDIA", "model": "H100 PCIe", "attributes": {"memory_gb": 80, "form_factor": "PCIe", "tdp_watts": 350}}, + { + "manufacturer": "NVIDIA", + "model": "RTX 4000 Ada", + "attributes": {"memory_gb": 20, "form_factor": "PCIe"}, + }, + { + "manufacturer": "NVIDIA", + "model": "RTX A5000", + "attributes": {"memory_gb": 24, "form_factor": "PCIe"}, + }, + { + "manufacturer": "NVIDIA", + "model": "RTX 6000 Ada", + "attributes": {"memory_gb": 48, "form_factor": "PCIe"}, + }, + { + "manufacturer": "NVIDIA", + "model": "RTX 4090", + "attributes": {"memory_gb": 24, "form_factor": "PCIe"}, + }, + { + "manufacturer": "NVIDIA", + "model": "RTX A6000", + "attributes": {"memory_gb": 48, "form_factor": "PCIe"}, + }, + { + "manufacturer": "NVIDIA", + "model": "H100 PCIe", + "attributes": {"memory_gb": 80, "form_factor": "PCIe", "tdp_watts": 350}, + }, ] # Seed known fleet types CPU_TYPES = [ - {"manufacturer": "Intel", "model": "Xeon Gold 6430", "attributes": {"core_count": 32, "base_frequency_ghz": 2.1, "architecture": "x86_64", "tdp_watts": 270}}, - {"manufacturer": "Intel", "model": "Xeon Gold 6448Y", "attributes": {"core_count": 32, "base_frequency_ghz": 2.1, "architecture": "x86_64", "tdp_watts": 225}}, - {"manufacturer": "Intel", "model": "Xeon Platinum 8480+", "attributes": {"core_count": 56, "base_frequency_ghz": 2.0, "architecture": "x86_64", "tdp_watts": 350}}, + { + "manufacturer": "Intel", + "model": "Xeon Gold 6430", + "attributes": { + "core_count": 32, + "base_frequency_ghz": 2.1, + "architecture": "x86_64", + "tdp_watts": 270, + }, + }, + { + "manufacturer": "Intel", + "model": "Xeon Gold 6448Y", + "attributes": { + "core_count": 32, + "base_frequency_ghz": 2.1, + "architecture": "x86_64", + "tdp_watts": 225, + }, + }, + { + "manufacturer": "Intel", + "model": "Xeon Platinum 8480+", + "attributes": { + "core_count": 56, + "base_frequency_ghz": 2.0, + "architecture": "x86_64", + "tdp_watts": 350, + }, + }, ] SSD_TYPES = [ - {"manufacturer": "Solidigm", "model": "D7-P5520 3.84TB", "attributes": {"size": 3840, "type": "NVME", "form_factor": "U.2", "interface": "Gen4"}}, - {"manufacturer": "Samsung", "model": "PM9A3 1.92TB", "attributes": {"size": 1920, "type": "NVME", "form_factor": "E1.S", "interface": "Gen4"}}, - {"manufacturer": "Samsung", "model": "PM9A3 3.84TB", "attributes": {"size": 3840, "type": "NVME", "form_factor": "U.2", "interface": "Gen4"}}, + { + "manufacturer": "Solidigm", + "model": "D7-P5520 3.84TB", + "attributes": {"size": 3840, "type": "NVME", "form_factor": "U.2", "interface": "Gen4"}, + }, + { + "manufacturer": "Samsung", + "model": "PM9A3 1.92TB", + "attributes": {"size": 1920, "type": "NVME", "form_factor": "E1.S", "interface": "Gen4"}, + }, + { + "manufacturer": "Samsung", + "model": "PM9A3 3.84TB", + "attributes": {"size": 3840, "type": "NVME", "form_factor": "U.2", "interface": "Gen4"}, + }, ] NIC_TYPES = [ - {"manufacturer": "Intel", "model": "E810-XXVDA2 25GbE", "attributes": {"port_count": 2, "speed_gbps": "25", "form_factor": "PCIe"}}, - {"manufacturer": "NVIDIA", "model": "ConnectX-6 Dx 100GbE", "attributes": {"port_count": 2, "speed_gbps": "100", "form_factor": "PCIe"}}, - {"manufacturer": "Broadcom", "model": "BCM57416 OCP 25GbE", "attributes": {"port_count": 2, "speed_gbps": "25", "form_factor": "OCP"}}, + { + "manufacturer": "Intel", + "model": "E810-XXVDA2 25GbE", + "attributes": {"port_count": 2, "speed_gbps": "25", "form_factor": "PCIe"}, + }, + { + "manufacturer": "NVIDIA", + "model": "ConnectX-6 Dx 100GbE", + "attributes": {"port_count": 2, "speed_gbps": "100", "form_factor": "PCIe"}, + }, + { + "manufacturer": "Broadcom", + "model": "BCM57416 OCP 25GbE", + "attributes": {"port_count": 2, "speed_gbps": "25", "form_factor": "OCP"}, + }, ] DIMM_TYPES = [ - {"manufacturer": "Samsung", "model": "M393A8G40AB2-CWE 64GB", "attributes": {"size": 64, "class": "DDR4", "data_rate": 3200, "ecc": True}}, - {"manufacturer": "Samsung", "model": "M321R8GA0BB0-CQKZJ 64GB", "attributes": {"size": 64, "class": "DDR5", "data_rate": 4800, "ecc": True}}, - {"manufacturer": "SK Hynix", "model": "HMCG94AEBRA109N 64GB", "attributes": {"size": 64, "class": "DDR5", "data_rate": 4800, "ecc": True}}, + { + "manufacturer": "Samsung", + "model": "M393A8G40AB2-CWE 64GB", + "attributes": {"size": 64, "class": "DDR4", "data_rate": 3200, "ecc": True}, + }, + { + "manufacturer": "Samsung", + "model": "M321R8GA0BB0-CQKZJ 64GB", + "attributes": {"size": 64, "class": "DDR5", "data_rate": 4800, "ecc": True}, + }, + { + "manufacturer": "SK Hynix", + "model": "HMCG94AEBRA109N 64GB", + "attributes": {"size": 64, "class": "DDR5", "data_rate": 4800, "ecc": True}, + }, ] PSU_TYPES = [ - {"manufacturer": "Delta", "model": "DPS-2400AB 2400W", "attributes": {"wattage": 2400, "input_current": "AC", "efficiency": "80+ Platinum"}}, - {"manufacturer": "Liteon", "model": "PS-2162-5L 1600W", "attributes": {"wattage": 1600, "input_current": "AC", "efficiency": "80+ Platinum"}}, + { + "manufacturer": "Delta", + "model": "DPS-2400AB 2400W", + "attributes": {"wattage": 2400, "input_current": "AC", "efficiency": "80+ Platinum"}, + }, + { + "manufacturer": "Liteon", + "model": "PS-2162-5L 1600W", + "attributes": {"wattage": 1600, "input_current": "AC", "efficiency": "80+ Platinum"}, + }, ] @@ -127,17 +228,29 @@ def run(nb): psu_profile = _get_profile(nb, "Power supply") for spec in GPU_TYPES: - _ensure_module_type(nb, spec["manufacturer"], spec["model"], gpu_profile, spec.get("attributes")) + _ensure_module_type( + nb, spec["manufacturer"], spec["model"], gpu_profile, spec.get("attributes") + ) for spec in CPU_TYPES: - _ensure_module_type(nb, spec["manufacturer"], spec["model"], cpu_profile, spec.get("attributes")) + _ensure_module_type( + nb, spec["manufacturer"], spec["model"], cpu_profile, spec.get("attributes") + ) for spec in SSD_TYPES: - _ensure_module_type(nb, spec["manufacturer"], spec["model"], disk_profile, spec.get("attributes")) + _ensure_module_type( + nb, spec["manufacturer"], spec["model"], disk_profile, spec.get("attributes") + ) for spec in NIC_TYPES: - _ensure_module_type(nb, spec["manufacturer"], spec["model"], nic_profile, spec.get("attributes")) + _ensure_module_type( + nb, spec["manufacturer"], spec["model"], nic_profile, spec.get("attributes") + ) for spec in DIMM_TYPES: - _ensure_module_type(nb, spec["manufacturer"], spec["model"], memory_profile, spec.get("attributes")) + _ensure_module_type( + nb, spec["manufacturer"], spec["model"], memory_profile, spec.get("attributes") + ) for spec in PSU_TYPES: - _ensure_module_type(nb, spec["manufacturer"], spec["model"], psu_profile, spec.get("attributes")) + _ensure_module_type( + nb, spec["manufacturer"], spec["model"], psu_profile, spec.get("attributes") + ) def main(): diff --git a/scripts/schema/06_update_device_types_module_bay_templates.py b/scripts/schema/06_update_device_types_module_bay_templates.py index bf2fa469..e93ee2c0 100644 --- a/scripts/schema/06_update_device_types_module_bay_templates.py +++ b/scripts/schema/06_update_device_types_module_bay_templates.py @@ -66,7 +66,8 @@ def _ensure_module_bay_templates(nb, device_type): if not STANDARD_BAY_PATTERN.match(t.name): logger.info( " Deleting legacy template '%s' from device type '%s'", - t.name, device_type.model, + t.name, + device_type.model, ) t.delete() deleted += 1 @@ -78,17 +79,21 @@ def _ensure_module_bay_templates(nb, device_type): bay_name = f"{category}-{i}" if bay_name in existing_names: continue - nb.dcim.module_bay_templates.create({ - "device_type": device_type.id, - "name": bay_name, - "position": bay_name, - }) + nb.dcim.module_bay_templates.create( + { + "device_type": device_type.id, + "name": bay_name, + "position": bay_name, + } + ) created += 1 if created > 0 or deleted > 0: logger.info( "Device type '%s': added %d, deleted %d legacy template(s)", - device_type.model, created, deleted, + device_type.model, + created, + deleted, ) else: logger.info("Device type '%s' — all bay templates present", device_type.model) @@ -115,11 +120,13 @@ def _backfill_device_module_bays(nb, device): for template in templates: if template.name in existing_names: continue - nb.dcim.module_bays.create({ - "device": device.id, - "name": template.name, - "position": template.name, - }) + nb.dcim.module_bays.create( + { + "device": device.id, + "name": template.name, + "position": template.name, + } + ) created += 1 return created + deleted # Return total changes for logging diff --git a/scripts/schema/07_create_spare_inventory_device.py b/scripts/schema/07_create_spare_inventory_device.py index 31e590e3..2666663e 100644 --- a/scripts/schema/07_create_spare_inventory_device.py +++ b/scripts/schema/07_create_spare_inventory_device.py @@ -40,7 +40,9 @@ def _get_or_create_manufacturer(nb, name): slug = re.sub(r"[^A-Za-z0-9]+", "-", name).lower().strip("-") mfr = nb.dcim.manufacturers.get(slug=slug) if mfr: - logger.info("Manufacturer '%s' found by slug '%s' (actual name='%s')", name, slug, mfr.name) + logger.info( + "Manufacturer '%s' found by slug '%s' (actual name='%s')", name, slug, mfr.name + ) else: mfr = nb.dcim.manufacturers.create(name=name, slug=slug) logger.info("Created manufacturer '%s'", name) @@ -53,11 +55,13 @@ def _get_or_create_device_type(nb, manufacturer, model): dt = nb.dcim.device_types.get(manufacturer_id=manufacturer.id, model=model) if not dt: slug = re.sub(r"[^A-Za-z0-9]+", "-", model).lower().strip("-") - dt = nb.dcim.device_types.create({ - "manufacturer": manufacturer.id, - "model": model, - "slug": slug, - }) + dt = nb.dcim.device_types.create( + { + "manufacturer": manufacturer.id, + "model": model, + "slug": slug, + } + ) logger.info("Created device type '%s'", model) else: logger.info("Device type '%s' already exists", model) @@ -70,11 +74,13 @@ def _get_or_create_device_type(nb, manufacturer, model): for i in range(count): bay_name = f"{category}-{i}" if bay_name not in existing_names: - nb.dcim.module_bay_templates.create({ - "device_type": dt.id, - "name": bay_name, - "position": bay_name, - }) + nb.dcim.module_bay_templates.create( + { + "device_type": dt.id, + "name": bay_name, + "position": bay_name, + } + ) created += 1 if created > 0: logger.info("Created %d module bay templates on '%s'", created, model) @@ -86,11 +92,13 @@ def _get_or_create_device_role(nb, name): slug = re.sub(r"[^A-Za-z0-9]+", "-", name).lower().strip("-") role = nb.dcim.device_roles.get(slug=slug) if not role: - role = nb.dcim.device_roles.create({ - "name": name, - "slug": slug, - "vm_role": False, - }) + role = nb.dcim.device_roles.create( + { + "name": name, + "slug": slug, + "vm_role": False, + } + ) logger.info("Created device role '%s'", name) else: logger.info("Device role '%s' already exists", name) @@ -113,27 +121,31 @@ def run(nb): if existing: logger.info("Device '%s' already exists (id=%d)", DEVICE_NAME, existing.id) else: - device = nb.dcim.devices.create({ - "name": DEVICE_NAME, - "device_type": device_type.id, - "role": device_role.id, - "site": site.id, - "status": "inventory", - "custom_fields": { - "owner": "FarmGPU", - }, - }) + device = nb.dcim.devices.create( + { + "name": DEVICE_NAME, + "device_type": device_type.id, + "role": device_role.id, + "site": site.id, + "status": "inventory", + "custom_fields": { + "owner": "FarmGPU", + }, + } + ) logger.info("Created device '%s' (id=%d)", DEVICE_NAME, device.id) # Backfill module bays on the device templates = list(nb.dcim.module_bay_templates.filter(device_type_id=device_type.id)) created = 0 for t in templates: - nb.dcim.module_bays.create({ - "device": device.id, - "name": t.name, - "position": t.name, - }) + nb.dcim.module_bays.create( + { + "device": device.id, + "name": t.name, + "position": t.name, + } + ) created += 1 logger.info("Created %d module bays on '%s'", created, DEVICE_NAME) diff --git a/scripts/schema/08_run_all_schema_setup.py b/scripts/schema/08_run_all_schema_setup.py index 6806b86a..f601e30c 100644 --- a/scripts/schema/08_run_all_schema_setup.py +++ b/scripts/schema/08_run_all_schema_setup.py @@ -22,7 +22,10 @@ ("03_create_custom_fields", "Create custom fields"), ("04_create_module_type_profiles", "Create module type profiles"), ("05_create_module_types", "Create module types"), - ("06_update_device_types_module_bay_templates", "Update device types with module bay templates"), + ( + "06_update_device_types_module_bay_templates", + "Update device types with module bay templates", + ), ("07_create_spare_inventory_device", "Create spare inventory device"), ] diff --git a/scripts/validation/validate_record_completeness.py b/scripts/validation/validate_record_completeness.py index 7e5e454f..e57666cb 100644 --- a/scripts/validation/validate_record_completeness.py +++ b/scripts/validation/validate_record_completeness.py @@ -105,11 +105,13 @@ def run(nb): device_complete += 1 else: device_incomplete += 1 - device_issues.append({ - "name": device.name, - "missing_required": missing, - "missing_preferred": preferred_missing, - }) + device_issues.append( + { + "name": device.name, + "missing_required": missing, + "missing_preferred": preferred_missing, + } + ) # Module validation module_complete = 0 @@ -132,11 +134,13 @@ def run(nb): module_complete += 1 else: module_incomplete += 1 - module_issues.append({ - "id": module.id, - "serial": module.serial, - "missing_required": missing, - }) + module_issues.append( + { + "id": module.id, + "serial": module.serial, + "missing_required": missing, + } + ) # Print summary total_devices = len(devices) @@ -172,7 +176,9 @@ def run(nb): print(f"\nINCOMPLETE MODULES ({len(module_issues)}):") print("-" * 70) for issue in module_issues[:50]: - print(f" Module {issue['id']} (serial={issue['serial']}): missing {issue['missing_required']}") + print( + f" Module {issue['id']} (serial={issue['serial']}): missing {issue['missing_required']}" + ) if len(module_issues) > 50: print(f" ... and {len(module_issues) - 50} more") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 5c26b323..faa3aba5 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -10,7 +10,6 @@ class TestDependencies: - def test_check_all_with_all_available(self): """All tools present → all True.""" with patch("netbox_agent.dependencies.which", return_value="/usr/bin/fake"): @@ -20,6 +19,7 @@ def test_check_all_with_all_available(self): def test_check_all_with_missing_tools(self): """Some tools missing → mixed True/False.""" + def mock_which(name): return "/usr/bin/fake" if name in ("dmidecode", "lshw") else None @@ -32,6 +32,7 @@ def mock_which(name): def test_get_missing_returns_correct_list(self): """get_missing() returns only missing tool names.""" + def mock_which(name): return "/usr/bin/fake" if name in ("dmidecode", "lshw", "lsblk") else None diff --git a/tests/test_fixture_integration.py b/tests/test_fixture_integration.py index f964e491..70bdf0f0 100644 --- a/tests/test_fixture_integration.py +++ b/tests/test_fixture_integration.py @@ -43,20 +43,33 @@ force_disk_refresh=False, dump_disks_map=None, log_level="debug", - netbox=SimpleNamespace(url="http://test", token="test", ssl_verify=True, ssl_ca_certs_file=None), - virtual=SimpleNamespace(enabled=False, cluster_name=None, hypervisor=False, list_guests_cmd=None), + netbox=SimpleNamespace( + url="http://test", token="test", ssl_verify=True, ssl_ca_certs_file=None + ), + virtual=SimpleNamespace( + enabled=False, cluster_name=None, hypervisor=False, list_guests_cmd=None + ), device=SimpleNamespace( - platform=None, tags="", custom_fields="", blade_role="Blade", - chassis_role="Server Chassis", server_role="Server", - default_owner="FarmGPU", asset_tag_cmd=None, + platform=None, + tags="", + custom_fields="", + blade_role="Blade", + chassis_role="Server Chassis", + server_role="Server", + default_owner="FarmGPU", + asset_tag_cmd=None, ), tenant=SimpleNamespace(driver=None, driver_file=None, regex=None), datacenter_location=SimpleNamespace(driver=None, driver_file=None, regex=None), rack_location=SimpleNamespace(driver=None, driver_file=None, regex=None), slot_location=SimpleNamespace(driver=None, driver_file=None, regex=None), network=SimpleNamespace( - ignore_interfaces="(dummy.*|docker.*)", ignore_ips="^(127\\.0\\.0\\..*)", - ipmi=True, lldp=None, nic_id="name", primary_mac="temp", + ignore_interfaces="(dummy.*|docker.*)", + ignore_ips="^(127\\.0\\.0\\..*)", + ipmi=True, + lldp=None, + nic_id="name", + primary_mac="temp", ), ) @@ -66,7 +79,9 @@ _mock_config_module.get_config = MagicMock(return_value=_mock_config) _mock_config_module.get_netbox_instance = MagicMock(return_value=_mock_nb) -if "netbox_agent.config" not in sys.modules or isinstance(sys.modules["netbox_agent.config"], MagicMock): +if "netbox_agent.config" not in sys.modules or isinstance( + sys.modules["netbox_agent.config"], MagicMock +): sys.modules["netbox_agent.config"] = _mock_config_module _mock_misc = MagicMock() @@ -77,11 +92,13 @@ _mock_misc.get_device_platform = MagicMock() _mock_misc.get_vendor = MagicMock(return_value="Unknown") -if "netbox_agent.misc" not in sys.modules or isinstance(sys.modules["netbox_agent.misc"], MagicMock): +if "netbox_agent.misc" not in sys.modules or isinstance( + sys.modules["netbox_agent.misc"], MagicMock +): sys.modules["netbox_agent.misc"] = _mock_misc -from netbox_agent.modules import ModuleManager -from netbox_agent.lshw import LSHW +from netbox_agent.modules import ModuleManager # noqa: E402 +from netbox_agent.lshw import LSHW # noqa: E402 # --------------------------------------------------------------------------- @@ -112,8 +129,9 @@ def _build_lshw_from_fixture(fixture): hw_data = hw_data[0] # Construct LSHW by patching subprocess and is_tool - with patch("netbox_agent.lshw.subprocess") as mock_sub, \ - patch("netbox_agent.lshw.is_tool", return_value=True): + with patch("netbox_agent.lshw.subprocess") as mock_sub, patch( + "netbox_agent.lshw.is_tool", return_value=True + ): mock_sub.getoutput.return_value = json.dumps(hw_data) lshw = LSHW() return lshw @@ -138,6 +156,7 @@ def _build_module_manager(fixture, lshw_instance): # Parametrized fixture discovery # --------------------------------------------------------------------------- + def _discover_fixtures(): """Find all fixture JSON files in tests/fixtures/.""" fixtures = [] @@ -154,6 +173,7 @@ def _discover_fixtures(): # Tests: LSHW Parsing with Real Data # --------------------------------------------------------------------------- + class TestLSHWFixtureParsing: """Test that LSHW correctly parses hardware from fixture data.""" @@ -192,17 +212,18 @@ def mock_check_output(cmd, **kwargs): return json.dumps(lscpu_data["data"]) raise FileNotFoundError(str(cmd)) - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", - side_effect=mock_check_output): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output + ): cpus = mm._get_local_cpus() else: # Fallback: lshw only with patch("netbox_agent.modules.is_tool", return_value=False): cpus = mm._get_local_cpus() - assert len(cpus) == expected["cpus"], \ + assert len(cpus) == expected["cpus"], ( f"Expected {expected['cpus']} CPUs, got {len(cpus)}: {[c['product'] for c in cpus]}" + ) @pytest.mark.parametrize("fixture_name", fixture_files) def test_gpu_count_matches_expected(self, fixture_name): @@ -220,8 +241,9 @@ def test_gpu_count_matches_expected(self, fixture_name): # Disable nvidia-smi in test with patch("netbox_agent.modules.is_tool", return_value=False): gpus = mm._get_local_gpus() - assert len(gpus) == expected["gpus"], \ + assert len(gpus) == expected["gpus"], ( f"Expected {expected['gpus']} GPUs, got {len(gpus)}: {[g['product'] for g in gpus]}" + ) @pytest.mark.parametrize("fixture_name", fixture_files) def test_dimm_count_matches_expected(self, fixture_name): @@ -237,8 +259,9 @@ def test_dimm_count_matches_expected(self, fixture_name): mm = _build_module_manager(fixture, lshw) dimms = mm._get_local_dimms() - assert len(dimms) == expected["dimms"], \ + assert len(dimms) == expected["dimms"], ( f"Expected {expected['dimms']} DIMMs, got {len(dimms)}" + ) @pytest.mark.parametrize("fixture_name", fixture_files) def test_nic_count_matches_expected(self, fixture_name): @@ -254,14 +277,16 @@ def test_nic_count_matches_expected(self, fixture_name): mm = _build_module_manager(fixture, lshw) nics = mm._get_local_nics() - assert len(nics) == expected["nics"], \ + assert len(nics) == expected["nics"], ( f"Expected {expected['nics']} NICs, got {len(nics)}: {[n['product'] for n in nics]}" + ) # --------------------------------------------------------------------------- # Tests: Storage Detection with lsblk Fixture Data # --------------------------------------------------------------------------- + class TestStorageFixtureDetection: """Test lsblk-based storage detection using fixture data.""" @@ -293,12 +318,14 @@ def mock_check_output(cmd, **kwargs): raise FileNotFoundError("nvme") raise FileNotFoundError(str(cmd)) - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output + ): storage = mm._get_local_ssds() - assert len(storage) == expected["storage"], \ + assert len(storage) == expected["storage"], ( f"Expected {expected['storage']} storage devices, got {len(storage)}: {[s['product'] for s in storage]}" + ) @pytest.mark.parametrize("fixture_name", fixture_files) def test_storage_interfaces_match_expected(self, fixture_name): @@ -328,13 +355,15 @@ def mock_check_output(cmd, **kwargs): raise FileNotFoundError("nvme") raise FileNotFoundError(str(cmd)) - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output + ): storage = mm._get_local_ssds() interfaces = [s.get("interface") for s in storage] - assert interfaces == expected["storage_interfaces"], \ + assert interfaces == expected["storage_interfaces"], ( f"Expected interfaces {expected['storage_interfaces']}, got {interfaces}" + ) @pytest.mark.parametrize("fixture_name", fixture_files) def test_nvme_enrichment_adds_vendor(self, fixture_name): @@ -358,23 +387,25 @@ def mock_check_output(cmd, **kwargs): return json.dumps(nvme_data["data"]) raise FileNotFoundError(str(cmd)) - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output + ): storage = mm._get_local_ssds() # Check NVMe devices have vendor populated for item in storage: if item.get("interface") == "NVMe": - assert item["vendor"] != "Unknown", \ + assert item["vendor"] != "Unknown", ( f"NVMe device {item['product']} should have vendor from nvme-cli" + ) # --------------------------------------------------------------------------- # Tests: GPU Serial Detection with nvidia-smi Fixture Data # --------------------------------------------------------------------------- -class TestGPUSerialFixtureDetection: +class TestGPUSerialFixtureDetection: @pytest.mark.parametrize("fixture_name", fixture_files) def test_gpu_serials_match_expected(self, fixture_name): """GPU serials from nvidia-smi should match expected.""" @@ -396,20 +427,23 @@ def test_gpu_serials_match_expected(self, fixture_name): def mock_is_tool(name): return name == "nvidia-smi" - with patch("netbox_agent.modules.is_tool", side_effect=mock_is_tool), \ - patch("netbox_agent.modules.subprocess.check_output", - return_value=nvidia.get("query_csv", "")): + with patch("netbox_agent.modules.is_tool", side_effect=mock_is_tool), patch( + "netbox_agent.modules.subprocess.check_output", + return_value=nvidia.get("query_csv", ""), + ): gpus = mm._get_local_gpus() serials = [g["serial"] for g in gpus if g.get("serial")] - assert serials == expected["gpu_serials"], \ + assert serials == expected["gpu_serials"], ( f"Expected GPU serials {expected['gpu_serials']}, got {serials}" + ) # --------------------------------------------------------------------------- # Tests: Interface Detection Unit Tests # --------------------------------------------------------------------------- + class TestInterfaceDetection: """Test _detect_storage_interface and _build_storage_description.""" @@ -429,44 +463,54 @@ def mm(self): with patch("netbox_agent.modules.LSHW", return_value=lshw): return ModuleManager(server=server, config=_mock_config) - @pytest.mark.parametrize("tran,name,expected", [ - ("nvme", "nvme0n1", "NVMe"), - ("sata", "sda", "SATA"), - ("sas", "sdb", "SAS"), - ("usb", "sdc", "USB"), - ("ata", "sdd", "SATA"), - ("fc", "sde", "FC"), - ("", "nvme0n1", "NVMe"), - ("", "sda", "SATA"), - ("", "hda", "IDE"), - (None, "xvda", None), - ]) + @pytest.mark.parametrize( + "tran,name,expected", + [ + ("nvme", "nvme0n1", "NVMe"), + ("sata", "sda", "SATA"), + ("sas", "sdb", "SAS"), + ("usb", "sdc", "USB"), + ("ata", "sdd", "SATA"), + ("fc", "sde", "FC"), + ("", "nvme0n1", "NVMe"), + ("", "sda", "SATA"), + ("", "hda", "IDE"), + (None, "xvda", None), + ], + ) def test_detect_storage_interface(self, mm, tran, name, expected): result = mm._detect_storage_interface(tran or "", name) - assert result == expected, f"tran={tran!r}, name={name!r}: expected {expected!r}, got {result!r}" - - @pytest.mark.parametrize("interface,rota,expected", [ - ("NVMe", "0", "NVMe SSD"), - ("NVMe", 0, "NVMe SSD"), - ("SATA", "0", "SATA SSD"), - ("SATA", "1", "SATA HDD"), - ("SATA", 1, "SATA HDD"), - ("SAS", "1", "SAS HDD"), - ("SAS", "0", "SAS SSD"), - (None, None, "disk"), - ("NVMe", None, "NVMe disk"), - ]) + assert result == expected, ( + f"tran={tran!r}, name={name!r}: expected {expected!r}, got {result!r}" + ) + + @pytest.mark.parametrize( + "interface,rota,expected", + [ + ("NVMe", "0", "NVMe SSD"), + ("NVMe", 0, "NVMe SSD"), + ("SATA", "0", "SATA SSD"), + ("SATA", "1", "SATA HDD"), + ("SATA", 1, "SATA HDD"), + ("SAS", "1", "SAS HDD"), + ("SAS", "0", "SAS SSD"), + (None, None, "disk"), + ("NVMe", None, "NVMe disk"), + ], + ) def test_build_storage_description(self, mm, interface, rota, expected): result = mm._build_storage_description(interface, rota) - assert result == expected, f"interface={interface!r}, rota={rota!r}: expected {expected!r}, got {result!r}" + assert result == expected, ( + f"interface={interface!r}, rota={rota!r}: expected {expected!r}, got {result!r}" + ) # --------------------------------------------------------------------------- # Tests: Vendor Guessing # --------------------------------------------------------------------------- -class TestVendorGuessing: +class TestVendorGuessing: @pytest.fixture def mm(self): lshw = MagicMock() @@ -482,18 +526,23 @@ def mm(self): with patch("netbox_agent.modules.LSHW", return_value=lshw): return ModuleManager(server=server, config=_mock_config) - @pytest.mark.parametrize("model,expected_vendor", [ - ("Samsung SSD 990 PRO 2TB", "Samsung"), - ("Solidigm D7-P5520", "Solidigm"), - ("Intel SSDPE2KX040T8", "Intel"), - ("Micron_5300_MTFDDAK960TDS", "Micron"), - ("WDC WD4003FFBX-68MU3N0", "Western Digital"), - ("ST4000NM000A-2HZ100", "Seagate"), - ("KIOXIA KCM61RUL3T84", "Kioxia"), - ("HGST HUS726040ALE614", "HGST"), - ("Hitachi HDS723020BLA642", "Hitachi"), - ("UNKNOWN-MODEL-XYZ", None), - ]) + @pytest.mark.parametrize( + "model,expected_vendor", + [ + ("Samsung SSD 990 PRO 2TB", "Samsung"), + ("Solidigm D7-P5520", "Solidigm"), + ("Intel SSDPE2KX040T8", "Intel"), + ("Micron_5300_MTFDDAK960TDS", "Micron"), + ("WDC WD4003FFBX-68MU3N0", "Western Digital"), + ("ST4000NM000A-2HZ100", "Seagate"), + ("KIOXIA KCM61RUL3T84", "Kioxia"), + ("HGST HUS726040ALE614", "HGST"), + ("Hitachi HDS723020BLA642", "Hitachi"), + ("UNKNOWN-MODEL-XYZ", None), + ], + ) def test_guess_vendor(self, mm, model, expected_vendor): result = mm._guess_vendor(model) - assert result == expected_vendor, f"model={model!r}: expected {expected_vendor!r}, got {result!r}" + assert result == expected_vendor, ( + f"model={model!r}: expected {expected_vendor!r}, got {result!r}" + ) diff --git a/tests/test_ipmi.py b/tests/test_ipmi.py index 9bf46571..73564650 100644 --- a/tests/test_ipmi.py +++ b/tests/test_ipmi.py @@ -30,7 +30,6 @@ class TestIPMI: - def test_ipmi_channel_fallback(self): """Channel 1 fails (0.0.0.0), channel 2 succeeds.""" call_count = [0] @@ -43,9 +42,9 @@ def mock_getstatusoutput(cmd): return (0, _SAMPLE_OUTPUT) return (1, "") - with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), \ - patch("netbox_agent.ipmi.subprocess.getstatusoutput", - side_effect=mock_getstatusoutput): + with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), patch( + "netbox_agent.ipmi.subprocess.getstatusoutput", side_effect=mock_getstatusoutput + ): ipmi = IPMI() assert ipmi.channel == 2 @@ -55,9 +54,9 @@ def mock_getstatusoutput(cmd): def test_ipmi_prefix_normalization_to_32(self): """OOB IP is always normalized to /32 regardless of subnet mask.""" - with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), \ - patch("netbox_agent.ipmi.subprocess.getstatusoutput", - return_value=(0, _SAMPLE_OUTPUT)): + with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), patch( + "netbox_agent.ipmi.subprocess.getstatusoutput", return_value=(0, _SAMPLE_OUTPUT) + ): ipmi = IPMI() result = ipmi.parse() @@ -74,12 +73,13 @@ def test_ipmi_unavailable_returns_empty(self): def test_ipmi_invalid_ip_skipped(self): """All channels return 0.0.0.0 → empty dict.""" + def mock_getstatusoutput(cmd): return (0, _INVALID_IP_OUTPUT) - with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), \ - patch("netbox_agent.ipmi.subprocess.getstatusoutput", - side_effect=mock_getstatusoutput): + with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), patch( + "netbox_agent.ipmi.subprocess.getstatusoutput", side_effect=mock_getstatusoutput + ): ipmi = IPMI() result = ipmi.parse() @@ -92,9 +92,9 @@ def test_ipmi_vlan_parsing(self): "802.1q VLAN ID : Disabled", "802.1q VLAN ID : 100", ) - with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), \ - patch("netbox_agent.ipmi.subprocess.getstatusoutput", - return_value=(0, vlan_output)): + with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), patch( + "netbox_agent.ipmi.subprocess.getstatusoutput", return_value=(0, vlan_output) + ): ipmi = IPMI() result = ipmi.parse() @@ -102,9 +102,9 @@ def test_ipmi_vlan_parsing(self): def test_ipmi_all_channels_fail(self): """All channels return non-zero exit code → empty dict.""" - with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), \ - patch("netbox_agent.ipmi.subprocess.getstatusoutput", - return_value=(1, "error")): + with patch("netbox_agent.ipmi.which", return_value="/usr/bin/ipmitool"), patch( + "netbox_agent.ipmi.subprocess.getstatusoutput", return_value=(1, "error") + ): ipmi = IPMI() result = ipmi.parse() diff --git a/tests/test_modules.py b/tests/test_modules.py index bbdc7bdf..f9321f50 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -38,20 +38,33 @@ force_disk_refresh=False, dump_disks_map=None, log_level="debug", - netbox=SimpleNamespace(url="http://test", token="test", ssl_verify=True, ssl_ca_certs_file=None), - virtual=SimpleNamespace(enabled=False, cluster_name=None, hypervisor=False, list_guests_cmd=None), + netbox=SimpleNamespace( + url="http://test", token="test", ssl_verify=True, ssl_ca_certs_file=None + ), + virtual=SimpleNamespace( + enabled=False, cluster_name=None, hypervisor=False, list_guests_cmd=None + ), device=SimpleNamespace( - platform=None, tags="", custom_fields="", blade_role="Blade", - chassis_role="Server Chassis", server_role="Server", - default_owner="FarmGPU", asset_tag_cmd=None, + platform=None, + tags="", + custom_fields="", + blade_role="Blade", + chassis_role="Server Chassis", + server_role="Server", + default_owner="FarmGPU", + asset_tag_cmd=None, ), tenant=SimpleNamespace(driver=None, driver_file=None, regex=None), datacenter_location=SimpleNamespace(driver=None, driver_file=None, regex=None), rack_location=SimpleNamespace(driver=None, driver_file=None, regex=None), slot_location=SimpleNamespace(driver=None, driver_file=None, regex=None), network=SimpleNamespace( - ignore_interfaces="(dummy.*|docker.*)", ignore_ips="^(127\\.0\\.0\\..*)", - ipmi=True, lldp=None, nic_id="name", primary_mac="temp", + ignore_interfaces="(dummy.*|docker.*)", + ignore_ips="^(127\\.0\\.0\\..*)", + ipmi=True, + lldp=None, + nic_id="name", + primary_mac="temp", ), ) @@ -74,13 +87,14 @@ sys.modules["netbox_agent.misc"] = _mock_misc # Now we can safely import -from netbox_agent.modules import ModuleManager, CATEGORIES +from netbox_agent.modules import ModuleManager, CATEGORIES # noqa: E402 # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- + @pytest.fixture(autouse=True) def reset_nb_mock(): """Reset the shared mock nb between tests, including nested side_effects.""" @@ -142,8 +156,8 @@ def mm(mock_server, mock_lshw): # Tests: Hardware Detection # --------------------------------------------------------------------------- -class TestModuleManagerDetection: +class TestModuleManagerDetection: def test_get_local_cpus_lscpu(self, mm, mock_lshw): """CPU detection via lscpu (primary path, informed by SILO cpu.py).""" lscpu_data = { @@ -156,9 +170,9 @@ def test_get_local_cpus_lscpu(self, mm, mock_lshw): {"field": "Thread(s) per core:", "data": "2"}, ] } - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", - return_value=json.dumps(lscpu_data)): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", return_value=json.dumps(lscpu_data) + ): cpus = mm._get_local_cpus() assert len(cpus) == 2 assert cpus[0]["product"] == "Intel(R) Xeon(R) Gold 6430" @@ -175,9 +189,9 @@ def test_get_local_cpus_lscpu_amd(self, mm, mock_lshw): {"field": "Vendor ID:", "data": "AuthenticAMD"}, ] } - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", - return_value=json.dumps(lscpu_data)): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", return_value=json.dumps(lscpu_data) + ): cpus = mm._get_local_cpus() assert len(cpus) == 2 assert cpus[0]["vendor"] == "AMD" # normalized from AuthenticAMD @@ -192,9 +206,9 @@ def test_get_local_cpus_lscpu_no_qat(self, mm, mock_lshw): {"field": "Vendor ID:", "data": "GenuineIntel"}, ] } - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", - return_value=json.dumps(lscpu_data)): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", return_value=json.dumps(lscpu_data) + ): cpus = mm._get_local_cpus() # lscpu reports exactly 2 sockets — no QAT noise assert len(cpus) == 2 @@ -205,8 +219,12 @@ def test_get_local_cpus_lshw_fallback_filters_qat(self, mm, mock_lshw): mock_lshw.get_hw_linux.return_value = [ {"product": "Xeon Gold 6430", "vendor": "Intel", "location": "CPU0"}, {"product": "Xeon Gold 6430", "vendor": "Intel", "location": "CPU1"}, - {"product": "C62x Chipset QuickAssist Technology", "vendor": "Intel", - "description": "Co-processor", "location": ""}, + { + "product": "C62x Chipset QuickAssist Technology", + "vendor": "Intel", + "description": "Co-processor", + "location": "", + }, {"product": "4xxx Series QAT", "vendor": "Intel Corporation", "description": ""}, {"product": "Intel Corporation", "vendor": "Intel Corporation", "description": ""}, ] @@ -231,8 +249,9 @@ def test_get_local_gpus_with_serials(self, mm, mock_lshw): {"product": "A100 80GB SXM4", "vendor": "NVIDIA", "description": "3D"}, ] nvidia_output = "0, 1324821038475\n1, 1324821038476" - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", return_value=nvidia_output): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", return_value=nvidia_output + ): gpus = mm._get_local_gpus() assert len(gpus) == 2 assert gpus[0]["serial"] == "1324821038475" @@ -240,10 +259,22 @@ def test_get_local_gpus_with_serials(self, mm, mock_lshw): def test_get_local_dimms(self, mm, mock_lshw): mock_lshw.memories = [ - {"product": "M393A8G40AB2-CWE", "vendor": "Samsung", "serial": "ABC123", - "slot": "DIMM_A1", "size": 64, "description": "DDR4"}, - {"product": "M393A8G40AB2-CWE", "vendor": "Samsung", "serial": "Not Specified", - "slot": "DIMM_A2", "size": 64, "description": "DDR4"}, + { + "product": "M393A8G40AB2-CWE", + "vendor": "Samsung", + "serial": "ABC123", + "slot": "DIMM_A1", + "size": 64, + "description": "DDR4", + }, + { + "product": "M393A8G40AB2-CWE", + "vendor": "Samsung", + "serial": "Not Specified", + "slot": "DIMM_A2", + "size": 64, + "description": "DDR4", + }, ] dimms = mm._get_local_dimms() assert len(dimms) == 2 @@ -254,26 +285,72 @@ def test_get_local_ssds_lsblk(self, mm, mock_lshw): """Storage detection via lsblk (primary path).""" lsblk_data = { "blockdevices": [ - {"name": "nvme0n1", "type": "disk", "size": 3840755982336, - "model": "D7-P5520", "serial": "SSD123", "vendor": None, - "tran": "nvme", "rota": "0", "hctl": None, "rev": "V1.0"}, - {"name": "sda", "type": "disk", "size": 960197124096, - "model": "Samsung SSD 870", "serial": "SSD456", "vendor": "ATA", - "tran": "sata", "rota": "0", "hctl": "0:0:0:0", "rev": "2B6Q"}, - {"name": "sdb", "type": "disk", "size": 4000787030016, - "model": "ST4000NM000A", "serial": "HDD789", "vendor": "ATA", - "tran": "sata", "rota": "1", "hctl": "1:0:0:0", "rev": None}, - {"name": "loop0", "type": "loop", "size": 0, - "model": None, "serial": None, "vendor": None, - "tran": None, "rota": "0", "hctl": None, "rev": None}, - {"name": "dm-0", "type": "disk", "size": 107374182400, - "model": None, "serial": None, "vendor": None, - "tran": None, "rota": "0", "hctl": None, "rev": None}, + { + "name": "nvme0n1", + "type": "disk", + "size": 3840755982336, + "model": "D7-P5520", + "serial": "SSD123", + "vendor": None, + "tran": "nvme", + "rota": "0", + "hctl": None, + "rev": "V1.0", + }, + { + "name": "sda", + "type": "disk", + "size": 960197124096, + "model": "Samsung SSD 870", + "serial": "SSD456", + "vendor": "ATA", + "tran": "sata", + "rota": "0", + "hctl": "0:0:0:0", + "rev": "2B6Q", + }, + { + "name": "sdb", + "type": "disk", + "size": 4000787030016, + "model": "ST4000NM000A", + "serial": "HDD789", + "vendor": "ATA", + "tran": "sata", + "rota": "1", + "hctl": "1:0:0:0", + "rev": None, + }, + { + "name": "loop0", + "type": "loop", + "size": 0, + "model": None, + "serial": None, + "vendor": None, + "tran": None, + "rota": "0", + "hctl": None, + "rev": None, + }, + { + "name": "dm-0", + "type": "disk", + "size": 107374182400, + "model": None, + "serial": None, + "vendor": None, + "tran": None, + "rota": "0", + "hctl": None, + "rev": None, + }, ] } - with patch("netbox_agent.modules.is_tool") as mock_is_tool, \ - patch("netbox_agent.modules.subprocess.check_output") as mock_subprocess: + with patch("netbox_agent.modules.is_tool") as mock_is_tool, patch( + "netbox_agent.modules.subprocess.check_output" + ) as mock_subprocess: mock_is_tool.side_effect = lambda t: t in ("lsblk",) mock_subprocess.return_value = json.dumps(lsblk_data) ssds = mm._get_local_ssds() @@ -295,20 +372,35 @@ def test_get_local_ssds_lsblk_nvme_enrichment(self, mm, mock_lshw): """NVMe devices should be enriched with nvme-cli data when available.""" lsblk_data = { "blockdevices": [ - {"name": "nvme0n1", "type": "disk", "size": 3840755982336, - "model": "D7-P5520", "serial": "SSD-NVM1", "vendor": None, - "tran": "nvme", "rota": "0", "hctl": None, "rev": None}, + { + "name": "nvme0n1", + "type": "disk", + "size": 3840755982336, + "model": "D7-P5520", + "serial": "SSD-NVM1", + "vendor": None, + "tran": "nvme", + "rota": "0", + "hctl": None, + "rev": None, + }, ] } nvme_data = { "Devices": [ - {"DevicePath": "/dev/nvme0n1", "ModelNumber": "Solidigm D7-P5520", - "SerialNumber": "SSD-NVM1", "Vendor": "Solidigm", - "PhysicalSize": 3840755982336, "Firmware": "V1.2.3"}, + { + "DevicePath": "/dev/nvme0n1", + "ModelNumber": "Solidigm D7-P5520", + "SerialNumber": "SSD-NVM1", + "Vendor": "Solidigm", + "PhysicalSize": 3840755982336, + "Firmware": "V1.2.3", + }, ] } call_count = [0] + def mock_check_output(cmd, **kwargs): call_count[0] += 1 if "lsblk" in cmd: @@ -317,8 +409,9 @@ def mock_check_output(cmd, **kwargs): return json.dumps(nvme_data) raise FileNotFoundError(cmd[0]) - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output + ): ssds = mm._get_local_ssds() assert len(ssds) == 1 @@ -328,12 +421,19 @@ def mock_check_output(cmd, **kwargs): def test_get_local_ssds_lshw_fallback(self, mm, mock_lshw): """Falls back to lshw when lsblk is not available.""" mock_lshw.get_hw_linux.return_value = [ - {"product": "D7-P5520", "vendor": "Solidigm", "serial": "SSD123", - "description": "NVMe disk"}, - {"product": None, "vendor": None, "serial": "SSD456", - "description": "NVMe disk"}, - {"product": "Virtual disk", "vendor": None, "serial": "VD001", - "description": "Virtual volume"}, + { + "product": "D7-P5520", + "vendor": "Solidigm", + "serial": "SSD123", + "description": "NVMe disk", + }, + {"product": None, "vendor": None, "serial": "SSD456", "description": "NVMe disk"}, + { + "product": "Virtual disk", + "vendor": None, + "serial": "VD001", + "description": "Virtual volume", + }, ] with patch("netbox_agent.modules.is_tool", return_value=False): ssds = mm._get_local_ssds() @@ -344,17 +444,36 @@ def test_get_local_ssds_dedup_serials(self, mm, mock_lshw): """Duplicate serials should be deduplicated.""" lsblk_data = { "blockdevices": [ - {"name": "nvme0n1", "type": "disk", "size": 100000, - "model": "TestDrive", "serial": "DUP-SERIAL", "vendor": "Test", - "tran": "nvme", "rota": "0", "hctl": None, "rev": None}, - {"name": "nvme1n1", "type": "disk", "size": 100000, - "model": "TestDrive", "serial": "DUP-SERIAL", "vendor": "Test", - "tran": "nvme", "rota": "0", "hctl": None, "rev": None}, + { + "name": "nvme0n1", + "type": "disk", + "size": 100000, + "model": "TestDrive", + "serial": "DUP-SERIAL", + "vendor": "Test", + "tran": "nvme", + "rota": "0", + "hctl": None, + "rev": None, + }, + { + "name": "nvme1n1", + "type": "disk", + "size": 100000, + "model": "TestDrive", + "serial": "DUP-SERIAL", + "vendor": "Test", + "tran": "nvme", + "rota": "0", + "hctl": None, + "rev": None, + }, ] } - with patch("netbox_agent.modules.is_tool") as mock_is_tool, \ - patch("netbox_agent.modules.subprocess.check_output") as mock_subprocess: + with patch("netbox_agent.modules.is_tool") as mock_is_tool, patch( + "netbox_agent.modules.subprocess.check_output" + ) as mock_subprocess: mock_is_tool.side_effect = lambda t: t == "lsblk" mock_subprocess.return_value = json.dumps(lsblk_data) ssds = mm._get_local_ssds() @@ -365,17 +484,36 @@ def test_get_local_ssds_vendor_guessing(self, mm, mock_lshw): """Vendor should be guessed from model when not provided by lsblk.""" lsblk_data = { "blockdevices": [ - {"name": "nvme0n1", "type": "disk", "size": 100000, - "model": "Samsung SSD 990 PRO", "serial": "S1", "vendor": None, - "tran": "nvme", "rota": "0", "hctl": None, "rev": None}, - {"name": "sda", "type": "disk", "size": 100000, - "model": "Solidigm D7-PS1010", "serial": "S2", "vendor": None, - "tran": "sata", "rota": "0", "hctl": None, "rev": None}, + { + "name": "nvme0n1", + "type": "disk", + "size": 100000, + "model": "Samsung SSD 990 PRO", + "serial": "S1", + "vendor": None, + "tran": "nvme", + "rota": "0", + "hctl": None, + "rev": None, + }, + { + "name": "sda", + "type": "disk", + "size": 100000, + "model": "Solidigm D7-PS1010", + "serial": "S2", + "vendor": None, + "tran": "sata", + "rota": "0", + "hctl": None, + "rev": None, + }, ] } - with patch("netbox_agent.modules.is_tool") as mock_is_tool, \ - patch("netbox_agent.modules.subprocess.check_output") as mock_subprocess: + with patch("netbox_agent.modules.is_tool") as mock_is_tool, patch( + "netbox_agent.modules.subprocess.check_output" + ) as mock_subprocess: mock_is_tool.side_effect = lambda t: t == "lsblk" mock_subprocess.return_value = json.dumps(lsblk_data) ssds = mm._get_local_ssds() @@ -385,12 +523,27 @@ def test_get_local_ssds_vendor_guessing(self, mm, mock_lshw): def test_get_local_nics(self, mm, mock_lshw): mock_lshw.interfaces = [ - {"product": "E810-XXVDA2", "vendor": "Intel", "serial": "aa:bb:cc:dd:ee:01", - "name": "eno1", "description": "Ethernet"}, - {"product": "E810-XXVDA2", "vendor": "Intel", "serial": "aa:bb:cc:dd:ee:02", - "name": "eno2", "description": "Ethernet"}, - {"product": "E810-XXVDA2", "vendor": "Intel", "serial": "aa:bb:cc:dd:ee:01", - "name": "eno1v1", "description": "Ethernet"}, + { + "product": "E810-XXVDA2", + "vendor": "Intel", + "serial": "aa:bb:cc:dd:ee:01", + "name": "eno1", + "description": "Ethernet", + }, + { + "product": "E810-XXVDA2", + "vendor": "Intel", + "serial": "aa:bb:cc:dd:ee:02", + "name": "eno2", + "description": "Ethernet", + }, + { + "product": "E810-XXVDA2", + "vendor": "Intel", + "serial": "aa:bb:cc:dd:ee:01", + "name": "eno1v1", + "description": "Ethernet", + }, ] nics = mm._get_local_nics() assert len(nics) == 2 @@ -444,8 +597,8 @@ def test_gpu_product_truncation(self, mm, mock_lshw): # Tests: Module Type Resolution # --------------------------------------------------------------------------- -class TestModuleManagerTypeResolution: +class TestModuleManagerTypeResolution: def test_resolve_existing_module_type(self, mm, nb): mock_mfr = MagicMock(id=10, name="Intel") nb.dcim.manufacturers.get.return_value = mock_mfr @@ -484,8 +637,8 @@ def test_manufacturer_caching(self, mm, nb): # Tests: Module Bay Management # --------------------------------------------------------------------------- -class TestModuleManagerBayManagement: +class TestModuleManagerBayManagement: def test_ensure_module_bays_creates_missing(self, mm, nb): device = SimpleNamespace(id=1, name="test-server") @@ -508,8 +661,8 @@ def test_ensure_module_bays_creates_missing(self, mm, nb): # Tests: Core Sync Algorithm # --------------------------------------------------------------------------- -class TestModuleManagerSync: +class TestModuleManagerSync: def test_sync_creates_new_modules(self, mm, nb): device = SimpleNamespace(id=1, name="test-server") mm.device = device @@ -586,8 +739,8 @@ def bays_filter_side_effect(**kwargs): # Tests: create_or_update Entry Point # --------------------------------------------------------------------------- -class TestModuleManagerCreateOrUpdate: +class TestModuleManagerCreateOrUpdate: def test_no_device_returns_false(self, mm, mock_server): mock_server.get_netbox_server.return_value = None result = mm.create_or_update() @@ -615,8 +768,8 @@ def test_create_or_update_calls_all_categories(self, mm, nb): # Tests: Asset Tag Validation (regex-only, no module import needed) # --------------------------------------------------------------------------- -class TestAssetTagParsing: +class TestAssetTagParsing: # Re-define the regex/constants here to avoid importing server.py # (which would trigger config import chain) _ASSET_TAG_RE = re.compile(r"^[0-9A-Z]{4}$", re.IGNORECASE) @@ -643,8 +796,8 @@ def test_placeholder_tags(self): # Tests: Base-36 Encoding # --------------------------------------------------------------------------- -class TestBase36Encoding: +class TestBase36Encoding: @staticmethod def int_to_base36(n, width=4): chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -684,16 +837,33 @@ def test_roundtrip(self): # These test the _is_valid_serial and _get_best_serial logic from server.py. # We re-implement the logic here to avoid importing server.py (config chain). + class TestSerialValidation: """Test DMI placeholder detection and serial cascade logic.""" _DMI_PLACEHOLDERS = { - "", "none", "n/a", "na", "not specified", "not available", - "not applicable", "to be filled by o.e.m.", "default string", - "0123456789", "..................", "system serial number", - "chassis serial number", "base board serial number", - "default", "unknown", "unspecified", "no asset information", - "empty", "xxxxxxxxxxxx", "0000000000", "____________", + "", + "none", + "n/a", + "na", + "not specified", + "not available", + "not applicable", + "to be filled by o.e.m.", + "default string", + "0123456789", + "..................", + "system serial number", + "chassis serial number", + "base board serial number", + "default", + "unknown", + "unspecified", + "no asset information", + "empty", + "xxxxxxxxxxxx", + "0000000000", + "____________", } def _is_valid_serial(self, value): @@ -764,11 +934,10 @@ def test_whitespace_handling(self): # Tests: API Retry # --------------------------------------------------------------------------- -from netbox_agent.modules import _api_retry, MAX_RETRIES, RETRY_BACKOFF +from netbox_agent.modules import _api_retry, MAX_RETRIES, RETRY_BACKOFF # noqa: E402 class TestApiRetry: - def test_api_retry_success_first_attempt(self): """Succeeds on first call — no retries needed.""" func = MagicMock(return_value="ok") @@ -799,13 +968,14 @@ def test_api_retry_backoff_timing(self): _api_retry(func) # First retry: 2 * 2^0 = 2s, second retry: 2 * 2^1 = 4s calls = [c.args[0] for c in mock_sleep.call_args_list] - assert calls == [RETRY_BACKOFF * (2 ** 0), RETRY_BACKOFF * (2 ** 1)] + assert calls == [RETRY_BACKOFF * (2**0), RETRY_BACKOFF * (2**1)] def test_api_retry_logs_warnings(self): """Retries log warning messages.""" func = MagicMock(side_effect=[Exception("oops"), "ok"]) - with patch("netbox_agent.modules.time.sleep"), \ - patch("netbox_agent.modules.logger") as mock_logger: + with patch("netbox_agent.modules.time.sleep"), patch( + "netbox_agent.modules.logger" + ) as mock_logger: _api_retry(func) mock_logger.warning.assert_called() @@ -814,8 +984,8 @@ def test_api_retry_logs_warnings(self): # Tests: Sync Algorithm (extended) # --------------------------------------------------------------------------- -class TestSyncAlgorithmExtended: +class TestSyncAlgorithmExtended: def test_sync_duplicate_serials_warns(self, mm, nb): """Duplicate serials in remote lookup should log warning.""" device = SimpleNamespace(id=1, name="test-server") @@ -839,9 +1009,12 @@ def test_sync_duplicate_serials_warns(self, mm, nb): nb.dcim.module_types.get.return_value = MagicMock(id=50) with patch("netbox_agent.modules.logger") as mock_logger: - mm._sync_category("gpu", [ - {"product": "A100", "vendor": "NVIDIA", "serial": "DUP-SN"}, - ]) + mm._sync_category( + "gpu", + [ + {"product": "A100", "vendor": "NVIDIA", "serial": "DUP-SN"}, + ], + ) # Should log warning about duplicate serials warning_calls = [str(c) for c in mock_logger.warning.call_args_list] assert any("Duplicate" in s or "duplicate" in s.lower() for s in warning_calls) @@ -899,9 +1072,12 @@ def test_sync_no_serial_positional_match(self, mm, nb): nb.dcim.module_type_profiles.get.return_value = MagicMock(id=1) nb.dcim.module_types.get.return_value = MagicMock(id=50) - mm._sync_category("cpu", [ - {"product": "Xeon Gold 6430", "vendor": "Intel", "serial": None}, - ]) + mm._sync_category( + "cpu", + [ + {"product": "Xeon Gold 6430", "vendor": "Intel", "serial": None}, + ], + ) # Should not create new — existing module is positionally matched nb.dcim.modules.create.assert_not_called() @@ -953,9 +1129,12 @@ def test_sync_module_type_resolution_failure_skips(self, mm, nb): try: with patch("netbox_agent.modules.time.sleep"): with pytest.raises(Exception, match="API error"): - mm._sync_category("gpu", [ - {"product": "BadGPU", "vendor": "Unknown", "serial": "SN1"}, - ]) + mm._sync_category( + "gpu", + [ + {"product": "BadGPU", "vendor": "Unknown", "serial": "SN1"}, + ], + ) finally: nb.dcim.manufacturers.get.side_effect = None nb.dcim.module_bays.filter.side_effect = None @@ -980,6 +1159,7 @@ def test_sync_bay_conflict_moves_occupant(self, mm, nb): spare_bay = SimpleNamespace(id=600, name="GPU-0") call_idx = [0] + def modules_filter_seq(**kwargs): call_idx[0] += 1 device_id = kwargs.get("device_id") @@ -1001,9 +1181,10 @@ def modules_filter_seq(**kwargs): def bays_filter_spare(**kwargs): return [spare_bay] + # Override for spare device bay lookups nb.dcim.module_bays.filter.side_effect = [ - [], # initial existing bays + [], # initial existing bays [bay0], # re-fetch [spare_bay], # spare bays ] @@ -1016,9 +1197,12 @@ def bays_filter_spare(**kwargs): new_mod = MagicMock(id=500) nb.dcim.modules.create.return_value = new_mod - mm._sync_category("gpu", [ - {"product": "H100", "vendor": "NVIDIA", "serial": "NEW-SN"}, - ]) + mm._sync_category( + "gpu", + [ + {"product": "H100", "vendor": "NVIDIA", "serial": "NEW-SN"}, + ], + ) # Occupant should have been moved (save called to re-parent) assert occupant.save.called @@ -1046,9 +1230,12 @@ def test_sync_serial_existing_on_device_correct_bay_noop(self, mm, nb): nb.dcim.module_type_profiles.get.return_value = MagicMock(id=1) nb.dcim.module_types.get.return_value = MagicMock(id=50) - mm._sync_category("gpu", [ - {"product": "A100", "vendor": "NVIDIA", "serial": "GPU-SN-1"}, - ]) + mm._sync_category( + "gpu", + [ + {"product": "A100", "vendor": "NVIDIA", "serial": "GPU-SN-1"}, + ], + ) # No save (no changes) and no create existing_mod.save.assert_not_called() nb.dcim.modules.create.assert_not_called() @@ -1058,8 +1245,8 @@ def test_sync_serial_existing_on_device_correct_bay_noop(self, mm, nb): # Tests: Re-parenting # --------------------------------------------------------------------------- -class TestReparenting: +class TestReparenting: def test_reparent_module_updates_device_and_bay(self, mm, nb): """reparent_module sets device and bay on the module.""" module = MagicMock(serial="GPU-001") @@ -1137,8 +1324,8 @@ def test_vacate_bay_empty_noop(self, mm, nb): # Tests: Spare Device # --------------------------------------------------------------------------- -class TestSpareDevice: +class TestSpareDevice: def test_get_spare_device_not_found_returns_none(self, mm, nb): """Spare device not in NetBox → returns None.""" nb.dcim.devices.get.return_value = None @@ -1163,8 +1350,8 @@ def test_find_module_by_serial_not_found_returns_none(self, mm, nb): # Tests: Module Type Resolution (extended) # --------------------------------------------------------------------------- -class TestModuleTypeResolutionExtended: +class TestModuleTypeResolutionExtended: def test_resolve_module_type_missing_profile_still_creates(self, mm, nb): """Module type created even when profile not found.""" mock_mfr = MagicMock(id=10) @@ -1220,13 +1407,14 @@ def test_manufacturer_cache_hit(self, mm, nb): # Tests: Edge Cases # --------------------------------------------------------------------------- -class TestEdgeCases: +class TestEdgeCases: def test_parse_lsblk_empty_blockdevices(self, mm, mock_lshw): """Empty blockdevices list → empty result.""" lsblk_data = {"blockdevices": []} - with patch("netbox_agent.modules.is_tool") as mock_is_tool, \ - patch("netbox_agent.modules.subprocess.check_output") as mock_subprocess: + with patch("netbox_agent.modules.is_tool") as mock_is_tool, patch( + "netbox_agent.modules.subprocess.check_output" + ) as mock_subprocess: mock_is_tool.side_effect = lambda t: t == "lsblk" mock_subprocess.return_value = json.dumps(lsblk_data) ssds = mm._get_local_ssds() @@ -1236,13 +1424,23 @@ def test_nvme_enrichment_fallback_on_failure(self, mm, mock_lshw): """NVMe enrichment failure falls back to lsblk data only.""" lsblk_data = { "blockdevices": [ - {"name": "nvme0n1", "type": "disk", "size": 100000, - "model": "TestNVMe", "serial": "SN-1", "vendor": None, - "tran": "nvme", "rota": "0", "hctl": None, "rev": None}, + { + "name": "nvme0n1", + "type": "disk", + "size": 100000, + "model": "TestNVMe", + "serial": "SN-1", + "vendor": None, + "tran": "nvme", + "rota": "0", + "hctl": None, + "rev": None, + }, ] } call_idx = [0] + def mock_check_output(cmd, **kwargs): call_idx[0] += 1 if "lsblk" in cmd: @@ -1251,8 +1449,9 @@ def mock_check_output(cmd, **kwargs): raise Exception("nvme-cli failed") raise FileNotFoundError(cmd[0]) - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", side_effect=mock_check_output + ): ssds = mm._get_local_ssds() assert len(ssds) == 1 @@ -1264,8 +1463,9 @@ def test_gpu_serial_placeholder_filtered(self, mm, mock_lshw): {"product": "A100", "vendor": "NVIDIA", "description": "3D"}, ] nvidia_output = "0, [N/A]" - with patch("netbox_agent.modules.is_tool", return_value=True), \ - patch("netbox_agent.modules.subprocess.check_output", return_value=nvidia_output): + with patch("netbox_agent.modules.is_tool", return_value=True), patch( + "netbox_agent.modules.subprocess.check_output", return_value=nvidia_output + ): gpus = mm._get_local_gpus() assert len(gpus) == 1 assert gpus[0]["serial"] is None @@ -1273,10 +1473,16 @@ def test_gpu_serial_placeholder_filtered(self, mm, mock_lshw): def test_gpu_mixed_discrete_and_onboard(self, mm, mock_lshw): """Only discrete GPUs are detected — onboard VGA filtered out.""" mock_lshw.get_hw_linux.return_value = [ - {"product": "ASPEED Graphics Family", "vendor": "ASPEED Technology, Inc.", - "description": "VGA compatible controller"}, - {"product": "H100 80GB HBM3", "vendor": "NVIDIA Corporation", - "description": "3D controller"}, + { + "product": "ASPEED Graphics Family", + "vendor": "ASPEED Technology, Inc.", + "description": "VGA compatible controller", + }, + { + "product": "H100 80GB HBM3", + "vendor": "NVIDIA Corporation", + "description": "3D controller", + }, ] with patch("netbox_agent.modules.is_tool", return_value=False): gpus = mm._get_local_gpus() @@ -1287,12 +1493,13 @@ def test_psu_serial_placeholder_filtered(self, mm): """PSU placeholder serials (Not Specified, etc.) become None.""" mock_dmidecode = MagicMock() mock_dmidecode.get_by_type.return_value = [ - {"Name": "PSU-1", "Manufacturer": "Supermicro", - "Serial Number": "Not Specified"}, - {"Name": "PSU-2", "Manufacturer": "Supermicro", - "Serial Number": "To Be Filled By O.E.M."}, - {"Name": "PSU-3", "Manufacturer": "Supermicro", - "Serial Number": "PSU-REAL-SN"}, + {"Name": "PSU-1", "Manufacturer": "Supermicro", "Serial Number": "Not Specified"}, + { + "Name": "PSU-2", + "Manufacturer": "Supermicro", + "Serial Number": "To Be Filled By O.E.M.", + }, + {"Name": "PSU-3", "Manufacturer": "Supermicro", "Serial Number": "PSU-REAL-SN"}, ] with patch.dict(sys.modules, {"netbox_agent.dmidecode": mock_dmidecode}): psus = mm._get_local_psus() @@ -1306,8 +1513,8 @@ def test_psu_serial_placeholder_filtered(self, mm): # Tests: create_or_update with deps and state # --------------------------------------------------------------------------- -class TestCreateOrUpdateWithDepsAndState: +class TestCreateOrUpdateWithDepsAndState: def test_create_or_update_skips_psu_when_dmidecode_missing(self, mm, nb): """When dmidecode unavailable, PSU detection is skipped.""" mm._get_local_cpus = MagicMock(return_value=[]) @@ -1327,9 +1534,9 @@ def test_create_or_update_skips_psu_when_dmidecode_missing(self, mm, nb): def test_create_or_update_with_state_skips_unchanged(self, mm, nb): """With state, unchanged categories are skipped.""" - mm._get_local_cpus = MagicMock(return_value=[ - {"product": "Xeon", "vendor": "Intel", "serial": None} - ]) + mm._get_local_cpus = MagicMock( + return_value=[{"product": "Xeon", "vendor": "Intel", "serial": None}] + ) mm._get_local_gpus = MagicMock(return_value=[]) mm._get_local_dimms = MagicMock(return_value=[]) mm._get_local_ssds = MagicMock(return_value=[]) @@ -1349,9 +1556,9 @@ def test_create_or_update_with_state_skips_unchanged(self, mm, nb): def test_create_or_update_with_state_syncs_changed(self, mm, nb): """With state, changed categories ARE synced.""" - mm._get_local_cpus = MagicMock(return_value=[ - {"product": "Xeon", "vendor": "Intel", "serial": None} - ]) + mm._get_local_cpus = MagicMock( + return_value=[{"product": "Xeon", "vendor": "Intel", "serial": None}] + ) mm._get_local_gpus = MagicMock(return_value=[]) mm._get_local_dimms = MagicMock(return_value=[]) mm._get_local_ssds = MagicMock(return_value=[]) diff --git a/tests/test_network_ip.py b/tests/test_network_ip.py index df0b98b6..06bb69ff 100644 --- a/tests/test_network_ip.py +++ b/tests/test_network_ip.py @@ -19,10 +19,16 @@ # --------------------------------------------------------------------------- _mock_nb = MagicMock(name="nb") _mock_config = SimpleNamespace( - update_all=True, update_network=True, register=False, + update_all=True, + update_network=True, + register=False, network=SimpleNamespace( - ignore_interfaces="(dummy.*|docker.*)", ignore_ips="^(127\\.0\\.0\\..*)", - ipmi=False, lldp=None, nic_id="name", primary_mac="temp", + ignore_interfaces="(dummy.*|docker.*)", + ignore_ips="^(127\\.0\\.0\\..*)", + ipmi=False, + lldp=None, + nic_id="name", + primary_mac="temp", ), ) _mock_config_module = MagicMock() @@ -68,7 +74,8 @@ def _run_ip_unassignment(device, nb_nics, netbox_ips, all_local_ips, nb_api): logging.info( "Unassigning IP %s from %s", - netbox_ip.address, netbox_ip.assigned_object, + netbox_ip.address, + netbox_ip.assigned_object, ) netbox_ip.assigned_object_type = None netbox_ip.assigned_object_id = None @@ -81,6 +88,7 @@ def _run_ip_unassignment(device, nb_nics, netbox_ips, all_local_ips, nb_api): # Helpers # --------------------------------------------------------------------------- + def _make_ip(ip_id, address): ip = MagicMock(name=f"ip-{ip_id}") ip.id = ip_id @@ -109,6 +117,7 @@ def _make_nic(nic_id): # Tests — these verify the fix logic directly # --------------------------------------------------------------------------- + class TestPrimaryIp4ClearingBeforeUnassign: """Test that primary_ip4 is cleared before unassigning IPs.""" @@ -153,7 +162,7 @@ def test_unassign_non_primary_ip_skips_clearing(self): When unassigning an IP that is NOT the device's primary, no clearing occurs. """ primary_ip = _make_ip(42, "10.100.200.50/24") # this is primary - other_ip = _make_ip(99, "10.100.200.60/24") # this will be unassigned + other_ip = _make_ip(99, "10.100.200.60/24") # this will be unassigned device = _make_device(1, "potato01", primary_ip4=primary_ip) nb_api = MagicMock(name="nb_api") diff --git a/tests/test_state.py b/tests/test_state.py index 6356c0d3..18f56002 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -11,7 +11,6 @@ class TestStateManager: - def test_save_and_load_state(self, tmp_path): """Round-trip: save → load returns same data.""" sm = StateManager(str(tmp_path))