diff --git a/test/integration/plugins/ontap/README.md b/test/integration/plugins/ontap/README.md index 6e0d0e7d6be5..5b1e2d62dd74 100644 --- a/test/integration/plugins/ontap/README.md +++ b/test/integration/plugins/ontap/README.md @@ -136,29 +136,39 @@ The test classes read `storageIP`, `svmName`, `username`, and `password` from th Run all suites for one protocol in a single batch, then inspect consolidated results: ```bash -# iSCSI only — 5 suites, ~30–45 min +# iSCSI only — 6 suites, ~40–60 min bash test/integration/plugins/ontap/run_tests.sh iscsi -# NFS3 only — 5 suites, ~30–45 min +# NFS3 only — 6 suites, ~40–60 min bash test/integration/plugins/ontap/run_tests.sh nfs3 -# Full plugin validation: iSCSI batch, then NFS3 batch (~60–90 min) +# Full plugin validation: iSCSI batch, then NFS3 batch bash test/integration/plugins/ontap/run_tests.sh both # Default (setup_zone + iscsi + nfs3; excludes cleanup_zone) bash test/integration/plugins/ontap/run_tests.sh bash test/integration/plugins/ontap/run_tests.sh all + +# Template-cache suites only +bash test/integration/plugins/ontap/run_tests.sh nfs3_template_cache +bash test/integration/plugins/ontap/run_tests.sh iscsi_template_cache +bash test/integration/plugins/ontap/run_tests.sh nfs3_template_cache_negative +bash test/integration/plugins/ontap/run_tests.sh iscsi_template_cache_negative ``` -Each protocol batch runs suites in this order: pool lifecycle → pool with volumes → volume lifecycle → zone-scoped pool → VM attach (last). +Each protocol batch runs suites in this order: pool lifecycle → pool with volumes → volume lifecycle → zone-scoped pool → VM attach → template cache (last). | Command | What it runs | |---------|--------------| -| `run_tests.sh iscsi` | All 5 iSCSI suites + unified iSCSI report | -| `run_tests.sh nfs3` | All 5 NFS3 suites + unified NFS3 report | +| `run_tests.sh iscsi` | All 6 iSCSI suites + unified iSCSI report | +| `run_tests.sh nfs3` | All 6 NFS3 suites + unified NFS3 report | | `run_tests.sh both` | iSCSI batch, then NFS3 batch + combined report | | `run_tests.sh all` | `setup_zone`, then `both` (iSCSI before NFS3) | | `run_tests.sh nfs3_workflow` | Single suite by tag (unchanged) | +| `run_tests.sh nfs3_template_cache` | NFS3 template-cache suite only | +| `run_tests.sh iscsi_template_cache` | iSCSI template-cache suite only | +| `run_tests.sh nfs3_template_cache_negative` | NFS3 template-cache boundary/negative suite | +| `run_tests.sh iscsi_template_cache_negative` | iSCSI template-cache boundary/negative suite | | `run_tests.sh setup_zone` | Zone setup only | | `run_tests.sh cleanup_zone` | Zone teardown (manual; destructive) | @@ -309,11 +319,15 @@ self.assertEqual(result.state, "Maintenance") | NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 4 | Zone scope — all hosts connected via `attachZone` | | NFS3 Volume Lifecycle | `nfs3/volume/test_volume_lifecycle.py` | 5 | Volume is metadata-only; FlexVol unchanged on delete | | NFS3 VM + Volume Attach | `nfs3/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle with hot-plug/detach | +| NFS3 Template Cache | `nfs3/template/test_template_cache.py` | 6 | ROOT on tagged pool; seed/reuse cache; survive VM delete | +| NFS3 Template Cache Negative | `nfs3/template/test_template_cache_negative.py` | 3 | Tag mismatch; undersized pool; out-of-band cache delete | | iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete + igroups | | iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 7 | Same + live LUN present; negative delete guard | | iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 4 | Zone scope | | iSCSI Volume Lifecycle | `iscsi/volume/test_volume_lifecycle.py` | 5 | LUN created per CS volume; LUN removed on delete | | iSCSI VM + Volume Attach | `iscsi/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle; LUN-maps on VM start/stop/detach | +| iSCSI Template Cache | `iscsi/template/test_template_cache.py` | 6 | ROOT on tagged pool; `cs_tmpl_*` LUN cache seed/reuse | +| iSCSI Template Cache Negative | `iscsi/template/test_template_cache_negative.py` | 3 | Tag mismatch; undersized pool; out-of-band cache delete | For the goal, dependencies, and exact success criteria of every individual test, see [TEST_CASES.md](TEST_CASES.md). diff --git a/test/integration/plugins/ontap/TEST_CASES.md b/test/integration/plugins/ontap/TEST_CASES.md index 73dc1990a5b6..7ef4963738e8 100644 --- a/test/integration/plugins/ontap/TEST_CASES.md +++ b/test/integration/plugins/ontap/TEST_CASES.md @@ -19,7 +19,7 @@ # ONTAP Integration Test Cases -Complete reference for all 62 test cases across 10 test suites. +Complete reference for all 74 test cases across 12 test suites. Each suite is sequential — tests must run in numbered order; each step builds on state created by the previous step. --- @@ -223,6 +223,72 @@ Each suite is sequential — tests must run in numbered order; each step builds --- +## Suite 11 — NFS3 Template Cache + +**File:** `nfs3/template/test_template_cache.py` +**Class:** `TestOntapNfs3TemplateCache` +**Tag:** `nfs3_template_cache` +**Total:** 6 tests | **Scope:** ROOT on tagged NFS3 ONTAP pool; primary template cache seed / reuse / survive VM delete + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_tagged_pool_and_service_offering` | Create NFS3 pool + SO sharing `templateCacheTags` | setUpClass | Pool `Up`; SO created | FlexVol `online` | positive | +| 02 | `test_02_deploy_vm1_seeds_template_cache` | Deploy VM-1 — ROOT on ONTAP; seeds cache | test_01 | VM Running; ROOT `storageid` = pool; `template_spool_ref` Ready/DOWNLOADED | Cache file present at spool `install_path` | positive | +| 03 | `test_03_assert_single_spool_ref_and_cache` | Exactly one spool_ref + cache object | test_02 | One `template_spool_ref` row | Cache file still present | positive | +| 04 | `test_04_deploy_vm2_reuses_cache` | Deploy VM-2 — reuses cache | test_03 | Still one spool_ref; VM-2 Running; ROOT on pool | Same cache file (no second cache) | positive | +| 05 | `test_05_destroy_vms_cache_survives` | Expunge VMs — cache must remain (lazy GC) | test_04 | spool_ref still Ready | Cache file still present | positive | +| 06 | `test_06_cleanup_pool_and_offering` | Delete SO; force-delete pool | test_05 | Pool gone | FlexVol deleted | cleanup | + +--- + +## Suite 12 — iSCSI Template Cache + +**File:** `iscsi/template/test_template_cache.py` +**Class:** `TestOntapIscsiTemplateCache` +**Tag:** `iscsi_template_cache` +**Total:** 6 tests | **Scope:** ROOT on tagged iSCSI ONTAP pool; `cs_tmpl_` LUN cache + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_tagged_pool_and_service_offering` | Create iSCSI pool + tagged SO | setUpClass | Pool `Up`; SO created | FlexVol `online` | positive | +| 02 | `test_02_deploy_vm1_seeds_template_cache` | Deploy VM-1 — seeds `cs_tmpl_*` + ROOT LUN | test_01 | VM Running; ROOT on pool; spool_ref Ready (`local_path` = LUN uuid) | LUN `/vol//cs_tmpl_` exists; ≥1 volume LUN | positive | +| 03 | `test_03_assert_single_spool_ref_and_cache` | Exactly one spool_ref + one cache LUN | test_02 | One spool_ref | Exactly one `cs_tmpl_*` LUN | positive | +| 04 | `test_04_deploy_vm2_reuses_cache` | Deploy VM-2 — reuse cache | test_03 | Still one spool_ref | Still one `cs_tmpl_*`; volume LUN count +1 | positive | +| 05 | `test_05_destroy_vms_cache_survives` | Expunge VMs — cache LUN remains | test_04 | spool_ref still Ready | Volume LUNs gone; `cs_tmpl_*` still present | positive | +| 06 | `test_06_cleanup_pool_and_offering` | Delete SO; force-delete pool | test_05 | Pool gone | FlexVol deleted | cleanup | + +--- + +## Suite 13 — NFS3 Template Cache Negative / Boundary + +**File:** `nfs3/template/test_template_cache_negative.py` +**Class:** `TestOntapNfs3TemplateCacheNegative` +**Tag:** `nfs3_template_cache_negative` +**Total:** 3 tests | **Scope:** Boundary conditions for NFS3 primary template cache (isolated from happy path) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_tag_mismatch_does_not_seed_cache` | SO tags ≠ pool tags | setUpClass | Deploy may succeed elsewhere; ROOT not on ONTAP pool; no `template_spool_ref` for pool | No cache file for template on FlexVol | negative | +| 02 | `test_02_undersized_pool_deploy_fails` | Matching tags but `capacitybytes` ≪ template size | setUpClass | Deploy fails / never Running; spool_ref not Ready/DOWNLOADED | No cache file | negative | +| 03 | `test_03_deleted_cache_blocks_reuse` | Seed cache, delete file out-of-band, redeploy | setUpClass | spool_ref still Ready after ONTAP delete; second deploy fails | Cache file absent after delete | negative | + +--- + +## Suite 14 — iSCSI Template Cache Negative / Boundary + +**File:** `iscsi/template/test_template_cache_negative.py` +**Class:** `TestOntapIscsiTemplateCacheNegative` +**Tag:** `iscsi_template_cache_negative` +**Total:** 3 tests | **Scope:** Boundary conditions for iSCSI primary template cache (isolated from happy path) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_tag_mismatch_does_not_seed_cache` | SO tags ≠ pool tags | setUpClass | ROOT not on ONTAP pool; no `template_spool_ref` | No `cs_tmpl_*` LUN | negative | +| 02 | `test_02_undersized_pool_deploy_fails` | Matching tags but undersized capacity | setUpClass | Deploy fails; spool_ref not Ready/DOWNLOADED | No `cs_tmpl_*` LUN | negative | +| 03 | `test_03_deleted_cache_blocks_reuse` | Seed cache, delete LUN out-of-band, redeploy | setUpClass | spool_ref still Ready; second deploy fails | `cs_tmpl_*` absent after delete | negative | + +--- + ## Cross-suite summary | Suite | Protocol | Scope | Tests | Status | @@ -232,9 +298,13 @@ Each suite is sequential — tests must run in numbered order; each step builds | NFS3 Zone-Scoped Pool | NFS3 | Zone | 4 | ✅ | | NFS3 Volume Lifecycle | NFS3 | Cluster | 5 | ✅ | | NFS3 VM + Volume Attach | NFS3 | Cluster | 8 | ✅ | +| NFS3 Template Cache | NFS3 | Cluster | 6 | 🆕 | +| NFS3 Template Cache Negative | NFS3 | Cluster | 3 | 🆕 | | iSCSI Pool Lifecycle | iSCSI | Cluster | 8 | ✅ | | iSCSI Pool with Volumes | iSCSI | Cluster | 7 | ✅ | | iSCSI Zone-Scoped Pool | iSCSI | Zone | 4 | ✅ | | iSCSI Volume Lifecycle | iSCSI | Cluster | 5 | ✅ | | iSCSI VM + Volume Attach | iSCSI | Cluster | 8 | ⚠️ 7/8 | -| **Total** | | | **62** | **61 passing** | +| iSCSI Template Cache | iSCSI | Cluster | 6 | 🆕 | +| iSCSI Template Cache Negative | iSCSI | Cluster | 3 | 🆕 | +| **Total** | | | **80** | | diff --git a/test/integration/plugins/ontap/helpers/__init__.py b/test/integration/plugins/ontap/helpers/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py b/test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py new file mode 100644 index 000000000000..47abd8e899eb --- /dev/null +++ b/test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py @@ -0,0 +1,637 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Negative / boundary workflow for ONTAP primary template-cache Marvin suites. + +Kept separate from the sequential happy-path suite so failures here cannot +cascade into seed/reuse/survive steps. + +Cases: + 01 Tag mismatch — SO tags do not match ONTAP pool; no spool_ref on pool + 02 Undersized pool — capacity << template size; deploy must fail + 03 Cache deleted out-of-band — Ready spool_ref but missing ONTAP object; + subsequent deploy must fail +""" + +from __future__ import print_function + +import copy +import logging +import random +import time + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + deleteServiceOffering as deleteServiceOfferingAPI, + deployVirtualMachine as deployVirtualMachineAPI, + enableStorageMaintenance, +) +from marvin.lib.base import ServiceOffering, StoragePool + +from helpers import template_cache_util as tcu +from helpers.template_cache_workflow import ( + OntapTemplateCacheWorkflow, + TemplateCacheTestData, +) + +logger = logging.getLogger("TemplateCacheNegativeWorkflow") + + +class OntapTemplateCacheNegativeWorkflow(OntapTemplateCacheWorkflow): + """ + Independent negative boundary tests. + + Reuses setUpClass (zone / template / ONTAP client) from the positive + workflow but creates per-test pools and offerings so state stays isolated. + """ + + NOSE_TAG = "template_cache_negative" + POOL_NAME_PREFIX = "OntapTmplCacheNeg" + + @classmethod + def setUpClass(cls): + super(OntapTemplateCacheNegativeWorkflow, cls).setUpClass() + # Lab public IP pool is tiny (often only 3 addresses: SSVM + CPVM + + # one SourceNat). Starting leftover VRs from prior runs consumes the + # last IP and then this suite's Allocated guest network cannot be + # implemented — seed deploy fails with "Unable to create a deployment". + cls._reclaim_stale_template_cache_networks() + cls._cleanup_stale_negative_pools() + cls._ensure_guest_network_router_running() + + @classmethod + def _cleanup_stale_negative_pools(cls): + """ + Force-delete leftover OntapTmplCacheNeg* pools from prior runs. + + Shared storage_tags on a leftover Up pool will steal ROOT placement + from the pool created by step_02/step_03 and make assertions fail. + """ + from marvin.cloudstackAPI import listStoragePools as listStoragePoolsAPI + + cmd = listStoragePoolsAPI.listStoragePoolsCmd() + cmd.zoneid = cls.zone.id + pools = cls.apiClient.listStoragePools(cmd) or [] + prefixes = ( + cls.POOL_NAME_PREFIX, + "OntapIscsiTmplNeg", + "OntapNfs3TmplNeg", + "OntapTmplCacheNeg", + ) + for pool in pools: + name = getattr(pool, "name", "") or "" + if not any(name.startswith(prefix) for prefix in prefixes): + continue + logger.info("Cleaning leftover negative-suite pool %s" % name) + try: + # Instance method helpers need a throwaway binder. + binder = cls.__new__(cls) + binder.apiClient = cls.apiClient + binder.ontap = cls.ontap + binder._force_delete_pool(pool) + except Exception as ex: + logger.warning("Could not delete leftover pool %s: %s" % (name, ex)) + + @classmethod + def _reclaim_stale_template_cache_networks(cls): + """ + Destroy leftover ontap-tmpl-cache-net-* networks (and their VRs/VMs) + from prior suite accounts so SourceNat public IPs are freed. + Keeps the network created for this class (cls.network_id). + """ + from marvin.cloudstackAPI import ( + deleteNetwork as deleteNetworkAPI, + destroyRouter as destroyRouterAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + listNetworks as listNetworksAPI, + listRouters as listRoutersAPI, + listVirtualMachines as listVirtualMachinesAPI, + stopVirtualMachine as stopVirtualMachineAPI, + ) + + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.listall = True + networks = cls.apiClient.listNetworks(net_cmd) or [] + for net in networks: + name = getattr(net, "name", "") or "" + # Match names from positive/negative suite network create + # (ontap-tmpl-cache-net-*). + if "tmpl-cache-net" not in name: + continue + if cls.network_id and str(net.id) == str(cls.network_id): + continue + + logger.info( + "Reclaiming stale template-cache network %s (%s)" + % (name, net.id) + ) + + vm_cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + vm_cmd.listall = True + vm_cmd.networkid = net.id + for vm in (cls.apiClient.listVirtualMachines(vm_cmd) or []): + try: + state = (vm.state or "").lower() + if state not in ( + "destroyed", "expunging", "error", "stopped" + ): + stop = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop.id = vm.id + cls.apiClient.stopVirtualMachine(stop) + dest = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest.id = vm.id + dest.expunge = True + cls.apiClient.destroyVirtualMachine(dest) + except Exception as ex: + logger.warning( + "Reclaim: destroy VM %s on %s: %s" % (vm.id, name, ex) + ) + + rtr_cmd = listRoutersAPI.listRoutersCmd() + rtr_cmd.zoneid = cls.zone.id + rtr_cmd.networkid = net.id + rtr_cmd.listall = True + for router in (cls.apiClient.listRouters(rtr_cmd) or []): + try: + dr = destroyRouterAPI.destroyRouterCmd() + dr.id = router.id + cls.apiClient.destroyRouter(dr) + logger.info( + "Reclaim: destroyed router %s for %s" + % (router.name, name) + ) + except Exception as ex: + logger.warning( + "Reclaim: destroy router %s: %s" % (router.id, ex) + ) + + try: + dn = deleteNetworkAPI.deleteNetworkCmd() + dn.id = net.id + cls.apiClient.deleteNetwork(dn) + logger.info("Reclaim: deleted network %s" % name) + except Exception as ex: + logger.warning( + "Reclaim: delete network %s: %s" % (net.id, ex) + ) + + @classmethod + def _ensure_guest_network_router_running(cls): + """ + Ensure the VR for *this suite's* guest network is Running. + + Do not start unrelated zone routers — that can exhaust the lab's + public IP range and block implementing cls.network_id. + """ + from marvin.cloudstackAPI import ( + listRouters as listRoutersAPI, + startRouter as startRouterAPI, + ) + if not cls.network_id: + logger.warning( + "No guest network_id — deploy may fail in Advanced zone" + ) + return + + cmd = listRoutersAPI.listRoutersCmd() + cmd.zoneid = cls.zone.id + cmd.networkid = cls.network_id + cmd.listall = True + routers = cls.apiClient.listRouters(cmd) or [] + if not routers: + # Allocated isolated network: VR is created on first deploy. + logger.info( + "No VR yet for network %s — will be created on first deploy" + % cls.network_id + ) + return + + for router in routers: + state = (router.state or "").lower() + if state == "running": + continue + logger.info( + "Starting guest-network router %s (was %s)" + % (router.name, router.state) + ) + start = startRouterAPI.startRouterCmd() + start.id = router.id + cls.apiClient.startRouter(start) + deadline = time.time() + 300 + while time.time() < deadline: + cur = cls.apiClient.listRouters(cmd) or [] + match = [r for r in cur if r.id == router.id] + if match and (match[0].state or "").lower() == "running": + logger.info("Router %s is Running" % router.name) + break + time.sleep(10) + else: + raise RuntimeError( + "Guest-network router %s did not reach Running" + % router.name + ) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _unique_storage_tags(self): + """Per-test tags so leftover pools with shared tags cannot steal ROOT.""" + return "%s-neg-%d" % (self.storage_tags, random.randint(0, 99999)) + + def _create_pool_with(self, tags=None, capacitybytes=None, name_prefix=None): + """Create an ONTAP pool, optionally overriding tags / capacity.""" + ps = copy.deepcopy(self.testdata[TemplateCacheTestData.primaryStorage]) + if tags is not None: + ps[TemplateCacheTestData.tags] = tags + if capacitybytes is not None: + ps["capacitybytes"] = int(capacitybytes) + + storage_ip = self.testdata[TemplateCacheTestData.ontap][ + TemplateCacheTestData.DETAIL_STORAGE_IP + ] + prefix = name_prefix or self.POOL_NAME_PREFIX + pool_name = "%s_%d" % (prefix, random.randint(0, 99999)) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "%s://%s/ontap" % (self.POOL_URL_SCHEME, storage_ip) + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TemplateCacheTestData.scope] + cmd.provider = ps[TemplateCacheTestData.provider] + cmd.tags = ps[TemplateCacheTestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + pool = StoragePool(response.__dict__) + self.assertEqual( + pool.state, "Up", + "Pool %s should be Up after create, got %s" % (pool.name, pool.state), + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "FlexVol missing for pool %s" % pool.name) + self.assertEqual(ontap_vol.get("state"), "online") + return pool + + def _create_service_offering_with(self, tags): + """Create a compute offering with the given storage tags.""" + so_data = copy.deepcopy( + self.testdata[TemplateCacheTestData.computeOffering] + ) + so_data["tags"] = tags + so_data["name"] = "OntapTmplCacheNegSO_%d" % random.randint(0, 99999) + so_data["displaytext"] = so_data["name"] + so = ServiceOffering.create(self.apiClient, so_data) + self.assertIsNotNone(so.id, "Service offering create failed") + return so + + def _deploy_vm_with_offering(self, name_suffix, service_offering_id): + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.name = "tmpl-cache-neg-%s-%d" % ( + name_suffix, random.randint(0, 9999) + ) + cmd.displayname = cmd.name + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + return self._poll_vm_state(vm.id, "Running", timeout=900) + + def _deploy_vm_expect_failure(self, name_suffix, service_offering_id, + timeout=300): + """Assert deploy does not reach Running (API error or Error state).""" + vm = None + try: + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.name = "tmpl-cache-neg-%s-%d" % ( + name_suffix, random.randint(0, 9999) + ) + cmd.displayname = cmd.name + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + vm = self.apiClient.deployVirtualMachine(cmd) + except Exception as ex: + logger.info( + "deployVirtualMachine failed as expected for %s: %s" + % (name_suffix, ex) + ) + return + + self.assertIsNotNone(vm, "deploy returned None without raising") + try: + self._poll_vm_state(vm.id, "Running", timeout=timeout) + self.fail( + "Deploy %s unexpectedly reached Running (boundary should fail)" + % name_suffix + ) + except Exception as ex: + logger.info( + "Deploy %s did not reach Running as expected: %s" + % (name_suffix, ex) + ) + finally: + try: + self._destroy_vm_static(vm.id) + except Exception: + pass + + def _force_delete_pool(self, pool): + if pool is None: + return + pool_name = pool.name + try: + mc = enableStorageMaintenance.enableStorageMaintenanceCmd() + mc.id = pool.id + self.apiClient.enableStorageMaintenance(mc) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + except Exception as ex: + logger.warning( + "enableStorageMaintenance for %s: %s" % (pool.id, ex) + ) + try: + self._delete_pool(pool.id, forced=True) + except Exception as ex: + logger.warning("force delete pool %s: %s" % (pool.id, ex)) + deadline = time.time() + 120 + while time.time() < deadline: + if self.ontap.get_volume(pool_name) is None: + break + time.sleep(5) + + def _delete_service_offering(self, offering): + if offering is None: + return + try: + cmd = deleteServiceOfferingAPI.deleteServiceOfferingCmd() + cmd.id = offering.id + self.apiClient.deleteServiceOffering(cmd) + except Exception as ex: + logger.warning("delete SO %s: %s" % (offering.id, ex)) + + def _delete_ontap_cache(self, pool, spool_ref): + """Remove ONTAP cache object while leaving CloudStack spool_ref.""" + if self.PROTOCOL.upper() == "ISCSI": + path = tcu.template_cache_lun_path( + pool.name, self.__class__.template_db_id + ) + deleted = self.ontap.delete_lun(self.svm_name, path) + self.assertTrue( + deleted, + "Failed to delete template cache LUN at %s" % path, + ) + tcu.assert_no_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + else: + install_path = spool_ref.get("install_path") + self.assertTrue(install_path, "spool_ref.install_path required") + deleted = self.ontap.delete_file_in_volume( + pool.name, install_path + ) + self.assertTrue( + deleted, + "Failed to delete NFS cache file %s in %s" + % (install_path, pool.name), + ) + tcu.assert_no_nfs_template_cache_file( + self, self.ontap, pool.name, install_path + ) + + # ------------------------------------------------------------------ + # Negative cases + # ------------------------------------------------------------------ + + def step_01_tag_mismatch_does_not_seed_cache(self): + """ + Pool has template-cache tags; SO uses a different tag. + + Valid outcomes: + a) deploy fails (no alternate tagged pool) — common in lab + b) deploy succeeds elsewhere — ROOT must not be on ONTAP pool + + In both cases there must be no spool_ref / cache on the ONTAP pool. + """ + pool = None + so = None + vm = None + mismatch_tag = "ontap-tmpl-cache-mismatch-%d" % random.randint(0, 99999) + try: + pool = self._create_pool_with(tags=self.storage_tags) + pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + so = self._create_service_offering_with(tags=mismatch_tag) + + try: + vm = self._deploy_vm_with_offering("tag-mismatch", so.id) + except Exception as ex: + logger.info( + "Deploy with mismatched tags failed (acceptable if no " + "alternate pool matches): %s" % ex + ) + vm = None + + if vm is not None: + root = self._root_volume_for_vm(vm.id) + self.assertNotEqual( + str(root.storageid), str(pool.id), + "ROOT should not land on ONTAP pool when SO tags mismatch " + "(storageid=%s pool=%s)" % (root.storageid, pool.id), + ) + + tcu.assert_no_spool_ref( + self, self.dbConnection, pool_db_id, + self.__class__.template_db_id, + ) + if self.PROTOCOL.upper() == "ISCSI": + tcu.assert_no_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + finally: + if vm is not None: + try: + self._destroy_vm_static(vm.id) + except Exception: + pass + self._delete_service_offering(so) + self._force_delete_pool(pool) + + def step_02_undersized_pool_deploy_fails(self): + """ + Matching tags but capacitybytes far below template virtual size. + + Deploy must fail; no Ready spool_ref / cache object should remain. + """ + pool = None + so = None + tmpl_size = tcu.get_template_size_bytes( + self.dbConnection, self.__class__.template_db_id + ) + if not tmpl_size or tmpl_size <= 0: + self.skipTest( + "Cannot determine vm_template.size for template_db_id=%s" + % self.__class__.template_db_id + ) + + # Lab templates can be small (~100–200 MiB); keep capacity under size. + undersized = max(tmpl_size // 4, 8 * 1024 * 1024) + if undersized >= tmpl_size: + undersized = max(tmpl_size // 2, 1024 * 1024) + self.assertLess( + undersized, tmpl_size, + "undersized capacity %s must be < template size %s" + % (undersized, tmpl_size), + ) + + try: + exclusive_tags = self._unique_storage_tags() + pool = self._create_pool_with( + tags=exclusive_tags, + capacitybytes=undersized, + name_prefix=self.POOL_NAME_PREFIX + "Tiny", + ) + pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + so = self._create_service_offering_with(tags=exclusive_tags) + + self._deploy_vm_expect_failure("undersized", so.id, timeout=300) + + spool = tcu.get_template_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id + ) + if spool is not None: + self.assertNotEqual( + str(spool.get("state")), "Ready", + "spool_ref must not be Ready after undersized deploy failure", + ) + self.assertNotEqual( + str(spool.get("download_state", "")).upper(), + "DOWNLOADED", + "spool_ref must not be DOWNLOADED after undersized failure", + ) + if self.PROTOCOL.upper() == "ISCSI": + tcu.assert_no_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + finally: + self._delete_service_offering(so) + self._force_delete_pool(pool) + + def step_03_deleted_cache_blocks_reuse(self): + """ + Seed cache, destroy VM, delete ONTAP cache out-of-band, redeploy. + + CloudStack still has Ready spool_ref but backend object is gone — + second deploy must fail (clone / missing cache path). + """ + self._ensure_guest_network_router_running() + pool = None + so = None + vm1 = None + try: + exclusive_tags = self._unique_storage_tags() + pool = self._create_pool_with(tags=exclusive_tags) + pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + so = self._create_service_offering_with(tags=exclusive_tags) + + # Seed deploy can flake on lab planner/VR; retry once after + # re-ensuring the guest network router is Running. + last_ex = None + for attempt in range(2): + try: + self._ensure_guest_network_router_running() + vm1 = self._deploy_vm_with_offering("neg-seed", so.id) + last_ex = None + break + except Exception as ex: + last_ex = ex + logger.warning( + "Seed deploy attempt %s failed: %s" + % (attempt + 1, ex) + ) + self._reclaim_stale_template_cache_networks() + time.sleep(15) + if last_ex is not None: + # Seed must succeed — this case is not optional. Failure here + # is usually exhausted public IPs / stale VR from prior runs. + self.fail( + "Could not seed template cache for out-of-band delete " + "case after retries (check public IP capacity / leftover " + "VRs): %s" % last_ex + ) + self._assert_root_on_pool(vm1.id, pool) + spool = tcu.wait_for_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id, + timeout=600, + ) + tcu.assert_spool_ref_ready( + self, spool, + expect_local_path=(self.PROTOCOL.upper() == "ISCSI"), + ) + self._assert_cache_on_ontap(pool, spool) + + self._destroy_vm_static(vm1.id) + vm1 = None + # Allow clone/ROOT LUN cleanup so the cache LUN is no longer a + # FlexClone parent before out-of-band delete. + time.sleep(30) + + self._delete_ontap_cache(pool, spool) + stale = tcu.get_template_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id + ) + tcu.assert_spool_ref_ready( + self, stale, + expect_local_path=(self.PROTOCOL.upper() == "ISCSI"), + ) + + self._deploy_vm_expect_failure("stale-cache", so.id, timeout=300) + finally: + if vm1 is not None: + try: + self._destroy_vm_static(vm1.id) + except Exception: + pass + self._delete_service_offering(so) + self._force_delete_pool(pool) diff --git a/test/integration/plugins/ontap/helpers/template_cache_util.py b/test/integration/plugins/ontap/helpers/template_cache_util.py new file mode 100644 index 000000000000..e733924ee940 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/template_cache_util.py @@ -0,0 +1,256 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Helpers for ONTAP primary template-cache Marvin assertions. + +Covers CloudStack DB (``template_spool_ref``) and ONTAP REST checks for the +cache LUN (iSCSI: ``/vol//cs_tmpl_``) or cache file (NFS). +""" + +from __future__ import print_function + +import logging +import time + +logger = logging.getLogger("template_cache_util") + +# Must stay in sync with OntapStorageConstants.TEMPLATE_LUN_PREFIX +TEMPLATE_LUN_PREFIX = "cs_tmpl_" + + +def get_db_id(db_connection, table, api_uuid): + """Resolve CloudStack API UUID to numeric DB id.""" + rows = db_connection.execute( + "SELECT id FROM `%s` WHERE uuid = '%s'" % (table, api_uuid) + ) + if not rows: + raise AssertionError( + "No row in %s for uuid=%s" % (table, api_uuid) + ) + return rows[0][0] + + +def get_template_spool_ref(db_connection, pool_db_id, template_db_id): + """ + Return template_spool_ref row as a dict, or None if absent. + + Columns: id, pool_id, template_id, download_state, local_path, + install_path, template_size, marked_for_gc, state + """ + rows = db_connection.execute( + "SELECT id, pool_id, template_id, download_state, local_path, " + "install_path, template_size, marked_for_gc, state " + "FROM template_spool_ref " + "WHERE pool_id = %s AND template_id = %s" + % (int(pool_db_id), int(template_db_id)) + ) + if not rows: + return None + row = rows[0] + return { + "id": row[0], + "pool_id": row[1], + "template_id": row[2], + "download_state": row[3], + "local_path": row[4], + "install_path": row[5], + "template_size": row[6], + "marked_for_gc": row[7], + "state": row[8], + } + + +def count_template_spool_refs(db_connection, pool_db_id, template_db_id): + """Return number of template_spool_ref rows for pool+template.""" + rows = db_connection.execute( + "SELECT COUNT(*) FROM template_spool_ref " + "WHERE pool_id = %s AND template_id = %s" + % (int(pool_db_id), int(template_db_id)) + ) + return int(rows[0][0]) if rows else 0 + + +def assert_spool_ref_ready(testcase, spool_ref, expect_local_path=True): + """Assert spool_ref is DOWNLOADED / Ready after a successful cache seed.""" + testcase.assertIsNotNone(spool_ref, "Expected template_spool_ref row") + testcase.assertEqual( + str(spool_ref["download_state"]).upper(), + "DOWNLOADED", + "download_state should be DOWNLOADED, got %s" + % spool_ref["download_state"], + ) + testcase.assertEqual( + str(spool_ref["state"]), + "Ready", + "state should be Ready, got %s" % spool_ref["state"], + ) + if expect_local_path: + testcase.assertTrue( + spool_ref.get("local_path"), + "local_path (LUN uuid / cache id) should be set", + ) + testcase.assertTrue( + spool_ref.get("install_path"), + "install_path should be set after template is cached", + ) + + +def assert_no_spool_ref(testcase, db_connection, pool_db_id, template_db_id): + """Assert there is no template_spool_ref for pool+template.""" + count = count_template_spool_refs( + db_connection, pool_db_id, template_db_id + ) + testcase.assertEqual( + count, 0, + "Expected no template_spool_ref for pool=%s template=%s, found %s" + % (pool_db_id, template_db_id, count), + ) + + +def template_cache_lun_path(flexvol_name, template_db_id): + """iSCSI cache LUN path: /vol//cs_tmpl_.""" + return "/vol/%s/%s%s" % ( + flexvol_name, TEMPLATE_LUN_PREFIX, int(template_db_id) + ) + + +def assert_iscsi_template_cache_lun(testcase, ontap, svm_name, flexvol_name, + template_db_id): + """Assert the deterministic template-cache LUN exists on ONTAP.""" + path = template_cache_lun_path(flexvol_name, template_db_id) + lun = ontap.get_lun(svm_name, path) + testcase.assertIsNotNone( + lun, + "Expected template cache LUN at %s on SVM %s" % (path, svm_name), + ) + return lun + + +def assert_no_iscsi_template_cache_lun(testcase, ontap, svm_name, flexvol_name, + template_db_id): + """Assert the template-cache LUN is absent.""" + path = template_cache_lun_path(flexvol_name, template_db_id) + lun = ontap.get_lun(svm_name, path) + testcase.assertIsNone( + lun, + "Template cache LUN should be absent at %s" % path, + ) + + +def count_iscsi_template_cache_luns(ontap, svm_name, flexvol_name, + template_db_id=None): + """Count cs_tmpl_* LUNs in the FlexVol (optionally for one template id).""" + luns = ontap.list_luns_in_volume(svm_name, flexvol_name) or [] + prefix = "/vol/%s/%s" % (flexvol_name, TEMPLATE_LUN_PREFIX) + matches = [l for l in luns if l.get("name", "").startswith(prefix)] + if template_db_id is not None: + exact = template_cache_lun_path(flexvol_name, template_db_id) + matches = [l for l in matches if l.get("name") == exact] + return len(matches) + + +def count_luns_excluding_template_cache(ontap, svm_name, flexvol_name): + """Count non-cache LUNs (volume clones / blank volumes) in the FlexVol.""" + luns = ontap.list_luns_in_volume(svm_name, flexvol_name) or [] + prefix = "/vol/%s/%s" % (flexvol_name, TEMPLATE_LUN_PREFIX) + return len([l for l in luns if not l.get("name", "").startswith(prefix)]) + + +def _normalize_nfs_path(install_path): + """Strip leading slash for OntapRestClient.list_files_in_volume paths.""" + if not install_path: + return "/" + path = install_path if install_path.startswith("/") else "/" + install_path + # Parent directory listing: if path is a file, list its parent + if path.endswith("/"): + return path.rstrip("/") or "/" + parent = path.rsplit("/", 1)[0] + return parent or "/" + + +def assert_nfs_template_cache_file(testcase, ontap, flexvol_name, install_path): + """ + Assert the NFS cache file referenced by install_path exists in the FlexVol. + + ``install_path`` comes from ``template_spool_ref.install_path`` (relative or + absolute path inside the FlexVol as reported by the KVM copy). + """ + testcase.assertTrue( + install_path, + "install_path is required to locate the NFS template cache file", + ) + file_name = install_path.rstrip("/").rsplit("/", 1)[-1] + parent = _normalize_nfs_path(install_path) + names = ontap.list_files_in_volume(flexvol_name, path=parent) + # Also try volume root if parent listing is empty (path styles differ) + if file_name not in names: + root_names = ontap.list_files_in_volume(flexvol_name, path="/") + testcase.assertIn( + file_name, root_names + names, + "Expected NFS template cache file '%s' under '%s' or '/' in " + "FlexVol '%s'; listed parent=%s root=%s" + % (file_name, parent, flexvol_name, names, root_names), + ) + return file_name + + +def wait_for_spool_ref(db_connection, pool_db_id, template_db_id, + timeout=600, interval=10): + """ + Poll until template_spool_ref exists and is Ready/DOWNLOADED. + + Returns the spool_ref dict, or None on timeout. + """ + deadline = time.time() + timeout + last = None + while time.time() < deadline: + last = get_template_spool_ref( + db_connection, pool_db_id, template_db_id + ) + if last and str(last.get("download_state", "")).upper() == "DOWNLOADED" \ + and str(last.get("state", "")) == "Ready": + return last + time.sleep(interval) + return last + + +def assert_no_nfs_template_cache_file(testcase, ontap, flexvol_name, install_path): + """Assert the NFS cache file referenced by install_path is absent.""" + testcase.assertTrue( + install_path, + "install_path is required to assert NFS cache file absence", + ) + file_name = install_path.rstrip("/").rsplit("/", 1)[-1] + parent = _normalize_nfs_path(install_path) + names = ontap.list_files_in_volume(flexvol_name, path=parent) or [] + root_names = ontap.list_files_in_volume(flexvol_name, path="/") or [] + testcase.assertNotIn( + file_name, names + root_names, + "NFS template cache file '%s' should be absent under '%s' or '/' in " + "FlexVol '%s'" % (file_name, parent, flexvol_name), + ) + + +def get_template_size_bytes(db_connection, template_db_id): + """Return vm_template.size (bytes) for the given DB id, or None.""" + rows = db_connection.execute( + "SELECT size FROM vm_template WHERE id = %s" % int(template_db_id) + ) + if not rows or rows[0][0] is None: + return None + return int(rows[0][0]) diff --git a/test/integration/plugins/ontap/helpers/template_cache_workflow.py b/test/integration/plugins/ontap/helpers/template_cache_workflow.py new file mode 100644 index 000000000000..afc7cae0e6f4 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/template_cache_workflow.py @@ -0,0 +1,674 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Shared sequential workflow for ONTAP primary template-cache Marvin suites. + +Protocol-specific subclasses set PROTOCOL / NOSE_TAG / pool URL details and +inherit the numbered steps: + + 01 Create tagged ONTAP pool + matching tagged service offering + 02 Deploy VM-1 (ROOT on ONTAP) — seeds template cache + 03 Assert template_spool_ref + ONTAP cache object + 04 Deploy VM-2 — reuses cache (no second cache object) + 05 Destroy both VMs — cache and spool_ref must survive + 06 Cleanup — delete SO, force-delete pool +""" + +from __future__ import print_function + +import base64 +import logging +import random +import time +import unittest + +from marvin.cloudstackAPI import ( + createNetwork as createNetworkAPI, + createStoragePool as createStoragePoolAPI, + deleteNetwork as deleteNetworkAPI, + deleteServiceOffering as deleteServiceOfferingAPI, + deployVirtualMachine as deployVirtualMachineAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + enableStorageMaintenance, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listTemplates as listTemplatesAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + stopVirtualMachine as stopVirtualMachineAPI, +) +from marvin.lib.base import ServiceOffering, StoragePool +from ontap_test_base import OntapRestClient, OntapTestBase, get_datacenter_config +from helpers import template_cache_util as tcu + +logger = logging.getLogger("TemplateCacheWorkflow") + + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + return cmd + + +def _list_vols_for_vm(vm_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.virtualmachineid = vm_id + cmd.listall = True + return cmd + + +class TemplateCacheTestData(object): + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + computeOffering = "computeOffering" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + # ~50 GiB — room for template cache + two ROOT clones + DEFAULT_CAPACITY_BYTES = 50 * 1024 * 1024 * 1024 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-tmpl-cache", capacitybytes=None, + pool_name_prefix="OntapTmplCache"): + if capacitybytes is None: + capacitybytes = self.DEFAULT_CAPACITY_BYTES + encoded_password = base64.b64encode(password.encode()).decode() + suffix = random.randint(0, 99999) + self.testdata = { + self.ontap: { + self.DETAIL_STORAGE_IP: storage_ip, + self.DETAIL_SVM_NAME: svm_name, + self.DETAIL_USERNAME: username, + self.DETAIL_PASSWORD: password, + }, + self.account: { + "email": "ontap-tmpl-cache@test.com", + "firstname": "ONTAP", + "lastname": "TmplCache", + "username": "ontap_tmpl_%d" % suffix, + "password": "password", + }, + self.primaryStorage: { + "name": "%s_%d" % (pool_name_prefix, suffix), + self.scope: scope, + self.provider: provider, + self.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + self.DETAIL_USERNAME: username, + self.DETAIL_PASSWORD: encoded_password, + self.DETAIL_SVM_NAME: svm_name, + self.DETAIL_PROTOCOL: protocol, + self.DETAIL_STORAGE_IP: storage_ip, + }, + }, + self.computeOffering: { + "name": "OntapTmplCacheSO_%d" % suffix, + "displaytext": "ONTAP template-cache SO (tagged)", + "cpunumber": 1, + "cpuspeed": 500, + "memory": 512, + "storagetype": "shared", + "tags": tags, + }, + } + + +class OntapTemplateCacheWorkflow(OntapTestBase): + """ + Protocol-parameterized template-cache suite. + + Subclasses must set: + PROTOCOL - "NFS3" or "ISCSI" + NOSE_TAG - nose attr tag string + PROTOCOL_CFG_KEY - "nfs3" or "iscsi" under storagePool.protocols + POOL_URL_SCHEME - used in createStoragePool url (e.g. nfs / iscsi) + """ + + PROTOCOL = "NFS3" + NOSE_TAG = "nfs3_template_cache" + PROTOCOL_CFG_KEY = "nfs3" + POOL_URL_SCHEME = "nfs" + POOL_NAME_PREFIX = "OntapNfsTmplCache" + + # Shared sequential state + template_id = None # API UUID + template_db_id = None # numeric DB id + pool_db_id = None + service_offering = None + network_id = None + _created_network_id = None + vm1 = None + vm2 = None + + @classmethod + def setUpClass(cls): + super(OntapTemplateCacheWorkflow, cls).setUpClass() + testclient = super( + OntapTemplateCacheWorkflow, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + + proto_cfg = pool_cfg.get("protocols", {}).get(cls.PROTOCOL_CFG_KEY, {}) + if not proto_cfg.get("enabled", True): + raise unittest.SkipTest( + "%s tests disabled in ontap.cfg " + "(set protocols.%s.enabled=true to enable)" + % (cls.PROTOCOL, cls.PROTOCOL_CFG_KEY) + ) + + # Dedicated tag so ROOT is forced onto this ONTAP pool (not defaultPrim) + default_tag = "ontap-%s-tmpl-cache" % cls.PROTOCOL_CFG_KEY + tags = proto_cfg.get("templateCacheTags") or default_tag + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + capacitybytes = pool_cfg.get( + "templateCacheCapacitybytes", + pool_cfg.get("capacitybytes", None), + ) + + cls.testdata = TemplateCacheTestData( + storage_ip, svm_name, username, password, + protocol=cls.PROTOCOL, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + pool_name_prefix=cls.POOL_NAME_PREFIX, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + cls.storage_tags = tags + + cls._setup_cloudstack_resources( + config, cls.testdata[TemplateCacheTestData.account] + ) + + # Ready user KVM template + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = cls.zone.id + templates = cls.apiClient.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + if not kvm_ready: + raise unittest.SkipTest( + "No ready user KVM template in zone '%s'" % cls.zone.name + ) + cls.template_id = kvm_ready[0].id + cls.template_db_id = tcu.get_db_id( + cls.dbConnection, "vm_template", cls.template_id + ) + + cls.network_id = None + cls._created_network_id = None + zone_type = getattr(cls.zone, "networktype", "Basic") + if zone_type.lower() == "advanced": + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + else: + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.supportedservices = "SourceNat" + offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + if not offerings: + raise unittest.SkipTest( + "No Isolated network offering for Advanced zone" + ) + cn = createNetworkAPI.createNetworkCmd() + cn.name = "ontap-tmpl-cache-net-%d" % random.randint(0, 9999) + cn.displaytext = cn.name + cn.networkofferingid = offerings[0].id + cn.zoneid = cls.zone.id + cn.account = cls.account.name + cn.domainid = cls.domain.id + created = cls.apiClient.createNetwork(cn) + cls.network_id = created.id + cls._created_network_id = created.id + + @classmethod + def tearDownClass(cls): + for vm in (cls.vm2, cls.vm1): + if vm is None: + continue + try: + cls._destroy_vm_static(vm.id) + except Exception as e: + logger.warning( + "tearDownClass: destroy VM %s failed: %s" % (vm.id, e) + ) + finally: + if vm is cls.vm1: + cls.vm1 = None + if vm is cls.vm2: + cls.vm2 = None + + if cls.service_offering is not None: + try: + cmd = deleteServiceOfferingAPI.deleteServiceOfferingCmd() + cmd.id = cls.service_offering.id + cls.apiClient.deleteServiceOffering(cmd) + except Exception as e: + logger.warning( + "tearDownClass: delete SO failed: %s" % e + ) + cls.service_offering = None + + if cls._created_network_id is not None: + try: + dn = deleteNetworkAPI.deleteNetworkCmd() + dn.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn) + except Exception as e: + logger.warning( + "tearDownClass: delete network failed: %s" % e + ) + cls._created_network_id = None + + super(OntapTemplateCacheWorkflow, cls).tearDownClass() + + @classmethod + def _destroy_vm_static(cls, vm_id): + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if not vms: + return + state = (vms[0].state or "").lower() + if state not in ("stopped", "destroyed", "expunging", "error"): + stop = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop.id = vm_id + cls.apiClient.stopVirtualMachine(stop) + deadline = time.time() + 300 + while time.time() < deadline: + cur = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if cur and cur[0].state.lower() == "stopped": + break + time.sleep(10) + dest = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest.id = vm_id + dest.expunge = True + cls.apiClient.destroyVirtualMachine(dest) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TemplateCacheTestData.primaryStorage] + storage_ip = self.testdata[TemplateCacheTestData.ontap][ + TemplateCacheTestData.DETAIL_STORAGE_IP + ] + pool_name = "%s_%d" % ( + self.POOL_NAME_PREFIX, random.randint(0, 99999) + ) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "%s://%s/ontap" % (self.POOL_URL_SCHEME, storage_ip) + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TemplateCacheTestData.scope] + cmd.provider = ps[TemplateCacheTestData.provider] + cmd.tags = ps[TemplateCacheTestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _poll_vm_state(self, vm_id, target_state, timeout=900, interval=10): + deadline = time.time() + timeout + current = "unknown" + while time.time() < deadline: + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms: + current = vms[0].state + if current.lower() == target_state.lower(): + return vms[0] + time.sleep(interval) + self.fail( + "VM %s did not reach '%s' within %ds (last='%s')" + % (vm_id, target_state, timeout, current) + ) + + def _deploy_vm(self, name_suffix): + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering.id + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.name = "tmpl-cache-%s-%d" % ( + name_suffix, random.randint(0, 9999) + ) + cmd.displayname = cmd.name + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + return self._poll_vm_state(vm.id, "Running", timeout=900) + + def _root_volume_for_vm(self, vm_id): + vols = self.apiClient.listVolumes(_list_vols_for_vm(vm_id)) or [] + roots = [ + v for v in vols + if str(getattr(v, "type", "")).upper() == "ROOT" + ] + self.assertTrue(roots, "No ROOT volume for VM %s" % vm_id) + return roots[0] + + def _assert_root_on_pool(self, vm_id, pool): + root = self._root_volume_for_vm(vm_id) + self.assertEqual( + str(root.storageid), str(pool.id), + "ROOT volume storageid=%s should equal ONTAP pool id=%s " + "(check service-offering / pool storage tags)" + % (root.storageid, pool.id), + ) + return root + + def _assert_cache_on_ontap(self, pool, spool_ref): + if self.PROTOCOL.upper() == "ISCSI": + tcu.assert_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + cache_count = tcu.count_iscsi_template_cache_luns( + self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + self.assertEqual( + cache_count, 1, + "Expected exactly one cs_tmpl_%s LUN, found %s" + % (self.__class__.template_db_id, cache_count), + ) + else: + tcu.assert_nfs_template_cache_file( + self, self.ontap, pool.name, spool_ref.get("install_path") + ) + + def _count_volume_objects(self, pool): + """Non-cache backend objects that represent deployed ROOT volumes.""" + if self.PROTOCOL.upper() == "ISCSI": + return tcu.count_luns_excluding_template_cache( + self.ontap, self.svm_name, pool.name + ) + # NFS: best-effort — count files under FlexVol root minus known dirs + names = self.ontap.list_files_in_volume(pool.name, path="/") or [] + ignore = {".", "..", ".snapshot"} + return len([n for n in names if n not in ignore]) + + # ================================================================== + # Step implementations (protocol suites expose tagged test_* wrappers) + # ================================================================== + + def step_01_create_tagged_pool_and_service_offering(self): + """ + Create ONTAP pool and a compute offering that share storage tags so + deployVirtualMachine places ROOT on this pool (managed cache path). + """ + pool = self._create_pool() + self.__class__.pool = pool + self.__class__.pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + + self.assertEqual(pool.state, "Up", "Pool should be Up") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "FlexVol missing for pool") + self.assertEqual(ontap_vol.get("state"), "online") + + so = ServiceOffering.create( + self.apiClient, + self.testdata[TemplateCacheTestData.computeOffering], + ) + self.__class__.service_offering = so + self.assertIsNotNone(so.id, "Service offering create failed") + + def step_02_deploy_vm1_seeds_template_cache(self): + """Deploy first VM — seeds primary template cache and clones ROOT.""" + self.assertIsNotNone(self.__class__.pool, "test_01 must pass first") + self.assertIsNotNone( + self.__class__.service_offering, "test_01 must pass first" + ) + + vm = self._deploy_vm("vm1") + self.__class__.vm1 = vm + self._assert_root_on_pool(vm.id, self.__class__.pool) + + spool = tcu.wait_for_spool_ref( + self.dbConnection, + self.__class__.pool_db_id, + self.__class__.template_db_id, + timeout=600, + ) + tcu.assert_spool_ref_ready( + self, spool, + expect_local_path=(self.PROTOCOL.upper() == "ISCSI"), + ) + self._assert_cache_on_ontap(self.__class__.pool, spool) + + def step_03_assert_single_spool_ref_and_cache(self): + """Exactly one template_spool_ref and one ONTAP cache object.""" + self.assertIsNotNone(self.__class__.vm1, "test_02 must pass first") + count = tcu.count_template_spool_refs( + self.dbConnection, + self.__class__.pool_db_id, + self.__class__.template_db_id, + ) + self.assertEqual( + count, 1, + "Expected one template_spool_ref, got %s" % count, + ) + + spool = tcu.get_template_spool_ref( + self.dbConnection, + self.__class__.pool_db_id, + self.__class__.template_db_id, + ) + tcu.assert_spool_ref_ready( + self, spool, + expect_local_path=(self.PROTOCOL.upper() == "ISCSI"), + ) + self._assert_cache_on_ontap(self.__class__.pool, spool) + + if self.PROTOCOL.upper() == "ISCSI": + vol_luns = tcu.count_luns_excluding_template_cache( + self.ontap, self.svm_name, self.__class__.pool.name + ) + self.assertGreaterEqual( + vol_luns, 1, + "Expected at least one volume LUN (ROOT clone) besides cache", + ) + + def step_04_deploy_vm2_reuses_cache(self): + """Second VM reuses the same cache — no second cs_tmpl_* / spool row.""" + self.assertIsNotNone(self.__class__.vm1, "test_02 must pass first") + pool = self.__class__.pool + + before_cache = 1 + before_vols = 0 + if self.PROTOCOL.upper() == "ISCSI": + before_cache = tcu.count_iscsi_template_cache_luns( + self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + before_vols = tcu.count_luns_excluding_template_cache( + self.ontap, self.svm_name, pool.name + ) + + vm = self._deploy_vm("vm2") + self.__class__.vm2 = vm + self._assert_root_on_pool(vm.id, pool) + + count = tcu.count_template_spool_refs( + self.dbConnection, + self.__class__.pool_db_id, + self.__class__.template_db_id, + ) + self.assertEqual( + count, 1, + "Second deploy must not create another template_spool_ref", + ) + + spool = tcu.get_template_spool_ref( + self.dbConnection, + self.__class__.pool_db_id, + self.__class__.template_db_id, + ) + self._assert_cache_on_ontap(pool, spool) + + if self.PROTOCOL.upper() == "ISCSI": + after_cache = tcu.count_iscsi_template_cache_luns( + self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + after_vols = tcu.count_luns_excluding_template_cache( + self.ontap, self.svm_name, pool.name + ) + self.assertEqual( + after_cache, before_cache, + "Cache LUN count must stay %s after second deploy" + % before_cache, + ) + self.assertEqual( + after_vols, before_vols + 1, + "Expected one additional volume LUN for VM-2 ROOT", + ) + + def step_05_destroy_vms_cache_survives(self): + """ + Destroy/expunge both VMs. Template cache must remain (lazy GC only). + + Verifies CloudStack does not delete primary template cache as part of + VM lifecycle — only storage GC / deleteTemplate / pool cleanup should. + """ + pool = self.__class__.pool + self.assertIsNotNone(pool, "test_01 must pass first") + + for attr_name in ("vm1", "vm2"): + vm = getattr(self.__class__, attr_name) + if vm is None: + continue + self._destroy_vm_static(vm.id) + setattr(self.__class__, attr_name, None) + + # Allow async expunge of volumes to settle + time.sleep(15) + + spool = tcu.get_template_spool_ref( + self.dbConnection, + self.__class__.pool_db_id, + self.__class__.template_db_id, + ) + tcu.assert_spool_ref_ready( + self, spool, + expect_local_path=(self.PROTOCOL.upper() == "ISCSI"), + ) + self._assert_cache_on_ontap(pool, spool) + + if self.PROTOCOL.upper() == "ISCSI": + vol_luns = tcu.count_luns_excluding_template_cache( + self.ontap, self.svm_name, pool.name + ) + self.assertEqual( + vol_luns, 0, + "Volume LUNs should be removed after VM expunge; found %s" + % vol_luns, + ) + self.assertEqual( + tcu.count_iscsi_template_cache_luns( + self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ), + 1, + "Template cache LUN must survive VM delete", + ) + + def step_06_cleanup_pool_and_offering(self): + """Delete service offering and force-delete the ONTAP pool.""" + if self.__class__.service_offering is not None: + cmd = deleteServiceOfferingAPI.deleteServiceOfferingCmd() + cmd.id = self.__class__.service_offering.id + self.apiClient.deleteServiceOffering(cmd) + self.__class__.service_offering = None + + pool = self.__class__.pool + self.assertIsNotNone(pool, "Pool absent") + pool_name = pool.name + + for attr_name in ("vm1", "vm2"): + vm = getattr(self.__class__, attr_name) + if vm is not None: + try: + self._destroy_vm_static(vm.id) + except Exception: + pass + setattr(self.__class__, attr_name, None) + + mc = enableStorageMaintenance.enableStorageMaintenanceCmd() + mc.id = pool.id + self.apiClient.enableStorageMaintenance(mc) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # Pool delete / FlexVol removal can take a moment + deadline = time.time() + 120 + while time.time() < deadline: + if self.ontap.get_volume(pool_name) is None: + break + time.sleep(5) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "FlexVol should be deleted after pool removal", + ) diff --git a/test/integration/plugins/ontap/iscsi/template/__init__.py b/test/integration/plugins/ontap/iscsi/template/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/template/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/template/test_template_cache.py b/test/integration/plugins/ontap/iscsi/template/test_template_cache.py new file mode 100644 index 000000000000..b70a69d8140f --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/template/test_template_cache.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +iSCSI primary template-cache workflow (Marvin). + +Seeds and reuses an ONTAP LUN cache named ``/vol//cs_tmpl_`` +when ROOT is placed on a tagged NetApp ONTAP iSCSI pool via a matching +service offering. + +Workflow (sequential — run full suite): + 01 Create tagged iSCSI pool + tagged service offering + 02 Deploy VM-1 — seeds cache LUN + clones ROOT LUN + 03 Assert single template_spool_ref + cs_tmpl_* LUN + 04 Deploy VM-2 — reuses cache (still one cs_tmpl_*, two volume LUNs) + 05 Destroy VMs — cache LUN survives (lazy GC) + 06 Cleanup pool + offering + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/template/test_template_cache.py \\ + -a tags=iscsi_template_cache -v + + # or via runner: + bash test/integration/plugins/ontap/run_tests.sh iscsi_template_cache +""" + +from nose.plugins.attrib import attr + +from helpers.template_cache_workflow import OntapTemplateCacheWorkflow + + +class TestOntapIscsiTemplateCache(OntapTemplateCacheWorkflow): + PROTOCOL = "ISCSI" + NOSE_TAG = "iscsi_template_cache" + PROTOCOL_CFG_KEY = "iscsi" + POOL_URL_SCHEME = "iscsi" + POOL_NAME_PREFIX = "OntapIscsiTmplCache" + + @attr(tags=["iscsi_template_cache"], required_hardware=True) + def test_01_create_tagged_pool_and_service_offering(self): + self.step_01_create_tagged_pool_and_service_offering() + + @attr(tags=["iscsi_template_cache"], required_hardware=True) + def test_02_deploy_vm1_seeds_template_cache(self): + self.step_02_deploy_vm1_seeds_template_cache() + + @attr(tags=["iscsi_template_cache"], required_hardware=True) + def test_03_assert_single_spool_ref_and_cache(self): + self.step_03_assert_single_spool_ref_and_cache() + + @attr(tags=["iscsi_template_cache"], required_hardware=True) + def test_04_deploy_vm2_reuses_cache(self): + self.step_04_deploy_vm2_reuses_cache() + + @attr(tags=["iscsi_template_cache"], required_hardware=True) + def test_05_destroy_vms_cache_survives(self): + self.step_05_destroy_vms_cache_survives() + + @attr(tags=["iscsi_template_cache"], required_hardware=True) + def test_06_cleanup_pool_and_offering(self): + self.step_06_cleanup_pool_and_offering() diff --git a/test/integration/plugins/ontap/iscsi/template/test_template_cache_negative.py b/test/integration/plugins/ontap/iscsi/template/test_template_cache_negative.py new file mode 100644 index 000000000000..33ee6f7a5a38 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/template/test_template_cache_negative.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +iSCSI primary template-cache negative / boundary suite (Marvin). + +Independent of the sequential happy-path suite. + + 01 Tag mismatch — no spool_ref / cache on ONTAP pool + 02 Undersized pool — deploy fails + 03 Out-of-band cache delete — reuse deploy fails + +Running: + bash test/integration/plugins/ontap/run_tests.sh iscsi_template_cache_negative +""" + +from nose.plugins.attrib import attr + +from helpers.template_cache_negative_workflow import ( + OntapTemplateCacheNegativeWorkflow, +) + + +class TestOntapIscsiTemplateCacheNegative(OntapTemplateCacheNegativeWorkflow): + PROTOCOL = "ISCSI" + NOSE_TAG = "iscsi_template_cache_negative" + PROTOCOL_CFG_KEY = "iscsi" + POOL_URL_SCHEME = "iscsi" + POOL_NAME_PREFIX = "OntapIscsiTmplNeg" + + @attr(tags=["iscsi_template_cache_negative"], required_hardware=True) + def test_01_tag_mismatch_does_not_seed_cache(self): + self.step_01_tag_mismatch_does_not_seed_cache() + + @attr(tags=["iscsi_template_cache_negative"], required_hardware=True) + def test_02_undersized_pool_deploy_fails(self): + self.step_02_undersized_pool_deploy_fails() + + @attr(tags=["iscsi_template_cache_negative"], required_hardware=True) + def test_03_deleted_cache_blocks_reuse(self): + self.step_03_deleted_cache_blocks_reuse() diff --git a/test/integration/plugins/ontap/nfs3/template/__init__.py b/test/integration/plugins/ontap/nfs3/template/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/template/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/template/test_template_cache.py b/test/integration/plugins/ontap/nfs3/template/test_template_cache.py new file mode 100644 index 000000000000..246f558c5cec --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/template/test_template_cache.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +NFS3 primary template-cache workflow (Marvin). + +Seeds and reuses an ONTAP FlexVol-backed template cache when ROOT is placed +on a tagged NetApp ONTAP NFS3 pool via a matching service offering. + +Workflow (sequential — run full suite): + 01 Create tagged NFS3 pool + tagged service offering + 02 Deploy VM-1 — seeds cache + clones ROOT + 03 Assert single template_spool_ref + NFS cache file + 04 Deploy VM-2 — reuses cache + 05 Destroy VMs — cache survives (lazy GC) + 06 Cleanup pool + offering + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/nfs3/template/test_template_cache.py \\ + -a tags=nfs3_template_cache -v + + # or via runner: + bash test/integration/plugins/ontap/run_tests.sh nfs3_template_cache +""" + +from nose.plugins.attrib import attr + +from helpers.template_cache_workflow import OntapTemplateCacheWorkflow + + +class TestOntapNfs3TemplateCache(OntapTemplateCacheWorkflow): + PROTOCOL = "NFS3" + NOSE_TAG = "nfs3_template_cache" + PROTOCOL_CFG_KEY = "nfs3" + POOL_URL_SCHEME = "nfs" + POOL_NAME_PREFIX = "OntapNfsTmplCache" + + @attr(tags=["nfs3_template_cache"], required_hardware=True) + def test_01_create_tagged_pool_and_service_offering(self): + self.step_01_create_tagged_pool_and_service_offering() + + @attr(tags=["nfs3_template_cache"], required_hardware=True) + def test_02_deploy_vm1_seeds_template_cache(self): + self.step_02_deploy_vm1_seeds_template_cache() + + @attr(tags=["nfs3_template_cache"], required_hardware=True) + def test_03_assert_single_spool_ref_and_cache(self): + self.step_03_assert_single_spool_ref_and_cache() + + @attr(tags=["nfs3_template_cache"], required_hardware=True) + def test_04_deploy_vm2_reuses_cache(self): + self.step_04_deploy_vm2_reuses_cache() + + @attr(tags=["nfs3_template_cache"], required_hardware=True) + def test_05_destroy_vms_cache_survives(self): + self.step_05_destroy_vms_cache_survives() + + @attr(tags=["nfs3_template_cache"], required_hardware=True) + def test_06_cleanup_pool_and_offering(self): + self.step_06_cleanup_pool_and_offering() diff --git a/test/integration/plugins/ontap/nfs3/template/test_template_cache_negative.py b/test/integration/plugins/ontap/nfs3/template/test_template_cache_negative.py new file mode 100644 index 000000000000..184b8ac84def --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/template/test_template_cache_negative.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +NFS3 primary template-cache negative / boundary suite (Marvin). + +Independent of the sequential happy-path suite. + + 01 Tag mismatch — no spool_ref / cache on ONTAP pool + 02 Undersized pool — deploy fails + 03 Out-of-band cache delete — reuse deploy fails + +Running: + bash test/integration/plugins/ontap/run_tests.sh nfs3_template_cache_negative +""" + +from nose.plugins.attrib import attr + +from helpers.template_cache_negative_workflow import ( + OntapTemplateCacheNegativeWorkflow, +) + + +class TestOntapNfs3TemplateCacheNegative(OntapTemplateCacheNegativeWorkflow): + PROTOCOL = "NFS3" + NOSE_TAG = "nfs3_template_cache_negative" + PROTOCOL_CFG_KEY = "nfs3" + POOL_URL_SCHEME = "nfs" + POOL_NAME_PREFIX = "OntapNfsTmplNeg" + + @attr(tags=["nfs3_template_cache_negative"], required_hardware=True) + def test_01_tag_mismatch_does_not_seed_cache(self): + self.step_01_tag_mismatch_does_not_seed_cache() + + @attr(tags=["nfs3_template_cache_negative"], required_hardware=True) + def test_02_undersized_pool_deploy_fails(self): + self.step_02_undersized_pool_deploy_fails() + + @attr(tags=["nfs3_template_cache_negative"], required_hardware=True) + def test_03_deleted_cache_blocks_reuse(self): + self.step_03_deleted_cache_blocks_reuse() diff --git a/test/integration/plugins/ontap/ontap.cfg b/test/integration/plugins/ontap/ontap.cfg index 64ee5174d379..41e3d6f42110 100644 --- a/test/integration/plugins/ontap/ontap.cfg +++ b/test/integration/plugins/ontap/ontap.cfg @@ -108,11 +108,13 @@ "protocols": { "iscsi": { "enabled": true, - "storagePoolTags": "ontap-iscsi" + "storagePoolTags": "ontap-iscsi", + "templateCacheTags": "ontap-iscsi-tmpl-cache" }, "nfs3": { "enabled": true, - "storagePoolTags": "ontap-nfs3" + "storagePoolTags": "ontap-nfs3", + "templateCacheTags": "ontap-nfs3-tmpl-cache" } } }, diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py index 4f60dbf9433f..73681b9eb701 100644 --- a/test/integration/plugins/ontap/ontap_test_base.py +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -144,7 +144,24 @@ def _delete(self, path, params=None): url = self._base + path resp = requests.delete(url, auth=self._auth, params=params, verify=False, timeout=30) + if not resp.ok: + raise requests.HTTPError( + "%s for url: %s body=%s" + % (resp.status_code, resp.url, resp.text), + response=resp, + ) + + def _patch(self, path, payload=None, params=None): + url = self._base + path + resp = requests.patch(url, auth=self._auth, params=params, + json=payload or {}, verify=False, timeout=30) resp.raise_for_status() + if resp.content: + try: + return resp.json() + except ValueError: + return None + return None def delete_volume(self, name): """Delete the ONTAP FlexVol with the given name. No-op if not found.""" @@ -274,6 +291,126 @@ def list_files_in_volume(self, vol_name, path="/"): return [r.get("name", "") for r in resp.get("records", []) if r.get("name") not in (".", "..")] + def _unmap_lun(self, svm_name, lun_uuid): + """Best-effort removal of all lun-maps for a LUN UUID.""" + try: + maps = self._get( + "/protocols/san/lun-maps", + params={ + "svm.name": svm_name, + "lun.uuid": lun_uuid, + "fields": "lun.uuid,igroup.uuid", + }, + ) + for lun_map in maps.get("records", []): + mapped_lun = lun_map.get("lun", {}).get("uuid") or lun_uuid + igroup_uuid = lun_map.get("igroup", {}).get("uuid") + if not igroup_uuid: + continue + try: + self._delete( + "/protocols/san/lun-maps/%s/%s" + % (mapped_lun, igroup_uuid) + ) + except Exception: + pass + except Exception: + pass + + def _offline_lun(self, svm_name, lun_path, lun_uuid): + """Take LUN offline. Lab ONTAP rejects allow_delete_online.""" + try: + self._patch( + "/storage/luns/%s" % lun_uuid, + payload={"enabled": False, "status": {"state": "offline"}}, + ) + except Exception: + try: + self._patch( + "/storage/luns/%s" % lun_uuid, + payload={"enabled": False}, + ) + except Exception: + return + deadline = time.time() + 60 + while time.time() < deadline: + cur = self.get_lun(svm_name, lun_path) + if not cur: + return + state = ( + (cur.get("status") or {}).get("state") + or ("online" if cur.get("enabled") else "offline") + ) + if str(state).lower() == "offline" or cur.get("enabled") is False: + return + time.sleep(2) + + def _delete_lun_clones(self, svm_name, parent_uuid): + """Delete FlexClone child LUNs so the parent cache LUN can be removed.""" + try: + data = self._get( + "/storage/luns", + params={ + "svm.name": svm_name, + "fields": "name,uuid,clone", + }, + ) + except Exception: + return + for child in data.get("records", []): + clone = child.get("clone") or {} + source = clone.get("source") or {} + if source.get("uuid") != parent_uuid: + continue + child_path = child.get("name") + child_uuid = child.get("uuid") + if not child_path or not child_uuid: + continue + self._unmap_lun(svm_name, child_uuid) + self._offline_lun(svm_name, child_path, child_uuid) + try: + self._delete("/storage/luns/%s" % child_uuid) + except Exception: + pass + + def delete_lun(self, svm_name, lun_path): + """Unmap + offline + delete a LUN by full path. No-op if missing.""" + lun = self.get_lun(svm_name, lun_path) + if not lun: + return False + uuid = lun.get("uuid") + if not uuid: + return False + + # Mapped LUNs / FlexClone parents cannot be deleted until dependents go. + self._unmap_lun(svm_name, uuid) + self._delete_lun_clones(svm_name, uuid) + self._offline_lun(svm_name, lun_path, uuid) + if not self.get_lun(svm_name, lun_path): + return True + # Do not pass allow_delete_online — rejected on this ONTAP build. + self._delete("/storage/luns/%s" % uuid) + return True + + def delete_file_in_volume(self, vol_name, file_path): + """ + Delete a file inside a FlexVol via ONTAP files API. + + ``file_path`` may be absolute (``/foo/bar``) or relative to volume root. + Returns True if a delete was attempted on an existing volume. + """ + vol = self.get_volume(vol_name) + if not vol: + return False + vol_uuid = vol.get("uuid", "") + if not vol_uuid: + return False + from urllib.parse import quote + path = file_path if file_path.startswith("/") else "/" + file_path + encoded_path = quote(path, safe="") + self._delete("/storage/volumes/%s/files/%s" % (vol_uuid, encoded_path)) + return True + # --------------------------------------------------------------------------- # Base test class diff --git a/test/integration/plugins/ontap/run_tests.sh b/test/integration/plugins/ontap/run_tests.sh index 8063fc3238e3..22f55657f96b 100755 --- a/test/integration/plugins/ontap/run_tests.sh +++ b/test/integration/plugins/ontap/run_tests.sh @@ -65,6 +65,8 @@ ISCSI_SUITES=( "iSCSI volume lifecycle|iscsi_volume|${ONTAP_DIR}/iscsi/volume/test_volume_lifecycle.py" "iSCSI zone-scoped pool|iscsi_zone_pool|${ONTAP_DIR}/iscsi/pool/test_zone_scoped_pool.py" "iSCSI VM volume workflow|iscsi_vm_workflow|${ONTAP_DIR}/iscsi/instance/test_vm_volume_attach.py" + "iSCSI template cache|iscsi_template_cache|${ONTAP_DIR}/iscsi/template/test_template_cache.py" + "iSCSI template cache negative|iscsi_template_cache_negative|${ONTAP_DIR}/iscsi/template/test_template_cache_negative.py" ) NFS3_SUITES=( @@ -73,6 +75,8 @@ NFS3_SUITES=( "NFS3 volume lifecycle|nfs3_volume|${ONTAP_DIR}/nfs3/volume/test_volume_lifecycle.py" "NFS3 zone-scoped pool|zone_pool|${ONTAP_DIR}/nfs3/pool/test_zone_scoped_pool.py" "NFS3 VM volume attach|vm_volume_workflow|${ONTAP_DIR}/nfs3/instance/test_vm_volume_attach.py" + "NFS3 template cache|nfs3_template_cache|${ONTAP_DIR}/nfs3/template/test_template_cache.py" + "NFS3 template cache negative|nfs3_template_cache_negative|${ONTAP_DIR}/nfs3/template/test_template_cache_negative.py" ) record_results() {