diff --git a/test/integration/smoke/test_backup_recovery_nas.py b/test/integration/smoke/test_backup_recovery_nas.py index 96021b9edff8..4046c13a638c 100644 --- a/test/integration/smoke/test_backup_recovery_nas.py +++ b/test/integration/smoke/test_backup_recovery_nas.py @@ -24,7 +24,11 @@ from marvin.lib.common import (get_domain, get_zone, get_template) from nose.plugins.attrib import attr from marvin.codes import FAILED +import os import time +from urllib.parse import urlsplit + +SUPPORTED_PRIMARY_STORAGE_POOL_TYPES = ['networkfilesystem', 'rbd'] class TestNASBackupAndRecovery(cloudstackTestCase): @@ -39,19 +43,56 @@ def setUpClass(cls): cls.services["mode"] = cls.zone.networktype cls.hypervisor = cls.testClient.getHypervisorInfo() cls.domain = get_domain(cls.api_client) + cls._cleanup = [] + + if cls.hypervisor.lower() != 'kvm': + cls.skipTest(cls, reason="Test can be run only on KVM hypervisor") + cls.template = get_template(cls.api_client, cls.zone.id, cls.services["ostype"]) if cls.template == FAILED: assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] cls.services["small"]["zoneid"] = cls.zone.id cls.services["small"]["template"] = cls.template.id - cls._cleanup = [] - if cls.hypervisor.lower() != 'kvm': - cls.skipTest(cls, reason="Test can be run only on KVM hypervisor") - - cls.storage_pool = StoragePool.list(cls.api_client)[0] - if cls.storage_pool.type.lower() != 'networkfilesystem': - cls.skipTest(cls, reason="Test can be run only if the primary storage is of type NFS") + # Pick a pool that's actually usable, not just list()[0] -- environments that + # added Ceph/RBD storage after the zone's original NFS primary storage keep the + # old NFS pool around in Disabled state, and it still sorts first. Falling back + # to index 0 there silently exercises the disabled NFS pool's path as if it were + # the primary storage in use, rather than the RBD pool VMs actually deploy on. + storage_pools = StoragePool.list(cls.api_client) + usable_pools = [p for p in storage_pools if getattr(p, 'state', 'Up') == 'Up'] + cls.storage_pool = usable_pools[0] if usable_pools else storage_pools[0] + if cls.storage_pool.type.lower() not in SUPPORTED_PRIMARY_STORAGE_POOL_TYPES: + cls.skipTest(cls, reason="Test can be run only if the primary storage is of type NFS or RBD (Ceph)") + + # The NAS backup repository needs an NFS export to mount. When the primary + # storage is itself NFS, its own path can double as that export (the + # historical behaviour). When the primary storage is Ceph/RBD, the primary + # storage location can't be reused as a NAS export, so fall back to the + # "nfs" test data entry -- the same NFS mount point test_primary_storage.py + # uses to create its temporary NFS primary storage pool, and something every + # marvin environment already has configured (services["nfs"]["url"], e.g. + # "nfs://nfs/export/automation/1/testprimary"). An explicit + # "nas_backup_repository_address" test data entry or NAS_BACKUP_REPO_ADDRESS + # environment variable, if set, takes precedence over both. + if cls.storage_pool.type.lower() == 'networkfilesystem': + default_nas_repository_address = cls.storage_pool.ipaddress + ":" + cls.storage_pool.path + else: + nfs_test_data = cls.services.get("nfs") + if nfs_test_data and nfs_test_data.get("url"): + nfs_url = urlsplit(nfs_test_data["url"]) + default_nas_repository_address = "%s:%s" % (nfs_url.hostname, nfs_url.path) + else: + default_nas_repository_address = None + cls.nas_repository_address = cls.services.get("nas_backup_repository_address") \ + or os.environ.get("NAS_BACKUP_REPO_ADDRESS") \ + or default_nas_repository_address + if not cls.nas_repository_address: + cls.skipTest(cls, reason="No NAS backup repository export configured. Set " + "'nas_backup_repository_address' in the test data, the " + "NAS_BACKUP_REPO_ADDRESS environment variable, or the standard " + "'nfs' test data entry, when the primary storage is not NFS " + "(e.g. Ceph/RBD)") # Check backup configuration values, set them to enable the nas provider backup_enabled_cfg = Configurations.list(cls.api_client, name='backup.framework.enabled') @@ -72,14 +113,22 @@ def setUpClass(cls): cls._cleanup = [cls.account] - # Create NAS backup repository and offering. Use the same directory as the storage pool + # Create NAS backup repository and offering. cls.backup_repository = BackupRepository.add(cls.api_client, zoneid=cls.zone.id, name="Nas", - address=cls.storage_pool.ipaddress + ":" + cls.storage_pool.path, + address=cls.nas_repository_address, provider="nas", type="nfs",) cls._cleanup.append(cls.backup_repository) + # Match the external offering to the repository just created above by externalid + # (== the repository's own id for the nas provider) rather than blindly taking + # index 0 -- a stray repository left over from an earlier interrupted run (e.g. + # one whose backups didn't get cleaned up, so its own teardown couldn't remove + # it either) sorts alongside the new one, and index 0 has no guarantee of being + # the one this run owns. cls.provider_offerings = BackupOffering.listExternal(cls.api_client, cls.zone.id) - cls.backup_offering = BackupOffering.importExisting(cls.api_client, cls.zone.id, cls.provider_offerings[0].externalid, - cls.provider_offerings[0].name, cls.provider_offerings[0].description) + matching_offerings = [o for o in cls.provider_offerings if o.externalid == cls.backup_repository.id] + provider_offering = matching_offerings[0] if matching_offerings else cls.provider_offerings[0] + cls.backup_offering = BackupOffering.importExisting(cls.api_client, cls.zone.id, provider_offering.externalid, + provider_offering.name, provider_offering.description) cls._cleanup.append(cls.backup_offering) cls.offering = ServiceOffering.create(cls.api_client,cls.services["service_offerings"]["small"]) @@ -303,12 +352,26 @@ def _backup_type(self, backup): # Backup objects expose `type`; for chained backups it's "INCREMENTAL", else "FULL". return getattr(backup, 'type', 'FULL') or 'FULL' + def _require_incremental_capable_storage(self): + """ + Incremental NAS backups rely on QEMU dirty bitmaps / libvirt checkpoints, which + only exist on file-based qcow2 storage -- see + NASBackupProvider.allVolumesOnCheckpointCapableStorage(), which forces every VM + on RBD/Ceph (or Linstor) onto the legacy full-only path server-side. On such + storage every backup comes back FULL regardless of cadence, so these chain/type + assertions can't pass (and some would pass vacuously without exercising the + chain logic at all). Skip rather than fail when running against RBD. + """ + if self.storage_pool.type.lower() == 'rbd': + self.skipTest("Incremental backups are not supported on RBD/Ceph primary Storage") + @attr(tags=["advanced", "backup"], required_hardware="true") def test_incremental_chain_cadence(self): """ With nas.backup.full.every=3, the sequence of backups should be FULL, INCREMENTAL, INCREMENTAL, FULL, INCREMENTAL, ... """ + self._require_incremental_capable_storage() self.backup_offering.assignOffering(self.apiclient, self.vm.id) original_full_every = self._get_full_every() self._set_full_every(3) @@ -358,6 +421,7 @@ def test_incremental_after_vm_restart(self): FULL + marker1 -> stop/start the VM (wipes the checkpoint registry) -> INCREMENTAL + marker2 -> restore the tip -> both markers present. """ + self._require_incremental_capable_storage() self.backup_offering.assignOffering(self.apiclient, self.vm.id) original_full_every = self._get_full_every() # High cadence so the post-restart backup is INCREMENTAL, not a periodic FULL. @@ -430,6 +494,7 @@ def test_restore_from_incremental(self): Take FULL + 2 INCREMENTAL backups, each with a marker file. Restore from the latest incremental and verify all three markers are present (chain flatten). """ + self._require_incremental_capable_storage() self.backup_offering.assignOffering(self.apiclient, self.vm.id) original_full_every = self._get_full_every() self._set_full_every(5) @@ -479,6 +544,7 @@ def test_delete_middle_incremental_repairs_chain(self): The chain repair should rebase INC2 onto FULL, and the final restore should still produce a working VM with all expected blocks. """ + self._require_incremental_capable_storage() self.backup_offering.assignOffering(self.apiclient, self.vm.id) original_full_every = self._get_full_every() self._set_full_every(5) @@ -531,6 +597,7 @@ def test_delete_full_with_children_is_deferred(self): FULL is hidden from the backup list while its child survives, and it is physically swept once the last descendant is deleted. """ + self._require_incremental_capable_storage() self.backup_offering.assignOffering(self.apiclient, self.vm.id) original_full_every = self._get_full_every() self._set_full_every(5) @@ -568,6 +635,7 @@ def test_stopped_vm_falls_back_to_full(self): would call for an incremental, the agent must fall back to a full and start a new chain. The incrementalFallback flag should be reflected in backup.type=FULL. """ + self._require_incremental_capable_storage() self.backup_offering.assignOffering(self.apiclient, self.vm.id) original_full_every = self._get_full_every() self._set_full_every(2) # next backup after the first should be incremental @@ -594,3 +662,79 @@ def test_stopped_vm_falls_back_to_full(self): finally: self._set_full_every(original_full_every) self.backup_offering.removeOffering(self.apiclient, self.vm.id) + + # ------------------------------------------------------------------ + # Restore-volume-and-attach regression (PR apache/cloudstack#14007) + # ------------------------------------------------------------------ + # This test exercises the fixed path end to end via restoreVolumeFromBackupAndAttachToVM, + # on whichever primary storage this environment is running with (NFS or Ceph/RBD). + + @attr(tags=["advanced", "backup"], required_hardware="true") + def test_restore_volume_and_attach_to_vm(self): + """ + Test restoring the ROOT and DATADISK volumes of a backup and attaching them + to a different Instance (restoreVolumeFromBackupAndAttachToVM). + """ + target_vm = None + self.backup_offering.assignOffering(self.apiclient, self.vm.id) + try: + ssh_client_vm = self.vm.get_ssh_client(reconnect=True) + ssh_client_vm.execute("echo restore-attach-marker > /root/restore_attach_marker.txt; sync") + + Backup.create(self.apiclient, self.vm.id, "restore_attach_backup") + + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(len(backups), 1, "There should exist only one backup for the VM") + backup = backups[0] + + volumes = Volume.list(self.apiclient, virtualmachineid=self.vm.id, listall=True) + self.assertTrue(isinstance(volumes, list), "List volumes should return a valid list") + root_disk_id = None + data_disk_id = None + for volume in volumes: + if volume.type == 'ROOT': + root_disk_id = volume.id + elif volume.type == 'DATADISK': + data_disk_id = volume.id + self.assertIsNotNone(root_disk_id, "The backed up VM should have a ROOT volume") + + # Target Instance that will receive the restored volumes. The nas provider + # (unlike KBOSS) requires the target Instance to be stopped before a volume + # can be restored and attached to it. + target_vm = VirtualMachine.create( + self.apiclient, self.services["small"], accountid=self.account.name, + domainid=self.account.domainid, serviceofferingid=self.offering.id, + mode=self.services["mode"] + ) + target_vm.stop(self.apiclient, forced=True) + + # Restore and attach the ROOT volume backup as an extra disk. + Backup.restoreVolumeFromBackupAndAttachToVM( + self.apiclient, backupid=backup.id, volumeid=root_disk_id, virtualmachineid=target_vm.id + ) + target_volumes = Volume.list(self.apiclient, virtualmachineid=target_vm.id, listall=True) + self.assertTrue(isinstance(target_volumes, list), "List volumes should return a valid list") + self.assertEqual(2, len(target_volumes), + "Target Instance should have its own ROOT volume plus the restored volume") + + if data_disk_id: + # Restore and attach the DATADISK volume backup as well. + Backup.restoreVolumeFromBackupAndAttachToVM( + self.apiclient, backupid=backup.id, volumeid=data_disk_id, virtualmachineid=target_vm.id + ) + target_volumes = Volume.list(self.apiclient, virtualmachineid=target_vm.id, listall=True) + self.assertEqual(3, len(target_volumes), + "Target Instance should have 3 volumes after restoring both the ROOT and DATADISK backups") + + # Start the target Instance to verify the restored disk(s) are actually + # usable and libvirt accepted the attach-device/attach-disk calls. + target_vm.start(self.apiclient) + + Backup.delete(self.apiclient, backup.id) + finally: + if target_vm is not None: + try: + target_vm.delete(self.apiclient) + except Exception: + pass + self.backup_offering.removeOffering(self.apiclient, self.vm.id) diff --git a/test/integration/smoke/test_direct_download.py b/test/integration/smoke/test_direct_download.py index e6960163bc22..0aeb4252f13d 100644 --- a/test/integration/smoke/test_direct_download.py +++ b/test/integration/smoke/test_direct_download.py @@ -169,9 +169,9 @@ def setUpClass(cls): cls._cleanup = [] cls.hypervisorSupported = False - cls.nfsStorageFound = False + cls.sharedStorageFound = False + cls.sharedPoolIds = {} cls.localStorageFound = False - cls.sharedMountPointFound = False if cls.hypervisor.lower() in ['kvm', 'lxc']: cls.hypervisorSupported = True @@ -218,19 +218,16 @@ def setUpClass(cls): zoneid=cls.zone.id ) for pool in storage_pools: - if not cls.nfsStorageFound and pool.type == "NetworkFilesystem": - cls.nfsStorageFound = True - cls.nfsPoolId = pool.id + if pool.type in ("NetworkFilesystem", "RBD", "SharedMountPoint"): + # the first pool of each supported shared storage type + cls.sharedStorageFound = True + cls.sharedPoolIds.setdefault(pool.type, pool.id) elif not cls.localStorageFound and pool.type == "Filesystem": cls.localStorageFound = True cls.localPoolId = pool.id - elif not cls.sharedMountPointFound and pool.type == "SharedMountPoint": - cls.sharedMountPointFound = True - cls.sharedPoolId = pool.id - cls.nfsKvmNotAvailable = not cls.hypervisorSupported or not cls.nfsStorageFound - cls.localStorageKvmNotAvailable = not cls.hypervisorSupported or not cls.localStorageFound - cls.sharedMountPointKvmNotAvailable = not cls.hypervisorSupported or not cls.sharedMountPointFound + cls.storageKvmNotAvailable = not cls.hypervisorSupported or not ( + cls.sharedStorageFound or cls.localStorageFound) return @classmethod @@ -295,82 +292,36 @@ def deployVM(self, offering) : self.cleanup.append(vm) return vm - @skipTestIf("nfsKvmNotAvailable") + @skipTestIf("storageKvmNotAvailable") @attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false") - def test_01_deploy_vm_from_direct_download_template_nfs_storage(self): - """Test Deploy VM from direct download template on NFS storage + def test_01_deploy_vm_from_direct_download_template(self): + """Test Deploy VM from direct download template on NFS, RBD, local storage and shared mount point """ - # Create service offering for local storage using storage tags - tags = self.getCurrentStoragePoolTags(self.nfsPoolId) - test_tag = "marvin_test_nfs_storage_direct_download" - self.updateStoragePoolTags(self.nfsPoolId, test_tag) - nfs_storage_offering = self.createServiceOffering("TestNFSStorageDirectDownload", "shared", test_tag) - - vm = self.deployVM(nfs_storage_offering) - self.assertEqual( - vm.state, - "Running", - "Check VM deployed from direct download template is running on NFS storage" - ) - - # Revert storage tags for the storage pool used in this test - self.updateStoragePoolTags(self.nfsPoolId, tags) - self.cleanup.append(nfs_storage_offering) - return - - @skipTestIf("localStorageKvmNotAvailable") - @attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false") - def test_02_deploy_vm_from_direct_download_template_local_storage(self): - """Test Deploy VM from direct download template on local storage - """ - - # Create service offering for local storage using storage tags - tags = self.getCurrentStoragePoolTags(self.localPoolId) - test_tag = "marvin_test_local_storage_direct_download" - self.updateStoragePoolTags(self.localPoolId, test_tag) - local_storage_offering = self.createServiceOffering("TestLocalStorageDirectDownload", "local", test_tag) - - # Deploy VM - vm = self.deployVM(local_storage_offering) - self.assertEqual( - vm.state, - "Running", - "Check VM deployed from direct download template is running on local storage" - ) - - # Revert storage tags for the storage pool used in this test - self.updateStoragePoolTags(self.localPoolId, tags) - self.cleanup.append(local_storage_offering) - return - - @skipTestIf("sharedMountPointKvmNotAvailable") - @attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false") - def test_03_deploy_vm_from_direct_download_template_shared_mount_point_storage(self): - """Test Deploy VM from direct download template on shared mount point - """ - - # Create service offering for local storage using storage tags - tags = self.getCurrentStoragePoolTags(self.sharedPoolId) - test_tag = "marvin_test_shared_mount_point_storage_direct_download" - self.updateStoragePoolTags(self.sharedPoolId, test_tag) - shared_offering = self.createServiceOffering("TestSharedMountPointStorageDirectDownload", "shared", test_tag) - - # Deploy VM - vm = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - serviceofferingid=shared_offering.id, - networkids=self.l2_network.id, - ) - self.assertEqual( - vm.state, - "Running", - "Check VM deployed from direct download template is running on shared mount point" - ) - - # Revert storage tags for the storage pool used in this test - self.updateStoragePoolTags(self.sharedPoolId, tags) - self.cleanup.append(vm) - self.cleanup.append(shared_offering) - return + storages = [] + for pool_type, pool_id in self.sharedPoolIds.items(): + storages.append((pool_type, pool_id, "shared")) + if self.localStorageFound: + storages.append(("Filesystem", self.localPoolId, "local")) + + for pool_type, pool_id, offering_type in storages: + self.debug("Deploying VM from direct download template on %s storage" % pool_type) + + # Create service offering for the storage using the storage pool type as storage tag + tags = self.getCurrentStoragePoolTags(pool_id) + test_tag = pool_type.lower() + self.updateStoragePoolTags(pool_id, test_tag) + offering = self.createServiceOffering("TestDirectDownload-%s" % test_tag, offering_type, test_tag) + try: + # Deploy VM + vm = self.deployVM(offering) + self.assertEqual( + vm.state, + "Running", + "Check VM deployed from direct download template is running on %s storage" % pool_type + ) + finally: + # Revert storage tags for the storage pool used in this test + self.updateStoragePoolTags(pool_id, tags) + # the offering is cleaned up after the VM using it + self.cleanup.append(offering) diff --git a/test/integration/smoke/test_host_tags.py b/test/integration/smoke/test_host_tags.py index b6bfe79148d1..7e6a0d0d67bb 100644 --- a/test/integration/smoke/test_host_tags.py +++ b/test/integration/smoke/test_host_tags.py @@ -63,7 +63,7 @@ def tearDownClass(cls): def update_host_tags_via_api(cls, hosttags): cmd = updateHost.updateHostCmd() cmd.id = cls.host.id - cmd.hosttags = hosttags + cmd.hosttags = hosttags if hosttags is not None else "" cls.apiclient.updateHost(cmd) @classmethod diff --git a/test/integration/smoke/test_list_storage_pools.py b/test/integration/smoke/test_list_storage_pools.py index c2c075da65a9..5d9aa1d84371 100644 --- a/test/integration/smoke/test_list_storage_pools.py +++ b/test/integration/smoke/test_list_storage_pools.py @@ -62,26 +62,31 @@ def test_01_list_storage_pools_clusterid_filter(self): isinstance(storage_pools, list), "Storage pool response type should be a list" ) - self.assertGreater( - len(storage_pools), - 0, - "Length of storage pools should greater than 0" - ) + # Filter may return zone-wide pools (with clusterid=None) alongside cluster-scoped pools + cluster_scoped_found = False for storage_pool in storage_pools: - self.assertEqual( - storage_pool.clusterid, - self.cluster.id, - "Cluster id should be equal to the cluster id passed in the filter" - ) + if storage_pool.clusterid is not None: + # Only assert clusterid for cluster-scoped pools + cluster_scoped_found = True + self.assertEqual( + storage_pool.clusterid, + self.cluster.id, + "Cluster id should be equal to the cluster id passed in the filter" + ) storage_pools = StoragePool.list( self.apiclient, clusterid="-1" ) - self.assertIsNone( - storage_pools, - "Response should be empty when invalid cluster id is passed" - ) + # Invalid clusterid may still return zone-wide pools (scope=ZONE) + if storage_pools: + # If pools are returned, they should all be zone-wide + for storage_pool in storage_pools: + self.assertEqual( + storage_pool.scope, + "ZONE", + "Invalid cluster id should only return zone-wide pools" + ) @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") def test_02_list_storage_pools_id_filter(self): @@ -264,26 +269,31 @@ def test_07_list_storage_pools_podid_filter(self): isinstance(storage_pools, list), "Storage pool response type should be a list" ) - self.assertGreater( - len(storage_pools), - 0, - "Length of storage pools should greater than 0" - ) + # Filter may return zone-wide pools (with podid=None) alongside pod-scoped pools + pod_scoped_found = False for storage_pool in storage_pools: - self.assertEqual( - storage_pool.podid, - self.cluster.podid, - "Pod id should be equal to the pod id passed in the filter" - ) + if storage_pool.podid is not None: + # Only assert podid for pod-scoped pools + pod_scoped_found = True + self.assertEqual( + storage_pool.podid, + self.cluster.podid, + "Pod id should be equal to the pod id passed in the filter" + ) storage_pools = StoragePool.list( self.apiclient, podid="-1" ) - self.assertIsNone( - storage_pools, - "Response should be empty when invalid pod id is passed" - ) + # Invalid podid may still return zone-wide pools (scope=ZONE) + if storage_pools: + # If pools are returned, they should all be zone-wide + for storage_pool in storage_pools: + self.assertEqual( + storage_pool.scope, + "ZONE", + "Invalid pod id should only return zone-wide pools" + ) @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") def test_08_list_storage_pools_scope_filter(self): @@ -394,3 +404,64 @@ def test_11_list_storage_pools_no_filter(self): 0, "Length of storage pools should greater than 0" ) + + @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") + def test_12_list_storage_pools_zone_wide(self): + """ Test listing zone-wide vs cluster-scoped storage pools + Zone-wide pools (scope=ZONE) are not bound to a specific cluster. + Cluster-scoped pools are bound to clusters. + """ + # List all storage pools and categorize them + storage_pools = StoragePool.list(self.apiclient) + self.assertTrue( + isinstance(storage_pools, list), + "Storage pool response type should be a list" + ) + + zone_wide_pools = [p for p in storage_pools if hasattr(p, 'scope') and p.scope == "ZONE"] + cluster_scoped_pools = [p for p in storage_pools if hasattr(p, 'scope') and p.scope != "ZONE"] + + # Verify zone-wide pools + if zone_wide_pools: + for pool in zone_wide_pools: + self.assertEqual( + pool.scope, + "ZONE", + "Pool scope should be ZONE" + ) + # Zone-wide pools should not have clusterid or podid + if hasattr(pool, 'clusterid'): + self.assertIsNone( + pool.clusterid, + "Zone-wide storage pool should not have a cluster id" + ) + if hasattr(pool, 'podid'): + self.assertIsNone( + pool.podid, + "Zone-wide storage pool should not have a pod id" + ) + # Verify they have zoneid + self.assertTrue( + hasattr(pool, 'zoneid') and pool.zoneid is not None, + "Zone-wide storage pool should have a zone id" + ) + self.debug("Verified %d zone-wide storage pools" % len(zone_wide_pools)) + else: + self.debug("No zone-wide storage pools found in this setup") + + # Verify cluster-scoped pools + if cluster_scoped_pools: + for pool in cluster_scoped_pools: + # Cluster-scoped pools should have clusterid + self.assertTrue( + hasattr(pool, 'clusterid') and pool.clusterid is not None, + "Cluster-scoped storage pool should have a cluster id" + ) + # Verify they have zoneid + self.assertTrue( + hasattr(pool, 'zoneid') and pool.zoneid is not None, + "Cluster-scoped storage pool should have a zone id" + ) + self.debug("Verified %d cluster-scoped storage pools" % len(cluster_scoped_pools)) + else: + self.debug("No cluster-scoped storage pools found in this setup") diff --git a/test/integration/smoke/test_over_provisioning.py b/test/integration/smoke/test_over_provisioning.py index c2b1a5ac2052..70a3b84b9cfb 100644 --- a/test/integration/smoke/test_over_provisioning.py +++ b/test/integration/smoke/test_over_provisioning.py @@ -60,10 +60,10 @@ def test_UpdateStorageOverProvisioningFactor(self): "The environment don't have storage pools required for test") for pool in storage_pools: - if pool.type == "NetworkFilesystem" or pool.type == "VMFS" or pool.type == "PowerFlex": + if pool.type in ("NetworkFilesystem", "VMFS", "PowerFlex", "RBD"): break - if pool.type != "NetworkFilesystem" and pool.type != "VMFS" and pool.type != "PowerFlex": + if pool.type not in ("NetworkFilesystem", "VMFS", "PowerFlex", "RBD"): raise self.skipTest("Storage overprovisioning currently not supported on " + pool.type + " pools") self.poolId = pool.id diff --git a/test/integration/smoke/test_snapshots.py b/test/integration/smoke/test_snapshots.py index b1a2569d9694..284d8959df56 100644 --- a/test/integration/smoke/test_snapshots.py +++ b/test/integration/smoke/test_snapshots.py @@ -262,7 +262,7 @@ def test_02_list_snapshots_with_removed_data_store(self): volume_pool_response = list_storage_pools(self.apiclient, id=vol_res[0].storageid) volume_pool = volume_pool_response[0] - if volume_pool.type.lower() != 'networkfilesystem': + if volume_pool.type not in ("NetworkFilesystem", "RBD"): self.skipTest("This test is not supported for volume created on storage pool type %s" % volume_pool.type) clusters = list_clusters( self.apiclient, @@ -291,9 +291,8 @@ def test_02_list_snapshots_with_removed_data_store(self): 'Up', "Check primary storage state" ) - self.assertEqual( - storage.type, - 'NetworkFilesystem', + self.assertTrue( + storage.type in ("NetworkFilesystem", "RBD"), "Check storage pool type" ) storage_pools_response = list_storage_pools(self.apiclient, diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 8df0b994a555..5ed35f81b912 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -48,6 +48,7 @@ get_suitable_test_template, get_test_ovf_templates, list_hosts, + list_storage_pools, get_vm_vapp_configs) from marvin.codes import FAILED, PASS from nose.plugins.attrib import attr @@ -1707,12 +1708,17 @@ def get_target_host(self, virtualmachineid): return target_hosts[0] - def get_target_pool(self, volid): - target_pools = StoragePool.listForMigration(self.apiclient, id=volid) + def get_target_pool(self, vol): + target_pools = StoragePool.listForMigration(self.apiclient, id=vol.id) if target_pools is None or len(target_pools) == 0: self.skipTest("Not enough storage pools found for migration") + volume = Volume.list(self.apiclient, id=vol.id)[0] + source_pool = list_storage_pools(self.apiclient, id=volume.storageid)[0] + if source_pool.type == 'RBD' and target_pools[0].type == 'RBD': + self.skipTest("Live VM migration between RBD pools is unsupported") + return target_pools[0] def get_vm_volumes(self, id): @@ -1751,7 +1757,7 @@ def test_01_migrate_VM_and_root_volume(self): root_volume = self.get_vm_volumes(vm.id)[0] - target_pool = self.get_target_pool(root_volume.id) + target_pool = self.get_target_pool(root_volume) target_host = self.get_target_host(vm.id) @@ -1789,9 +1795,9 @@ def test_02_migrate_VM_with_two_data_disks(self): root_volume = self.get_vm_volumes(vm.id)[0] - target_pool = self.get_target_pool(root_volume.id) - volume1.target_pool = self.get_target_pool(volume1.id) - volume2.target_pool = self.get_target_pool(volume2.id) + target_pool = self.get_target_pool(root_volume) + volume1.target_pool = self.get_target_pool(volume1) + volume2.target_pool = self.get_target_pool(volume2) target_host = self.get_target_host(vm.id) diff --git a/test/integration/smoke/test_vm_lifecycle_unmanage_import.py b/test/integration/smoke/test_vm_lifecycle_unmanage_import.py index 838794716742..25ffc31ddf8b 100644 --- a/test/integration/smoke/test_vm_lifecycle_unmanage_import.py +++ b/test/integration/smoke/test_vm_lifecycle_unmanage_import.py @@ -59,7 +59,7 @@ def setUpClass(cls): if cls.template == FAILED: assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"] - cls.hypervisorNotSupported = cls.hypervisor.lower() != "vmware" + cls.hypervisorNotSupported = cls.hypervisor.lower() not in ("vmware", "kvm") if cls.hypervisorNotSupported: return diff --git a/test/integration/smoke/test_vm_snapshot_kvm.py b/test/integration/smoke/test_vm_snapshot_kvm.py index 9dd7c529de5e..59c487066ae0 100644 --- a/test/integration/smoke/test_vm_snapshot_kvm.py +++ b/test/integration/smoke/test_vm_snapshot_kvm.py @@ -186,6 +186,8 @@ def test_01_create_vm_snapshots(self): cmds = [ "echo %s > %s/%s" % (self.random_data_0, self.test_dir, self.random_data), + # make sure the data is on disk, a disk-only snapshot does not contain the page cache + "sync", "cat %s/%s" % (self.test_dir, self.random_data)] diff --git a/test/integration/smoke/test_vm_snapshots.py b/test/integration/smoke/test_vm_snapshots.py index c64de12b1c31..7c7c4a52104a 100644 --- a/test/integration/smoke/test_vm_snapshots.py +++ b/test/integration/smoke/test_vm_snapshots.py @@ -47,6 +47,11 @@ def setUpClass(cls): cls.unsupportedHypervisor = True return + list_volume_pool_response = list_storage_pools(cls.apiclient) + volume_pool = list_volume_pool_response[0] + if volume_pool.type == "RBD": + cls.skipTest(cls, reason="VM snapshot is unsupported for VMs on RBD storage pool") + cls.services = testClient.getParsedTestDataConfig() # Get Zone, Domain and templates cls.domain = get_domain(cls.apiclient) diff --git a/test/integration/smoke/test_volumes.py b/test/integration/smoke/test_volumes.py index ad1c359c27b5..ec8cf0030696 100644 --- a/test/integration/smoke/test_volumes.py +++ b/test/integration/smoke/test_volumes.py @@ -825,30 +825,36 @@ def test_10_list_volumes(self): self.assertTrue(hasattr(root_volume, "vmname")) self.assertEqual(root_volume.vmname, list_vm.name) - self.assertTrue(hasattr(root_volume, "clustername")) - self.assertTrue(root_volume.clustername is not None) - - self.assertTrue(hasattr(root_volume, "clusterid")) - self.assertTrue(root_volume.clusterid is not None) - self.assertTrue(hasattr(root_volume, "storageid")) self.assertTrue(root_volume.storageid is not None) self.assertTrue(hasattr(root_volume, "storage")) self.assertTrue(root_volume.storage is not None) + volume_pool = list_storage_pools(self.apiclient, id=root_volume.storageid)[0] + if volume_pool.scope == "ZONE": + # A zone-wide storage pool isn't bound to a single pod/cluster, + # so volumes on it have no pod/cluster association. + self.debug("Skipping pod/cluster checks: storage pool %s is zone-wide" % volume_pool.id) + else: + self.assertTrue(hasattr(root_volume, "clustername")) + self.assertTrue(root_volume.clustername is not None) + + self.assertTrue(hasattr(root_volume, "clusterid")) + self.assertTrue(root_volume.clusterid is not None) + + self.assertTrue(hasattr(root_volume, "podid")) + self.assertEqual(root_volume.podid, list_pods.id) + + self.assertTrue(hasattr(root_volume, "podname")) + self.assertEqual(root_volume.podname, list_pods.name) + self.assertTrue(hasattr(root_volume, "zoneid")) self.assertEqual(root_volume.zoneid, self.zone.id) self.assertTrue(hasattr(root_volume, "zonename")) self.assertEqual(root_volume.zonename, self.zone.name) - self.assertTrue(hasattr(root_volume, "podid")) - self.assertEqual(root_volume.podid, list_pods.id) - - self.assertTrue(hasattr(root_volume, "podname")) - self.assertEqual(root_volume.podname, list_pods.name) - @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="true") def test_11_attach_volume_with_unstarted_vm(self): """Attach a created Volume to a unstarted VM @@ -1111,6 +1117,12 @@ def setUpClass(cls): cls.unsupportedHypervisor = True return + list_volume_pool_response = list_storage_pools(cls.apiclient) + volume_pool = list_volume_pool_response[0] + # RBD pools are encrypted natively by librbd (LUKS2, engine='librbd'); any other pool type + # goes through qemu-native encryption (LUKS1, no engine attribute). + cls.storage_pool_type = volume_pool.type + # Get Zone and Domain cls.domain = get_domain(cls.apiclient) cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) @@ -1447,12 +1459,7 @@ def test_02_data_volume_encryption(self): list_volume_response = Volume.list( self.apiclient, id=volume.id) - vol_sz = str(list_volume_response[0].size) - list_volume_pool_response = list_storage_pools(self.apiclient, id=list_volume_response[0].storageid) - volume_pool = list_volume_pool_response[0] - if volume_pool.type.lower() == "powerflex": - vol_sz = int(vol_sz) - vol_sz = str(vol_sz - (128 << 20) - ((vol_sz >> 30) * 200704)) + vol_sz = self.get_expected_guest_visible_size(list_volume_response[0], list_volume_response[0].size) volume_name = "/dev/vd" + chr(ord('a') + int(list_volume_response[0].deviceid)) self.debug(" Using KVM volume_name: %s" % (volume_name)) @@ -1467,13 +1474,21 @@ def test_02_data_volume_encryption(self): @attr(tags=["advanced", "smoke", "diskencrypt"], required_hardware="true") def test_03_root_and_data_volume_encryption(self): - """Test Root and Data Volumes Encryption + """Test Root and Data Volumes Encryption, plus resize of the encrypted data volume # Validate the following # 1. Create VM using the service offering with encryptroot true # 2. Verify VM created and Root Volume # 3. Create Data Volume using the disk offering with encrypt true # 4. Verify Data Volume + # 5. Resize the encrypted data volume online (VM stays running) -> for RBD this is + # routed through virsh blockresize; for any other pool type it goes through the + # same virsh blockresize NOTIFYONLY path used for qemu-native LUKS. + # 6. Detach and resize it again offline -> for RBD this uses `rbd resize + # --encryption-passphrase-file`; for any other pool type it uses qemu-img + # (LibvirtResizeVolumeCommandWrapper.resizeEncryptedQcowFile). + # Resize is folded into this test (reusing its VM and volume) rather than a separate + # test, since spinning up another VM just for resize coverage would cost a lot more time. """ virtual_machine = VirtualMachine.create( @@ -1585,12 +1600,7 @@ def test_03_root_and_data_volume_encryption(self): list_volume_response = Volume.list( self.apiclient, id=volume.id) - vol_sz = str(list_volume_response[0].size) - list_volume_pool_response = list_storage_pools(self.apiclient, id=list_volume_response[0].storageid) - volume_pool = list_volume_pool_response[0] - if volume_pool.type.lower() == "powerflex": - vol_sz = int(vol_sz) - vol_sz = str(vol_sz - (128 << 20) - ((vol_sz >> 30) * 200704)) + vol_sz = self.get_expected_guest_visible_size(list_volume_response[0], list_volume_response[0].size) volume_name = "/dev/vd" + chr(ord('a') + int(list_volume_response[0].deviceid)) self.debug(" Using KVM volume_name: %s" % (volume_name)) @@ -1599,10 +1609,90 @@ def test_03_root_and_data_volume_encryption(self): self.check_volume_encryption(virtual_machine, 3) + # Resize the attached encrypted data volume, online then offline, reusing this VM + # instead of provisioning a separate one just for resize coverage. + # + # Online (VM stays running): for RBD this is routed through virsh blockresize; for + # any other pool type it goes through the same virsh blockresize NOTIFYONLY path used + # for qemu-native LUKS. + disk_offering_encrypt_2gb = DiskOffering.create( + self.apiclient, + self.services["disk_offering"], + name="Encrypted-2GB", + disksize=2, + encrypt=True + ) + self.cleanup.append(disk_offering_encrypt_2gb) + + volume.resize(self.apiclient, diskofferingid=disk_offering_encrypt_2gb.id) + + # Online resize only grows the block device live; picking up the new capacity in the + # guest still needs a rescan or reboot depending on the guest OS/kernel (this template's + # CentOS 5.5 predates reliable virtio-blk live-capacity notification), so verify success + # through the volume's own reported size here. Guest-visible size is verified below after + # the offline resize, once the volume is freshly re-attached. + list_volume_response = Volume.list(self.apiclient, id=volume.id) + expected_raw_size = disk_offering_encrypt_2gb.disksize << 30 + if self.storage_pool_type == "RBD" and getattr(list_volume_response[0], "encryptformat", None): + expected_raw_size += (16 << 20) + self.assertEqual( + int(list_volume_response[0].size), + expected_raw_size, + "Check that the online resize grew the volume to the expected size" + ) + self.assertEqual(list_volume_response[0].state, 'Ready', "Check that the online-resized volume is Ready") + + # Offline (volume detached first): for RBD this uses `rbd resize + # --encryption-passphrase-file`; for any other pool type it uses qemu-img + # (LibvirtResizeVolumeCommandWrapper.resizeEncryptedQcowFile). + virtual_machine.detach_volume(self.apiclient, volume) + + disk_offering_encrypt_3gb = DiskOffering.create( + self.apiclient, + self.services["disk_offering"], + name="Encrypted-3GB", + disksize=3, + encrypt=True + ) + self.cleanup.append(disk_offering_encrypt_3gb) + + volume.resize(self.apiclient, diskofferingid=disk_offering_encrypt_3gb.id) + + virtual_machine.attach_volume(self.apiclient, volume) + self.check_volume_encryption(virtual_machine, 3) + + ssh = virtual_machine.get_ssh_client(reconnect=True) + list_volume_response = Volume.list(self.apiclient, id=volume.id) + vol_sz = self.get_expected_guest_visible_size(list_volume_response[0], list_volume_response[0].size) + # After a detach + reattach, libvirt is not guaranteed to reuse the same target device + # letter the volume had before (it may pick a fresh one), so the offering's deviceid no + # longer reliably maps to /dev/vdX here. Match by the reported byte count instead. + fdisk_output = ssh.execute("/sbin/fdisk -l | grep Disk") + self.debug("Offline resize - Volume Size Expected %s - guest disks: %s" % (vol_sz, fdisk_output)) + self.assertTrue( + any(" %s bytes" % vol_sz in line for line in fdisk_output), + "Check if the offline-resized encrypted volume size is available in the guest" + ) + virtual_machine.detach_volume(self.apiclient, volume) - self.assertEqual(ret[0], SUCCESS, "Check if promised disk size actually available") time.sleep(self.services["sleep"]) + def get_expected_guest_visible_size(self, volume, vol_sz): + """Some pool types/encryption mechanisms reserve overhead out of the requested size, so the + guest sees fewer usable bytes than the volume's nominal size. Returns the byte count the + guest should actually report for the given volume. + """ + list_volume_pool_response = list_storage_pools(self.apiclient, id=volume.storageid) + volume_pool = list_volume_pool_response[0] + vol_sz = int(vol_sz) + if volume_pool.type.lower() == "powerflex": + vol_sz = vol_sz - (128 << 20) - ((vol_sz >> 30) * 200704) + elif volume_pool.type == "RBD" and getattr(volume, "encryptformat", None): + # RBD volumes are encrypted natively by librbd: the LUKS2 header/keyslots (16 MiB) are + # reserved out of the volume's own size, so the guest sees size - header. + vol_sz = vol_sz - (16 << 20) + return str(vol_sz) + def does_host_with_encryption_support_exists(self): hosts = Host.list( self.apiclient, @@ -1611,6 +1701,19 @@ def does_host_with_encryption_support_exists(self): hypervisor='KVM', state='Up') + if self.storage_pool_type == "RBD": + # host.volume.encryption ("encryptionsupported" in the API response) only reflects + # qemu-native LUKS support; librbd RBD encryption is tracked separately in + # host_details as host.volume.encryption.rbd, which is not surfaced on HostResponse. + for host in hosts: + rbd_supported = self.dbclient.execute( + "select hd.value from host_details hd join host h on h.id=hd.host_id " + "where h.uuid='%s' and hd.name='host.volume.encryption.rbd'" % host.id + ) + if rbd_supported and rbd_supported[0][0].lower() == 'true': + return True + return False + for host in hosts: if host.encryptionsupported: return True @@ -1640,14 +1743,25 @@ def check_volume_encryption(self, virtual_machine, volumes_count): parser = etree.XMLParser(remove_blank_text=True) virshxml_root = ET.fromstring(xml_as_str, parser=parser) - encryption_format = virshxml_root.findall(".devices/disk/encryption[@format='luks']") - self.assertIsNotNone(encryption_format, "The volume encryption format is not luks") + # RBD volumes are encrypted natively by librbd: . + # Any other pool type keeps using qemu-native LUKS1: (no engine). + encryption_format_value = 'luks2' if self.storage_pool_type == "RBD" else 'luks' + encryption_format = virshxml_root.findall(".devices/disk/encryption[@format='%s']" % encryption_format_value) + self.assertIsNotNone(encryption_format, "The volume encryption format is not %s" % encryption_format_value) self.assertEqual( len(encryption_format), volumes_count, - "Check the number of volumes encrypted with luks format" + "Check the number of volumes encrypted with %s format" % encryption_format_value ) + if self.storage_pool_type == "RBD": + librbd_engine = [e for e in encryption_format if e.get('engine') == 'librbd'] + self.assertEqual( + len(librbd_engine), + volumes_count, + "Check the number of RBD volumes encrypted with the librbd engine" + ) + secret_type = virshxml_root.findall(".devices/disk/encryption/secret[@type='passphrase']") self.assertIsNotNone(secret_type, "The volume encryption secret type is not passphrase") self.assertEqual(