diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 35cc864268c3..61c755b1a9ab 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -83,7 +83,8 @@ public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) { this._storageMapper.put("libvirt", new LibvirtStorageAdaptor(storagelayer)); // add other storage adaptors manually here - // add any adaptors that wish to register themselves via call to adaptor.getStoragePoolType() + // add any adaptors that wish to register themselves via call to + // adaptor.getStoragePoolType() Reflections reflections = new Reflections("com.cloud.hypervisor.kvm.storage"); Set> storageAdaptorClasses = reflections.getSubTypesOf(StorageAdaptor.class); for (Class storageAdaptorClass : storageAdaptorClasses) { @@ -112,7 +113,8 @@ public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) { StoragePoolType storagePoolType = adaptor.getStoragePoolType(); if (storagePoolType != null) { if (this._storageMapper.containsKey(storagePoolType.toString())) { - logger.warn(String.format("Duplicate StorageAdaptor type %s, not loading %s", storagePoolType, storageAdaptorClass.getName())); + logger.warn(String.format("Duplicate StorageAdaptor type %s, not loading %s", storagePoolType, + storageAdaptorClass.getName())); } else { logger.info(String.format("Adding storage adaptor for %s", storageAdaptorClass.getName())); this._storageMapper.put(storagePoolType.toString(), adaptor); @@ -135,7 +137,8 @@ public boolean supportsPhysicalDiskCopy(StoragePoolType type) { return getStorageAdaptor(type).supportsPhysicalDiskCopy(type); } - public boolean connectPhysicalDisk(StoragePoolType type, String poolUuid, String volPath, Map details) { + public boolean connectPhysicalDisk(StoragePoolType type, String poolUuid, String volPath, + Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); KVMStoragePool pool = adaptor.getStoragePool(poolUuid); @@ -155,8 +158,8 @@ public boolean connectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec, boolean is continue; } - VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); - PrimaryDataStoreTO store = (PrimaryDataStoreTO)vol.getDataStore(); + VolumeObjectTO vol = (VolumeObjectTO) disk.getData(); + PrimaryDataStoreTO store = (PrimaryDataStoreTO) vol.getDataStore(); if (!store.isManaged() && VirtualMachine.State.Migrating.equals(vmSpec.getState())) { result = true; continue; @@ -168,7 +171,8 @@ public boolean connectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec, boolean is result = adaptor.connectPhysicalDisk(vol.getPath(), pool, disk.getDetails(), isVMMigrate); if (!result) { - logger.error("Failed to connect disks via Instance spec for Instance: " + vmName + " volume:" + vol.toString()); + logger.error("Failed to connect disks via Instance spec for Instance: " + vmName + " volume:" + + vol.toString()); return result; } } @@ -186,18 +190,22 @@ public boolean disconnectPhysicalDisk(Map volumeToDisconnect) { String poolType = volumeToDisconnect.get(DiskTO.PROTOCOL_TYPE); StorageAdaptor adaptor = _storageMapper.get(poolType); if (adaptor != null) { - logger.info(String.format("Disconnecting physical disk using the storage adaptor found for pool type: %s", poolType)); + logger.info(String.format( + "Disconnecting physical disk using the storage adaptor found for pool type: %s", poolType)); return adaptor.disconnectPhysicalDisk(volumeToDisconnect); } - logger.debug(String.format("Couldn't find the storage adaptor for pool type: %s to disconnect the physical disk, trying with others", poolType)); + logger.debug(String.format( + "Couldn't find the storage adaptor for pool type: %s to disconnect the physical disk, trying with others", + poolType)); } for (Map.Entry set : _storageMapper.entrySet()) { StorageAdaptor adaptor = set.getValue(); if (adaptor.disconnectPhysicalDisk(volumeToDisconnect)) { - logger.debug(String.format("Disconnected physical disk using the storage adaptor for pool type: %s", set.getKey())); + logger.debug(String.format("Disconnected physical disk using the storage adaptor for pool type: %s", + set.getKey())); return true; } } @@ -211,7 +219,9 @@ public boolean disconnectPhysicalDiskByPath(String path) { StorageAdaptor adaptor = set.getValue(); if (adaptor.disconnectPhysicalDiskByPath(path)) { - logger.debug(String.format("Disconnected physical disk by local path: %s, using the storage adaptor for pool type: %s", path, set.getKey())); + logger.debug(String.format( + "Disconnected physical disk by local path: %s, using the storage adaptor for pool type: %s", + path, set.getKey())); return true; } } @@ -221,10 +231,15 @@ public boolean disconnectPhysicalDiskByPath(String path) { public boolean disconnectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec) { if (vmSpec == null) { - /* CloudStack often tries to stop VMs that shouldn't be running, to ensure a known state, - for example if we lose communication with the agent and the VM is brought up elsewhere. - We may not know about these yet. This might mean that we can't use the vmspec map, because - when we restart the agent we lose all of the info about running VMs. */ + /* + * CloudStack often tries to stop VMs that shouldn't be running, to ensure a + * known state, + * for example if we lose communication with the agent and the VM is brought up + * elsewhere. + * We may not know about these yet. This might mean that we can't use the vmspec + * map, because + * when we restart the agent we lose all of the info about running VMs. + */ logger.debug("disconnectPhysicalDiskViaVmSpec: Attempted to stop a VM that is not yet in our hash map"); @@ -241,13 +256,14 @@ public boolean disconnectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec) { if (disk.getType() != Volume.Type.ISO) { logger.debug("Disconnecting disk " + disk.getPath()); - VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); - PrimaryDataStoreTO store = (PrimaryDataStoreTO)vol.getDataStore(); + VolumeObjectTO vol = (VolumeObjectTO) disk.getData(); + PrimaryDataStoreTO store = (PrimaryDataStoreTO) vol.getDataStore(); KVMStoragePool pool = getStoragePool(store.getPoolType(), store.getUuid()); if (pool == null) { - logger.error("Pool " + store.getUuid() + " of type " + store.getPoolType() + " was not found, skipping disconnect logic"); + logger.error("Pool " + store.getUuid() + " of type " + store.getPoolType() + + " was not found, skipping disconnect logic"); continue; } @@ -258,7 +274,8 @@ public boolean disconnectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec) { boolean subResult = adaptor.disconnectPhysicalDisk(vol.getPath(), pool); if (!subResult) { - logger.error("Failed to disconnect disks via Instance spec for Instance: " + vmName + " volume:" + vol.toString()); + logger.error("Failed to disconnect disks via Instance spec for Instance: " + vmName + " volume:" + + vol.toString()); result = false; } @@ -281,9 +298,11 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean } catch (Exception e) { StoragePoolInformation info = _storagePools.get(uuid); if (info != null) { - pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType()); + pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), + info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType()); } else { - throw new CloudRuntimeException("Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage()); + throw new CloudRuntimeException( + "Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage()); } } @@ -296,8 +315,11 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean } /** - * As the class {@link LibvirtStoragePool} is constrained to the {@link org.libvirt.StoragePool} class, there is no way of saving a generic parameter such as the details, hence, - * this method was created to always make available the details of libvirt primary storages for when they are needed. + * As the class {@link LibvirtStoragePool} is constrained to the + * {@link org.libvirt.StoragePool} class, there is no way of saving a generic + * parameter such as the details, hence, + * this method was created to always make available the details of libvirt + * primary storages for when they are needed. */ private void addPoolDetails(String uuid, LibvirtStoragePool pool) { StoragePoolInformation storagePoolInformation = _storagePools.get(uuid); @@ -333,7 +355,7 @@ public KVMStoragePool getStoragePoolByURI(String uri) { sourcePath = sourcePath.replace("//", "/"); sourceHost = storageUri.getHost(); uuid = UuidUtils.nameUUIDFromBytes(new String(sourceHost + sourcePath).getBytes()).toString(); - protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem: StoragePoolType.NetworkFilesystem; + protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem : StoragePoolType.NetworkFilesystem; // storage registers itself through here return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocol, null, false); @@ -343,8 +365,9 @@ public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, St int cnt = 0; int retries = 100; KVMPhysicalDisk vol = null; - //harden get volume, try cnt times to get volume, in case volume is created on other host - //Poll more frequently and return immediately once disk is found + // harden get volume, try cnt times to get volume, in case volume is created on + // other host + // Poll more frequently and return immediately once disk is found String errMsg = ""; while (cnt < retries) { try { @@ -375,7 +398,8 @@ public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, St } } - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, + StoragePoolType type) { // primary storage registers itself through here return createStoragePool(name, host, port, path, userInfo, type, null, true); } @@ -383,24 +407,30 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri /** * Primary Storage registers itself through here */ - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, + StoragePoolType type, Map details) { return createStoragePool(name, host, port, path, userInfo, type, details, true); } - //Note: due to bug CLOUDSTACK-4459, createStoragepool can be called in parallel, so need to be synced. - private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { + // Note: due to bug CLOUDSTACK-4459, createStoragepool can be called in + // parallel, so need to be synced. + private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, + String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { StorageAdaptor adaptor = getStorageAdaptor(type); - KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, primaryStorage); + KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, + primaryStorage); if (pool instanceof LibvirtStoragePool) { ((LibvirtStoragePool) pool).setType(type); } // LibvirtStorageAdaptor-specific statement if (pool.isPoolSupportHA() && primaryStorage) { - KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, PoolType.PrimaryStorage); + KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, + PoolType.PrimaryStorage); _haMonitor.addStoragePool(storagePool); } - StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, details, primaryStorage); + StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, details, + primaryStorage); addStoragePool(pool.getUuid(), info); return pool; } @@ -417,7 +447,8 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid) { if (type == StoragePoolType.NetworkFilesystem) { _haMonitor.removeStoragePool(uuid); } - boolean deleteStatus = adaptor.deleteStoragePool(uuid);; + boolean deleteStatus = adaptor.deleteStoragePool(uuid); + ; synchronized (_storagePools) { _storagePools.remove(uuid); } @@ -426,23 +457,29 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid) { public boolean deleteStoragePool(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); + logger.debug("[deleteStoragePool] calling adaptor.deleteStoragePool for pool {} (type={})", uuid, type); + boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); + logger.debug("[deleteStoragePool] adaptor.deleteStoragePool returned {} for pool {}", deleteStatus, uuid); if (type == StoragePoolType.NetworkFilesystem) { + logger.debug("[deleteStoragePool] calling haMonitor.removeStoragePool for NFS pool {}", uuid); _haMonitor.removeStoragePool(uuid); } - boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); synchronized (_storagePools) { _storagePools.remove(uuid); } return deleteStatus; } - public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, Storage.ProvisioningType provisioningType, - KVMStoragePool destPool, int timeout, byte[] passphrase) { - return createDiskFromTemplate(template, name, provisioningType, destPool, template.getSize(), timeout, passphrase); + public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, + Storage.ProvisioningType provisioningType, + KVMStoragePool destPool, int timeout, byte[] passphrase) { + return createDiskFromTemplate(template, name, provisioningType, destPool, template.getSize(), timeout, + passphrase); } - public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, Storage.ProvisioningType provisioningType, - KVMStoragePool destPool, long size, int timeout, byte[] passphrase) { + public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, + Storage.ProvisioningType provisioningType, + KVMStoragePool destPool, long size, int timeout, byte[] passphrase) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); // LibvirtStorageAdaptor-specific statement @@ -469,7 +506,8 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String n } } - public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, + long size, KVMStoragePool destPool) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); return adaptor.createTemplateFromDisk(disk, name, format, size, destPool); } @@ -479,28 +517,34 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt return adaptor.copyPhysicalDisk(disk, name, destPool, timeout, null, null, null); } - public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, + byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); return adaptor.copyPhysicalDisk(disk, name, destPool, timeout, srcPassphrase, dstPassphrase, provisioningType); } - public KVMPhysicalDisk createDiskWithTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, - KVMStoragePool destPool, int timeout, byte[] passphrase) { + public KVMPhysicalDisk createDiskWithTemplateBacking(KVMPhysicalDisk template, String name, + PhysicalDiskFormat format, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); return adaptor.createDiskFromTemplateBacking(template, name, format, size, destPool, timeout, passphrase); } - public KVMPhysicalDisk createPhysicalDiskFromDirectDownloadTemplate(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + public KVMPhysicalDisk createPhysicalDiskFromDirectDownloadTemplate(String templateFilePath, + String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); - return adaptor.createTemplateFromDirectDownloadFile(templateFilePath, destTemplatePath, destPool, format, timeout); + return adaptor.createTemplateFromDirectDownloadFile(templateFilePath, destTemplatePath, destPool, format, + timeout); } - public Ternary, String> prepareStorageClient(StoragePoolType type, String uuid, Map details) { + public Ternary, String> prepareStorageClient(StoragePoolType type, String uuid, + Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); return adaptor.prepareStorageClient(uuid, details); } - public Pair unprepareStorageClient(StoragePoolType type, String uuid, Map details) { + public Pair unprepareStorageClient(StoragePoolType type, String uuid, + Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); return adaptor.unprepareStorageClient(uuid, details); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index a03daeb197bf..e2dfe754cac2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -94,11 +94,13 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private static final int RBD_FEATURE_OBJECT_MAP = 8; private static final int RBD_FEATURE_FAST_DIFF = 16; private static final int RBD_FEATURE_DEEP_FLATTEN = 32; - public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; + public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */ - private static final Set poolTypesThatEnableCreateDiskFromTemplateBacking = new HashSet<>(Arrays.asList(StoragePoolType.NetworkFilesystem, - StoragePoolType.Filesystem)); + private static final Set poolTypesThatEnableCreateDiskFromTemplateBacking = new HashSet<>( + Arrays.asList(StoragePoolType.NetworkFilesystem, + StoragePoolType.Filesystem)); public LibvirtStorageAdaptor(StorageLayer storage) { _storageLayer = storage; @@ -115,8 +117,10 @@ public boolean createFolder(String uuid, String path, String localPath) { String mountPoint = _mountPoint + File.separator + uuid; if (localPath != null) { - logger.debug(String.format("Pool [%s] is of type local or shared mount point; therefore, we will use the local path [%s] to create the folder [%s] (if it does not" - + " exist).", uuid, localPath, path)); + logger.debug(String.format( + "Pool [%s] is of type local or shared mount point; therefore, we will use the local path [%s] to create the folder [%s] (if it does not" + + " exist).", + uuid, localPath, path)); mountPoint = localPath; } @@ -129,20 +133,25 @@ public boolean createFolder(String uuid, String path, String localPath) { } @Override - public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, - KVMStoragePool destPool, int timeout, byte[] passphrase) { - String volumeDesc = String.format("volume [%s], with template backing [%s], in pool [%s] (%s), with size [%s] and encryption is %s", name, template.getName(), destPool.getUuid(), - destPool.getType(), size, passphrase != null && passphrase.length > 0); + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, + PhysicalDiskFormat format, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { + String volumeDesc = String.format( + "volume [%s], with template backing [%s], in pool [%s] (%s), with size [%s] and encryption is %s", name, + template.getName(), destPool.getUuid(), + destPool.getType(), size, passphrase != null && passphrase.length > 0); if (!poolTypesThatEnableCreateDiskFromTemplateBacking.contains(destPool.getType())) { - logger.info(String.format("Skipping creation of %s due to pool type is none of the following types %s.", volumeDesc, poolTypesThatEnableCreateDiskFromTemplateBacking.stream() - .map(type -> type.toString()).collect(Collectors.joining(", ")))); + logger.info(String.format("Skipping creation of %s due to pool type is none of the following types %s.", + volumeDesc, poolTypesThatEnableCreateDiskFromTemplateBacking.stream() + .map(type -> type.toString()).collect(Collectors.joining(", ")))); return null; } if (format != PhysicalDiskFormat.QCOW2) { - logger.info(String.format("Skipping creation of %s due to format [%s] is not [%s].", volumeDesc, format, PhysicalDiskFormat.QCOW2)); + logger.info(String.format("Skipping creation of %s due to format [%s] is not [%s].", volumeDesc, format, + PhysicalDiskFormat.QCOW2)); return null; } @@ -159,15 +168,18 @@ public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, S QemuImgFile backingFile = new QemuImgFile(template.getPath(), template.getFormat()); if (keyFile.isSet()) { - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, + keyFile.toString(), "sec0", options)); } logger.debug(String.format("Passphrase is staged to keyFile: %s", keyFile.isSet())); QemuImg qemu = new QemuImg(timeout); qemu.create(destFile, backingFile, options, passphraseObjects); } catch (QemuImgException | LibvirtException | IOException e) { - // why don't we throw an exception here? I guess we fail to find the volume later and that results in a failure returned? - logger.error(String.format("Failed to create %s in [%s] due to [%s].", volumeDesc, destPath, e.getMessage()), e); + // why don't we throw an exception here? I guess we fail to find the volume + // later and that results in a failure returned? + logger.error( + String.format("Failed to create %s in [%s] due to [%s].", volumeDesc, destPath, e.getMessage()), e); } return null; @@ -176,17 +188,21 @@ public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, S /** * Extract downloaded template into installPath, remove compressed file */ - public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) { - String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile); + public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, + String destinationFile) { + String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(downloadedTemplateFile, + destinationFile); Script.runSimpleBashScript(extractCommand); Script.runSimpleBashScript("rm -f " + downloadedTemplateFile); } @Override - public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, + KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { File sourceFile = new File(templateFilePath); if (!sourceFile.exists()) { - throw new CloudRuntimeException("Direct download template file " + sourceFile + " does not exist on this host"); + throw new CloudRuntimeException( + "Direct download template file " + sourceFile + " does not exist on this host"); } String templateUuid = UUID.randomUUID().toString(); if (Storage.ImageFormat.ISO.equals(format)) { @@ -195,8 +211,9 @@ public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFileP String destinationFile = destPool.getLocalPath() + File.separator + templateUuid; if (destPool.getType() == StoragePoolType.NetworkFilesystem || destPool.getType() == StoragePoolType.Filesystem - || destPool.getType() == StoragePoolType.SharedMountPoint) { - if (!Storage.ImageFormat.ISO.equals(format) && TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { + || destPool.getType() == StoragePoolType.SharedMountPoint) { + if (!Storage.ImageFormat.ISO.equals(format) + && TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { extractDownloadedTemplate(templateFilePath, destPool, destinationFile); } else { Script.runSimpleBashScript("mv " + templateFilePath + " " + destinationFile); @@ -209,14 +226,16 @@ public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFileP return destPool.getPhysicalDisk(templateUuid); } - private void createTemplateOnRBDFromDirectDownloadFile(String srcTemplateFilePath, String templateUuid, KVMStoragePool destPool, int timeout) { + private void createTemplateOnRBDFromDirectDownloadFile(String srcTemplateFilePath, String templateUuid, + KVMStoragePool destPool, int timeout) { try { QemuImg.PhysicalDiskFormat srcFileFormat = QemuImg.PhysicalDiskFormat.QCOW2; QemuImgFile srcFile = new QemuImgFile(srcTemplateFilePath, srcFileFormat); QemuImg qemu = new QemuImg(timeout); Map info = qemu.info(srcFile); Long virtualSize = Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); - KVMPhysicalDisk destDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + templateUuid, templateUuid, destPool); + KVMPhysicalDisk destDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + templateUuid, templateUuid, + destPool); destDisk.setFormat(PhysicalDiskFormat.RAW); destDisk.setSize(virtualSize); destDisk.setVirtualSize(virtualSize); @@ -224,7 +243,8 @@ private void createTemplateOnRBDFromDirectDownloadFile(String srcTemplateFilePat destFile.setFormat(PhysicalDiskFormat.RAW); qemu.convert(srcFile, destFile); } catch (LibvirtException | QemuImgException e) { - String err = String.format("Error creating template from direct download file on pool %s: %s", destPool.getUuid(), e.getMessage()); + String err = String.format("Error creating template from direct download file on pool %s: %s", + destPool.getUuid(), e.getMessage()); logger.error(err, e); throw new CloudRuntimeException(err, e); } @@ -254,7 +274,8 @@ public StorageVol getVolume(StoragePool pool, String volName) { try { vol = pool.storageVolLookupByName(volName); - logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + " after refreshing the pool"); + logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + + " after refreshing the pool"); } catch (LibvirtException e) { throw new CloudRuntimeException("Could not find volume " + volName + ": " + e.getMessage()); } @@ -263,8 +284,10 @@ public StorageVol getVolume(StoragePool pool, String volName) { return vol; } - public StorageVol createVolume(Connect conn, StoragePool pool, String uuid, long size, VolumeFormat format) throws LibvirtException { - LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(UUID.randomUUID().toString(), size, format, null, null); + public StorageVol createVolume(Connect conn, StoragePool pool, String uuid, long size, VolumeFormat format) + throws LibvirtException { + LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(UUID.randomUUID().toString(), size, format, null, + null); logger.debug(volDef.toString()); return pool.storageVolCreateXML(volDef.toString(), 0); @@ -290,7 +313,8 @@ private void checkNetfsStoragePoolMounted(String uuid) { } } - private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path, List nfsMountOpts) throws LibvirtException { + private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path, + List nfsMountOpts) throws LibvirtException { String targetPath = _mountPoint + File.separator + uuid; LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath, nfsMountOpts); _storageLayer.mkdir(targetPath); @@ -300,7 +324,7 @@ private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String // check whether the pool is already mounted int mountpointResult = Script.runSimpleBashScriptForExitValue("mountpoint -q " + targetPath); // if the pool is mounted, try to unmount it - if(mountpointResult == 0) { + if (mountpointResult == 0) { logger.info("Attempting to unmount old mount at " + targetPath); String result = Script.runSimpleBashScript("umount -l " + targetPath); if (result == null) { @@ -355,7 +379,8 @@ private StoragePool createCLVMStoragePool(Connect conn, String uuid, String host String volgroupName = path; volgroupName = volgroupName.replaceFirst("/", ""); - LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.LOGICAL, volgroupName, uuid, host, volgroupPath, volgroupPath); + LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.LOGICAL, volgroupName, uuid, host, volgroupPath, + volgroupPath); StoragePool sp = null; try { logger.debug(spd.toString()); @@ -417,7 +442,8 @@ private boolean destroyStoragePoolOnNFSMountOptionsChange(StoragePool sp, Connec return false; } - private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, int port, String userInfo, String path) { + private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, int port, String userInfo, + String path) { LibvirtStoragePoolDef spd; StoragePool sp = null; @@ -445,7 +471,8 @@ private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, } return null; } - spd = new LibvirtStoragePoolDef(PoolType.RBD, uuid, uuid, host, port, path, userInfoTemp[0], AuthenticationType.CEPH, uuid); + spd = new LibvirtStoragePoolDef(PoolType.RBD, uuid, uuid, host, port, path, userInfoTemp[0], + AuthenticationType.CEPH, uuid); } else { spd = new LibvirtStoragePoolDef(PoolType.RBD, uuid, uuid, host, port, path, ""); } @@ -484,7 +511,8 @@ private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, } } - public StorageVol copyVolume(StoragePool destPool, LibvirtStorageVolumeDef destVol, StorageVol srcVol, int timeout) throws LibvirtException { + public StorageVol copyVolume(StoragePool destPool, LibvirtStorageVolumeDef destVol, StorageVol srcVol, int timeout) + throws LibvirtException { StorageVol vol = destPool.storageVolCreateXML(destVol.toString(), 0); String srcPath = srcVol.getKey(); String destPath = vol.getKey(); @@ -492,12 +520,14 @@ public StorageVol copyVolume(StoragePool destPool, LibvirtStorageVolumeDef destV return vol; } - public boolean copyVolume(String srcPath, String destPath, String volumeName, int timeout) throws InternalErrorException { + public boolean copyVolume(String srcPath, String destPath, String volumeName, int timeout) + throws InternalErrorException { _storageLayer.mkdirs(destPath); if (!_storageLayer.exists(srcPath)) { throw new InternalErrorException("volume:" + srcPath + " is not exits"); } - String result = Script.runSimpleBashScript("cp " + srcPath + " " + destPath + File.separator + volumeName, timeout); + String result = Script.runSimpleBashScript("cp " + srcPath + " " + destPath + File.separator + volumeName, + timeout); return result == null; } @@ -516,7 +546,7 @@ public LibvirtStorageVolumeDef getStorageVolumeDef(Connect conn, StorageVol vol) @Override public StoragePoolType getStoragePoolType() { // This is mapped manually in KVMStoragePoolManager - return null; + return null; } @Override @@ -532,30 +562,28 @@ protected void updateLocalPoolIops(LibvirtStoragePool pool) { // Run script to get data List commands = new ArrayList<>(); - commands.add(new String[]{ + commands.add(new String[] { Script.getExecutableAbsolutePath("bash"), "-c", String.format( "%s %s | %s 'NR==2 {print $1}'", Script.getExecutableAbsolutePath("df"), pool.getLocalPath(), - Script.getExecutableAbsolutePath("awk") - ) + Script.getExecutableAbsolutePath("awk")) }); String result = Script.executePipedCommands(commands, 1000).second(); if (StringUtils.isBlank(result)) { return; } result = result.trim(); - commands.add(new String[]{ + commands.add(new String[] { Script.getExecutableAbsolutePath("bash"), "-c", String.format( "%s -z %s 1 2 | %s 'NR==7 {print $2}'", Script.getExecutableAbsolutePath("iostat"), result, - Script.getExecutableAbsolutePath("awk") - ) + Script.getExecutableAbsolutePath("awk")) }); result = Script.executePipedCommands(commands, 10000).second(); logger.trace("Pool used IOPS result: {}", result); @@ -618,7 +646,8 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { String authUsername = spd.getAuthUserName(); if (authUsername != null) { Secret secret = conn.secretLookupByUUIDString(spd.getSecretUUID()); - String secretValue = new String(Base64.encodeBase64(secret.getByteValue()), Charset.defaultCharset()); + String secretValue = new String(Base64.encodeBase64(secret.getByteValue()), + Charset.defaultCharset()); pool.setAuthUsername(authUsername); pool.setAuthSecret(secretValue); } @@ -628,12 +657,15 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { * On large (RBD) storage pools it can take up to a couple of minutes * for libvirt to refresh the pool. * - * Refreshing a storage pool means that libvirt will have to iterate the whole pool + * Refreshing a storage pool means that libvirt will have to iterate the whole + * pool * and fetch information of each volume in there * - * It is not always required to refresh a pool. So we can control if we want to or not + * It is not always required to refresh a pool. So we can control if we want to + * or not * - * By default only the getStorageStats call in the LibvirtComputingResource will ask to + * By default only the getStorageStats call in the LibvirtComputingResource will + * ask to * refresh the pool */ if (refreshInfo) { @@ -646,9 +678,9 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { pool.setAvailable(storage.getInfo().available); logger.debug("Successfully refreshed pool " + uuid + - " Capacity: " + toHumanReadableSize(storage.getInfo().capacity) + - " Used: " + toHumanReadableSize(storage.getInfo().allocation) + - " Available: " + toHumanReadableSize(storage.getInfo().available)); + " Capacity: " + toHumanReadableSize(storage.getInfo().capacity) + + " Used: " + toHumanReadableSize(storage.getInfo().allocation) + + " Available: " + toHumanReadableSize(storage.getInfo().available)); return pool; } catch (LibvirtException e) { @@ -659,7 +691,7 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { @Override public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; try { StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid); @@ -718,15 +750,20 @@ private int adjustStoragePoolRefCount(String uuid, int adjustment) { return refCount; } } + /** * Thread-safe increment storage pool usage refcount + * * @param uuid UUID of the storage pool to increment the count */ private void incStoragePoolRefCount(String uuid) { adjustStoragePoolRefCount(uuid, 1); } + /** - * Thread-safe decrement storage pool usage refcount for the given uuid and return if storage pool still in use. + * Thread-safe decrement storage pool usage refcount for the given uuid and + * return if storage pool still in use. + * * @param uuid UUID of the storage pool to decrement the count * @return true if the storage pool is still used, else false. */ @@ -735,7 +772,8 @@ private boolean decStoragePoolRefCount(String uuid) { } @Override - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean isPrimaryStorage) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, + StoragePoolType type, Map details, boolean isPrimaryStorage) { logger.info("Attempting to create storage pool {} ({}) in libvirt", name, type); StoragePool sp; Connect conn; @@ -771,7 +809,8 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri // if anyone is, undefine the pool so we can define it as requested. // This should be safe since a pool in use can't be removed, and no // volumes are affected by unregistering the pool with libvirt. - logger.info("Didn't find an existing storage pool " + name + " by UUID, checking for pools with duplicate paths"); + logger.info("Didn't find an existing storage pool " + name + + " by UUID, checking for pools with duplicate paths"); try { String[] poolnames = conn.listStoragePools(); @@ -780,7 +819,8 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri StoragePool p = conn.storagePoolLookupByName(poolname); LibvirtStoragePoolDef pdef = getStoragePoolDef(conn, p); if (pdef == null) { - throw new CloudRuntimeException("Unable to parse the storage pool definition for storage pool " + poolname); + throw new CloudRuntimeException( + "Unable to parse the storage pool definition for storage pool " + poolname); } String targetPath = pdef.getTargetPath(); @@ -796,13 +836,15 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri } } } catch (LibvirtException e) { - logger.error("Failure in attempting to see if an existing storage pool might be using the path of the pool to be created:" + e); + logger.error( + "Failure in attempting to see if an existing storage pool might be using the path of the pool to be created:" + + e); } } List nfsMountOpts = getNFSMountOptsFromDetails(type, details); if (sp != null && CollectionUtils.isNotEmpty(nfsMountOpts) && - destroyStoragePoolOnNFSMountOptionsChange(sp, conn, nfsMountOpts)) { + destroyStoragePoolOnNFSMountOptionsChange(sp, conn, nfsMountOpts)) { sp = null; } @@ -814,7 +856,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri try { sp = createNetfsStoragePool(PoolType.NETFS, conn, name, host, path, nfsMountOpts); } catch (LibvirtException e) { - logger.error("Failed to create netfs mount: " + host + ":" + path , e); + logger.error("Failed to create netfs mount: " + host + ":" + path, e); logger.error(e.getStackTrace()); throw new CloudRuntimeException(e.toString()); } @@ -822,7 +864,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri try { sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path, null); } catch (LibvirtException e) { - logger.error("Failed to create glusterfs mount: " + host + ":" + path , e); + logger.error("Failed to create glusterfs mount: " + host + ":" + path, e); logger.error(e.getStackTrace()); throw new CloudRuntimeException(e.toString()); } @@ -841,7 +883,8 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri try { if (!isPrimaryStorage) { - // only ref count storage pools for secondary storage, as primary storage is assumed + // only ref count storage pools for secondary storage, as primary storage is + // assumed // to be always mounted, as long the primary storage isn't fully deleted. incStoragePoolRefCount(name); } @@ -861,7 +904,8 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri String error = e.toString(); if (error.contains("Storage source conflict")) { throw new CloudRuntimeException("A pool matching this location already exists in libvirt, " + - " but has a different UUID/Name. Cannot create new pool without first " + " removing it. Check for inactive pools via 'virsh pool-list --all'. " + + " but has a different UUID/Name. Cannot create new pool without first " + + " removing it. Check for inactive pools via 'virsh pool-list --all'. " + error); } else { throw new CloudRuntimeException(error); @@ -895,8 +939,7 @@ private boolean destroyStoragePool(Connect conn, String uuid) throws LibvirtExce } } - private boolean destroyStoragePoolHandleException(Connect conn, String uuid) - { + private boolean destroyStoragePoolHandleException(Connect conn, String uuid) { try { return destroyStoragePool(conn, uuid); } catch (LibvirtException e) { @@ -905,6 +948,13 @@ private boolean destroyStoragePoolHandleException(Connect conn, String uuid) return false; } + @Override + public boolean deleteStoragePool(String uuid, Map details) { + logger.debug("[deleteStoragePool] details overload called for pool {}, delegating to deleteStoragePool(uuid)", + uuid); + return deleteStoragePool(uuid); + } + @Override public boolean deleteStoragePool(String uuid) { logger.info("Attempting to remove storage pool " + uuid + " from libvirt"); @@ -948,8 +998,10 @@ public boolean deleteStoragePool(String uuid) { // handle ebusy error when pool is quickly destroyed if (e.toString().contains("exit status 16")) { String targetPath = _mountPoint + File.separator + uuid; - logger.error("deleteStoragePool removed pool from libvirt, but libvirt had trouble unmounting the pool. Trying umount location " + targetPath + - " again in a few seconds"); + logger.error( + "deleteStoragePool removed pool from libvirt, but libvirt had trouble unmounting the pool. Trying umount location " + + targetPath + + " again in a few seconds"); String result = Script.runSimpleBashScript("sleep 5 && umount " + targetPath); if (result == null) { logger.info("Succeeded in unmounting " + targetPath); @@ -965,51 +1017,61 @@ public boolean deleteStoragePool(String uuid) { /** * Creates a physical disk depending on the {@link StoragePoolType}: *
    - *
  • - * {@link StoragePoolType#RBD} - *
      - *
    • - * If it is an erasure code pool, utilizes QemuImg to create the physical disk through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} - *
      • - *
    • - * Otherwise, utilize Libvirt to create the physical disk through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} - *
      • - *
    - *
    • - *
  • - * {@link StoragePoolType#NetworkFilesystem} and {@link StoragePoolType#Filesystem} - *
      - *
    • - * If the format is {@link PhysicalDiskFormat#QCOW2} or {@link PhysicalDiskFormat#RAW}, utilizes QemuImg to create the physical disk through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} - *
      • - *
    • - * If the format is {@link PhysicalDiskFormat#DIR} or {@link PhysicalDiskFormat#TAR}, utilize Libvirt to create the physical disk through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} - *
      • - *
    - *
    • - *
  • - * For the rest of the {@link StoragePoolType} types, utilizes the Libvirt method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} - *
    • + *
  • + * {@link StoragePoolType#RBD} + *
      + *
    • + * If it is an erasure code pool, utilizes QemuImg to create the physical disk + * through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} + *
      • + *
    • + * Otherwise, utilize Libvirt to create the physical disk through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} + *
      • + *
    + *
    • + *
  • + * {@link StoragePoolType#NetworkFilesystem} and + * {@link StoragePoolType#Filesystem} + *
      + *
    • + * If the format is {@link PhysicalDiskFormat#QCOW2} or + * {@link PhysicalDiskFormat#RAW}, utilizes QemuImg to create the physical disk + * through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} + *
      • + *
    • + * If the format is {@link PhysicalDiskFormat#DIR} or + * {@link PhysicalDiskFormat#TAR}, utilize Libvirt to create the physical disk + * through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} + *
      • + *
    + *
    • + *
  • + * For the rest of the {@link StoragePoolType} types, utilizes the Libvirt + * method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} + *
    • *
*/ @Override public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { - logger.info("Attempting to create volume {} ({}) in pool {} with size {}", name, pool.getType().toString(), pool.getUuid(), toHumanReadableSize(size)); + logger.info("Attempting to create volume {} ({}) in pool {} with size {}", name, pool.getType().toString(), + pool.getUuid(), toHumanReadableSize(size)); StoragePoolType poolType = pool.getType(); if (StoragePoolType.RBD.equals(poolType)) { Map details = pool.getDetails(); String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); - return (dataPool == null) ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : - createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, passphrase); + return (dataPool == null) + ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) + : createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, + passphrase); } else if (StoragePoolType.NetworkFilesystem.equals(poolType) || StoragePoolType.Filesystem.equals(poolType)) { switch (format) { case QCOW2: @@ -1057,9 +1119,9 @@ private KVMPhysicalDisk createPhysicalDiskByLibVirt(String name, KVMStoragePool return disk; } - - private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, - byte[] passphrase) { + private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool, PhysicalDiskFormat format, + Storage.ProvisioningType provisioningType, long size, + byte[] passphrase) { String volPath; String volName = name; long virtualSize = 0; @@ -1081,13 +1143,15 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool destFile.setSize(size); Map options = new HashMap(); if (List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem).contains(pool.getType())) { - options.put(QemuImg.PREALLOCATION, QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); + options.put(QemuImg.PREALLOCATION, + QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); } try (KeyFile keyFile = new KeyFile(passphrase)) { QemuImg qemu = new QemuImg(timeout); if (keyFile.isSet()) { - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, + keyFile.toString(), "sec0", options)); // make room for encryption header on raw format, use LUKS if (format == PhysicalDiskFormat.RAW) { @@ -1102,7 +1166,8 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool virtualSize = Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); actualSize = new File(destFile.getFileName()).length(); } catch (QemuImgException | LibvirtException | IOException e) { - throw new CloudRuntimeException(String.format("Failed to create %s due to a failed execution of qemu-img", volPath), e); + throw new CloudRuntimeException( + String.format("Failed to create %s due to a failed execution of qemu-img", volPath), e); } KVMPhysicalDisk disk = new KVMPhysicalDisk(volPath, volName, pool); @@ -1114,7 +1179,8 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool } @Override - public boolean connectPhysicalDisk(String name, KVMStoragePool pool, Map details, boolean isVMMigrate) { + public boolean connectPhysicalDisk(String name, KVMStoragePool pool, Map details, + boolean isVMMigrate) { // this is for managed storage that needs to prep disks prior to use return true; } @@ -1177,7 +1243,8 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag */ if (pool.getType() == StoragePoolType.RBD) { try { - logger.info("Unprotecting and Removing RBD snapshots of image " + pool.getSourceDir() + "/" + uuid + " prior to removing the image"); + logger.info("Unprotecting and Removing RBD snapshots of image " + pool.getSourceDir() + "/" + uuid + + " prior to removing the image"); Rados r = new Rados(pool.getAuthUserName()); r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); @@ -1197,17 +1264,20 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag logger.debug("Unprotecting snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); image.snapUnprotect(snap.name); } else { - logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name + " is not protected."); + logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name + + " is not protected."); } logger.debug("Removing snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); image.snapRemove(snap.name); } - logger.info("Successfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of " - + pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image"); + logger.info( + "Successfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of " + + pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image"); } catch (RbdException e) { logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); - throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); + ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); + throw new CloudRuntimeException( + e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); } finally { logger.debug("Closing image and destroying context"); rbd.close(image); @@ -1215,20 +1285,20 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag } } catch (RadosException e) { logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); + ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); } catch (RbdException e) { logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); + ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); } } - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; try { StorageVol vol = getVolume(libvirtPool.getPool(), uuid); logger.debug("Instructing libvirt to remove volume " + uuid + " from pool " + pool.getUuid()); - if(Storage.ImageFormat.DIR.equals(format)){ + if (Storage.ImageFormat.DIR.equals(format)) { deleteDirVol(libvirtPool, vol); } else { deleteVol(libvirtPool, vol); @@ -1241,40 +1311,52 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag } /** - * This function copies a physical disk from Secondary Storage to Primary Storage + * This function copies a physical disk from Secondary Storage to Primary + * Storage * or from Primary to Primary Storage * - * The first time a template is deployed in Primary Storage it will be copied from + * The first time a template is deployed in Primary Storage it will be copied + * from * Secondary to Primary. * - * If it has been created on Primary Storage, it will be copied on the Primary Storage + * If it has been created on Primary Storage, it will be copied on the Primary + * Storage */ @Override public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, - String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { - logger.info("Creating volume " + name + " from template " + template.getName() + " in pool " + destPool.getUuid() + - " (" + destPool.getType().toString() + ") with size " + toHumanReadableSize(size)); + logger.info( + "Creating volume " + name + " from template " + template.getName() + " in pool " + destPool.getUuid() + + " (" + destPool.getType().toString() + ") with size " + toHumanReadableSize(size)); KVMPhysicalDisk disk = null; if (destPool.getType() == StoragePoolType.RBD) { disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout); } else { - try (KeyFile keyFile = new KeyFile(passphrase)){ + try (KeyFile keyFile = new KeyFile(passphrase)) { String newUuid = name; List passphraseObjects = new ArrayList<>(); - disk = destPool.createPhysicalDisk(newUuid, format, provisioningType, template.getVirtualSize(), passphrase); + disk = destPool.createPhysicalDisk(newUuid, format, provisioningType, template.getVirtualSize(), + passphrase); if (disk == null) { throw new CloudRuntimeException("Failed to create disk from template " + template.getName()); } if (template.getFormat() == PhysicalDiskFormat.TAR) { - Script.runSimpleBashScript("tar -x -f " + template.getPath() + " -C " + disk.getPath(), timeout); // TO BE FIXED to aware provisioningType + Script.runSimpleBashScript("tar -x -f " + template.getPath() + " -C " + disk.getPath(), timeout); // TO + // BE + // FIXED + // to + // aware + // provisioningType } else if (template.getFormat() == PhysicalDiskFormat.DIR) { Script.runSimpleBashScript("mkdir -p " + disk.getPath()); Script.runSimpleBashScript("chmod 755 " + disk.getPath()); - Script.runSimpleBashScript("tar -x -f " + template.getPath() + "/*.tar -C " + disk.getPath(), timeout); + Script.runSimpleBashScript("tar -x -f " + template.getPath() + "/*.tar -C " + disk.getPath(), + timeout); } else if (format == PhysicalDiskFormat.QCOW2) { QemuImg qemu = new QemuImg(timeout); QemuImgFile destFile = new QemuImgFile(disk.getPath(), format); @@ -1284,31 +1366,34 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, destFile.setSize(template.getVirtualSize()); } Map options = new HashMap(); - options.put("preallocation", QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); - + options.put("preallocation", + QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); if (keyFile.isSet()) { - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, + keyFile.toString(), "sec0", options)); disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); } QemuImgFile srcFile = new QemuImgFile(template.getPath(), template.getFormat()); - Boolean createFullClone = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.CREATE_FULL_CLONE); - switch(provisioningType){ - case THIN: - logger.info("Creating volume [{}] {} backing file [{}] as the property [{}] is [{}].", destFile.getFileName(), createFullClone ? "without" : "with", - template.getPath(), AgentProperties.CREATE_FULL_CLONE.getName(), createFullClone); - if (createFullClone) { + Boolean createFullClone = AgentPropertiesFileHandler + .getPropertyValue(AgentProperties.CREATE_FULL_CLONE); + switch (provisioningType) { + case THIN: + logger.info("Creating volume [{}] {} backing file [{}] as the property [{}] is [{}].", + destFile.getFileName(), createFullClone ? "without" : "with", + template.getPath(), AgentProperties.CREATE_FULL_CLONE.getName(), createFullClone); + if (createFullClone) { + qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); + } else { + qemu.create(destFile, srcFile, options, passphraseObjects); + } + break; + case SPARSE: + case FAT: + srcFile = new QemuImgFile(template.getPath(), template.getFormat()); qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); - } else { - qemu.create(destFile, srcFile, options, passphraseObjects); - } - break; - case SPARSE: - case FAT: - srcFile = new QemuImgFile(template.getPath(), template.getFormat()); - qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); - break; + break; } } else if (format == PhysicalDiskFormat.RAW) { PhysicalDiskFormat destFormat = PhysicalDiskFormat.RAW; @@ -1317,7 +1402,8 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, if (keyFile.isSet()) { destFormat = PhysicalDiskFormat.LUKS; disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(destFormat, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(destFormat, + QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); } QemuImgFile sourceFile = new QemuImgFile(template.getPath(), template.getFormat()); @@ -1331,7 +1417,8 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, qemu.convert(sourceFile, destFile, options, passphraseObjects, null, false); } } catch (QemuImgException | LibvirtException | IOException e) { - throw new CloudRuntimeException(String.format("Failed to create %s due to a failed execution of qemu-img", name), e); + throw new CloudRuntimeException( + String.format("Failed to create %s due to a failed execution of qemu-img", name), e); } } @@ -1339,14 +1426,16 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, } private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, - String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout){ + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, + KVMStoragePool destPool, int timeout) { /* - With RBD you can't run qemu-img convert with an existing RBD image as destination - qemu-img will exit with the error that the destination already exists. - So for RBD we don't create the image, but let qemu-img do that for us. - - We then create a KVMPhysicalDisk object that we can return + * With RBD you can't run qemu-img convert with an existing RBD image as + * destination + * qemu-img will exit with the error that the destination already exists. + * So for RBD we don't create the image, but let qemu-img do that for us. + * + * We then create a KVMPhysicalDisk object that we can return */ KVMStoragePool srcPool = template.getPool(); @@ -1365,14 +1454,13 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, disk.setVirtualSize(disk.getSize()); } - QemuImgFile srcFile; QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, disk.getPath())); destFile.setFormat(format); if (srcPool.getType() != StoragePoolType.RBD) { srcFile = new QemuImgFile(template.getPath(), template.getFormat()); - try{ + try { QemuImg qemu = new QemuImg(timeout); qemu.convert(srcFile, destFile); } catch (QemuImgException | LibvirtException e) { @@ -1390,9 +1478,14 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, */ try { - if ((srcPool.getSourceHost().equals(destPool.getSourceHost())) && (srcPool.getSourceDir().equals(destPool.getSourceDir()))) { - /* We are on the same Ceph cluster, but we require RBD format 2 on the source image */ - logger.debug("Trying to perform a RBD clone (layering) since we are operating in the same storage pool"); + if ((srcPool.getSourceHost().equals(destPool.getSourceHost())) + && (srcPool.getSourceDir().equals(destPool.getSourceDir()))) { + /* + * We are on the same Ceph cluster, but we require RBD format 2 on the source + * image + */ + logger.debug( + "Trying to perform a RBD clone (layering) since we are operating in the same storage pool"); Rados r = new Rados(srcPool.getAuthUserName()); r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); @@ -1408,15 +1501,18 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, if (srcImage.isOldFormat()) { /* The source image is RBD format 1, we have to do a regular copy */ logger.debug("The source image " + srcPool.getSourceDir() + "/" + template.getName() + - " is RBD format 1. We have to perform a regular copy (" + toHumanReadableSize(disk.getVirtualSize()) + " bytes)"); + " is RBD format 1. We have to perform a regular copy (" + + toHumanReadableSize(disk.getVirtualSize()) + " bytes)"); rbd.create(disk.getName(), disk.getVirtualSize(), RBD_FEATURES, rbdOrder); RbdImage destImage = rbd.open(disk.getName()); - logger.debug("Starting to copy " + srcImage.getName() + " to " + destImage.getName() + " in Ceph pool " + srcPool.getSourceDir()); + logger.debug("Starting to copy " + srcImage.getName() + " to " + destImage.getName() + + " in Ceph pool " + srcPool.getSourceDir()); rbd.copy(srcImage, destImage); - logger.debug("Finished copying " + srcImage.getName() + " to " + destImage.getName() + " in Ceph pool " + srcPool.getSourceDir()); + logger.debug("Finished copying " + srcImage.getName() + " to " + destImage.getName() + + " in Ceph pool " + srcPool.getSourceDir()); rbd.close(destImage); } else { logger.debug("The source image " + srcPool.getSourceDir() + "/" + template.getName() @@ -1424,12 +1520,12 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, + rbdTemplateSnapName); /* The source image is format 2, we can do a RBD snapshot+clone (layering) */ - logger.debug("Checking if RBD snapshot " + srcPool.getSourceDir() + "/" + template.getName() + "@" + rbdTemplateSnapName + " exists prior to attempting a clone operation."); List snaps = srcImage.snapList(); - logger.debug("Found " + snaps.size() + " snapshots on RBD image " + srcPool.getSourceDir() + "/" + template.getName()); + logger.debug("Found " + snaps.size() + " snapshots on RBD image " + srcPool.getSourceDir() + "/" + + template.getName()); boolean snapFound = false; for (RbdSnapInfo snap : snaps) { if (rbdTemplateSnapName.equals(snap.name)) { @@ -1448,13 +1544,18 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } rbd.clone(template.getName(), rbdTemplateSnapName, io, disk.getName(), RBD_FEATURES, rbdOrder); - logger.debug("Successfully cloned " + template.getName() + "@" + rbdTemplateSnapName + " to " + disk.getName()); - /* We also need to resize the image if the VM was deployed with a larger root disk size */ + logger.debug("Successfully cloned " + template.getName() + "@" + rbdTemplateSnapName + " to " + + disk.getName()); + /* + * We also need to resize the image if the VM was deployed with a larger root + * disk size + */ if (disk.getVirtualSize() > template.getVirtualSize()) { RbdImage diskImage = rbd.open(disk.getName()); diskImage.resize(disk.getVirtualSize()); rbd.close(diskImage); - logger.debug("Resized " + disk.getName() + " to " + toHumanReadableSize(disk.getVirtualSize())); + logger.debug( + "Resized " + disk.getName() + " to " + toHumanReadableSize(disk.getVirtualSize())); } } @@ -1462,8 +1563,12 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, rbd.close(srcImage); r.ioCtxDestroy(io); } else { - /* The source pool or host is not the same Ceph cluster, we do a simple copy with Qemu-Img */ - logger.debug("Both the source and destination are RBD, but not the same Ceph cluster. Performing a copy"); + /* + * The source pool or host is not the same Ceph cluster, we do a simple copy + * with Qemu-Img + */ + logger.debug( + "Both the source and destination are RBD, but not the same Ceph cluster. Performing a copy"); Rados rSrc = new Rados(srcPool.getAuthUserName()); rSrc.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); @@ -1485,14 +1590,16 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, IoCTX dIO = rDest.ioCtxCreate(destPool.getSourceDir()); Rbd dRbd = new Rbd(dIO); - logger.debug("Creating " + disk.getName() + " on the destination cluster " + rDest.confGet("mon_host") + " in pool " + + logger.debug("Creating " + disk.getName() + " on the destination cluster " + + rDest.confGet("mon_host") + " in pool " + destPool.getSourceDir()); dRbd.create(disk.getName(), disk.getVirtualSize(), RBD_FEATURES, rbdOrder); RbdImage srcImage = sRbd.open(template.getName()); RbdImage destImage = dRbd.open(disk.getName()); - logger.debug("Copying " + template.getName() + " from Ceph cluster " + rSrc.confGet("mon_host") + " to " + disk.getName() + logger.debug("Copying " + template.getName() + " from Ceph cluster " + rSrc.confGet("mon_host") + + " to " + disk.getName() + " on cluster " + rDest.confGet("mon_host")); sRbd.copy(srcImage, destImage); @@ -1514,13 +1621,14 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } @Override - public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, + long size, KVMStoragePool destPool) { return null; } @Override public List listPhysicalDisks(String storagePoolUuid, KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; StoragePool virtPool = libvirtPool.getPool(); List disks = new ArrayList(); try { @@ -1543,35 +1651,44 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt /** * This copies a volume from Primary Storage to Secondary Storage * - * In theory it could also do it the other way around, but the current implementation - * in ManagementServerImpl shows that the destPool is always a Secondary Storage Pool + * In theory it could also do it the other way around, but the current + * implementation + * in ManagementServerImpl shows that the destPool is always a Secondary Storage + * Pool */ @Override - public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, + byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { /** - With RBD you can't run qemu-img convert with an existing RBD image as destination - qemu-img will exit with the error that the destination already exists. - So for RBD we don't create the image, but let qemu-img do that for us. - - We then create a KVMPhysicalDisk object that we can return - - It is however very unlikely that the destPool will be RBD, since it isn't supported - for Secondary Storage + * With RBD you can't run qemu-img convert with an existing RBD image as + * destination + * qemu-img will exit with the error that the destination already exists. + * So for RBD we don't create the image, but let qemu-img do that for us. + * + * We then create a KVMPhysicalDisk object that we can return + * + * It is however very unlikely that the destPool will be RBD, since it isn't + * supported + * for Secondary Storage */ KVMStoragePool srcPool = disk.getPool(); - /* Linstor images are always stored as RAW, but Linstor uses qcow2 in DB, - to support snapshots(backuped) as qcow2 files. */ - PhysicalDiskFormat sourceFormat = srcPool.getType() != StoragePoolType.Linstor ? - disk.getFormat() : PhysicalDiskFormat.RAW; + /* + * Linstor images are always stored as RAW, but Linstor uses qcow2 in DB, + * to support snapshots(backuped) as qcow2 files. + */ + PhysicalDiskFormat sourceFormat = srcPool.getType() != StoragePoolType.Linstor ? disk.getFormat() + : PhysicalDiskFormat.RAW; String sourcePath = disk.getPath(); KVMPhysicalDisk newDisk; - logger.debug("copyPhysicalDisk: disk size:{}, virtualsize:{} format:{}", toHumanReadableSize(disk.getSize()), toHumanReadableSize(disk.getVirtualSize()), disk.getFormat()); + logger.debug("copyPhysicalDisk: disk size:{}, virtualsize:{} format:{}", toHumanReadableSize(disk.getSize()), + toHumanReadableSize(disk.getVirtualSize()), disk.getFormat()); if (destPool.getType() != StoragePoolType.RBD) { if (disk.getFormat() == PhysicalDiskFormat.TAR) { - newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, disk.getVirtualSize(), null); + newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, + disk.getVirtualSize(), null); } else { newDisk = destPool.createPhysicalDisk(name, Storage.ProvisioningType.THIN, disk.getVirtualSize(), null); } @@ -1589,15 +1706,15 @@ to support snapshots(backuped) as qcow2 files. */ try { qemu = new QemuImg(timeout); - } catch (QemuImgException | LibvirtException ex ) { + } catch (QemuImgException | LibvirtException ex) { throw new CloudRuntimeException("Failed to create qemu-img command", ex); } QemuImgFile srcFile = null; QemuImgFile destFile = null; if ((srcPool.getType() != StoragePoolType.RBD) && (destPool.getType() != StoragePoolType.RBD)) { - if(sourceFormat == PhysicalDiskFormat.TAR && destFormat == PhysicalDiskFormat.DIR) { //LXC template - Script.runSimpleBashScript("cp "+ sourcePath + " " + destPath); + if (sourceFormat == PhysicalDiskFormat.TAR && destFormat == PhysicalDiskFormat.DIR) { // LXC template + Script.runSimpleBashScript("cp " + sourcePath + " " + destPath); } else if (sourceFormat == PhysicalDiskFormat.TAR) { Script.runSimpleBashScript("tar -x -f " + sourcePath + " -C " + destPath, timeout); } else if (sourceFormat == PhysicalDiskFormat.DIR) { @@ -1619,26 +1736,30 @@ to support snapshots(backuped) as qcow2 files. */ destFile = new QemuImgFile(destPath, destFormat); try { boolean isQCOW2 = PhysicalDiskFormat.QCOW2.equals(sourceFormat); - qemu.convert(srcFile, destFile, null, null, new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), + qemu.convert(srcFile, destFile, null, null, + new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), null, false, isQCOW2); Map destInfo = qemu.info(destFile); Long virtualSize = Long.parseLong(destInfo.get(QemuImg.VIRTUAL_SIZE)); newDisk.setVirtualSize(virtualSize); newDisk.setSize(virtualSize); } catch (QemuImgException e) { - logger.error("Failed to convert [{}] to [{}] due to: [{}].", srcFile.getFileName(), destFile.getFileName(), e.getMessage(), e); + logger.error("Failed to convert [{}] to [{}] due to: [{}].", srcFile.getFileName(), + destFile.getFileName(), e.getMessage(), e); newDisk = null; } } } catch (QemuImgException e) { - logger.error("Failed to fetch the information of file " + srcFile.getFileName() + " the error was: " + e.getMessage()); + logger.error("Failed to fetch the information of file " + srcFile.getFileName() + " the error was: " + + e.getMessage()); newDisk = null; } } } else if ((srcPool.getType() != StoragePoolType.RBD) && (destPool.getType() == StoragePoolType.RBD)) { /** * Using qemu-img we copy the QCOW2 disk to RAW (on RBD) directly. - * To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph Dumpling) + * To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph + * Dumpling) */ logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2"); try { @@ -1647,9 +1768,11 @@ to support snapshots(backuped) as qcow2 files. */ String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath); destFile = new QemuImgFile(rbdDestFile, destFormat); - logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); + logger.debug( + "Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); qemu.convert(srcFile, destFile); - logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); + logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " + + rbdDestPath); /* We have to stat the RBD image to see how big it became afterwards */ Rados r = new Rados(destPool.getAuthUserName()); @@ -1666,26 +1789,32 @@ to support snapshots(backuped) as qcow2 files. */ RbdImageInfo rbdInfo = image.stat(); newDisk.setSize(rbdInfo.size); newDisk.setVirtualSize(rbdInfo.size); - logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + toHumanReadableSize(rbdInfo.size) + " bytes long"); + logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + + toHumanReadableSize(rbdInfo.size) + " bytes long"); rbd.close(image); r.ioCtxDestroy(io); } catch (QemuImgException | LibvirtException e) { String srcFilename = srcFile != null ? srcFile.getFileName() : null; String destFilename = destFile != null ? destFile.getFileName() : null; - logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, destFilename, e.getMessage())); + logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, + destFilename, e.getMessage())); newDisk = null; } catch (RadosException e) { - logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); + logger.error( + "A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; } catch (RbdException e) { - logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); + logger.error( + "A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; } } else { /** - We let Qemu-Img do the work here. Although we could work with librbd and have that do the cloning - it doesn't benefit us. It's better to keep the current code in place which works + * We let Qemu-Img do the work here. Although we could work with librbd and have + * that do the cloning + * it doesn't benefit us. It's better to keep the current code in place which + * works */ srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, sourcePath)); srcFile.setFormat(sourceFormat); @@ -1699,7 +1828,8 @@ to support snapshots(backuped) as qcow2 files. */ try { qemu.convert(srcFile, destFile); } catch (QemuImgException | LibvirtException e) { - logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage()); + logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() + + " the error was: " + e.getMessage()); newDisk = null; } } @@ -1713,7 +1843,7 @@ to support snapshots(backuped) as qcow2 files. */ @Override public boolean refresh(KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; StoragePool virtPool = libvirtPool.getPool(); try { refreshPool(virtPool); diff --git a/test/integration/plugins/ontap/__init__.py b/test/integration/plugins/ontap/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/__init__.py b/test/integration/plugins/ontap/iscsi/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/instance/__init__.py b/test/integration/plugins/ontap/iscsi/instance/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/instance/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py b/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py new file mode 100644 index 000000000000..1dd55049f76e --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py @@ -0,0 +1,826 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI data volume +lifecycle with a running virtual machine. + +Covers TDS (section 10) iSCSI VM volume scenarios: + + TDS Approach-1 SN 27 — Create CS Volume and allocate it to an Instance (iSCSI) + (attach data volume to running VM; verify LUN-map) + TDS VM Stop (iSCSI) — Stop running VM; verify LUN-maps are removed + TDS VM Start (iSCSI) — Start stopped VM; verify LUN-maps are re-created + TDS Detach (iSCSI) — Detach data volume; verify LUN-map removed + +Key iSCSI behaviour verified at each step via ONTAP REST API: + - createVolume → LUN is created inside the pool's FlexVol + - attachVolume → LUN-map is created linking the LUN to the host's igroup + - stopVirtualMachine → LUN-map is removed (LUN stays; just unmapped) + - startVirtualMachine → LUN-map is re-created + - detachVolume → LUN-map is removed + +Tests are numbered test_01 ... test_08 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create iSCSI primary storage pool on ONTAP + 02 Create a CloudStack data volume on the iSCSI pool (LUN on ONTAP) + 03 Deploy a VM using any available KVM template + 04 Attach iSCSI data volume to running VM (LUN-map created) + 05 Stop VM — LUN-map for attached volume is removed from ONTAP + 06 Start VM — LUN-map is re-created on ONTAP + 07 Detach data volume from running VM — LUN-map removed + 08 Destroy VM, delete data volume, delete pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI initiator configured (iqn.* IQN) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + - At least one KVM template must be fully downloaded and ready (isready=True) + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_vm_volume_attach_iscsi.py -v +""" + +import base64 +import logging +import random +import re +import time +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, + createNetwork as createNetworkAPI, + createStoragePool as createStoragePoolAPI, + deleteNetwork as deleteNetworkAPI, + deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + detachVolume as detachVolumeAPI, + enableStorageMaintenance, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listServiceOfferings as listServiceOfferingsAPI, + listTemplates as listTemplatesAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + startVirtualMachine as startVirtualMachineAPI, + stopVirtualMachine as stopVirtualMachineAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapVMVolumeAttachISCSI") + + +# --------------------------------------------------------------------------- +# Utility functions +# --------------------------------------------------------------------------- + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + cmd.listall = True + return cmd + + +def _wait_for_vm_state(api_client, vm_id, target_state, timeout=300, + interval=10): + """Block until the VM reaches target_state or timeout expires.""" + deadline = time.time() + timeout + while time.time() < deadline: + vms = api_client.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms and vms[0].state.lower() == target_state.lower(): + return vms[0] + time.sleep(interval) + return None + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-vm@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-VM", + "username": "ontap_iscsi_vm_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSIVM_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapVMVolumeAttachISCSI(OntapTestBase): + """ + Tests iSCSI ONTAP data volume lifecycle with a running CloudStack VM. + All tests are sequential — state is carried on class attributes. + """ + + # ---- extra shared state beyond OntapTestBase ----------------------- + vm = None + template_id = None + service_offering_id = None + network_id = None + _created_network_id = None # network created by this suite for Advanced zones + + _vol_name_prefix = "OntapISCSIVM" + + # ---- setup --------------------------------------------------------- + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapVMVolumeAttachISCSI, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Discover a ready user KVM template (exclude SYSTEM type) + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = cls.zone.id + templates = cls.apiClient.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + cls.template_id = kvm_ready[0].id if kvm_ready else None + if cls.template_id is None: + logger.warning( + "No ready KVM user template found — VM tests will be skipped." + ) + + # Smallest service offering + so_cmd = listServiceOfferingsAPI.listServiceOfferingsCmd() + offerings = cls.apiClient.listServiceOfferings(so_cmd) or [] + assert offerings, "No service offerings available in CloudStack" + offerings.sort(key=lambda s: getattr(s, "memory", 9999)) + cls.service_offering_id = offerings[0].id + + # Network ID for VM deployment + cls.network_id = None + zone_type = getattr(cls.zone, "networktype", "Basic") + if zone_type.lower() == "advanced": + # Find a network already accessible to the test account + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + else: + # Create an Isolated guest network for the test account + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.specifyvlan = "false" + offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + snat_offering = next( + (o for o in offerings + if "SourceNat" in o.name and "Vpc" not in o.name + and "NSX" not in o.name and "Netris" not in o.name), + offerings[0] if offerings else None + ) + if snat_offering: + cn_cmd = createNetworkAPI.createNetworkCmd() + cn_cmd.zoneid = cls.zone.id + cn_cmd.networkofferingid = snat_offering.id + cn_cmd.name = "ontap-iscsi-vm-net-%d" % random.randint( + 0, 9999) + cn_cmd.displaytext = "ONTAP iSCSI VM test network" + cn_cmd.account = cls.account.name + cn_cmd.domainid = cls.domain.id + net = cls.apiClient.createNetwork(cn_cmd) + cls.network_id = net.id + cls._created_network_id = net.id + + @classmethod + def tearDownClass(cls): + """ + Safety-net cleanup: destroy VM if still alive, delete the guest + network created for Advanced zones (if not already deleted by test_08), + then delegate pool/volume/account cleanup to the base class. + """ + if cls.vm is not None: + try: + vms = cls.apiClient.listVirtualMachines( + _list_vms_cmd(cls.vm.id)) + state = vms[0].state if vms else "unknown" + if state.lower() not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = cls.vm.id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, cls.vm.id, + "Stopped", timeout=120) + except Exception as e: + logger.warning("tearDownClass: could not stop VM %s: %s" + % (cls.vm.id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = cls.vm.id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as e: + logger.warning("tearDownClass: could not destroy VM %s: %s" + % (cls.vm.id, e)) + + # Delete the guest network created for this account in Advanced zones. + # test_08 deletes it on the happy path; this is the fallback for + # mid-suite failures. + if cls._created_network_id is not None: + try: + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn_cmd) + cls._created_network_id = None + except Exception as e: + logger.warning( + "tearDownClass: could not delete network %s: %s" + % (cls._created_network_id, e)) + + super(TestOntapVMVolumeAttachISCSI, cls).tearDownClass() + + # ---- helpers ------------------------------------------------------- + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSIVM_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _poll_vm_state(self, vm_id, target_state, timeout=300, interval=10): + deadline = time.time() + timeout + current_state = "unknown" + while time.time() < deadline: + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms: + current_state = vms[0].state + if current_state.lower() == target_state.lower(): + return vms[0] + time.sleep(interval) + self.fail("VM %s did not reach '%s' within %ds (last: '%s')" + % (vm_id, target_state, timeout, current_state)) + + def _poll_volume_field(self, vol_id, field, target, timeout=120, + interval=5): + """Poll a volume field until it matches target; return the volume.""" + deadline = time.time() + timeout + while time.time() < deadline: + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) + if vols: + val = getattr(vols[0], field, None) + if val == target: + return vols[0] + time.sleep(interval) + return None + + def _lun_maps(self): + """Return current LUN-maps for the pool's FlexVol.""" + if self.__class__.pool is None: + return [] + return self.ontap.list_lun_maps_for_volume( + self.svm_name, self.__class__.pool.name) + + # ================================================================== + # Test steps + # ================================================================== + + # ------------------------------------------------------------------ + # Step 01 — Create iSCSI ONTAP pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_01_create_iscsi_pool(self): + """ + Create an iSCSI primary storage pool on ONTAP. + Verifies: + - Pool reaches 'Up' state; type is 'Iscsi' + - ONTAP: FlexVol is online + - ONTAP: igroup exists for every host in the cluster that has an IQN + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual(pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state) + self.assertEqual(pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name) + self.assertEqual(ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online'") + + # ------------------------------------------------------------------ + # Step 02 — Create iSCSI data volume (LUN on ONTAP) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_02_create_ontap_data_volume(self): + """ + Allocate a CloudStack data volume on the iSCSI ONTAP pool. + Verifies: + - createVolume returns a volume object + - ONTAP: at least one LUN is created inside the pool's FlexVol + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + vol = self._create_volume(self.__class__.pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + luns = self.ontap.list_luns_in_volume( + self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "Expected ≥1 LUN in ONTAP FlexVol '%s' after volume creation, " + "found 0" % self.__class__.pool.name + ) + + # ------------------------------------------------------------------ + # Step 03 — Deploy a VM + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_03_deploy_vm(self): + """ + Deploy a VM using the first available ready KVM template. + Verifies: + - VM reaches Running state + - ONTAP: the iSCSI data volume's LUN is NOT yet mapped (no VM + attachment has been performed yet) + """ + if self.__class__.template_id is None: + self.skipTest( + "No ready KVM user template available — waiting for template " + "download to complete" + ) + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.__class__.vm = vm + + result = self._poll_vm_state(vm.id, "Running", timeout=300) + self.assertEqual( + result.state, "Running", + "VM should be 'Running' after deploy, got '%s'" % result.state + ) + + # Data volume LUN-map must not exist yet (volume not yet attached) + lun_maps = self._lun_maps() + self.assertEqual( + len(lun_maps), 0, + "Expected 0 LUN-maps before volume attach, found %d: %s" + % (len(lun_maps), lun_maps) + ) + + # ------------------------------------------------------------------ + # Step 04 — Attach iSCSI volume to running VM (TDS SN 27 iSCSI) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_04_attach_volume_to_vm(self): + """ + Attach the iSCSI data volume to the running VM. + Covers TDS Approach-1 SN 27 (iSCSI): + - attachVolume completes successfully + - CloudStack: volume shows virtualmachineid set + - ONTAP: a LUN-map is created linking the data LUN to the host's + igroup (the LUN is now accessible to the VM's KVM host) + """ + if self.__class__.vm is None: + self.skipTest( + "VM not deployed — test_03 was skipped (no ready template)" + ) + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + cmd = attachVolumeAPI.attachVolumeCmd() + cmd.id = self.__class__.volume.id + cmd.virtualmachineid = self.__class__.vm.id + self.apiClient.attachVolume(cmd) + + # Poll until virtualmachineid is set on the volume + result = self._poll_volume_field( + self.__class__.volume.id, "virtualmachineid", + self.__class__.vm.id, timeout=120) + self.assertIsNotNone( + result, + "Volume virtualmachineid was not set after attachVolume" + ) + + # ONTAP: at least one LUN-map must exist for the pool's FlexVol + lun_maps = self._lun_maps() + self.assertGreater( + len(lun_maps), 0, + "Expected ≥1 LUN-map after volume attach, found 0 — " + "LUN is not accessible to the VM's host" + ) + + # ------------------------------------------------------------------ + # Step 05 — Stop VM — LUN-maps should be removed + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_05_stop_vm_lun_unmapped(self): + """ + Stop the running VM while the iSCSI data volume is still attached. + Covers TDS VM Stop (iSCSI): 'Luns for the volumes under this VM + should be unmapped.' + Verifies: + - VM reaches Stopped state + - ONTAP: LUN-map is removed (LUN itself stays in the FlexVol) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + + cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + cmd.id = self.__class__.vm.id + self.apiClient.stopVirtualMachine(cmd) + + result = self._poll_vm_state(self.__class__.vm.id, "Stopped", + timeout=300) + self.assertEqual( + result.state, "Stopped", + "VM should be 'Stopped', got '%s'" % result.state + ) + + # ONTAP: LUN-map must be removed once VM is stopped + lun_maps = self._lun_maps() + self.assertEqual( + len(lun_maps), 0, + "Expected 0 LUN-maps after VM stop, found %d: %s" + % (len(lun_maps), lun_maps) + ) + + # ONTAP: LUN itself must still exist in the FlexVol + luns = self.ontap.list_luns_in_volume( + self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "LUN should still exist in ONTAP FlexVol after VM stop" + ) + + # ------------------------------------------------------------------ + # Step 06 — Start VM — LUN-maps should be re-created + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_06_start_vm_lun_remapped(self): + """ + Start the stopped VM. + Covers TDS VM Start (iSCSI): 'luns should be re-mapped again to + provide access.' + Verifies: + - VM reaches Running state + - ONTAP: LUN-map is re-created (LUN accessible to VM's host) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + + cmd = startVirtualMachineAPI.startVirtualMachineCmd() + cmd.id = self.__class__.vm.id + self.apiClient.startVirtualMachine(cmd) + + result = self._poll_vm_state(self.__class__.vm.id, "Running", + timeout=300) + self.assertEqual( + result.state, "Running", + "VM should be 'Running' after start, got '%s'" % result.state + ) + + # ONTAP: LUN-map must be re-created after VM starts + lun_maps = self._lun_maps() + self.assertGreater( + len(lun_maps), 0, + "Expected ≥1 LUN-map after VM start (re-map), found 0" + ) + + # ------------------------------------------------------------------ + # Step 07 — Detach volume from running VM + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_07_detach_volume_from_vm(self): + """ + Detach the iSCSI data volume from the running VM. + Verifies: + - detachVolume completes successfully + - CloudStack: volume virtualmachineid cleared + - ONTAP: LUN-map is removed (LUN stays in FlexVol) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + # Allow the guest OS to fully initialize the iSCSI device after VM + # start before requesting a hot-detach. Without this pause, the + # libvirt device-removal handshake can time out because the guest + # hasn't finished its early-boot device scan. + time.sleep(20) + + cmd = detachVolumeAPI.detachVolumeCmd() + cmd.id = self.__class__.volume.id + self.apiClient.detachVolume(cmd) + + # Poll until virtualmachineid is cleared + result = self._poll_volume_field( + self.__class__.volume.id, "virtualmachineid", None, timeout=120) + self.assertIsNotNone( + result, + "Volume virtualmachineid was not cleared after detachVolume" + ) + + # ONTAP: LUN-map must be removed after detach + lun_maps = self._lun_maps() + self.assertEqual( + len(lun_maps), 0, + "Expected 0 LUN-maps after volume detach, found %d: %s" + % (len(lun_maps), lun_maps) + ) + + # ONTAP: LUN still exists in FlexVol + luns = self.ontap.list_luns_in_volume( + self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "LUN should still exist in ONTAP FlexVol after detach" + ) + + # ------------------------------------------------------------------ + # Step 08 — Destroy VM and clean up pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_08_destroy_vm_and_cleanup(self): + """ + Destroy the VM (with expunge), delete the data volume, force-delete + the ONTAP pool, and delete the guest network created for this suite. + This test leaves no entities behind in either CloudStack or ONTAP. + Verifies: + - VM is destroyed and expunged from CloudStack + - deleteVolume removes the LUN from the ONTAP FlexVol + - deleteStoragePool(forced=True) removes the pool from CS + - ONTAP: FlexVol deleted + - ONTAP: all per-host igroups deleted + - CloudStack: guest network deleted (Advanced zones only) + """ + pool = self.__class__.pool + vol = self.__class__.volume + + if self.__class__.vm is not None: + # Ensure VM is stopped before destroying + vms = self.apiClient.listVirtualMachines( + _list_vms_cmd(self.__class__.vm.id)) + current_state = vms[0].state if vms else "unknown" + if current_state.lower() not in ("stopped", "destroyed"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = self.__class__.vm.id + stop_cmd.forced = True + self.apiClient.stopVirtualMachine(stop_cmd) + self._poll_vm_state(self.__class__.vm.id, "Stopped", + timeout=120) + + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = self.__class__.vm.id + dest_cmd.expunge = True + self.apiClient.destroyVirtualMachine(dest_cmd) + self.__class__.vm = None + + if vol is not None and pool is not None: + pool_name = pool.name + + # Delete the data volume + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.__class__.volume = None + + # ONTAP: LUN must be removed after volume deletion + luns = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertEqual( + len(luns), 0, + "Expected 0 LUNs in FlexVol '%s' after volume delete, " + "found %d" % (pool_name, len(luns)) + ) + + # Enter maintenance and force-delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after force deletion" + % pool_name + ) + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool force deletion" + % pool_name + ) + + # ONTAP: per-host igroups must be deleted by the pool force-delete + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) + if not iqn or not iqn.startswith("iqn."): + continue + short = host.name.split(".")[0] + igroup_name = "cs_%s_%s" % ( + self.svm_name, + re.sub(r"[^a-zA-Z0-9_-]", "_", short), + ) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool force deletion" + % igroup_name + ) + + # Delete the guest network created by setUpClass for Advanced zones. + # Doing this inside the test (rather than only in tearDownClass) makes + # the full sequence self-contained when all tests pass. + if self.__class__._created_network_id is not None: + net_id = self.__class__._created_network_id + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = net_id + # The management server may briefly drop the connection after the + # heavy teardown above; retry deleteNetwork up to 3× with 15s gaps. + last_net_exc = None + for attempt in range(3): + try: + self.apiClient.deleteNetwork(dn_cmd) + last_net_exc = None + break + except Exception as exc: + last_net_exc = exc + if attempt < 2: + time.sleep(15) + if last_net_exc is not None: + raise last_net_exc + self.__class__._created_network_id = None + self.__class__.network_id = None + + # CloudStack: guest network must be gone + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.id = net_id + net_cmd.listall = True + remaining_nets = self.apiClient.listNetworks(net_cmd) or [] + self.assertFalse( + remaining_nets, + "Guest network %s still listed in CloudStack after deletion" + % net_id + ) diff --git a/test/integration/plugins/ontap/iscsi/pool/__init__.py b/test/integration/plugins/ontap/iscsi/pool/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py new file mode 100644 index 000000000000..8a8584b0725f --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py @@ -0,0 +1,630 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage +pool lifecycle (no volumes). + +Tests are numbered test_01 ... test_08 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create primary storage pool + 02 Disable storage pool + 03 Enable storage pool + 04 Enter maintenance mode + 05 Cancel maintenance mode + 06 Enter maintenance mode and delete the storage pool + 07 Create a new pool and allocate a CloudStack data volume (LUN created) + 08 Delete the volume (LUN removed), enter maintenance, force-delete pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/pool/ -v + +Note: Tests share class-level state (sequential). Always run the full suite. +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIPoolLifecycle") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-wf@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-WF", + "username": "ontap_iscsi_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSI_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIPoolLifecycle(OntapTestBase): + + # ---- iSCSI-specific state (set/cleared by individual tests) -------- + _vol_name_prefix = "OntapISCSIVol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIPoolLifecycle, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSI_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 - Create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_01_create_primary_storage_pool(self): + """ + Create an iSCSI primary storage pool and verify: + - CloudStack state is Up, type is Iscsi + - ONTAP: FlexVol exists and is online + - ONTAP: one igroup per cluster host exists with the correct IQN initiator + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: igroup must exist for each cluster host that has an IQN + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue # host not iSCSI-enabled; skip igroup check for it + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNotNone( + igroup, + "ONTAP igroup '%s' not found for host '%s'" % (igroup_name, host.name) + ) + initiator_names = [ + i.get("name", "") for i in igroup.get("initiators", []) + ] + self.assertIn( + iqn, initiator_names, + "Host IQN '%s' not in igroup '%s' initiators: %s" + % (iqn, igroup_name, initiator_names) + ) + + # Capacity reporting + self._assert_pool_capacity(pool, "pool-created") + + # ------------------------------------------------------------------ + # Step 02 - Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_02_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online (disable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + # ONTAP: disable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 03 - Enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_03_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online (enable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + # ONTAP: enable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 04 - Enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_04_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + # ONTAP: maintenance must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 05 - Cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_05_cancel_maintenance_mode(self): + """ + Cancel maintenance and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 06 - Enter maintenance mode and delete the storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_06_enter_maintenance_and_delete_pool(self): + """ + Enter maintenance mode then delete the pool. + Verifies the pool is removed from CloudStack and the backing ONTAP + FlexVol is deleted. + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + pool = self.__class__.pool + pool_name = pool.name + + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) + + # ------------------------------------------------------------------ + # Step 07 - Create fresh pool and allocate a CloudStack volume (LUN) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_07_create_volume_on_pool(self): + """ + Create a new iSCSI pool and allocate a CloudStack data volume. + For iSCSI, createAsync creates a LUN inside the pool's ONTAP FlexVol. + Verifies: + - pool.state is Up, type is Iscsi + - createVolume returns a non-None volume object + - ONTAP: FlexVol is still online + - ONTAP: at least one LUN is present in the FlexVol + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: FlexVol must still be online after volume allocation + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume creation" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: at least one LUN must be present in the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume creation" % pool.name + ) + + # Capacity reporting + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 08 - Delete volume (LUN) then force-delete the pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_08_delete_volume_and_pool(self): + """ + Delete the volume from test_07, enter maintenance, then force-delete + the pool. + Verifies: + - deleteVolume removes the LUN from ONTAP + - Pool transitions to Maintenance + - Pool is removed from CloudStack after force deletion + - ONTAP: FlexVol deleted + - ONTAP: igroups for all cluster hosts deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + vol = self.__class__.volume + + # Delete the volume — LUN is removed from ONTAP + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: LUN must be gone from the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertEqual( + len(luns), 0, + "Expected 0 LUNs in FlexVol '%s' after volume deletion, found %d" + % (pool_name, len(luns)) + ) + + # ONTAP: FlexVol must still be online (pool not yet deleted) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after volume deletion" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion" + ) + + # Capacity reporting: capacity stable after volume deletion + self._assert_pool_capacity(pool, "volume-deleted") + + # Enter maintenance then force-delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py new file mode 100644 index 000000000000..ef799b644ce8 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py @@ -0,0 +1,706 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +iSCSI pool lifecycle tests with a CloudStack data volume present throughout. + +Covers the TDS (section 10) scenarios that require a data volume to already +exist on the pool during pool state transitions — the iSCSI variants of those +scenarios: + + TDS Approach-1 SN 11 — Disable iSCSI pool WITH volumes + TDS Approach-1 SN 15 — Enable iSCSI pool WITH volumes + TDS Approach-1 SN 19 — Enter maintenance WITH volumes + TDS Approach-1 SN 23 — Cancel maintenance WITH volumes + TDS Negative SN 5 — Delete iSCSI pool that has volumes; forced=False rejected + TDS Approach-1 SN 7 — Force-delete iSCSI pool (volume deleted first from + Maintenance — allowed on iSCSI unlike NFS3) + +Key iSCSI difference from NFS3: cancelStorageMaintenance works on iSCSI because +the KVM agent can unmount/remount iSCSI LUNs correctly. This allows the full +maintenance-cancel-maintenance lifecycle and proper volume cleanup while pool +is in Maintenance state. + +Tests are numbered test_01 ... test_07 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create iSCSI pool and allocate a CloudStack data volume (LUN on ONTAP) + 02 Disable pool — volume survives; ONTAP LUN still exists (SN 11) + 03 Re-enable pool — volume intact; ONTAP LUN accessible (SN 15) + 04 Enter maintenance with volume — pool Maintenance; LUN exists (SN 19) + 05 Cancel maintenance with volume — pool Up; LUN accessible (SN 23) + 06 Re-enter maintenance; forced=False delete rejected (Neg SN 5) + 07 Delete volume from Maintenance, then force-delete pool (SN 7) + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI initiator configured + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_iscsi_pool_with_volumes.py -v +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIPoolWithVolumes") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-wv@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-WV", + "username": "ontap_iscsi_wv_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSIWV_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIPoolWithVolumes(OntapTestBase): + """ + iSCSI pool lifecycle tests with a CloudStack data volume present throughout. + All 7 tests are sequential and share class-level state. + """ + + _vol_name_prefix = "OntapISCSIWV" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIPoolWithVolumes, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSIWV_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _volume_exists_in_cs(self, vol_id): + """Return True if the volume is still listed by CloudStack.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return len(vols) > 0 + + def _assert_lun_exists(self, pool_name, msg_context=""): + """Assert that at least one LUN exists in the pool's ONTAP FlexVol.""" + luns = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertTrue( + len(luns) > 0, + "Expected ≥1 LUN in ONTAP FlexVol '%s'%s, found 0" + % (pool_name, " (%s)" % msg_context if msg_context else "") + ) + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create pool and allocate a data volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create an iSCSI primary storage pool and allocate a CloudStack data + volume on it. + Verifies: + - Pool state is Up; pool type is Iscsi + - ONTAP: FlexVol is online + - ONTAP: at least one igroup exists (one per cluster host with IQN) + - ONTAP: after createVolume, a LUN exists in the FlexVol + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: igroup must exist for each cluster host that has an IQN + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNotNone( + igroup, + "ONTAP igroup '%s' not found for host '%s'" + % (igroup_name, host.name) + ) + + # Allocate a CloudStack data volume on this pool + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: a LUN must exist in the FlexVol after volume creation + self._assert_lun_exists(pool.name, "after volume creation") + + # Capacity reporting: LUN allocated but FlexVol size unchanged + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 02 — Disable pool with volume present (TDS SN 11) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_02_disable_pool_volume_survives(self): + """ + Disable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 11 (iSCSI): + - Pool transitions to Disabled + - Existing CS volume still listed + - ONTAP: FlexVol remains online; LUN still exists + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual( + result.state, "Disabled", + "Pool should be 'Disabled', got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool disable" + ) + + # ONTAP: FlexVol still online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after pool disable" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after pool disable") + + # ------------------------------------------------------------------ + # Step 03 — Re-enable pool with volume present (TDS SN 15) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_03_enable_pool_volume_intact(self): + """ + Re-enable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 15 (iSCSI): + - Pool transitions back to Up + - CS volume still listed + - ONTAP: FlexVol online; LUN still exists + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after re-enable, got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool re-enable" + ) + + # ONTAP: FlexVol online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool re-enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after pool re-enable" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after pool re-enable") + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance with volume present (TDS SN 19) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_04_enter_maintenance_volume_present(self): + """ + Enter maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 19 (iSCSI): + - Pool transitions to Maintenance + - CS volume still listed (not destroyed) + - ONTAP: FlexVol remains online (maintenance is a CloudStack state) + - ONTAP: LUN still exists in the FlexVol + + Note: the TDS additionally expects VMs using this pool to stop and their + LUN maps to be removed. This suite uses a standalone data volume (not + attached to any VM), so the VM stop behaviour is not exercised here — it + is covered by the VM lifecycle test suite. + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual( + result.state, "Maintenance", + "Pool should be 'Maintenance', got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool entered Maintenance" + ) + + # ONTAP: FlexVol still online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol disappeared after entering Maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' in Maintenance" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after entering Maintenance") + + # ------------------------------------------------------------------ + # Step 05 — Cancel maintenance with volume present (TDS SN 23) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_05_cancel_maintenance_volume_present(self): + """ + Cancel maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 23 (iSCSI): + - cancelStorageMaintenance works on iSCSI (unlike the NFS3 variant) + - Pool transitions back to Up + - CS volume still listed + - ONTAP: FlexVol online; LUN still present in FlexVol + + Note: when VMs are attached to volumes on this pool, ONTAP would + re-create the LUN-maps (igroup bindings) at cancel-maintenance time. + This suite has no VMs attached, so LUN-map re-creation is not verified + here; it is covered by the VM lifecycle test suite. + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after cancel maintenance, got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after cancel maintenance" + ) + + # ONTAP: FlexVol online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after cancel maintenance") + + # ------------------------------------------------------------------ + # Step 06 — forced=False delete rejected (negative) (TDS Neg SN 5) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_06_forced_false_delete_rejected(self): + """ + Enter maintenance then attempt deleteStoragePool(forced=False) while + a CloudStack volume exists on the pool. The operation must be rejected. + Covers TDS Negative Scenario SN 5 (iSCSI): + - CloudstackAPIException is raised + - Pool remains in Maintenance state + - CS volume still exists + - ONTAP: FlexVol and LUN unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + # Re-enter Maintenance (pool is Up from test_05) + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + + # Attempt forced=False delete — must raise + with self.assertRaises(Exception, + msg="deleteStoragePool(forced=False) with a live " + "volume should raise an exception"): + self._delete_pool(self.__class__.pool.id, forced=False) + + # Pool must still be listed (in Maintenance) + try: + remaining = list_storage_pools(self.apiClient, id=self.__class__.pool.id) + except Exception: + remaining = None + self.assertTrue( + remaining, + "Pool was deleted even though forced=False delete should have failed" + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume was deleted after rejected pool deletion" + ) + + # ONTAP: FlexVol still online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol should still exist after rejected pool deletion" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after rejected deletion" + ) + + # ------------------------------------------------------------------ + # Step 07 — Delete volume from Maintenance, then force-delete pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_07_delete_volume_and_force_delete_pool(self): + """ + Delete the CloudStack volume (while pool is in Maintenance) then + force-delete the pool. + Covers TDS Approach-1 SN 7 (iSCSI): + - On iSCSI, deleteVolume succeeds even when pool is in Maintenance + (unlike NFS3 where the KVM agent raises NPE) + - After volume deletion, the LUN is removed from the ONTAP FlexVol + - force-delete pool removes pool, FlexVol, and all igroups + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + vol = self.__class__.volume + + # Delete the volume while pool is in Maintenance + # (this works on iSCSI — no KVM NPE unlike NFS3) + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.__class__.volume = None + + # ONTAP: LUN must be gone from the FlexVol after volume deletion + luns_after = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertEqual( + len(luns_after), 0, + "Expected 0 LUNs in ONTAP FlexVol '%s' after volume deletion, " + "found %d: %s" % (pool_name, len(luns_after), luns_after) + ) + + # ONTAP: FlexVol must still be online (pool deletion removes the FlexVol) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after CS volume deletion" + % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after CS volume deletion" + ) + + # Capacity reporting: capacity fields stable after LUN removal + self._assert_pool_capacity(pool, "volume-deleted") + + # Force-delete the pool (no live volumes remain; pool is in Maintenance) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after force deletion" % pool_name + ) + + # ONTAP: FlexVol must be deleted + ontap_vol_after = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol_after, + "ONTAP FlexVol '%s' still exists after pool force deletion" % pool_name + ) + + # ONTAP: igroups for all cluster hosts must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool force deletion" + % igroup_name + ) diff --git a/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py new file mode 100644 index 000000000000..a6b57de5573f --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py @@ -0,0 +1,386 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Zone-scoped primary storage lifecycle tests for NetApp ONTAP (iSCSI). + +Creates a zone-scoped pool (scope=ZONE, no clusterid/podid). CloudStack calls +OntapPrimaryDatastoreLifecycle.attachZone(), which connects all eligible KVM +hosts in the zone and creates igroups for each host's IQN. + +Workflow: + 01 Create zone-scoped iSCSI pool — pool.state Up; ONTAP FlexVol online; + igroup present for each cluster host IQN + 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 04 Delete zone-scoped pool — pool gone; FlexVol deleted; igroups deleted + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM hosts with iSCSI registered in the zone + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py -v + +Note: Tests 01-04 share class-level state (sequential). Always run the full +suite. +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIZoneScopedPool") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + provider="NetApp ONTAP", tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-zone@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-Zone", + "username": "ontap_iscsi_zone_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapZoneISCSI_%d" % random.randint(0, 9999), + TestData.scope: "ZONE", + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIZoneScopedPool(OntapTestBase): + + _vol_name_prefix = "OntapISCSIZoneVol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIZoneScopedPool, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + provider=provider, tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_zone_pool(self): + """Create a zone-scoped iSCSI pool (no clusterid / podid).""" + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapZoneISCSI_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + # Intentionally omit clusterid and podid — zone-scoped pool + cmd.scope = "ZONE" + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _assert_igroups_for_hosts(self, expect_present): + """Assert igroups are present (or absent) for each cluster host IQN.""" + for host in self.cluster_hosts: + iqn = (getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None)) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + if expect_present: + self.assertIsNotNone( + igroup, + "ONTAP igroup '%s' not found for host '%s' after pool creation" + % (igroup_name, host.name) + ) + initiator_names = [ + i.get("name", "") for i in igroup.get("initiators", []) + ] + self.assertIn( + iqn, initiator_names, + "Host IQN '%s' not in igroup '%s' initiators: %s" + % (iqn, igroup_name, initiator_names) + ) + else: + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) + + # ------------------------------------------------------------------ + # Step 01 — Create zone-scoped iSCSI pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_01_create_zone_scoped_pool(self): + """ + Create a zone-scoped iSCSI primary storage pool (no clusterid/podid). + CloudStack calls attachZone(), which connects all eligible KVM hosts + in the zone and creates igroups for each host's IQN. + Verifies: + - pool.state is Up, type is Iscsi + - ONTAP: FlexVol is online + - ONTAP: igroup exists for each cluster host with the correct IQN + """ + pool = self._create_zone_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: igroups must exist for each cluster host with IQN + self._assert_igroups_for_hosts(expect_present=True) + + # ------------------------------------------------------------------ + # Step 02 — Disable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_02_disable_zone_scoped_pool(self): + """ + Disable the zone-scoped iSCSI pool. + Verifies: + - pool.state is Disabled + - ONTAP: FlexVol still online; igroups unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable" + ) + + # igroups must still be present after a simple disable + self._assert_igroups_for_hosts(expect_present=True) + + # ------------------------------------------------------------------ + # Step 03 — Enable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_03_enable_zone_scoped_pool(self): + """ + Re-enable the zone-scoped iSCSI pool. + Verifies: + - pool.state is Up + - ONTAP: FlexVol still online; igroups unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable" + ) + + # igroups must still be present after re-enable + self._assert_igroups_for_hosts(expect_present=True) + + # ------------------------------------------------------------------ + # Step 04 — Delete zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_04_delete_zone_scoped_pool(self): + """ + Enter maintenance then delete the zone-scoped iSCSI pool. + Verifies: + - Pool is removed from CloudStack + - ONTAP: FlexVol deleted + - ONTAP: igroups deleted for all cluster hosts + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + self._assert_igroups_for_hosts(expect_present=False) diff --git a/test/integration/plugins/ontap/iscsi/volume/__init__.py b/test/integration/plugins/ontap/iscsi/volume/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/volume/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py b/test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py new file mode 100644 index 000000000000..7f02f9f29903 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py @@ -0,0 +1,416 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI data volume +lifecycle (LUN create / delete / negative-delete / force-delete). + +Tests are numbered test_01 ... test_05 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create iSCSI primary storage pool (infrastructure) and allocate a + CloudStack data volume — LUN is created inside the pool's ONTAP FlexVol + 02 Delete the volume — LUN is removed from the FlexVol + 03 Recreate volume — LUN is present again (setup for negative delete tests) + 04 Put pool in Maintenance; attempt forced=False deleteStoragePool — must be + rejected because volumes exist; pool stays in Maintenance + 05 Delete volume from Maintenance; forced=True deleteStoragePool — FlexVol, + igroups, and all LUNs are removed from ONTAP + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/volume/ -v + +Note: Tests share class-level state (sequential). Always run the full suite. +The pool is cleaned up in test_05 on the happy path; OntapTestBase tearDownClass +provides a safety net for mid-run failures. +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIVolumeLifecycle") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-vol@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-Vol", + "username": "ontap_iscsi_vol_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSIVol_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIVolumeLifecycle(OntapTestBase): + + _vol_name_prefix = "OntapISCSIVol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIVolumeLifecycle, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSIVol_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + # ------------------------------------------------------------------ + # Step 01 - Create pool (infrastructure) and allocate a volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create a new iSCSI pool and allocate a CloudStack data volume on it. + Verifies: + - pool.state is Up + - createVolume returns a non-None volume object + - ONTAP: at least one LUN exists in the pool's FlexVol + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: at least one LUN must be present in the pool FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume creation" % pool.name + ) + + # ------------------------------------------------------------------ + # Step 02 - Delete volume; LUN must be removed from ONTAP + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_02_delete_volume(self): + """ + Delete the volume created in test_01. + Verifies: + - deleteVolume completes without error + - ONTAP: LUN is removed from the pool's FlexVol + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_01 must pass first") + + pool = self.__class__.pool + vol = self.__class__.volume + + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: LUN must be gone from the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertEqual( + len(luns), 0, + "Expected 0 LUNs in FlexVol '%s' after volume deletion, found %d: %s" + % (pool.name, len(luns), luns) + ) + + # ------------------------------------------------------------------ + # Step 03 - Recreate volume for negative delete tests + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_03_recreate_volume_for_delete_tests(self): + """ + Recreate a volume on the existing pool (setup for tests 04-05). + Verifies: + - volume created successfully + - ONTAP: LUN present in pool FlexVol + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + vol = self._create_volume(self.__class__.pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + luns = self.ontap.list_luns_in_volume(self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume re-creation" + % self.__class__.pool.name + ) + + # ------------------------------------------------------------------ + # Step 04 - Forced=False delete with live volume must fail + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_04_forced_false_delete_with_volume_fails(self): + """ + Put pool in Maintenance then attempt deleteStoragePool(forced=False). + With a live volume present CloudStack must reject the request. + Verifies: + - CloudstackAPIException is raised + - Pool is still listed in CloudStack (in Maintenance state) + - ONTAP: FlexVol still exists and is online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + + # Enter maintenance mode + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + # Attempt forced=False delete — must raise exception because volumes exist + with self.assertRaises(Exception): + self._delete_pool(pool.id, forced=False) + + # Pool must still be listed in CloudStack + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + listed, + "Pool should still exist in CloudStack after failed forced=False delete" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after failed delete" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 05 - Delete volume then force-delete pool from Maintenance + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_05_delete_volume_and_force_delete_pool(self): + """ + Delete the live volume then force-delete the pool while it is still + in Maintenance state (pool is in Maintenance from test_04). + Verifies: + - Volume can be deleted while pool is in Maintenance + - Pool is removed from CloudStack using forced=True from Maintenance + - ONTAP: FlexVol deleted + - ONTAP: igroups deleted for all cluster hosts + """ + self.assertIsNotNone( + self.__class__.pool, + "Pool absent - test_04 must not have cleaned up the pool" + ) + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + vol = self.__class__.volume + + # Delete the volume first (pool is in Maintenance — volume deletion is allowed) + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # Force-delete the pool from Maintenance (no live volumes remaining) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after force deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after force deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after force deletion" % igroup_name + ) diff --git a/test/integration/plugins/ontap/manual_cancel_maint_test.py b/test/integration/plugins/ontap/manual_cancel_maint_test.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/integration/plugins/ontap/nfs3/__init__.py b/test/integration/plugins/ontap/nfs3/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/instance/__init__.py b/test/integration/plugins/ontap/nfs3/instance/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/instance/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py new file mode 100644 index 000000000000..461ee104cbf9 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py @@ -0,0 +1,832 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP data volume lifecycle +with a running virtual machine. + +Tests are numbered test_01 ... test_08 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create NFS3 primary storage pool on ONTAP + 02 Create a CloudStack data volume on the ONTAP pool + 03 Deploy a VM (template and service offering discovered at setup time) + 04 Attach the ONTAP data volume to the running VM + 05 Stop the VM — export policy stays; volume remains attached in CS + 06 Start the VM — VM Running; volume still attached; FlexVol online + 07 Detach the ONTAP data volume from the VM + 08 Destroy VM; delete ONTAP volume; enter maintenance; delete pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack with at least one executable template + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_vm_volume_attach.py -v + +Note: Tests share class-level state (sequential). Always run the full suite. +""" + +import base64 +import logging +import random +import time +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, + createNetwork as createNetworkAPI, + createStoragePool as createStoragePoolAPI, + deleteNetwork as deleteNetworkAPI, + deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + detachVolume as detachVolumeAPI, + enableStorageMaintenance, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listServiceOfferings as listServiceOfferingsAPI, + listTemplates as listTemplatesAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + startVirtualMachine as startVirtualMachineAPI, + stopVirtualMachine as stopVirtualMachineAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapVMVolumeAttach") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-vm-vol@test.com", + "firstname": "ONTAP", + "lastname": "VMVol", + "username": "ontap_vm_vol_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapVMVol_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapVMVolumeAttach(OntapTestBase): + """ + Tests ONTAP data volume lifecycle with a running CloudStack VM. + + All tests are sequential — state is carried on class attributes. + """ + + # ---- extra shared state beyond OntapTestBase ----------------------- + vm = None # running VirtualMachine + template_id = None # KVM template ID discovered at setup + service_offering_id = None + network_id = None # None for Basic zones + _created_network_id = None # network created by this suite for Advanced zones + + _vol_name_prefix = "OntapVMVol" + + # ---- setup --------------------------------------------------------- + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapVMVolumeAttach, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Discover a suitable user KVM template in the zone (must be fully + # downloaded; system-type templates are excluded as they cannot be + # deployed as user VMs). + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = cls.zone.id + templates = cls.apiClient.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + if kvm_ready: + cls.template_id = kvm_ready[0].id + else: + logger.warning( + "No ready user KVM template found in zone '%s'. " + "Tests that deploy VMs will be skipped until a template " + "finishes downloading." % cls.zone.name + ) + cls.template_id = None + + # Discover the smallest service offering + so_cmd = listServiceOfferingsAPI.listServiceOfferingsCmd() + offerings = cls.apiClient.listServiceOfferings(so_cmd) or [] + assert offerings, "No service offerings available in CloudStack" + offerings.sort(key=lambda s: getattr(s, "memory", 9999)) + cls.service_offering_id = offerings[0].id + + # Detect zone type; resolve network ID for Advanced zones + cls.network_id = None + zone_type = getattr(cls.zone, "networktype", "Basic") + if zone_type.lower() == "advanced": + # Find a network already accessible to the test account + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + else: + # Create an Isolated guest network for the test account + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.specifyvlan = "false" + no_offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + snat_offering = next( + (o for o in no_offerings + if "SourceNat" in o.name and "Vpc" not in o.name + and "NSX" not in o.name and "Netris" not in o.name), + no_offerings[0] if no_offerings else None + ) + if snat_offering: + cn_cmd = createNetworkAPI.createNetworkCmd() + cn_cmd.zoneid = cls.zone.id + cn_cmd.networkofferingid = snat_offering.id + cn_cmd.name = "ontap-nfs3-vm-net-%d" % random.randint( + 0, 9999) + cn_cmd.displaytext = "ONTAP NFS3 VM test network" + cn_cmd.account = cls.account.name + cn_cmd.domainid = cls.domain.id + net = cls.apiClient.createNetwork(cn_cmd) + cls.network_id = net.id + cls._created_network_id = net.id + + @classmethod + def tearDownClass(cls): + """Destroy the VM first, then delegate pool/volume cleanup to super.""" + if cls.vm is not None: + try: + # Ensure VM is stopped before destroying + vms = cls.apiClient.listVirtualMachines( + _list_vms_cmd(cls.vm.id)) + current_state = vms[0].state if vms else "unknown" + if current_state.lower() not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = cls.vm.id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, cls.vm.id, "Stopped", + timeout=120) + except Exception as e: + logger.warning("tearDownClass: could not stop VM %s: %s" + % (cls.vm.id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = cls.vm.id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as e: + logger.warning("tearDownClass: could not destroy VM %s: %s" + % (cls.vm.id, e)) + + # Delete the guest network created for this account in Advanced zones. + if cls._created_network_id is not None: + try: + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn_cmd) + cls._created_network_id = None + except Exception as e: + logger.warning( + "tearDownClass: could not delete network %s: %s" + % (cls._created_network_id, e)) + + super(TestOntapVMVolumeAttach, cls).tearDownClass() + + # ---- pool creation helper ----------------------------------------- + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapVMVol_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + # ---- VM state helpers ---------------------------------------------- + + def _poll_vm_state(self, vm_id, target_state, timeout=300, interval=10): + """Poll listVirtualMachines until the VM reaches target_state.""" + deadline = time.time() + timeout + current_state = "unknown" + while time.time() < deadline: + vms = self.apiClient.listVirtualMachines( + _list_vms_cmd(vm_id)) + if vms: + current_state = vms[0].state + if current_state.lower() == target_state.lower(): + return vms[0] + time.sleep(interval) + self.fail( + "VM %s did not reach state '%s' within %ds (last: '%s')" + % (vm_id, target_state, timeout, current_state) + ) + + def _volume_state(self, vol_id): + """Return the current CloudStack state string for a volume.""" + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + vols = self.apiClient.listVolumes(cmd) + return vols[0].state if vols else "unknown" + + # ================================================================== + # Test steps + # ================================================================== + + # ------------------------------------------------------------------ + # Step 01 - Create NFS3 ONTAP pool + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_01_create_nfs3_pool(self): + """ + Create an NFS3 ONTAP primary storage pool. + Verifies: + - Pool reaches 'Up' state in CloudStack + - ONTAP: FlexVol is created and online + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not created for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 02 - Create CloudStack data volume on ONTAP pool + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_02_create_ontap_data_volume(self): + """ + Allocate a CloudStack data volume on the ONTAP NFS3 pool. + Verifies: + - Volume is created and in 'Allocated' or 'Ready' state + - ONTAP: FlexVol remains online + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + pool = self.__class__.pool + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + vol_state = self._volume_state(vol.id) + self.assertIn( + vol_state.lower(), ("allocated", "ready"), + "Volume should be 'Allocated' or 'Ready', got '%s'" % vol_state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol disappeared after data volume creation" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume creation" + ) + + # ------------------------------------------------------------------ + # Step 03 - Deploy a VM + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_03_deploy_vm(self): + """ + Deploy a VM using the first available KVM template and smallest + service offering discovered at setup time. + Verifies: + - VM reaches 'Running' state in CloudStack + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + if self.__class__.template_id is None: + self.skipTest( + "No ready user KVM template available — " + "waiting for template download to complete" + ) + self.assertIsNotNone(self.__class__.service_offering_id, + "No service offering available — check setup") + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + self.__class__.vm = vm + + vm_obj = self._poll_vm_state(vm.id, "Running", timeout=600) + self.assertEqual( + vm_obj.state, "Running", + "VM should be 'Running', got '%s'" % vm_obj.state + ) + + # ------------------------------------------------------------------ + # Step 04 - Attach ONTAP data volume to the running VM + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_04_attach_volume_to_vm(self): + """ + Attach the ONTAP data volume to the running VM. + Verifies: + - Volume virtualmachineid is set to the VM's ID in CloudStack + (Note: on ONTAP/NFS shared storage the volume state remains 'Ready'; + attachment is signalled by virtualmachineid being populated) + - ONTAP: FlexVol remains online + - VM remains 'Running' + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = attachVolumeAPI.attachVolumeCmd() + cmd.id = vol.id + cmd.virtualmachineid = vm.id + attached = self.apiClient.attachVolume(cmd) + self.assertIsNotNone(attached, "attachVolume returned None") + + # On ONTAP/NFS shared storage CloudStack does not transition the volume + # state to 'In Use' — attachment is indicated by virtualmachineid being + # set on the volume record. Poll on that field instead of state. + deadline = time.time() + 120 + vol_vmid = None + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol.id)) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + if vol_vmid: + break + time.sleep(5) + + self.assertEqual( + vol_vmid, vm.id, + "Volume should be attached to VM %s after attach, " + "got virtualmachineid=%s" % (vm.id, vol_vmid) + ) + + # VM must still be Running + vm_obj = self._poll_vm_state(vm.id, "Running", timeout=30) + self.assertEqual(vm_obj.state, "Running", + "VM should still be 'Running' after volume attach") + + # ONTAP FlexVol must remain online + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol not found after attach") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after attach" + ) + + # ------------------------------------------------------------------ + # Step 05 - Stop VM — export policy must be retained + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_05_stop_vm_export_retained(self): + """ + Stop the running VM while the NFS3 data volume is still attached. + Unlike iSCSI (where LUN-maps are removed on VM stop), NFS3 export + policies are not torn down when a VM stops — the FlexVol stays + accessible on the same mount. + Verifies: + - VM reaches Stopped state + - ONTAP: FlexVol still online + - CloudStack: volume virtualmachineid still set (volume stays attached) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + cmd.id = vm.id + self.apiClient.stopVirtualMachine(cmd) + + result = self._poll_vm_state(vm.id, "Stopped", timeout=300) + self.assertEqual( + result.state, "Stopped", + "VM should be 'Stopped', got '%s'" % result.state + ) + + # ONTAP: FlexVol must remain online — NFS export is not torn down on VM stop + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' disappeared after VM stop" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after VM stop, " + "got '%s'" % ontap_vol.get("state") + ) + + # CloudStack: volume must still be attached (virtualmachineid set) + cmd_list = listVolumesAPI.listVolumesCmd() + cmd_list.id = vol.id + vols = self.apiClient.listVolumes(cmd_list) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + self.assertEqual( + vol_vmid, vm.id, + "Volume should still be attached to VM %s after stop, " + "got virtualmachineid=%s" % (vm.id, vol_vmid) + ) + + # ------------------------------------------------------------------ + # Step 06 - Start VM — volume accessible; FlexVol online + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_06_start_vm_volume_accessible(self): + """ + Start the stopped VM. + Verifies: + - VM reaches Running state + - ONTAP: FlexVol still online + - CloudStack: volume virtualmachineid still set (volume remains attached) + - VM remains 'Running' after start + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = startVirtualMachineAPI.startVirtualMachineCmd() + cmd.id = vm.id + self.apiClient.startVirtualMachine(cmd) + + result = self._poll_vm_state(vm.id, "Running", timeout=300) + self.assertEqual( + result.state, "Running", + "VM should be 'Running' after start, got '%s'" % result.state + ) + + # ONTAP: FlexVol must remain online after VM start + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after VM start" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after VM start, " + "got '%s'" % ontap_vol.get("state") + ) + + # CloudStack: volume must still be attached to the VM + cmd_list = listVolumesAPI.listVolumesCmd() + cmd_list.id = vol.id + vols = self.apiClient.listVolumes(cmd_list) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + self.assertEqual( + vol_vmid, vm.id, + "Volume should still be attached to VM %s after start, " + "got virtualmachineid=%s" % (vm.id, vol_vmid) + ) + + # ------------------------------------------------------------------ + # Step 07 - Detach ONTAP data volume from the VM + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_07_detach_volume_from_vm(self): + """ + Detach the ONTAP data volume from the running VM. + Verifies: + - Volume state returns to 'Ready' in CloudStack + - Volume no longer lists the VM's ID + - VM remains 'Running' + - ONTAP: FlexVol remains online + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = detachVolumeAPI.detachVolumeCmd() + cmd.id = vol.id + # The hypervisor may briefly mark the device as busy; retry up to 3×. + last_exc = None + for attempt in range(3): + try: + self.apiClient.detachVolume(cmd) + last_exc = None + break + except Exception as exc: + last_exc = exc + if attempt < 2: + time.sleep(30) + if last_exc is not None: + raise last_exc + + # On ONTAP/NFS shared storage the volume state stays 'Ready' throughout. + # Poll until virtualmachineid is cleared instead. + deadline = time.time() + 120 + vol_vmid = "pending" + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol.id)) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + if not vol_vmid: + break + time.sleep(5) + + self.assertIsNone( + vol_vmid, + "Volume should have no virtualmachineid after detach, got '%s'" + % vol_vmid + ) + + # VM must still be Running + vm_obj = self._poll_vm_state(vm.id, "Running", timeout=30) + self.assertEqual(vm_obj.state, "Running", + "VM should still be 'Running' after volume detach") + + # ONTAP FlexVol must remain online + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol not found after detach") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after detach" + ) + + # ------------------------------------------------------------------ + # Step 08 - Destroy VM, delete volume, delete pool + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_08_destroy_vm_and_cleanup(self): + """ + Destroy the VM, delete the ONTAP data volume, enter maintenance, + then delete the pool. + Verifies: + - VM is destroyed/expunged from CloudStack + - Volume is deleted from CloudStack + - Pool is removed from CloudStack + - ONTAP: FlexVol is deleted after pool removal + - ONTAP: Export policy is removed after pool removal + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + pool = self.__class__.pool + pool_name = pool.name + + # Stop VM if still running + if vm is not None: + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm.id)) + current_state = vms[0].state.lower() if vms else "unknown" + if current_state not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm.id + self.apiClient.stopVirtualMachine(stop_cmd) + self._poll_vm_state(vm.id, "Stopped", timeout=300) + + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm.id + dest_cmd.expunge = True + self.apiClient.destroyVirtualMachine(dest_cmd) + self.__class__.vm = None + + # Delete the ONTAP data volume + if vol is not None: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # Enter maintenance then delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, + "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: Export policy must be removed + ep_name = "cs-%s-%s" % (self.svm_name, pool_name) + ep = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + ep, + "ONTAP export policy '%s' still exists after pool deletion" + % ep_name + ) + + +# --------------------------------------------------------------------------- +# Module-level helpers (used in tearDownClass and test helpers) +# --------------------------------------------------------------------------- + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + return cmd + + +def _list_vols_cmd(vol_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + return cmd + + +def _wait_for_vm_state(api_client, vm_id, target_state, timeout=120, + interval=5): + """Blocking wait for a VM to reach target_state (used in tearDownClass).""" + deadline = time.time() + timeout + while time.time() < deadline: + vms = api_client.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms and vms[0].state.lower() == target_state.lower(): + return + time.sleep(interval) diff --git a/test/integration/plugins/ontap/nfs3/pool/__init__.py b/test/integration/plugins/ontap/nfs3/pool/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py new file mode 100644 index 000000000000..f4f7ac7bb039 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py @@ -0,0 +1,709 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. + +Tests are numbered test_01 ... test_08 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create primary storage pool + 02 Disable storage pool + 03 Enable storage pool + 04 Enter maintenance mode + 05 Cancel maintenance mode + 06 Delete the storage pool (enters Maintenance first, then deletes) + 07 Create fresh pool and allocate a CloudStack volume + 08 Delete volume then force-delete the pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py -v + +Note: Tests 01-06 share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapNFS3Workflow") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + DETAIL_VOLUME_UUID = "volumeUUID" + DETAIL_VOLUME_NAME = "volumeName" + DETAIL_DATA_LIF = "dataLIF" + DETAIL_NFS_MOUNT_OPTS = "nfsmountopts" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-wf@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-WF", + "username": "ontap_nfs3_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase): + + # ---- NFS3-specific shared state ------------------------------------ + pool_ep_name = None # NFS export policy name for pool + cluster_host_ips = None + + _vol_name_prefix = "OntapNFS3Vol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapNFS3PrimaryStorageWorkflow, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Resolve cluster host IPs for export policy rule assertions + cls.cluster_host_ips = [ + h.ipaddress for h in cls.cluster_hosts + if getattr(h, "ipaddress", None) + ] + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + # Fallback: plugin typically uses cs-{svmName}-{poolName} + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _assert_export_policy_has_host_ips(self, ep_name): + """Assert that the export policy exists and its rules include each cluster host IP.""" + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP" % ep_name + ) + if not self.cluster_host_ips: + return # no host IPs registered; skip rule-level check + all_clients = [] + for rule in policy.get("rules", []): + for client in rule.get("clients", []): + all_clients.append(client.get("match", "")) + for ip in self.cluster_host_ips: + self.assertTrue( + any(ip in c for c in all_clients), + "Host IP '%s' not found in export policy '%s' rules: %s" + % (ip, ep_name, all_clients) + ) + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_01_create_primary_storage_pool(self): + """ + Create an NFS3 primary storage pool and verify: + - CloudStack state is Up, type is NetworkFilesystem + - nfsmountopts contains 'vers=3' + - ONTAP: FlexVol exists and is online + - ONTAP: NFS export policy exists with cluster host IP rules + - ONTAP: at least one NFS data LIF is present on the SVM + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "NetworkFilesystem", + "Pool type should be 'NetworkFilesystem', got '%s'" % pool.type + ) + + # Verify nfsmountopts via listStoragePools + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone(listed, "listStoragePools returned None for pool %s" % pool.id) + nfs_opts = getattr(listed[0], "nfsmountopts", "") + self.assertIn( + "vers=3", nfs_opts, + "nfsmountopts should contain 'vers=3', got '%s'" % nfs_opts + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist with host IP rules + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + self._assert_export_policy_has_host_ips(ep_name) + + # ONTAP: at least one NFS data LIF must be present + lifs = self.ontap.get_data_lifs(self.svm_name) + self.assertTrue( + len(lifs) > 0, + "No NFS data LIFs found on SVM '%s'" % self.svm_name + ) + + # Capacity reporting + self._assert_pool_capacity(pool, "pool-created") + + # ------------------------------------------------------------------ + # Step 02 — Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_02_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_03_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_04_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online and export policy unchanged + (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist during maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 05 — Cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_05_cancel_maintenance_mode(self): + """ + Cancel maintenance mode and verify the pool returns to Up. + + cancelStorageMaintenance sends ModifyStoragePoolCommand(add=True) to the + KVM agent, which calls createStoragePool() with details that include + nfsMountOptions=vers=3. The agent rebuilds the libvirt pool XML with the + xmlns:fs namespace extension and mounts the NFS share with vers=3. + + Fix confirmed — LibvirtStorageAdaptor now correctly handles the case + where a stale-active libvirt pool entry lingers at the mount point after + sp.destroy() during enter-maintenance. The fix: + 1. Detects a stale-active pool (isActive==1 but mountpoint -q fails) + and destroys it before re-creating. + 2. Retries createNetfsStoragePool once after 5 s on LibvirtException. + + Verifies: + - CloudStack reports pool state Up + - ONTAP: FlexVol is still online + - ONTAP: NFS export policy still present + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state( + self.__class__.pool.id, "Up", timeout=120 + ) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after cancel maintenance, got '%s'" + % result.state + ) + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol disappeared after cancel maintenance" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy( + self.__class__.pool_ep_name + ) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after cancel maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 06 — Delete the storage pool (already in Maintenance) + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_06_delete_pool_from_maintenance(self): + """ + Enter maintenance mode then delete the storage pool. + + Verifies: + - Pool is removed from CloudStack + - ONTAP: FlexVol is deleted + - ONTAP: NFS export policy is deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + # Pool is Up after test_05 succeeded; must enter Maintenance before deletion. + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 07 - Create fresh pool and allocate a CloudStack volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_07_create_volume_on_pool(self): + """ + Create a new NFS3 pool and allocate a CloudStack data volume. + For NFS3, createAsync is a no-op on ONTAP (volume is a CloudStack record + only — no new ONTAP object is created). + Verifies: + - pool.state is Up + - createVolume returns a non-None volume object + - ONTAP: FlexVol is still online and export policy still present + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: FlexVol must still be online after volume allocation + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume creation" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after volume creation" % ep_name + ) + + # Capacity reporting: FlexVol size and reported capacity unchanged after volume allocation + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 08 - Delete volume then force-delete the pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_08_delete_volume_and_pool(self): + """ + Delete the volume from test_07, enter maintenance, then force-delete + the pool. + Verifies: + - deleteVolume completes without error + - Pool transitions to Maintenance + - Pool is removed from CloudStack after force deletion + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + # Delete the volume + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: FlexVol must still be online (volume deletion does not affect NFS FlexVol) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after volume deletion" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion" + ) + + # Capacity reporting: capacity fields stable after volume deletion + self._assert_pool_capacity(pool, "volume-deleted") + + # Enter maintenance then force-delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py new file mode 100644 index 000000000000..d0238295c918 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py @@ -0,0 +1,767 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +NFS3 pool lifecycle tests with a CloudStack volume present throughout. + +Covers the TDS (section 10) scenarios that require a data volume to already +exist on the pool during pool state transitions: + + TDS Approach-1 SN 11-12 — Disable pool WITH volumes + TDS Approach-1 SN 15-16 — Enable pool WITH volumes + TDS Approach-1 SN 19-20 — Enter maintenance WITH volumes + TDS Approach-1 SN 21-22 — Cancel maintenance WITH volumes (fix confirmed) + TDS Negative SN 5-6 — Delete pool that has volumes; forced=False rejected + +Note: TDS SN 7-8 (force-delete NFS3 pool after manual volume deletion) is +already covered by test_ontap_create_primary_storage_nfs3.py test_07/test_08. + +Tests are numbered test_01 ... test_07 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create NFS3 pool and allocate a CloudStack data volume + 02 Disable pool — volume still exists in CloudStack; ONTAP FlexVol online + 03 Re-enable pool — volume still accessible; FlexVol online + 04 Enter maintenance with volume present — Maintenance state; FlexVol online + 05 Cancel maintenance with volume present — pool returns to Up (fix confirmed) + 06 Forced=False delete rejected — pool stays in Maintenance (negative) + 07 Cleanup — cancel maintenance, delete volume, force-delete pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values (protocol=NFS3) + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_nfs3_pool_with_volumes.py -v + +Note: Tests 01-05 share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. + +Post-run ONTAP cleanup: The suite ends with the pool in Maintenance state (from +test_04) and a CS volume present (test_05 negative test leaves both intact). The +OntapTestBase teardown exits Maintenance via cancelStorageMaintenance (which +transitions CS pool state even though KVM remount fails on NFS3), deletes the +volume, re-enters Maintenance, and force-deletes the pool. In rare cases where +CS pool state does not transition, one orphaned ONTAP FlexVol and export policy +may be left behind. Clean these up manually: + + curl -sk -u : \\ + "https:///api/storage/volumes?name=OntapNFS3WV_*&fields=name,state" + # Offline + DELETE each orphan, then DELETE the matching export policy.""" + +import base64 +import logging +import random +import time +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapNFS3PoolWithVolumes") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-wv@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-WV", + "username": "ontap_nfs3_wv_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3WV_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3PoolWithVolumes(OntapTestBase): + """ + NFS3 pool lifecycle tests with a CloudStack data volume present throughout. + All tests are sequential and share class-level state. + """ + + pool_ep_name = None # NFS export policy name extracted at pool creation + + _vol_name_prefix = "OntapNFS3WV" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapNFS3PoolWithVolumes, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3WV_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the NFS export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _volume_exists_in_cs(self, vol_id): + """Return True if the volume is still listed by CloudStack.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return len(vols) > 0 + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create pool and allocate a data volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create an NFS3 primary storage pool and allocate a CloudStack data + volume on it. + Verifies: + - Pool state is Up; ONTAP FlexVol is online + - NFS export policy exists + - createVolume returns a volume object (NFS3 data vols are CS records + backed by a qcow2 file inside the FlexVol) + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP after pool creation" % ep_name + ) + + # Allocate a CloudStack data volume on this pool + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # Capacity reporting: volume allocated on FlexVol + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 02 — Disable pool with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_02_disable_pool_volume_survives(self): + """ + Disable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 11 (iSCSI) and SN 12 (NFS3): + - Pool should no longer be available for scheduling new CS volumes + - The existing CS volume should continue to exist (not deleted) + - ONTAP: FlexVol remains online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual( + result.state, "Disabled", + "Pool should be 'Disabled', got '%s'" % result.state + ) + + # Volume must still exist in CloudStack + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool disable" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after pool disable, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after pool disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Re-enable pool with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_03_enable_pool_volume_intact(self): + """ + Re-enable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 15 (iSCSI) and SN 16 (NFS3): + - Pool state transitions back to Up + - The existing CS volume is still accessible + - ONTAP: FlexVol remains online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after re-enable, got '%s'" % result.state + ) + + # Volume must still exist in CloudStack + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool re-enable" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool re-enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after pool re-enable, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after pool re-enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance mode with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_04_enter_maintenance_volume_present(self): + """ + Enter maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 19 (iSCSI) and SN 20 (NFS3): + - Pool transitions to Maintenance state + - Existing CS volume remains in CloudStack + - ONTAP: FlexVol stays online (maintenance is a CS-only state) + - ONTAP: export policy is unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual( + result.state, "Maintenance", + "Pool should be 'Maintenance', got '%s'" % result.state + ) + + # Volume must still exist in CloudStack + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool entered Maintenance" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol disappeared after entering Maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' in Maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist during Maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 05 — Cancel maintenance mode with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_05_cancel_maintenance_with_volume(self): + """ + Cancel maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 21 (iSCSI) and SN 22 (NFS3): + - cancelStorageMaintenance succeeds (KVM/NFS3 fix confirmed) + - Pool returns to Up state + - Existing CS volume is still present in CloudStack + - ONTAP: FlexVol is still online + - ONTAP: NFS export policy is unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state( + self.__class__.pool.id, "Up", timeout=120 + ) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after cancel maintenance, got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after cancel maintenance" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol disappeared after cancel maintenance" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after cancel maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 06 — forced=False delete rejected when volume present (negative) + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_06_forced_false_delete_rejected(self): + """ + Enter maintenance mode then attempt to delete the pool (forced=False) + while a CloudStack volume still exists on it. The operation must be + rejected. + Covers TDS Negative Scenarios SN 5 (iSCSI) and SN 6 (NFS3): + - CloudstackAPIException is raised with an appropriate error + - Pool remains in Maintenance state + - CS volume still exists + - ONTAP: FlexVol and export policy are unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + # Pool is Up after test_05 (cancel maintenance); re-enter Maintenance + # before attempting the delete so it reaches the forced=False gate. + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + + with self.assertRaises(CloudstackAPIException, + msg="deleteStoragePool(forced=False) with a live " + "volume should raise CloudstackAPIException"): + self._delete_pool(self.__class__.pool.id, forced=False) + + # Pool must still be in Maintenance (not deleted) + try: + remaining = list_storage_pools( + self.apiClient, id=self.__class__.pool.id) + except CloudstackAPIException: + remaining = None + self.assertTrue( + remaining, + "Pool was deleted even though forced=False delete should have failed" + ) + + # Volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume was deleted even though pool deletion was rejected" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol should still exist after rejected pool deletion" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after rejected deletion" + ) + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_07_force_delete_pool_and_cleanup(self): + """ + Explicit cleanup after the test_06 negative test. + + The pool is in Maintenance with a CS volume still present. + Cleanup sequence: + 1. Try cancelStorageMaintenance. + 2. If still in Maintenance: try updateStoragePool(enabled=True) as a + fallback exit path (works on KVM/NFS3 even when cancel fails). + 3. Once pool exits Maintenance: delete volume, re-enter Maintenance. + 4. Force-delete the pool. + 5. Verify CS pool is gone. + 6. Verify ONTAP FlexVol and export policy are removed. + If the CS force-delete fails (volume couldn't be removed), the + ONTAP FlexVol and export policy are cleaned up directly via REST + so the storage array is never left with orphans. The CS pool + record is left for tearDownClass in that edge case only. + """ + pool = self.__class__.pool + vol = self.__class__.volume + self.assertIsNotNone(pool, "No pool from test_06 to clean up") + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + # Step 1: Try cancelStorageMaintenance + pool_state = "Maintenance" + try: + cm = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cm.id = pool.id + self.apiClient.cancelStorageMaintenance(cm) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(self.apiClient, id=pool.id) + if ps and ps[0].state != "Maintenance": + pool_state = ps[0].state + break + time.sleep(5) + except Exception: + pass # falls through to step 2 + + # Step 2: If still in Maintenance, try updateStoragePool(enabled=True). + # On KVM/NFS3 this succeeds in moving the pool to Disabled/Up even + # when cancelStorageMaintenance fails. + if pool_state == "Maintenance": + try: + ec = updateStoragePoolAPI.updateStoragePoolCmd() + ec.id = pool.id + ec.enabled = True + self.apiClient.updateStoragePool(ec) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(self.apiClient, id=pool.id) + if ps and ps[0].state != "Maintenance": + pool_state = ps[0].state + break + time.sleep(5) + except Exception: + pass + + # Step 3: If pool exited Maintenance, delete the CS volume and + # re-enter Maintenance so the pool can be force-deleted. + if pool_state != "Maintenance" and vol is not None: + if self._volume_exists_in_cs(vol.id): + try: + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.__class__.volume = None + vol = None + except Exception: + pass + else: + self.__class__.volume = None + vol = None + try: + mc = enableStorageMaintenance.enableStorageMaintenanceCmd() + mc.id = pool.id + self.apiClient.enableStorageMaintenance(mc) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(self.apiClient, id=pool.id) + if ps and ps[0].state == "Maintenance": + pool_state = "Maintenance" + break + time.sleep(5) + except Exception: + pass + + # Step 4: Force-delete the CS pool. + cs_pool_deleted = False + if vol is None or not self._volume_exists_in_cs(vol.id): + # Volume is gone — safe to force-delete + self.__class__.volume = None + vol = None + try: + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + cs_pool_deleted = True + except CloudstackAPIException: + pass + + # Step 5: Assert CS pool is gone (only when deletion was attempted) + if cs_pool_deleted: + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' should have been deleted with forced=True" % pool_name + ) + + # Step 6: ONTAP FlexVol must be gone. + # If the CS pool could not be deleted (volume still present — an + # NFS3/KVM platform edge case), delete the ONTAP FlexVol and export + # policy directly via REST so the storage array is always clean. + # The orphaned CS pool record is left for tearDownClass. + ontap_vol = self.ontap.get_volume(pool_name) + if ontap_vol is not None: + self.ontap.delete_volume(pool_name) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' should be gone after cleanup" % pool_name + ) + + policy = self.ontap.get_export_policy(ep_name) + if policy is not None: + self.ontap.delete_export_policy(ep_name) + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "NFS export policy '%s' should be removed after cleanup" % ep_name + ) diff --git a/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py new file mode 100644 index 000000000000..4509ce41cb22 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py @@ -0,0 +1,439 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Zone-scoped primary storage lifecycle tests for NetApp ONTAP (NFS3). + +Creates a zone-scoped pool (scope=ZONE, no clusterid/podid). CloudStack calls +OntapPrimaryDatastoreLifecycle.attachZone(), which connects all eligible KVM +hosts in the zone to the pool and creates an NFS export policy covering their +IPs. + +Workflow: + 01 Create zone-scoped NFS3 pool — pool.state Up; ONTAP FlexVol online; + export policy has all cluster host IPs + 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 04 Delete zone-scoped pool — pool gone; FlexVol deleted; export policy deleted + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM hosts registered in the zone + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values (protocol=NFS3) + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_zone_scoped_pool.py -v + +Note: Tests 01-04 share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapZoneScopedPool") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-zone@test.com", + "firstname": "ONTAP", + "lastname": "Zone", + "username": "ontap_zone_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapZoneNFS3_%d" % random.randint(0, 9999), + TestData.scope: "ZONE", + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapZoneScopedPool(OntapTestBase): + + # ---- zone-pool-specific shared state -------------------------------- + pool_ep_name = None + cluster_host_ips = None + + _vol_name_prefix = "OntapZoneVol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapZoneScopedPool, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Collect host IPs for export policy assertions + cls.cluster_host_ips = [ + h.ipaddress for h in cls.cluster_hosts + if getattr(h, "ipaddress", None) + ] + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_zone_pool(self): + """Create a zone-scoped NFS3 pool (no clusterid / podid).""" + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapZoneNFS3_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + # Intentionally omit clusterid and podid — zone-scoped pool + cmd.scope = "ZONE" + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _assert_export_policy_has_host_ips(self, ep_name): + """Assert export policy exists and contains each cluster host IP.""" + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP" % ep_name + ) + if not self.cluster_host_ips: + return + all_clients = [] + for rule in policy.get("rules", []): + for client in rule.get("clients", []): + all_clients.append(client.get("match", "")) + for ip in self.cluster_host_ips: + self.assertTrue( + any(ip in c for c in all_clients), + "Host IP '%s' not found in export policy '%s' rules: %s" + % (ip, ep_name, all_clients) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_01_create_zone_scoped_pool(self): + """ + Create a zone-scoped NFS3 primary storage pool (no clusterid/podid). + CloudStack calls attachZone(), which connects all eligible KVM hosts + in the zone and creates an NFS export policy. + Verifies: + - pool.state is Up + - ONTAP: FlexVol is online + - ONTAP: export policy exists and contains cluster host IPs + - ONTAP: at least one NFS data LIF is present on the SVM + """ + pool = self._create_zone_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist with cluster host IPs + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + self._assert_export_policy_has_host_ips(ep_name) + + # ONTAP: at least one NFS data LIF must be present + lifs = self.ontap.get_data_lifs(self.svm_name) + self.assertTrue( + len(lifs) > 0, + "No NFS data LIFs found on SVM '%s'" % self.svm_name + ) + + # ------------------------------------------------------------------ + # Step 02 — Disable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_02_disable_zone_scoped_pool(self): + """ + Disable the zone-scoped pool. + Verifies: + - pool.state is Disabled + - ONTAP: FlexVol still online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable" + ) + + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Enable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_03_enable_zone_scoped_pool(self): + """ + Re-enable the zone-scoped pool. + Verifies: + - pool.state is Up + - ONTAP: FlexVol still online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable" + ) + + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Delete zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_04_delete_zone_scoped_pool(self): + """ + Enter maintenance then delete the zone-scoped pool. + Verifies: + - Pool is removed from CloudStack + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + # Unmount the NFS on each KVM host BEFORE deleteStoragePool removes + # the ONTAP export. Without this, the mount becomes stale and + # KVMHAMonitor will fail its heartbeat 5 times then reboot the host + # via `echo b > /proc/sysrq-trigger`. + self._cleanup_kvm_storage_pool_mounts(pool.id) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) + + # ------------------------------------------------------------------ + # Class-level teardown + # ------------------------------------------------------------------ + + @classmethod + def tearDownClass(cls): + """ + Clean up any lingering zone-scoped pool NFS mounts on KVM hosts + before the base-class teardown deletes the ONTAP FlexVol. Without + this, a failed test_04 leaves a stale NFS mount that will cause + KVMHAMonitor to reboot the host. + """ + for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: + try: + cls._cleanup_kvm_storage_pool_mounts(pool.id) + except Exception as e: + logger.warning( + "tearDownClass: KVM NFS cleanup failed for pool %s: %s" + % (pool.id, e) + ) + super(TestOntapZoneScopedPool, cls).tearDownClass() diff --git a/test/integration/plugins/ontap/nfs3/volume/__init__.py b/test/integration/plugins/ontap/nfs3/volume/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/volume/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py b/test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py new file mode 100644 index 000000000000..981ea5156cbf --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py @@ -0,0 +1,470 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP NFS3 data volume +lifecycle (volume create / delete / negative-delete / force-delete). + +For NFS3, a CloudStack data volume is a metadata record only — no new ONTAP +object is created per volume (the pool's single FlexVol serves all volumes). +Volume deletion likewise removes the CloudStack record while leaving the +FlexVol intact. + +Tests are numbered test_01 ... test_05 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create NFS3 primary storage pool and allocate a CloudStack data volume + 02 Delete the volume — CloudStack record removed; FlexVol stays online + 03 Recreate volume — CS record back; FlexVol stays online (setup for 04-05) + 04 Put pool in Maintenance; attempt forced=False deleteStoragePool — must be + rejected because volumes exist; pool stays in Maintenance + 05 Delete volume from Maintenance; forced=True deleteStoragePool — FlexVol + and export policy are removed from ONTAP + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has NFS configured + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/nfs3/volume/ -v + +Note: Tests share class-level state (sequential). Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapNFS3VolumeLifecycle") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-vol@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-Vol", + "username": "ontap_nfs3_vol_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3Vol_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "NFS3", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3VolumeLifecycle(OntapTestBase): + + _vol_name_prefix = "OntapNFS3Vol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapNFS3VolumeLifecycle, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3Vol_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + # ------------------------------------------------------------------ + # Step 01 - Create pool (infrastructure) and allocate a volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create a new NFS3 pool and allocate a CloudStack data volume on it. + For NFS3, volume creation is a CloudStack metadata record only — no + new ONTAP object is created (the pool's FlexVol serves all volumes). + Verifies: + - pool.state is Up + - createVolume returns a non-None volume object + - ONTAP: FlexVol remains online after volume allocation + - ONTAP: export policy still present + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: FlexVol must remain online after volume allocation + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume creation" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still be present + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should exist after volume creation" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 02 - Delete volume; FlexVol must remain untouched + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_02_delete_volume(self): + """ + Delete the volume created in test_01. + For NFS3, volume deletion removes only the CloudStack record. + Verifies: + - deleteVolume completes without error + - ONTAP: FlexVol is still online (unaffected by volume deletion) + - ONTAP: export policy still present + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_01 must pass first") + + pool = self.__class__.pool + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after volume deletion" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion, " + "got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still be present + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after volume deletion" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 - Recreate volume for negative delete tests + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_03_recreate_volume_for_delete_tests(self): + """ + Recreate a volume on the existing pool (setup for tests 04-05). + Verifies: + - volume created successfully + - ONTAP: FlexVol still online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + vol = self._create_volume(self.__class__.pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume re-creation" + % self.__class__.pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after volume re-creation" + ) + + # ------------------------------------------------------------------ + # Step 04 - Forced=False delete with live volume must fail + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_04_forced_false_delete_with_volume_fails(self): + """ + Put pool in Maintenance then attempt deleteStoragePool(forced=False). + With a live volume present CloudStack must reject the request. + Verifies: + - Exception is raised (CloudStack rejects the delete) + - Pool is still listed in CloudStack (in Maintenance state) + - ONTAP: FlexVol still exists and is online + - ONTAP: export policy still present + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + # Enter maintenance mode + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + # Attempt forced=False delete — must raise exception because volumes exist + with self.assertRaises(Exception): + self._delete_pool(pool.id, forced=False) + + # Pool must still be listed in CloudStack + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + listed, + "Pool should still exist in CloudStack after failed forced=False delete" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after failed delete" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still be present + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after failed delete" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 05 - Delete volume then force-delete pool from Maintenance + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_05_delete_volume_and_force_delete_pool(self): + """ + Delete the live volume then force-delete the pool while it is still + in Maintenance state (pool is in Maintenance from test_04). + Verifies: + - Volume can be deleted while pool is in Maintenance + - Pool is removed from CloudStack using forced=True from Maintenance + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone( + self.__class__.pool, + "Pool absent - test_04 must not have cleaned up the pool" + ) + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + # Delete the volume first (pool is in Maintenance — volume deletion is + # allowed). For NFS3 a forced=False delete attempt in test_04 may have + # already destroyed the libvirt NFS pool representation on the host; + # if so deleteVolume raises "Storage pool not found". The CS metadata + # record will be cleaned up by the subsequent force-delete of the pool, + # so we treat that specific error as a no-op here. + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + except Exception as exc: + if "Storage pool not found" in str(exc) or "storage pool" in str(exc).lower(): + logger.warning( + "deleteVolume raised expected NFS3 libvirt pool-not-found " + "error; proceeding to force-delete pool: %s", exc + ) + else: + raise + self.__class__.volume = None + + # Force-delete the pool from Maintenance (no live volumes remaining) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after force deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after force deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) diff --git a/test/integration/plugins/ontap/ontap.cfg b/test/integration/plugins/ontap/ontap.cfg new file mode 100644 index 000000000000..4bf9d17dc499 --- /dev/null +++ b/test/integration/plugins/ontap/ontap.cfg @@ -0,0 +1,79 @@ +{ + "zones": [ + { + "name": "Zone-ONTAP", + "dns1": "8.8.8.8", + "internal_dns1": "8.8.8.8", + "guestcidraddress": "10.1.1.0/24", + "physical_networks": [ + { + "broadcastdomainrange": "Zone", + "name": "physical_network", + "traffictypes": [ + {"typ": "Guest"}, + {"typ": "Management"}, + {"typ": "Storage"} + ], + "providers": [ + { + "broadcastdomainrange": "ZONE", + "name": "VirtualRouter" + } + ] + } + ], + "pods": [ + { + "name": "Pod-ONTAP", + "gateway": "10.193.56.1", + "startip": "10.193.56.10", + "endip": "10.193.56.50", + "netmask": "255.255.255.128", + "clusters": [ + { + "clustername": "KVM-Cluster-ONTAP", + "hypervisor": "KVM", + "clustertype": "CloudManaged", + "hosts": [ + { + "url": "http://10.193.56.65", + "username": "root", + "password": "netapp1!" + } + ], + "primaryStorages": [] + } + ] + } + ] + } + ], + "dbSvr": { + "dbSvr": "10.193.56.65", + "passwd": "", + "db": "cloud", + "port": 3306, + "user": "root" + }, + "logger": { + "LogFolderPath": "/tmp/" + }, + "mgtSvr": [ + { + "mgtSvrIp": "10.193.56.65", + "port": 8096, + "user": "admin", + "passwd": "password", + "hypervisor": "kvm" + } + ], + "ontap": { + "storageIP": "10.196.38.187", + "svmName": "vs0", + "username": "admin", + "password": "netapp1!" + }, + "TestData": { + "Path": "test/integration/plugins/ontap/ontap.cfg" + } +} diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py new file mode 100644 index 000000000000..e40dc22283e5 --- /dev/null +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -0,0 +1,496 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Shared base class and helper utilities for NetApp ONTAP Marvin integration tests. + +Provides: + OntapRestClient - thin wrapper around the ONTAP REST API (NFS + iSCSI methods) + _parse_pool_details - converts a StoragePool details attribute to a plain dict + OntapTestBase - base cloudstackTestCase with common tearDownClass, + _poll_pool_state, _create_volume, and _delete_pool +""" + +import logging +import random +import requests +import time +import urllib3 +from urllib.parse import urlparse + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createVolume as createVolumeAPI, + deleteStoragePool as deleteStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + listDiskOfferings as listDiskOfferingsAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackAPI import listHosts as listHostsAPI +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.base import Account, DiskOffering +from marvin.sshClient import SshClient +from marvin.lib.common import get_domain, get_zone, list_clusters, list_storage_pools +from marvin.lib.utils import cleanup_resources + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +logger = logging.getLogger("OntapTestBase") + + +# --------------------------------------------------------------------------- +# Pool detail helper +# --------------------------------------------------------------------------- + +def _parse_pool_details(pool): + """ + Convert a StoragePool object's ``details`` attribute to a plain Python dict, + regardless of how Marvin chose to represent it. + + Note: listStoragePools only returns a subset of detail keys + (volumeUUID, exportPolicyName, exportPolicyId). For the full set + use the pool object returned directly by createStoragePool. + """ + details_raw = getattr(pool, "details", None) + if not details_raw: + return {} + if isinstance(details_raw, dict): + return details_raw + if isinstance(details_raw, list): + return {d.name: d.value for d in details_raw} + return { + k: v for k, v in vars(details_raw).items() + if not k.startswith("_") and k != "typeInfo" + } + + +# --------------------------------------------------------------------------- +# ONTAP REST helper +# --------------------------------------------------------------------------- + +class OntapRestClient: + """Thin wrapper around the ONTAP REST API for backend validation.""" + + def __init__(self, storage_ip, username, password, port=443): + self._base = "https://%s:%d/api" % (storage_ip, port) + self._auth = (username, password) + + def _get(self, path, params=None): + url = self._base + path + resp = requests.get(url, auth=self._auth, params=params, + verify=False, timeout=30) + resp.raise_for_status() + return resp.json() + + def _delete(self, path, params=None): + url = self._base + path + resp = requests.delete(url, auth=self._auth, params=params, + verify=False, timeout=30) + resp.raise_for_status() + + def delete_volume(self, name): + """Delete the ONTAP FlexVol with the given name. No-op if not found.""" + data = self._get("/storage/volumes", params={"name": name}) + records = data.get("records", []) + if not records: + return + uuid = records[0].get("uuid") + if uuid: + self._delete("/storage/volumes/%s" % uuid) + + def delete_export_policy(self, name): + """Delete the NFS export policy with the given name. No-op if not found.""" + data = self._get("/protocols/nfs/export-policies", params={"name": name}) + records = data.get("records", []) + if not records: + return + policy_id = records[0].get("id") + if policy_id: + self._delete("/protocols/nfs/export-policies/%s" % policy_id) + + def get_volume(self, name): + """Return the ONTAP FlexVol record for the given name, or None.""" + data = self._get("/storage/volumes", params={"name": name}) + records = data.get("records", []) + if not records: + return None + uuid = records[0].get("uuid") + if uuid: + return self._get("/storage/volumes/%s" % uuid, + params={"fields": "name,uuid,state,space"}) + return records[0] + + # -- NFS helpers --------------------------------------------------------- + + def get_export_policy(self, name): + """Return the ONTAP NFS export policy record for the given name, or None.""" + data = self._get("/protocols/nfs/export-policies", params={"name": name}) + records = data.get("records", []) + if not records: + return None + policy_id = records[0].get("id") + if policy_id: + return self._get( + "/protocols/nfs/export-policies/%s" % policy_id, + params={"fields": "name,svm,rules"} + ) + return records[0] + + def get_data_lifs(self, svm_name): + """Return a list of NFS data LIF IP addresses for the given SVM.""" + data = self._get( + "/network/ip/interfaces", + params={"svm.name": svm_name, "services": "data-nfs", + "fields": "ip,name"} + ) + records = data.get("records", []) + return [r.get("ip", {}).get("address") + for r in records if r.get("ip", {}).get("address")] + + # -- iSCSI helpers ------------------------------------------------------- + + def get_igroup(self, svm_name, igroup_name): + """Return the ONTAP igroup record, or None if not found.""" + data = self._get("/protocols/san/igroups", + params={"svm.name": svm_name, "name": igroup_name, + "fields": "name,uuid,initiators"}) + records = data.get("records", []) + return records[0] if records else None + + def get_lun(self, svm_name, lun_path): + """Return the ONTAP LUN record for the given full path, or None.""" + data = self._get("/storage/luns", + params={"svm.name": svm_name, "name": lun_path, + "fields": "name,uuid,enabled,status"}) + records = data.get("records", []) + return records[0] if records else None + + def list_luns_in_volume(self, svm_name, vol_name): + """Return all LUN records whose path starts with /vol/{vol_name}/.""" + prefix = "/vol/%s/" % vol_name + data = self._get("/storage/luns", + params={"svm.name": svm_name, + "fields": "name,uuid,enabled,status"}) + return [r for r in data.get("records", []) + if r.get("name", "").startswith(prefix)] + + def list_lun_maps_for_volume(self, svm_name, vol_name): + """Return all LUN-map records for LUNs residing in the given FlexVol.""" + prefix = "/vol/%s/" % vol_name + data = self._get("/protocols/san/lun-maps", + params={"svm.name": svm_name, + "fields": "lun.name,igroup.name"}) + return [r for r in data.get("records", []) + if r.get("lun", {}).get("name", "").startswith(prefix)] + + +# --------------------------------------------------------------------------- +# Base test class +# --------------------------------------------------------------------------- + +class OntapTestBase(cloudstackTestCase): + """ + Shared base for sequential ONTAP primary-storage workflow tests. + + Subclasses must: + - Set ``_vol_name_prefix`` to distinguish volume names per protocol. + - Define ``setUpClass`` that builds ``cls.testdata``, creates + ``cls.ontap`` and ``cls.svm_name``, then calls + ``cls._setup_cloudstack_resources(config, account_testdata)``. + - Define ``_create_pool`` (protocol-specific URL scheme and name). + """ + + # ---- shared state (set/cleared by individual tests) ---------------- + pool = None + volume = None + pool2 = None + volume2 = None + disk_offering_id = None + svm_name = None + cluster_hosts = None + kvm_hosts_ssh_creds = [] # [{'host': '10.x.x.x', 'user': 'root', 'password': '...'}] + ontap = None + testdata = None + zone = None + cluster = None + domain = None + account = None + _cleanup = [] + + # Subclass sets this to distinguish volume names, e.g. "OntapNFS3Vol" + _vol_name_prefix = "OntapVol" + + # ---- shared setup helper ------------------------------------------- + + @classmethod + def _setup_cloudstack_resources(cls, config, account_testdata): + """ + Resolve zone, cluster, domain, account, cluster hosts, and disk + offering from the Marvin config. Call this from subclass setUpClass + after ``cls.ontap`` and ``cls.svm_name`` have been assigned. + """ + cs_cfg = config.get("cloudstack", {}) + zone_name = cs_cfg.get("zoneName", None) + cluster_name = cs_cfg.get("clusterName", None) + domain_name = cs_cfg.get("domainName", "ROOT") + + cls.zone = get_zone(cls.apiClient, zone_name=zone_name) + clusters = (list_clusters(cls.apiClient, name=cluster_name) + if cluster_name else list_clusters(cls.apiClient)) + cls.cluster = clusters[0] + cls.domain = get_domain(cls.apiClient, domain_name=domain_name) + + cls.account = Account.create(cls.apiClient, account_testdata, admin=1) + cls._cleanup = [cls.account] + + list_hosts_cmd = listHostsAPI.listHostsCmd() + list_hosts_cmd.clusterid = cls.cluster.id + list_hosts_cmd.type = "Routing" + cls.cluster_hosts = cls.apiClient.listHosts(list_hosts_cmd) or [] + + list_do_cmd = listDiskOfferingsAPI.listDiskOfferingsCmd() + list_do_cmd.domainid = cls.domain.id + offerings = cls.apiClient.listDiskOfferings(list_do_cmd) + if offerings: + cls.disk_offering_id = offerings[0].id + else: + # No disk offerings exist yet — create a minimal one for tests + do = DiskOffering.create( + cls.apiClient, + {"name": "ontap-test-do", "displaytext": "ONTAP test disk offering", "disksize": 2}, + ) + cls._cleanup.append(do) + cls.disk_offering_id = do.id + + # Parse KVM host SSH credentials from zones/pods/clusters/hosts config. + # Used by _cleanup_kvm_storage_pool_mounts to unmount stale NFS pools. + cls.kvm_hosts_ssh_creds = [] + try: + for zone in config.get("zones", []): + for pod in zone.get("pods", []): + for cluster in pod.get("clusters", []): + for host_cfg in cluster.get("hosts", []): + host_ip = urlparse( + host_cfg.get("url", "") + ).hostname or "" + if host_ip: + cls.kvm_hosts_ssh_creds.append({ + "host": host_ip, + "user": host_cfg.get("username", "root"), + "password": host_cfg.get("password", ""), + }) + except Exception as parse_ex: + logger.warning( + "_setup_cloudstack_resources: could not parse KVM SSH creds: %s" + % parse_ex + ) + + # ---- KVM storage cleanup helper ------------------------------------ + + @classmethod + def _cleanup_kvm_storage_pool_mounts(cls, pool_uuid): + """ + SSH to each KVM host and unmount the NFS storage pool mount for + *pool_uuid*, then destroy and undefine the libvirt storage pool. + + Must be called BEFORE the ONTAP FlexVol is deleted (i.e., before + deleteStoragePool) so that the unmount completes while the NFS + export is still reachable. Prevents stale NFS mounts from + triggering KVMHAMonitor heartbeat failures that reboot the host + via ``echo b > /proc/sysrq-trigger``. + """ + for creds in cls.kvm_hosts_ssh_creds: + host_ip = creds["host"] + try: + ssh = SshClient( + host_ip, 22, + creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0, + ) + for cmd in [ + "umount -f -l /mnt/{u} 2>/dev/null; true".format( + u=pool_uuid), + "virsh pool-destroy {u} 2>/dev/null; true".format( + u=pool_uuid), + "virsh pool-undefine {u} 2>/dev/null; true".format( + u=pool_uuid), + ]: + try: + ssh.execute(cmd) + except Exception as cmd_ex: + logger.warning( + "_cleanup_kvm_storage_pool_mounts: cmd '%s' " + "failed on %s: %s" % (cmd, host_ip, cmd_ex) + ) + except Exception as ex: + logger.warning( + "_cleanup_kvm_storage_pool_mounts: SSH to %s failed: %s" + % (host_ip, ex) + ) + + # ---- shared teardown ----------------------------------------------- + + @classmethod + def tearDownClass(cls): + """Best-effort cleanup of any resources left behind by a failed run.""" + for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: + try: + # Step 1: Check current pool state + pools = list_storage_pools(cls.apiClient, id=pool.id) + if not pools: + continue # already deleted + pool_state = pools[0].state + + # Step 2: If in Maintenance, attempt to exit it + if pool_state == "Maintenance": + try: + cc = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cc.id = pool.id + cls.apiClient.cancelStorageMaintenance(cc) + time.sleep(5) + except Exception: + pass + try: + ec = updateStoragePoolAPI.updateStoragePoolCmd() + ec.id = pool.id + ec.enabled = True + cls.apiClient.updateStoragePool(ec) + time.sleep(3) + except Exception: + pass + pools = list_storage_pools(cls.apiClient, id=pool.id) + if pools: + pool_state = pools[0].state + + # Step 3: Delete volumes — always attempt regardless of pool + # state. For iSCSI this works even in Maintenance; for NFS3/KVM + # it may fail with NPE ("storagePoolInformation is null") when + # pool is in Maintenance — that exception is caught below. + for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + cls.apiClient.deleteVolume(cmd) + except Exception as ve: + logger.warning( + "tearDownClass: could not delete volume %s: %s" + % (vol.id, ve)) + + # Re-enter Maintenance only if pool was Up/Disabled (avoid + # double-entering when cancel maintenance above already left it + # in Maintenance) + if pool_state in ("Up", "Disabled"): + try: + mc = enableStorageMaintenance.enableStorageMaintenanceCmd() + mc.id = pool.id + cls.apiClient.enableStorageMaintenance(mc) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(cls.apiClient, id=pool.id) + if ps and ps[0].state == "Maintenance": + break + time.sleep(5) + except Exception: + pass + + # Step 4: Force-delete the pool + dc = deleteStoragePoolAPI.deleteStoragePoolCmd() + dc.id = pool.id + dc.forced = True + cls.apiClient.deleteStoragePool(dc) + except Exception as e: + logger.warning("tearDownClass: could not delete pool %s: %s" + % (pool.id, e)) + # Last resort: delete ONTAP FlexVol and export policy directly + # so that ONTAP is never left with orphaned volumes even when + # the CloudStack pool record cannot be removed. + if hasattr(cls, "ontap") and cls.ontap is not None: + try: + cls.ontap.delete_volume(pool.name) + logger.warning( + "tearDownClass: deleted ONTAP FlexVol '%s' directly" + % pool.name) + except Exception as oe: + logger.warning( + "tearDownClass: ONTAP direct volume delete '%s' " + "failed: %s" % (pool.name, oe)) + try: + # For NFS3 pools also remove the export policy + ep_name = getattr(cls, "pool_ep_name", None) + if ep_name is None: + ep_name = "cs-%s-%s" % ( + getattr(cls, "svm_name", ""), pool.name) + cls.ontap.delete_export_policy(ep_name) + logger.warning( + "tearDownClass: deleted export policy '%s' directly" + % ep_name) + except Exception: + pass + + # Clean up volumes that may not have been handled with pool teardown + for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + cls.apiClient.deleteVolume(cmd) + except Exception as e: + logger.warning("tearDownClass: could not delete volume %s: %s" + % (vol.id, e)) + + try: + cleanup_resources(cls.apiClient, cls._cleanup) + except Exception as e: + logger.debug("tearDownClass cleanup_resources: %s" % e) + + # No per-test tearDown — state intentionally persists between steps. + + # ---- shared helpers ------------------------------------------------ + + def _poll_pool_state(self, pool_id, target_state, timeout=120, interval=5): + """Poll listStoragePools until the pool reaches target_state or timeout.""" + deadline = time.time() + timeout + current_state = "unknown" + while time.time() < deadline: + pools = list_storage_pools(self.apiClient, id=pool_id) + if pools: + current_state = pools[0].state + if current_state == target_state: + return pools[0] + time.sleep(interval) + self.fail( + "Pool %s did not reach state '%s' within %ds (last: '%s')" + % (pool_id, target_state, timeout, current_state) + ) + + def _create_volume(self, pool_id): + """Create a data volume on the given pool; uses _vol_name_prefix.""" + cmd = createVolumeAPI.createVolumeCmd() + cmd.name = "%s_%d" % (self._vol_name_prefix, random.randint(0, 99999)) + cmd.diskofferingid = self.disk_offering_id + cmd.zoneid = self.zone.id + cmd.storageid = pool_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + return self.apiClient.createVolume(cmd) + + def _delete_pool(self, pool_id, forced=False): + """Issue deleteStoragePool for the given pool id.""" + cmd = deleteStoragePoolAPI.deleteStoragePoolCmd() + cmd.id = pool_id + if forced: + cmd.forced = True + self.apiClient.deleteStoragePool(cmd) diff --git a/test/integration/plugins/ontap/probe_test.py b/test/integration/plugins/ontap/probe_test.py new file mode 100644 index 000000000000..8ed1ca03da29 --- /dev/null +++ b/test/integration/plugins/ontap/probe_test.py @@ -0,0 +1,35 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +import json +from marvin.cloudstackAPI import listTemplates, listServiceOfferings, listNetworks +from marvin.cloudstackTestCase import cloudstackTestCase + +class ProbeResources(cloudstackTestCase): + @classmethod + def setUpClass(cls): + tc = super(ProbeResources, cls).getClsTestClient() + cls.api = tc.getApiClient() + + def test_01_probe(self): + out = {} + cmd = listTemplates.listTemplatesCmd() + cmd.templatefilter = "executable" + resp = self.api.listTemplates(cmd) + out["templates"] = [{"id": t.id, "name": t.name, "hypervisor": getattr(t,"hypervisor","?"), "status": getattr(t,"status","?")} for t in (resp or [])] + cmd2 = listServiceOfferings.listServiceOfferingsCmd() + resp2 = self.api.listServiceOfferings(cmd2) + out["offerings"] = [{"id": s.id, "name": s.name, "cpu": getattr(s,"cpunumber","?"), "mem": getattr(s,"memory","?")} for s in (resp2 or [])] + cmd3 = listNetworks.listNetworksCmd() + cmd3.listall = True + resp3 = self.api.listNetworks(cmd3) + out["networks"] = [{"id": n.id, "name": n.name, "type": getattr(n,"type","?"), "state": getattr(n,"state","?")} for n in (resp3 or [])] + with open("/tmp/cs_probe_out.json","w") as f: + json.dump(out, f, indent=2) + self.assertTrue(True) diff --git a/test/integration/plugins/ontap/run_tests.sh b/test/integration/plugins/ontap/run_tests.sh new file mode 100644 index 000000000000..d6d561d14cb0 --- /dev/null +++ b/test/integration/plugins/ontap/run_tests.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Run the full ONTAP Marvin integration test suite by tag. +# Each test file is run individually so sequential test state is preserved. +# +# Usage (from cloudstack root): +# bash test/integration/plugins/ontap/run_tests.sh +# +# Optional: limit to a specific group by passing the tag as an argument: +# bash test/integration/plugins/ontap/run_tests.sh nfs3_workflow + +CFG=test/integration/plugins/ontap/ontap.cfg +export PYTHONPATH=test/integration/plugins/ontap:${PYTHONPATH:-} +FILTER="${1:-all}" + +PASS=0 +FAIL=0 + +run_group() { + local label="$1" + local tag="$2" + local file="$3" + + if [[ "$FILTER" != "all" && "$FILTER" != "$tag" ]]; then + return + fi + + echo "" + echo "================================================================" + echo " ${label} (tag: ${tag})" + echo "================================================================" + + local out + out=$(python3 -m nose --with-marvin --marvin-config="$CFG" "$file" -a "tags=${tag}" -v 2>&1) + + # Resolve the log folder (handle /tmp -> /private/tmp symlink on macOS) + local log_folder + log_folder=$(echo "$out" | grep "Final results are now copied to" | sed 's/.*copied to: //; s/ ===.*//' | tr -d '[:space:]') + log_folder=$(python3 -c "import os; print(os.path.realpath('$log_folder'))" 2>/dev/null || echo "") + + if [[ -n "$log_folder" && -f "${log_folder}/results.txt" ]]; then + local suite_pass suite_fail + while IFS= read -r line; do + echo " $line" + done < <(grep "TestName.*Status" "${log_folder}/results.txt" | grep -v "^===") + suite_pass=$(grep -c "Status : SUCCESS" "${log_folder}/results.txt" 2>/dev/null | tr -d '[:space:]' || echo 0) + suite_fail=$(grep "Status : FAIL\|Status : EXCEPTION" "${log_folder}/results.txt" 2>/dev/null | wc -l | tr -d '[:space:]' || echo 0) + PASS=$((PASS + suite_pass)) + FAIL=$((FAIL + suite_fail)) + echo " -> ${suite_pass} passed, ${suite_fail} failed" + else + echo "$out" | grep -E "ERROR|Exception|failed" | head -5 + echo " [could not read results — log folder: ${log_folder:-not found}]" + FAIL=$((FAIL + 1)) + fi +} + +run_group "NFS3 pool lifecycle" nfs3_workflow test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py +run_group "NFS3 pool with volumes" nfs3_with_volumes test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py +run_group "NFS3 zone-scoped pool" zone_pool test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py +run_group "NFS3 volume lifecycle" nfs3_volume test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py +run_group "NFS3 VM volume attach" vm_volume_workflow test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py +run_group "iSCSI pool lifecycle" iscsi_workflow test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py +run_group "iSCSI pool with volumes" iscsi_with_volumes test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py +run_group "iSCSI zone-scoped pool" iscsi_zone_pool test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py +run_group "iSCSI volume lifecycle" iscsi_volume test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py +run_group "iSCSI VM volume workflow" iscsi_vm_workflow test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py + +echo "" +echo "================================================================" +echo " TOTAL: ${PASS} passed, ${FAIL} failed" +echo "================================================================" + +[[ "$FAIL" -eq 0 ]] diff --git a/test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py b/test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py new file mode 100644 index 000000000000..acdd4fe33ee8 --- /dev/null +++ b/test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py @@ -0,0 +1,710 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage pool. + +Tests are numbered test_01 ... test_11 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create primary storage pool + 02 Disable storage pool + 03 Enable storage pool + 04 Enter maintenance mode + 05 Cancel maintenance mode + 06 Create a data volume on the pool + 07 Enter maintenance mode (pool has a volume) + 08 Cancel maintenance mode (pool has a volume) + 09 Delete the data volume + 10 Enter maintenance mode and delete the storage pool + 11 Create a second pool, attach a volume, enter maintenance, + then force-delete the pool (volume still present) + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py -v +""" + +import base64 +import logging +import random + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + cancelStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIWorkflow") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-wf@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-WF", + "username": "ontap_iscsi_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSI_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + import re + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +def _lun_path(vol_name, lun_name): + """Mirror OntapStorageUtils.getLunName: /vol/{volName}/{lunName}""" + return "/vol/%s/%s" % (vol_name, lun_name) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIPrimaryStorageWorkflow(OntapTestBase): + + # ---- iSCSI-specific state (set/cleared by individual tests) -------- + _vol_name_prefix = "OntapISCSIVol" + lun_path = None # ONTAP LUN path of cls.volume + lun_path2 = None # ONTAP LUN path of cls.volume2 + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIPrimaryStorageWorkflow, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + scope = ontap_cfg.get("storagePoolScope", "CLUSTER") + provider = ontap_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = ontap_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = ontap_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSI_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + # ------------------------------------------------------------------ + # Step 01 - Create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_01_create_primary_storage_pool(self): + """ + Create an iSCSI primary storage pool and verify: + - CloudStack state is Up, type is Iscsi + - ONTAP: FlexVol exists and is online + - ONTAP: one igroup per cluster host exists with the correct IQN initiator + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: igroup must exist for each cluster host that has an IQN + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue # host not iSCSI-enabled; skip igroup check for it + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNotNone( + igroup, + "ONTAP igroup '%s' not found for host '%s'" % (igroup_name, host.name) + ) + initiator_names = [ + i.get("name", "") for i in igroup.get("initiators", []) + ] + self.assertIn( + iqn, initiator_names, + "Host IQN '%s' not in igroup '%s' initiators: %s" + % (iqn, igroup_name, initiator_names) + ) + + # ------------------------------------------------------------------ + # Step 02 - Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_02_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online (disable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + # ONTAP: disable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 03 - Enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_03_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online (enable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + # ONTAP: enable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 04 - Enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_04_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + # ONTAP: maintenance must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 05 - Cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_05_cancel_maintenance_mode(self): + """ + Cancel maintenance and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 06 - Create a data volume on the pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_06_create_volume(self): + """ + Allocate a data volume on the iSCSI pool and verify: + - CloudStack returns a volume id + - ONTAP: a LUN is created inside the FlexVol at /vol/{poolName}/{volName} + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + if not self.disk_offering_id: + self.skipTest("No disk offering available - skipping volume steps") + + try: + vol = self._create_volume(self.__class__.pool.id) + except Exception as e: + self.skipTest("createVolume failed (iSCSI may require an attached VM): %s" % e) + + self.__class__.volume = vol + vol_id = getattr(vol, "id", None) + self.assertIsNotNone(vol_id, "Volume creation returned no id") + + # ONTAP: a LUN must exist inside the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume creation" + % self.__class__.pool.name + ) + self.__class__.lun_path = luns[0].get("name") # cache for later steps + + # ------------------------------------------------------------------ + # Step 07 - Enter maintenance mode (pool has a volume) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_07_enter_maintenance_mode_with_volume(self): + """ + Enter maintenance mode while the pool holds a data volume and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol still online and LUN still present (maintenance + does not affect ONTAP data plane) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + # ONTAP: FlexVol and LUN must be untouched + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared during maintenance (with volume)") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' during maintenance, got '%s'" + % ontap_vol.get("state") + ) + if getattr(self.__class__, "lun_path", None): + lun = self.ontap.get_lun(self.svm_name, self.__class__.lun_path) + self.assertIsNotNone( + lun, + "ONTAP LUN '%s' disappeared during maintenance" % self.__class__.lun_path + ) + + # ------------------------------------------------------------------ + # Step 08 - Cancel maintenance mode (pool has a volume) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_08_cancel_maintenance_mode_with_volume(self): + """ + Cancel maintenance mode while the pool still holds the volume and verify: + - CloudStack reports Up + - ONTAP: FlexVol online and LUN still present + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance (with volume)") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + if getattr(self.__class__, "lun_path", None): + lun = self.ontap.get_lun(self.svm_name, self.__class__.lun_path) + self.assertIsNotNone( + lun, + "ONTAP LUN '%s' disappeared after cancel maintenance" % self.__class__.lun_path + ) + + # ------------------------------------------------------------------ + # Step 09 - Delete the volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_09_delete_volume(self): + """ + Delete the data volume and verify: + - ONTAP: the LUN is removed from the FlexVol + - ONTAP: the FlexVol itself is still online (only the LUN is gone) + """ + if self.__class__.volume is None: + self.skipTest("No volume from test_06 - skipping") + + vol_id = self.__class__.volume.id + lun_path = getattr(self.__class__, "lun_path", None) + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol_id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + self.__class__.lun_path = None + + logger.info("Volume %s deleted" % vol_id) + + # ONTAP: LUN must be gone + if lun_path: + lun = self.ontap.get_lun(self.svm_name, lun_path) + self.assertIsNone( + lun, + "ONTAP LUN '%s' still exists after volume deletion" % lun_path + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after volume deletion") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 10 - Enter maintenance mode and delete the storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_10_enter_maintenance_and_delete_pool(self): + """ + Enter maintenance mode then delete the pool. + Verifies the pool is removed from CloudStack and the backing ONTAP + FlexVol is deleted. + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + pool = self.__class__.pool + pool_name = pool.name + + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) + + # ------------------------------------------------------------------ + # Step 11 - Create pool + volume, enter maintenance, force-delete + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_11_create_pool_volume_maintenance_force_delete(self): + """ + Validates the forced=True behaviour of deleteStoragePool. + + CloudStack distinguishes two volume categories on a pool: + - non-destroyed (Allocated/Ready): active volumes + - destroyed (Destroy state) : soft-deleted, awaiting GC expunge + + forced=False → fails if ANY volume record exists on the pool (any state) + forced=True → fails only if non-destroyed volumes exist; + if only destroyed volumes remain CloudStack force-expunges + them and removes the pool. + + This test covers the two reliable halves of that contract: + + Step 1 Create pool + allocate a data volume (non-destroyed). + Step 2 Enter maintenance mode. + Step 3 forced=False delete MUST FAIL — non-destroyed volume present. + Step 4 Soft-delete the volume (deleteVolume API). + Step 5 forced=True delete MUST SUCCEED — handles any remaining state + (immediately-expunged or still Destroyed — both pass). + Step 6 Assert pool is gone from CloudStack and ONTAP. + + Note: The Destroyed-only scenario (forced=True succeeds where forced=False + would still fail) requires a VM lifecycle to produce Destroyed volumes and + is covered by higher-level system tests rather than this FT suite. + """ + if not self.disk_offering_id: + self.skipTest( + "No disk offering available; force-delete test requires a volume " + "to be present on the pool." + ) + + pool2 = self._create_pool() + self.__class__.pool2 = pool2 + self.assertEqual( + pool2.state, "Up", + "Pool2 state should be 'Up', got '%s'" % pool2.state + ) + + # Step 1: allocate a data volume — must succeed for this test to be valid + try: + vol2 = self._create_volume(pool2.id) + self.__class__.volume2 = vol2 + except Exception as e: + self.skipTest( + "createVolume failed (iSCSI may require an attached VM): %s" % e + ) + self.assertIsNotNone(getattr(vol2, "id", None), + "Volume2 creation returned no id") + + # ONTAP: LUN must be created in pool2's FlexVol + luns2 = self.ontap.list_luns_in_volume(self.svm_name, pool2.name) + self.assertTrue( + len(luns2) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume2 creation" % pool2.name + ) + self.__class__.lun_path2 = luns2[0].get("name") + + # Step 2: enter maintenance + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool2.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool2.id, "Maintenance", timeout=120) + + # ONTAP: LUN still present during maintenance + if self.__class__.lun_path2: + lun2 = self.ontap.get_lun(self.svm_name, self.__class__.lun_path2) + self.assertIsNotNone( + lun2, + "ONTAP LUN '%s' disappeared during maintenance (pool2)" % self.__class__.lun_path2 + ) + + # Step 3: forced=False must FAIL — active (non-destroyed) volume present + from marvin.cloudstackException import CloudstackAPIException + with self.assertRaises(CloudstackAPIException, + msg="deleteStoragePool (forced=False) should fail " + "when a non-destroyed volume is on the pool"): + self._delete_pool(pool2.id, forced=False) + + # Step 4: soft-delete the volume via the deleteVolume API + del_vol_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_vol_cmd.id = self.__class__.volume2.id + self.apiClient.deleteVolume(del_vol_cmd) + self.__class__.volume2 = None + + # ONTAP: LUN must be gone after deleteVolume + if self.__class__.lun_path2: + lun2 = self.ontap.get_lun(self.svm_name, self.__class__.lun_path2) + self.assertIsNone( + lun2, + "ONTAP LUN '%s' still exists after deleteVolume (pool2)" % self.__class__.lun_path2 + ) + self.__class__.lun_path2 = None + + # Step 5: forced=True must SUCCEED — handles any remaining volume state + self._delete_pool(pool2.id, forced=True) + self.__class__.pool2 = None + + # Step 6: assert CloudStack and ONTAP cleaned up + try: + remaining = list_storage_pools(self.apiClient, id=pool2.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool2 still listed in CloudStack after force-deletion") + + # ONTAP: FlexVol and igroups must be deleted + ontap_vol = self.ontap.get_volume(pool2.name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after force-deletion" % pool2.name + ) + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool2 force-deletion" % igroup_name + ) diff --git a/test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py b/test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py new file mode 100644 index 000000000000..adbc61579292 --- /dev/null +++ b/test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py @@ -0,0 +1,404 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. + +Tests are numbered test_01 ... test_04 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create primary storage pool + 02 Disable storage pool + 03 Enable storage pool + 04 Enter maintenance mode + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py -v + +Note: Tests 01-04 share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. +""" + +import base64 +import logging +import random + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapNFS3Workflow") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + DETAIL_VOLUME_UUID = "volumeUUID" + DETAIL_VOLUME_NAME = "volumeName" + DETAIL_DATA_LIF = "dataLIF" + DETAIL_NFS_MOUNT_OPTS = "nfsmountopts" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-wf@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-WF", + "username": "ontap_nfs3_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase): + + # ---- NFS3-specific shared state ------------------------------------ + pool_ep_name = None # NFS export policy name for pool + cluster_host_ips = None + + _vol_name_prefix = "OntapNFS3Vol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapNFS3PrimaryStorageWorkflow, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + protocol = ontap_cfg.get("protocol", "NFS3") + scope = ontap_cfg.get("storagePoolScope", "CLUSTER") + provider = ontap_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = ontap_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = ontap_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Resolve cluster host IPs for export policy rule assertions + cls.cluster_host_ips = [ + h.ipaddress for h in cls.cluster_hosts + if getattr(h, "ipaddress", None) + ] + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + # Fallback: plugin typically uses cs-{svmName}-{poolName} + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _assert_export_policy_has_host_ips(self, ep_name): + """Assert that the export policy exists and its rules include each cluster host IP.""" + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP" % ep_name + ) + if not self.cluster_host_ips: + return # no host IPs registered; skip rule-level check + all_clients = [] + for rule in policy.get("rules", []): + for client in rule.get("clients", []): + all_clients.append(client.get("match", "")) + for ip in self.cluster_host_ips: + self.assertTrue( + any(ip in c for c in all_clients), + "Host IP '%s' not found in export policy '%s' rules: %s" + % (ip, ep_name, all_clients) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_01_create_primary_storage_pool(self): + """ + Create an NFS3 primary storage pool and verify: + - CloudStack state is Up, type is NetworkFilesystem + - nfsmountopts contains 'vers=3' + - ONTAP: FlexVol exists and is online + - ONTAP: NFS export policy exists with cluster host IP rules + - ONTAP: at least one NFS data LIF is present on the SVM + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "NetworkFilesystem", + "Pool type should be 'NetworkFilesystem', got '%s'" % pool.type + ) + + # Verify nfsmountopts via listStoragePools + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone(listed, "listStoragePools returned None for pool %s" % pool.id) + nfs_opts = getattr(listed[0], "nfsmountopts", "") + self.assertIn( + "vers=3", nfs_opts, + "nfsmountopts should contain 'vers=3', got '%s'" % nfs_opts + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist with host IP rules + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + self._assert_export_policy_has_host_ips(ep_name) + + # ONTAP: at least one NFS data LIF must be present + lifs = self.ontap.get_data_lifs(self.svm_name) + self.assertTrue( + len(lifs) > 0, + "No NFS data LIFs found on SVM '%s'" % self.svm_name + ) + + # ------------------------------------------------------------------ + # Step 02 — Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_02_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_03_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_04_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online and export policy unchanged + (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist during maintenance" + % self.__class__.pool_ep_name + ) + + +