diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 93dee22e4df7..4cba045763ca 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -110,6 +110,7 @@ public class EventTypes { public static final String EVENT_VM_IMPORT = "VM.IMPORT"; public static final String EVENT_VM_UNMANAGE = "VM.UNMANAGE"; public static final String EVENT_VM_RECOVER = "VM.RECOVER"; + public static final String EVENT_VM_CLONE = "VM.CLONE"; // Domain Router public static final String EVENT_ROUTER_CREATE = "ROUTER.CREATE"; diff --git a/api/src/main/java/com/cloud/network/NetworkModel.java b/api/src/main/java/com/cloud/network/NetworkModel.java index 9fd4fcb98621..025a5f32446f 100644 --- a/api/src/main/java/com/cloud/network/NetworkModel.java +++ b/api/src/main/java/com/cloud/network/NetworkModel.java @@ -122,6 +122,10 @@ public interface NetworkModel { List listNetworksUsedByVm(long vmId, boolean isSystem); + default List listNetworksUsedByVm(long vmId) { + throw new UnsupportedOperationException(); + } + Nic getNicInNetwork(long vmId, long networkId); List getNicsForTraffic(long vmId, TrafficType type); diff --git a/api/src/main/java/com/cloud/storage/Snapshot.java b/api/src/main/java/com/cloud/storage/Snapshot.java index 5b25843f48b0..a3837c2dfa42 100644 --- a/api/src/main/java/com/cloud/storage/Snapshot.java +++ b/api/src/main/java/com/cloud/storage/Snapshot.java @@ -26,7 +26,8 @@ public interface Snapshot extends ControlledEntity, Identity, InternalIdentity, StateObject { public enum Type { - MANUAL, RECURRING, TEMPLATE, HOURLY, DAILY, WEEKLY, MONTHLY, GROUP; + MANUAL, RECURRING, TEMPLATE, HOURLY, DAILY, WEEKLY, MONTHLY, GROUP, INTERNAL; + // New types should be defined after INTERNAL, and change the max value private int max = 8; public void setMax(int max) { @@ -71,6 +72,7 @@ enum LocationType { } public static final long MANUAL_POLICY_ID = 0L; + public static final long INTERNAL_POLICY_ID = 8L; @Override long getAccountId(); diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 986326454b05..1ccb7bed1909 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -21,6 +21,7 @@ import java.net.MalformedURLException; import java.util.Map; +import com.cloud.exception.StorageUnavailableException; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ChangeOfferingForVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; @@ -101,6 +102,8 @@ public interface VolumeApiService { Volume detachVolumeViaDestroyVM(long vmId, long volumeId); + Volume cloneDataVolume(long vmId, long snapshotId, Volume volume) throws StorageUnavailableException; + Volume detachVolumeFromVM(DetachVolumeCmd cmd); Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) @@ -110,6 +113,8 @@ Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account acc Volume updateVolume(long volumeId, String path, String state, Long storageId, Boolean displayVolume, String customId, long owner, String chainInfo, String name); + Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId); + /** * Extracts the volume to a particular location. * diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index 5b494c308c3c..87a7655e9f84 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -20,6 +20,9 @@ import java.net.URISyntaxException; import java.util.List; +import com.cloud.storage.Snapshot; +import com.cloud.storage.VolumeApiService; +import com.cloud.uservm.UserVm; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; @@ -40,6 +43,7 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; import org.apache.cloudstack.api.response.GetUploadParamsResponse; @@ -102,6 +106,15 @@ public interface TemplateApiService { boolean updateTemplateOrIsoPermissions(BaseUpdateTemplateOrIsoPermissionsCmd cmd); + Snapshot createSnapshotFromTemplateOwner(long vmId, UserVm curVm, Account templateOwner, VolumeApiService volumeService) throws ResourceAllocationException; + + /** + * create a template record for later usage of creating a real template by createPrivateTemplate + * */ + VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner, VolumeApiService serviceObj, Snapshot snapshot) throws ResourceAllocationException; + + VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd, long snapshotId, long templateId) throws CloudRuntimeException; + VirtualMachineTemplate createPrivateTemplateRecord(CreateTemplateCmd cmd, Account templateOwner) throws ResourceAllocationException; VirtualMachineTemplate createPrivateTemplate(CreateTemplateCmd command) throws CloudRuntimeException; diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 258e87002dab..e8bb70591f84 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -18,11 +18,15 @@ import java.util.List; import java.util.Map; +import java.util.Optional; +import com.cloud.storage.VolumeApiService; +import com.cloud.storage.snapshot.SnapshotApiService; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; @@ -86,6 +90,18 @@ public interface UserVmService { */ UserVm destroyVm(long vmId, boolean expunge) throws ResourceUnavailableException, ConcurrentOperationException; + /** + * Clone a specific VM (full clone) + * + * @param cmd + * - the command specifying vmId to be cloned + * @return the VM if cloneVM operation is successful + * */ + Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService, SnapshotApiService snapshotService) throws ResourceUnavailableException, ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; + + void validateCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, ResourceAllocationException; + + void prepareCloneVirtualMachine(CloneVMCmd cmd) throws ResourceAllocationException, InsufficientCapacityException, ResourceUnavailableException; /** * Resets the password of a virtual machine. * @@ -432,6 +448,9 @@ UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serviceOffe UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, StorageUnavailableException, ResourceAllocationException; + UserVm recordVirtualMachineToDB(CloneVMCmd cmd, long templateId) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, + StorageUnavailableException, ResourceAllocationException; + UserVm getUserVm(long vmId); VirtualMachine getVm(long vmId); diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java index 60c2a183ad33..10fee857dfcf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java @@ -48,5 +48,4 @@ public String getCreateEventType() { public String getCreateEventDescription() { return null; } - } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java new file mode 100644 index 000000000000..33314ac7ac89 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.vm; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import java.util.Optional; + +@APICommand(name = "cloneVirtualMachine", responseObject = UserVmResponse.class, description = "clone a virtual VM", + responseView = ResponseObject.ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, entityType = {VirtualMachine.class}, since="4.16.0") +public class CloneVMCmd extends BaseAsyncCreateCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(CloneVMCmd.class.getName()); + private static final String s_name = "clonevirtualmachineresponse"; + private static final String CLONE_IDENTIFIER = "Clone"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @ACL(accessType = AccessType.OperateEntry) + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType=UserVmResponse.class, + required = true, description = "The ID of the virtual machine") + private Long virtualmachineid; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "name of the cloned virtual machine") + private String name; + + //Owner information + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") + private String accountName; + + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used.") + private Long domainId; + + public String getAccountName() { + return accountName; + } + + public Long getDomainId() { + return domainId; + } + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public Long getId() { + return this.virtualmachineid; + } + @Override + public String getEventType() { + return EventTypes.EVENT_VM_CLONE; + } + + @Override + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.VirtualMachine; + } + + @Override + public String getEventDescription() { + return "Cloning user VM: " + this._uuidMgr.getUuid(VirtualMachine.class, getId()); + } + + @Override + public void create() throws ResourceAllocationException { + try { + _userVmService.validateCloneCondition(this); + _userVmService.prepareCloneVirtualMachine(this); + } + catch (ResourceUnavailableException | InsufficientCapacityException e) { + s_logger.warn("Exception: ", e); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); + } catch (InvalidParameterValueException e) { + s_logger.warn("Exception: ", e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } catch (ServerApiException e) { + throw new ServerApiException(e.getErrorCode(), e.getDescription()); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + public boolean isPublic() { + return false; + } + + public String getVMName() { + if (getName() == null) { + return getTargetVM().getInstanceName() + "-" + CLONE_IDENTIFIER; + } + return getName(); + } + + public String getTemplateName() { + return (getVMName() + "-" + _uuidMgr.generateUuid(VirtualMachineTemplate.class, null)).substring(0, 32); + } + + @Override + public void execute() { + Optional result; + try { + CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); + s_logger.info("starting actual VM id: " + getEntityId()); + result = _userVmService.cloneVirtualMachine(this, _volumeService, _snapshotService); + } catch (ResourceUnavailableException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); + } catch (ConcurrentOperationException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + catch (ResourceAllocationException | InsufficientCapacityException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage()); + } + result.ifPresentOrElse((userVm)-> { + UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", result.get()).get(0); + response.setResponseName("full_clone"); + setResponseObject(response); + }, ()-> { + throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, "failed to clone VM: " + getId()); + }); + } + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public long getEntityOwnerId() { + UserVm vm = this._responseGenerator.findUserVmById(getId()); + if (vm != null) { + return vm.getAccountId(); + } + return Account.ACCOUNT_ID_SYSTEM; + } + + public UserVm getTargetVM() { + return this._userVmService.getUserVm(getId()); + } +} diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 4d106f5c4f6a..a20399ee944c 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -221,8 +221,11 @@ public SnapshotResult takeSnapshot(SnapshotInfo snap) { try { result = future.get(); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_ON_PRIMARY, snap.getAccountId(), snap.getDataCenterId(), snap.getId(), - snap.getName(), null, null, snapshotOnPrimary.getSize(), snapshotOnPrimary.getSize(), snap.getClass().getName(), snap.getUuid()); + SnapshotVO snapVO = _snapshotDao.findById(snap.getId()); + if (snapVO == null || snapVO.getSnapshotType() != Snapshot.Type.INTERNAL.ordinal()) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_ON_PRIMARY, snap.getAccountId(), snap.getDataCenterId(), snap.getId(), + snap.getName(), null, null, snapshotOnPrimary.getSize(), snapshotOnPrimary.getSize(), snap.getClass().getName(), snap.getUuid()); + } return result; } catch (InterruptedException e) { s_logger.debug("Failed to create snapshot", e); diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index 8dabe5ce172f..53f89ccb82e6 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -845,6 +846,15 @@ public List listNetworksUsedByVm(long vmId, boolean isSystem) { return networks; } + @Override + public List listNetworksUsedByVm(long vmId) { + return listNetworksUsedByVm(vmId, false). + stream(). + mapToLong(NetworkVO::getId). + boxed(). + collect(Collectors.toList()); + } + @Override public Nic getNicInNetwork(long vmId, long networkId) { return _nicDao.findByNtwkIdAndInstanceId(networkId, vmId); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index aa362b2dc636..d9d1f50191e9 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -521,6 +521,7 @@ import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd; import org.apache.cloudstack.api.command.user.vm.AddIpToVmNicCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd; @@ -3486,6 +3487,7 @@ public List> getCommands() { cmdList.add(ExpungeVMCmd.class); cmdList.add(GetVMPasswordCmd.class); cmdList.add(ListVMsCmd.class); + cmdList.add(CloneVMCmd.class); cmdList.add(ScaleVMCmd.class); cmdList.add(RebootVMCmd.class); cmdList.add(RemoveNicFromVMCmd.class); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 86849a87a910..ce1c0cb1e8ea 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -1003,6 +1003,11 @@ public VolumeVO createVolume(CreateVolumeCmd cmd) { } } + @Override + public Volume cloneDataVolume(long vmId, long snapshotId, Volume volume) throws StorageUnavailableException { + return createVolumeFromSnapshot((VolumeVO) volume, snapshotId, vmId); + } + protected VolumeVO createVolumeFromSnapshot(VolumeVO volume, long snapshotId, Long vmId) throws StorageUnavailableException { VolumeInfo createdVolume = null; SnapshotVO snapshot = _snapshotDao.findById(snapshotId); @@ -2234,6 +2239,7 @@ private Volume orchestrateAttachVolumeToVM(Long vmId, Long volumeId, Long device return newVol; } + @Override public Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId) { Account caller = CallContext.current().getCallingAccount(); diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 6b66b999eadd..23f9b9beea43 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -338,7 +338,7 @@ public Snapshot revertSnapshot(Long snapshotId) { boolean result = snapshotStrategy.revertSnapshot(snapshotInfo); if (result) { // update volume size and primary storage count - _resourceLimitMgr.decrementResourceCount(snapshot.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize() - snapshot.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshot.getAccountId(), ResourceType.primary_storage, volume.getSize() - snapshot.getSize()); volume.setSize(snapshot.getSize()); _volsDao.update(volume.getId(), volume); return snapshotInfo; @@ -504,7 +504,7 @@ public Snapshot backupSnapshotFromVmSnapshot(Long snapshotId, Long vmId, Long vo } catch (Exception e) { s_logger.debug("Failed to backup snapshot from vm snapshot", e); _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.secondary_storage, volume.getSize()); throw new CloudRuntimeException("Failed to backup snapshot from vm snapshot", e); } return snapshotInfo; @@ -614,8 +614,11 @@ public boolean deleteSnapshot(long snapshotId) { annotationDao.removeByEntityType(AnnotationService.EntityType.SNAPSHOT.name(), snapshotCheck.getUuid()); if (snapshotCheck.getState() == Snapshot.State.BackedUp) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_DELETE, snapshotCheck.getAccountId(), snapshotCheck.getDataCenterId(), snapshotId, - snapshotCheck.getName(), null, null, 0L, snapshotCheck.getClass().getName(), snapshotCheck.getUuid()); + SnapshotVO snapVO = _snapshotDao.findById(snapshotId); + if (snapVO == null || snapVO.getSnapshotType() != Type.INTERNAL.ordinal()) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_DELETE, snapshotCheck.getAccountId(), snapshotCheck.getDataCenterId(), snapshotId, + snapshotCheck.getName(), null, null, 0L, snapshotCheck.getClass().getName(), snapshotCheck.getUuid()); + } } if (snapshotCheck.getState() != Snapshot.State.Error && snapshotCheck.getState() != Snapshot.State.Destroyed) { @@ -624,7 +627,7 @@ public boolean deleteSnapshot(long snapshotId) { if (snapshotCheck.getState() == Snapshot.State.BackedUp) { if (snapshotStoreRef != null) { - _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.secondary_storage, new Long(snapshotStoreRef.getPhysicalSize())); + _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.secondary_storage, snapshotStoreRef.getPhysicalSize()); } } } @@ -691,7 +694,7 @@ public Pair, Integer> listSnapshots(ListSnapshotsCmd cm sb.and("snapshotTypeEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.IN); sb.and("snapshotTypeNEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.NIN); sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); - + sb.and("snapshotTypeInternal", sb.entity().getSnapshotType(), SearchCriteria.Op.NEQ); if (tags != null && !tags.isEmpty()) { SearchBuilder tagSearch = _resourceTagDao.createSearchBuilder(); for (int count = 0; count < tags.size(); count++) { @@ -763,7 +766,7 @@ public Pair, Integer> listSnapshots(ListSnapshotsCmd cm // Show only MANUAL and RECURRING snapshot types sc.setParameters("snapshotTypeNEQ", Snapshot.Type.TEMPLATE.ordinal(), Snapshot.Type.GROUP.ordinal()); } - + sc.setParameters("snapshotTypeInternal", Type.INTERNAL.ordinal()); Pair, Integer> result = _snapshotDao.searchAndCount(sc, searchFilter); return new Pair, Integer>(result.first(), result.second()); } @@ -824,7 +827,7 @@ public boolean deleteSnapshotDirsForAccount(long accountId) { if (Type.MANUAL == snapshot.getRecurringType()) { _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.snapshot); if (snapshotStoreRef != null) { - _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, new Long(snapshotStoreRef.getPhysicalSize())); + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, snapshotStoreRef.getPhysicalSize()); } } @@ -1119,7 +1122,13 @@ public List findRecurringSnapshotSchedule(ListRecurringSnaps private Type getSnapshotType(Long policyId) { if (policyId.equals(Snapshot.MANUAL_POLICY_ID)) { return Type.MANUAL; - } else { + } + + else if (policyId.equals(Snapshot.INTERNAL_POLICY_ID)) { + return Type.INTERNAL; + } + + else { SnapshotPolicyVO spstPolicyVO = _snapshotPolicyDao.findById(policyId); IntervalType intvType = DateUtil.getIntervalType(spstPolicyVO.getInterval()); return getSnapshotType(intvType); @@ -1268,11 +1277,14 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc if (snapshotStoreRef == null) { throw new CloudRuntimeException(String.format("Could not find snapshot %s [%s] on [%s]", snapshot.getName(), snapshot.getUuid(), snapshot.getLocationType())); } - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, - snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); + SnapshotVO snapInstance = _snapshotDao.findById(snapshot.getId()); + if (snapInstance == null || snapInstance.getSnapshotType() != Type.INTERNAL.ordinal()) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, + snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); + } // Correct the resource count of snapshot in case of delta snapshots. - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, volume.getSize() - snapshotStoreRef.getPhysicalSize()); } catch (Exception e) { s_logger.debug("post process snapshot failed", e); } @@ -1281,14 +1293,14 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc s_logger.debug("Failed to create snapshot" + cre.getLocalizedMessage()); } _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, volume.getSize()); throw cre; } catch (Exception e) { if (s_logger.isDebugEnabled()) { s_logger.debug("Failed to create snapshot", e); } _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, volume.getSize()); throw new CloudRuntimeException("Failed to create snapshot", e); } return snapshot; @@ -1489,7 +1501,7 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, try { _resourceLimitMgr.checkResourceLimit(owner, ResourceType.snapshot); - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.secondary_storage, new Long(volume.getSize()).longValue()); + _resourceLimitMgr.checkResourceLimit(owner, ResourceType.secondary_storage, volume.getSize().longValue()); } catch (ResourceAllocationException e) { if (snapshotType != Type.MANUAL) { String msg = "Snapshot resource limit exceeded for account id : " + owner.getId() + ". Failed to create recurring snapshots"; @@ -1540,7 +1552,7 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, } CallContext.current().putContextParameter(Snapshot.class, snapshot.getUuid()); _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.snapshot); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, volume.getSize()); return snapshot; } diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java old mode 100755 new mode 100644 index 1a360c88edb8..2c197ac50e96 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -34,8 +34,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.user.UserData; -import com.cloud.storage.VolumeApiService; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; @@ -57,6 +55,7 @@ import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.context.CallContext; @@ -164,7 +163,7 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; import com.cloud.storage.Volume; -import com.cloud.storage.VolumeVO; +import com.cloud.storage.VolumeApiService; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.SnapshotDao; @@ -181,6 +180,7 @@ import com.cloud.user.AccountService; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; +import com.cloud.user.UserData; import com.cloud.user.dao.AccountDao; import com.cloud.uservm.UserVm; import com.cloud.utils.DateUtil; @@ -1740,6 +1740,252 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } } + @Override + @DB + @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating actual private template", create = true) + public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd, long snapshotId, long templateId) throws CloudRuntimeException { + UserVm curVm = cmd.getTargetVM(); + final Long accountId = curVm.getAccountId(); + Account caller = CallContext.current().getCallingAccount(); + List volumes = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); + VolumeVO targetVolume = volumes.get(0); + long volumeId = targetVolume.getId(); + VMTemplateVO finalTmpProduct = null; + SnapshotVO snapshot = null; + try { + TemplateInfo cloneTempalateInfp = _tmplFactory.getTemplate(templateId, DataStoreRole.Image); + long zoneId = curVm.getDataCenterId(); + AsyncCallFuture future = null; + VolumeInfo vInfo = _volFactory.getVolume(volumeId); + DataStore store = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + snapshot = _snapshotDao.findById(snapshotId); + // create template from snapshot + DataStoreRole dataStoreRole = ApiResponseHelper.getDataStoreRole(snapshot, _snapshotStoreDao, _dataStoreMgr); + SnapshotInfo snapInfo = _snapshotFactory.getSnapshot(snapshotId, dataStoreRole); + if (dataStoreRole == DataStoreRole.Image) { + if (snapInfo == null) { + snapInfo = _snapshotFactory.getSnapshot(snapshotId, DataStoreRole.Primary); + if(snapInfo == null) { + throw new CloudRuntimeException("Cannot find snapshot "+snapshotId); + } + // We need to copy the snapshot onto secondary. + SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.BACKUP); + snapshotStrategy.backupSnapshot(snapInfo); + + // Attempt to grab it again. + snapInfo = _snapshotFactory.getSnapshot(snapshotId, dataStoreRole); + if(snapInfo == null) { + throw new CloudRuntimeException("Cannot find snapshot " + snapshotId + " on secondary and could not create backup"); + } + } + _accountMgr.checkAccess(caller, null, true, snapInfo); + DataStore snapStore = snapInfo.getDataStore(); + + if (snapStore != null) { + store = snapStore; // pick snapshot image store to create template + } + } + future = _tmpltSvr.createTemplateFromSnapshotAsync(snapInfo, cloneTempalateInfp, store); + // wait for the result to converge + CommandResult result = null; + try { + result = future.get(); + + if (result.isFailed()) { + finalTmpProduct = null; + s_logger.warn("Failed to create template: " + result.getResult()); + throw new CloudRuntimeException("Failed to create template: " + result.getResult()); + } + if (_dataStoreMgr.isRegionStore(store)) { + _tmpltSvr.associateTemplateToZone(templateId, null); + } else { + // Already done in the record to db step + VMTemplateZoneVO templateZone = new VMTemplateZoneVO(zoneId, templateId, new Date()); + _tmpltZoneDao.persist(templateZone); + } + s_logger.info("successfully created the template with Id: " + templateId); + finalTmpProduct = _tmpltDao.findById(templateId); + TemplateDataStoreVO srcTmpltStore = _tmplStoreDao.findByStoreTemplate(store.getId(), templateId); + try { + srcTmpltStore.getSize(); + } catch (NullPointerException e) { + srcTmpltStore.setSize(0L); + _tmplStoreDao.update(srcTmpltStore.getId(), srcTmpltStore); + } + UsageEventVO usageEvent = + new UsageEventVO(EventTypes.EVENT_TEMPLATE_CREATE, finalTmpProduct.getAccountId(), zoneId, finalTmpProduct.getId(), finalTmpProduct.getName(), null, + finalTmpProduct.getSourceTemplateId(), srcTmpltStore.getPhysicalSize(), finalTmpProduct.getSize()); + _usageEventDao.persist(usageEvent); + } catch (InterruptedException e) { + s_logger.debug("Failed to create template for id: " + templateId, e); + throw new CloudRuntimeException("Failed to create template" , e); + } catch (ExecutionException e) { + s_logger.debug("Failed to create template for id: " + templateId, e); + throw new CloudRuntimeException("Failed to create template ", e); + } + + } finally { + finalTmpProduct = _tmpltDao.findById(templateId); + if (finalTmpProduct == null) { + final VolumeVO volumeFinal = targetVolume; + final SnapshotVO snapshotFinal = snapshot; + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + // template_store_ref entries should have been removed using our + // DataObject.processEvent command in case of failure, but clean + // it up here to avoid + // some leftovers which will cause removing template from + // vm_template table fail. + _tmplStoreDao.deletePrimaryRecordsForTemplate(templateId); + // Remove the template_zone_ref record + _tmpltZoneDao.deletePrimaryRecordsForTemplate(templateId); + // Remove the template record + _tmpltDao.expunge(templateId); + + // decrement resource count + if (accountId != null) { + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template); + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, new Long(volumeFinal != null ? volumeFinal.getSize() + : snapshotFinal.getSize())); + } + } + }); + + } + } + return null; + } + + @Override + public Snapshot createSnapshotFromTemplateOwner(long vmId, UserVm curVm, Account templateOwner, VolumeApiService volumeService) throws ResourceAllocationException { + Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, templateOwner); +// UserVm curVm = cmd.getTargetVM(); + Long nextSnapId = _tmpltDao.getNextInSequence(Long.class, "id"); + Long volumeId = _volumeDao.findByInstanceAndType(vmId, Volume.Type.ROOT).get(0).getId(); + VolumeVO volume = _volumeDao.findById(volumeId); + if (volume == null) { + throw new InvalidParameterValueException("Failed to create private template record, unable to find root volume " + volumeId); + } + + // check permissions + _accountMgr.checkAccess(caller, null, true, volume); + s_logger.info("Creating snapshot for the tempalte creation"); + SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, Snapshot.INTERNAL_POLICY_ID, curVm.getDisplayName() + "-Clone-" + nextSnapId, null); + if (snapshot == null) { + throw new CloudRuntimeException("Unable to create a snapshot during the template creation recording"); + } + Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, Snapshot.INTERNAL_POLICY_ID, snapshot.getId(), caller, false, null, false, new HashMap<>()); + if (snapshotEntity == null) { + throw new CloudRuntimeException("Error when creating the snapshot entity"); + } + if (snapshotEntity.getState() != Snapshot.State.BackedUp) { + throw new CloudRuntimeException("Async backup of snapshot happens during the clone for snapshot id: " + snapshot.getId()); + } + return snapshot; + } + @Override + @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template from clone", create = true) + public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner, VolumeApiService volumeService, Snapshot snapshot) throws ResourceAllocationException { + Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, templateOwner); + String name = cmd.getTemplateName(); + if (name.length() > 32) { + + name = name.substring(5) + "-QA-Clone"; + } + + boolean featured = false; + boolean isPublic = cmd.isPublic(); + UserVm curVm = cmd.getTargetVM(); + long zoneId = curVm.getDataCenterId(); + Long volumeId = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT).get(0).getId(); + HypervisorType hyperType = null; + VolumeVO volume = _volumeDao.findById(volumeId); + if (volume == null) { + throw new InvalidParameterValueException("Failed to create private template record, unable to find root volume " + volumeId); + } + // check permissions + _accountMgr.checkAccess(caller, null, true, volume); + hyperType = _volumeDao.getHypervisorType(volumeId); + if (HypervisorType.LXC.equals(hyperType)) { + throw new InvalidParameterValueException("Template creation is not supported for LXC volume: " + volumeId); + } + + _resourceLimitMgr.checkResourceLimit(templateOwner, ResourceType.template); + _resourceLimitMgr.checkResourceLimit(templateOwner, ResourceType.secondary_storage, volume.getSize()); + + Long guestOSId = cmd.getTargetVM().getGuestOSId(); + GuestOSVO guestOS = _guestOSDao.findById(guestOSId); + if (guestOS == null) { + throw new InvalidParameterValueException("GuestOS with ID: " + guestOSId + " does not exist."); + } + // get snapshot from this step + Long nextTemplateId = _tmpltDao.getNextInSequence(Long.class, "id"); + String description = ""; // TODO: add this to clone parameter in the future + boolean isExtractable = false; + Long sourceTemplateId = null; + if (volume != null) { + VMTemplateVO template = ApiDBUtils.findTemplateById(volume.getTemplateId()); + isExtractable = template != null && template.isExtractable() && template.getTemplateType() != Storage.TemplateType.SYSTEM; + if (volume.getIsoId() != null && volume.getIsoId() != 0) { + sourceTemplateId = volume.getIsoId(); + } else if (volume.getTemplateId() != null) { + sourceTemplateId = volume.getTemplateId(); + } + } + + VMTemplateVO privateTemplate = null; + privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable, + TemplateType.USER, null, true, 64, templateOwner.getId(), null, description, + false, guestOS.getId(), true, hyperType, null, new HashMap<>(){{put("template to be cleared", "yes");}}, false, false, false, false); + List stores = _imgStoreDao.findRegionImageStores(); + if (!CollectionUtils.isEmpty(stores)) { + privateTemplate.setCrossZones(true); + } + + privateTemplate.setSourceTemplateId(sourceTemplateId); + VMTemplateVO template = _tmpltDao.persist(privateTemplate); + if (template != null) { + Map details = new HashMap(); + + if (sourceTemplateId != null) { + VMTemplateVO sourceTemplate = _tmpltDao.findById(sourceTemplateId); + if (sourceTemplate != null && sourceTemplate.getDetails() != null) { + details.putAll(sourceTemplate.getDetails()); + } + } + + if (volume != null) { + Long vmId = volume.getInstanceId(); + if (vmId != null) { + UserVmVO userVm = _userVmDao.findById(vmId); + if (userVm != null) { + _userVmDao.loadDetails(userVm); + Map vmDetails = userVm.getDetails(); + vmDetails = vmDetails.entrySet() + .stream() + .filter(map -> map.getValue() != null) + .collect(Collectors.toMap(map -> map.getKey(), map -> map.getValue())); + details.putAll(vmDetails); + } + } + } + + if (!details.isEmpty()) { + privateTemplate.setDetails(details); + _tmpltDao.saveDetails(privateTemplate); + } + + _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.template); + _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.secondary_storage, + ((SnapshotVO) snapshot).getSize()); + } + + return template; + } + @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template", create = true) public VMTemplateVO createPrivateTemplateRecord(CreateTemplateCmd cmd, Account templateOwner) throws ResourceAllocationException { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 2a95f26c1910..ff037a7225f4 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -45,13 +46,11 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; - import javax.inject.Inject; import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.resourcelimit.CheckedReservation; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -69,6 +68,7 @@ import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; @@ -139,7 +139,6 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; - import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -267,6 +266,7 @@ import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.security.SecurityGroup; import com.cloud.network.security.SecurityGroupManager; +import com.cloud.network.security.SecurityGroupVO; import com.cloud.network.security.dao.SecurityGroupDao; import com.cloud.network.vpc.VpcManager; import com.cloud.offering.DiskOffering; @@ -279,6 +279,7 @@ import com.cloud.org.Grouping; import com.cloud.resource.ResourceManager; import com.cloud.resource.ResourceState; +import com.cloud.resourcelimit.CheckedReservation; import com.cloud.server.ManagementService; import com.cloud.server.ResourceTag; import com.cloud.server.StatsCollector; @@ -312,6 +313,7 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.snapshot.SnapshotApiService; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.TemplateApiService; @@ -320,6 +322,7 @@ import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; +import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; import com.cloud.user.SSHKeyPairVO; import com.cloud.user.User; @@ -347,6 +350,7 @@ import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; @@ -556,6 +560,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject private BackupManager backupManager; @Inject + private SnapshotApiService _snapshotService; + @Inject private AnnotationDao annotationDao; @Inject private VmStatsDao vmStatsDao; @@ -1307,14 +1313,14 @@ private UserVm upgradeStoppedVirtualMachine(Long vmId, Long svcOffId, Map currentCpu) { - _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); + _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.cpu, (long) (newCpu - currentCpu)); } else if (currentCpu > newCpu) { - _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.cpu, new Long(currentCpu - newCpu)); + _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.cpu, (long) (currentCpu - newCpu)); } if (newMemory > currentMemory) { - _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.memory, new Long(newMemory - currentMemory)); + _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.memory, (long) (newMemory - currentMemory)); } else if (currentMemory > newMemory) { - _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.memory, new Long(currentMemory - newMemory)); + _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.memory, (long) (currentMemory - newMemory)); } } @@ -2047,11 +2053,11 @@ private boolean upgradeRunningVirtualMachine(Long vmId, Long newServiceOfferingI // Increment CPU and Memory count accordingly. if (newCpu > currentCpu) { - _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); + _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.cpu, (long) (newCpu - currentCpu)); } if (memoryDiff > 0) { - _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(memoryDiff)); + _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.memory, (long) memoryDiff); } // #1 Check existing host has capacity @@ -2083,11 +2089,11 @@ private boolean upgradeRunningVirtualMachine(Long vmId, Long newServiceOfferingI if (!success) { // Decrement CPU and Memory count accordingly. if (newCpu > currentCpu) { - _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); + _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.cpu, (long) (newCpu - currentCpu)); } if (memoryDiff > 0) { - _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(memoryDiff)); + _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, (long) memoryDiff); } } } @@ -2315,7 +2321,7 @@ public UserVm recoverVirtualMachine(RecoverVMCmd cmd) throws ResourceAllocationE // First check that the maximum number of UserVMs, CPU and Memory limit for the given // accountId will not be exceeded if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceLimitCheck(account, vm.isDisplayVm(), new Long(serviceOffering.getCpu()), new Long(serviceOffering.getRamSize())); + resourceLimitCheck(account, vm.isDisplayVm(), serviceOffering.getCpu().longValue(), serviceOffering.getRamSize().longValue()); } _haMgr.cancelDestroy(vm, vm.getHostId()); @@ -2339,7 +2345,7 @@ public UserVm recoverVirtualMachine(RecoverVMCmd cmd) throws ResourceAllocationE } //Update Resource Count for the given account - resourceCountIncrement(account.getId(), vm.isDisplayVm(), new Long(serviceOffering.getCpu()), new Long(serviceOffering.getRamSize())); + resourceCountIncrement(account.getId(), vm.isDisplayVm(), serviceOffering.getCpu().longValue(), serviceOffering.getRamSize().longValue()); } }); @@ -2632,7 +2638,7 @@ private void updateVmStateForFailedVmCreation(Long vmId, Long hostId) { ServiceOfferingVO offering = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); // Update Resource Count for the given account - resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); } } } @@ -3989,6 +3995,7 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); volumesSize += verifyAndGetDiskSize(diskOffering, diskSize); } + UserVm vm = getCheckedUserVmResource(zone, hostName, displayName, owner, diskOfferingId, diskSize, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, caller, requestedIps, defaultIps, isDisplayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, template, hypervisorType, accountId, offering, isIso, rootDiskOfferingId, volumesSize); _securityGroupMgr.addInstanceToGroups(vm.getId(), securityGroupIdList); @@ -4576,7 +4583,7 @@ public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCap } //Update Resource Count for the given account - resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountIncrement(accountId, isDisplayVm, offering.getCpu().longValue(), offering.getRamSize().longValue()); } return vm; } @@ -4904,6 +4911,238 @@ protected String validateUserData(String userData, HTTPMethod httpmethod) { return null; } + @Override + public void validateCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueException, ResourceUnavailableException, CloudRuntimeException, ResourceAllocationException { + + if (cmd.getAccountName() != null && cmd.getDomainId() == null) { + throw new InvalidParameterValueException("You must input the domainId together with the account name"); + } + + final DomainVO domain = cmd.getDomainId() == null ? null : _domainDao.findById(cmd.getDomainId()); + final Account account = cmd.getAccountName() == null ? null : _accountService.getActiveAccountByName(cmd.getAccountName(), cmd.getDomainId()); + if (domain != null && account != null) { + if (account.getType() == Account.ACCOUNT_TYPE_PROJECT) { + throw new InvalidParameterValueException("Invalid user type: project to clone the VM"); + } + if (account.getState() != Account.State.enabled) { + throw new InvalidParameterValueException("User is not enabled to clone this VM"); + } + } + UserVm curVm = cmd.getTargetVM(); + if (curVm == null) { + throw new CloudRuntimeException("the VM doesn't exist or not registered in management server!"); + } + UserVmVO vmStatus = _vmDao.findById(cmd.getId()); + if (vmStatus.getHypervisorType() != HypervisorType.KVM && vmStatus.getHypervisorType() != HypervisorType.Simulator) { + throw new CloudRuntimeException("The clone operation is only supported on KVM and Simulator!"); + } + String kvmEnabled = _configDao.getValue("kvm.snapshot.enabled"); + if (kvmEnabled == null || !kvmEnabled.equalsIgnoreCase("true")) { + throw new CloudRuntimeException("Clone VM is not supported, as snapshots are disabled"); + } + Long accountId = curVm.getAccountId(); + Account vmOwner = _accountDao.findById(accountId); + if (vmOwner == null) { + throw new CloudRuntimeException("This VM doesn't have an owner account, please assign one to it"); + } + List volumes = _volsDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); + if (CollectionUtils.isEmpty(volumes)) { + throw new CloudRuntimeException("The VM to copy does not have a Volume attached!"); + } + // verify that the VM doesn't expire + Map details = curVm.getDetails(); + verifyDetails(details); + long zoneId = curVm.getDataCenterId(); + DataCenter zone = _entityMgr.findById(DataCenter.class, zoneId); + if (zone == null) { + throw new InvalidParameterValueException("Unable to find a zone in the current VM by zone id=" + zoneId); + } + // service offering check + long serviceOfferingId = curVm.getServiceOfferingId(); + ServiceOffering serviceOffering = _entityMgr.findById(ServiceOffering.class, serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Service offering Id for this VM: " + serviceOfferingId + " doesn't exist now"); + } + if (!serviceOffering.isDynamic() && details != null) { + for(String detail: details.keySet()) { + if(detail.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || detail.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || detail.equalsIgnoreCase(VmDetailConstants.MEMORY)) { + throw new InvalidParameterValueException("cpuNumber or cpuSpeed or memory should not be specified for static service offering"); + } + } + } + // disk offering check + VolumeVO rootDisk = volumes.get(0); + Long diskOfferingID = rootDisk.getDiskOfferingId(); + DiskOfferingVO diskOffering =null; + if (diskOfferingID != null) { + diskOffering = _diskOfferingDao.findById(diskOfferingID); + if (diskOffering == null) { + throw new CloudRuntimeException("Unable to find disk offering " + diskOfferingID); + } + } + if (!zone.isLocalStorageEnabled()) { + if (diskOffering != null && diskOffering.isUseLocalStorage()) { + throw new CloudRuntimeException("Zone is not configured to use local storage but disk offering " + diskOffering.getName() + " uses it"); + } + } + // resource limit checks & account check + AccountVO activeOwner = _accountDao.findById(cmd.getEntityOwnerId()); + List totalVolumes = _volsDao.findByInstance(cmd.getId()); + _resourceLimitMgr.checkResourceLimit(activeOwner, ResourceType.volume, totalVolumes.size()); + Long totalSize = 0L; + for (VolumeVO volumeToCheck : totalVolumes) { + totalSize += volumeToCheck.getSize(); + } + _resourceLimitMgr.checkResourceLimit(activeOwner, ResourceType.primary_storage, totalSize); + } + + private VolumeVO saveDataDiskVolumeFromSnapShot(final Account owner, final Boolean displayVolume, final Long zoneId, final Long diskOfferingId, + final Storage.ProvisioningType provisioningType, final Long size, final Long minIops, final Long maxIops, final VolumeVO parentVolume, final String volumeName, final String uuid, final Map details) { + return Transaction.execute((TransactionCallback) status -> { + VolumeVO volume = new VolumeVO(volumeName, -1, -1, -1, -1, Long.valueOf(-1), null, null, provisioningType, 0, Volume.Type.DATADISK); + volume.setPoolId(null); + volume.setUuid(uuid); + volume.setDataCenterId(zoneId); + volume.setPodId(null); + volume.setAccountId(owner.getId()); + volume.setDomainId(owner.getDomainId()); + volume.setDiskOfferingId(diskOfferingId); + volume.setSize(size); + volume.setMinIops(minIops); + volume.setMaxIops(maxIops); + volume.setInstanceId(null); + volume.setUpdated(new Date()); + volume.setDisplayVolume(displayVolume); + if (parentVolume != null) { + volume.setTemplateId(parentVolume.getTemplateId()); + volume.setFormat(parentVolume.getFormat()); + } else { + volume.setTemplateId(null); + } + + volume = _volsDao.persist(volume); + CallContext.current().setEventDetails("Volume Id: " + volume.getUuid()); + + // Increment resource count during allocation; if actual creation fails, + // decrement it + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume, displayVolume); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, displayVolume, volume.getSize()); + return volume; + }); + } + + @Override + public void prepareCloneVirtualMachine(CloneVMCmd cmd) throws ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + Long temporarySnapshotId = null; + try { + Account owner = _accountService.getAccount(cmd.getEntityOwnerId()); + Snapshot snapshot = _tmplService.createSnapshotFromTemplateOwner(cmd.getId(), cmd.getTargetVM(), owner, _volumeService); + temporarySnapshotId = snapshot.getId(); + VirtualMachineTemplate template = _tmplService.createPrivateTemplateRecord(cmd, owner, _volumeService, snapshot); + if (template == null) { + throw new CloudRuntimeException("failed to create a template to db"); + } + s_logger.info("The template id recorded is: " + template.getId()); + _tmplService.createPrivateTemplate(cmd, snapshot.getId(), template.getId()); + UserVm vmRecord = recordVirtualMachineToDB(cmd, template.getId()); + if (vmRecord == null) { + throw new CloudRuntimeException("Unable to record the VM to DB!"); + } + cmd.setEntityUuid(vmRecord.getUuid()); + cmd.setEntityId(vmRecord.getId()); + } finally { + if (temporarySnapshotId != null) { + _snapshotService.deleteSnapshot(temporarySnapshotId); + s_logger.warn("clearing the temporary snapshot: " + temporarySnapshotId); + } + } + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_VM_CLONE, eventDescription = "clone vm", async = true) + public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService, SnapshotApiService snapshotService) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException, InsufficientCapacityException, ResourceAllocationException { + long vmId = cmd.getEntityId(); + UserVmVO curVm = _vmDao.findById(vmId); + Account curVmAccount = _accountDao.findById(curVm.getAccountId()); + // create and attach data disk + long targetClonedVmId = cmd.getId(); + Account caller = CallContext.current().getCallingAccount(); + List dataDisks = _volsDao.findByInstanceAndType(targetClonedVmId, Volume.Type.DATADISK); + List createdSnapshots = new ArrayList<>(); + List createdVolumes = new ArrayList<>(); + long zoneId = cmd.getTargetVM().getDataCenterId(); + s_logger.info("Trying to attach data disk before starting the VM..."); + if (dataDisks.size() > 0) { + VolumeVO newDatadisk = null; + try { + for (VolumeVO dataDisk : dataDisks) { + long diskId = dataDisk.getId(); + SnapshotVO dataSnapShot = (SnapshotVO) volumeService.allocSnapshot(diskId, Snapshot.INTERNAL_POLICY_ID, "DataDisk-Clone" + dataDisk.getName(), null); + if (dataSnapShot == null) { + throw new CloudRuntimeException("Unable to allocate snapshot of data disk: " + dataDisk.getId() + " name: " + dataDisk.getName()); + } + createdSnapshots.add(dataSnapShot); + SnapshotVO snapshotEntity = (SnapshotVO) volumeService.takeSnapshot(diskId, Snapshot.INTERNAL_POLICY_ID, dataSnapShot.getId(), caller, false, null, false, new HashMap<>()); + if (snapshotEntity == null) { + throw new CloudRuntimeException("Error when creating the snapshot entity"); + } + if (snapshotEntity.getState() != Snapshot.State.BackedUp) { + throw new CloudRuntimeException("Async backup of snapshot happens during the clone for snapshot id: " + dataSnapShot.getId()); + } + long diskOfferingId = snapshotEntity.getDiskOfferingId(); + DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); + Long minIops = snapshotEntity.getMinIops(); + Long maxIops = snapshotEntity.getMaxIops(); + Long size = snapshotEntity.getSize(); + Storage.ProvisioningType provisioningType = diskOffering.getProvisioningType(); + DataCenterVO dataCenter = _dcDao.findById(zoneId); + String volumeName = snapshotEntity.getName() + "-DataDisk-Volume"; + VolumeVO parentVolume = _volsDao.findByIdIncludingRemoved(snapshotEntity.getVolumeId()); + newDatadisk = saveDataDiskVolumeFromSnapShot(curVmAccount, true, zoneId, + diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, volumeName, _uuidMgr.generateUuid(Volume.class, null), new HashMap<>()); + VolumeVO volumeEntity = (VolumeVO) volumeService.cloneDataVolume(cmd.getEntityId(), snapshotEntity.getId(), newDatadisk); + createdVolumes.add(volumeEntity); + } + + for (VolumeVO createdVol : createdVolumes) { +// volumeService.attachVolumeToVm(cmd, createdVol.getId(), createdVol.getDeviceId()); + volumeService.attachVolumeToVM(cmd.getEntityId(), createdVol.getId(), createdVol.getDeviceId()); + } + } catch (CloudRuntimeException e){ + s_logger.warn("data disk process failed during clone, clearing the temporary resources..."); + for (VolumeVO dataDiskToClear : createdVolumes) { + volumeService.destroyVolume(dataDiskToClear.getId(), caller, true, false); + } + // clear the created disks + if (newDatadisk != null) { + volumeService.destroyVolume(newDatadisk.getId(), caller, true, false); + } + destroyVm(vmId, true); + throw new CloudRuntimeException(e.getMessage()); + } finally { + // clear the temporary data snapshots + for (Snapshot snapshotLeftOver : createdSnapshots) { + snapshotService.deleteSnapshot(snapshotLeftOver.getId()); + } + } + } + + // start the VM if successfull + Long podId = curVm.getPodIdToDeployIn(); + Long clusterId = null; + Long hostId = curVm.getHostId(); + Map additonalParams = new HashMap<>(); + Map diskOfferingMap = new HashMap<>(); + if (MapUtils.isNotEmpty(curVm.getDetails()) && curVm.getDetails().containsKey(ApiConstants.BootType.UEFI.toString())) { + Map map = curVm.getDetails(); + additonalParams.put(VirtualMachineProfile.Param.UefiFlag, "Yes"); + additonalParams.put(VirtualMachineProfile.Param.BootType, ApiConstants.BootType.UEFI.toString()); + additonalParams.put(VirtualMachineProfile.Param.BootMode, map.get(ApiConstants.BootType.UEFI.toString())); + } + + return Optional.of(startVirtualMachine(vmId, podId, clusterId, hostId, diskOfferingMap, additonalParams, null)); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CREATE, eventDescription = "starting Vm", async = true) public UserVm startVirtualMachine(DeployVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException { @@ -5588,7 +5827,7 @@ public UserVm destroyVm(long vmId, boolean expunge) throws ResourceUnavailableEx ServiceOfferingVO offering = serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()); //Update Resource Count for the given account - resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); } return _vmDao.findById(vmId); } else { @@ -6033,6 +6272,74 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE return vm; } + @Override + public UserVm recordVirtualMachineToDB(CloneVMCmd cmd, long templateId) throws ConcurrentOperationException, ResourceAllocationException, InsufficientCapacityException, ResourceUnavailableException { + //network configurations and check, then create the template + UserVm curVm = cmd.getTargetVM(); + // check if host is available + Long hostId = curVm.getHostId(); + getDestinationHost(hostId, true, false); + Long zoneId = curVm.getDataCenterId(); + DataCenter dataCenter = _entityMgr.findById(DataCenter.class, zoneId); + Map vmProperties = curVm.getDetails() != null ? curVm.getDetails() : new HashMap<>(); + String keyboard = vmProperties.get(VmDetailConstants.KEYBOARD); + HypervisorType hypervisorType = curVm.getHypervisorType(); + Account curAccount = _accountDao.findById(curVm.getAccountId()); + String ipv6Address = null; + String macAddress = null; + IpAddresses addr = new IpAddresses(null, ipv6Address, macAddress); + long serviceOfferingId = curVm.getServiceOfferingId(); + ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); + List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); + List securityGroupIdList = securityGroupList.stream().map(SecurityGroupVO::getId).collect(Collectors.toList()); + String uuidName = _uuidMgr.generateUuid(UserVm.class, null); + String hostName = generateHostName(uuidName); + String displayName = hostName + "-Clone"; + VolumeVO curVolume = _volsDao.findByInstance(curVm.getId()).get(0); + Long diskOfferingId = curVolume.getDiskOfferingId(); + Long size = null; // mutual exclusive with disk offering id + String userData = curVm.getUserData(); + String sshKeyPair = null; + Map ipToNetoworkMap = null; // Since we've specified Ip + boolean isDisplayVM = curVm.isDisplayVm(); + boolean dynamicScalingEnabled = curVm.isDynamicallyScalable(); + VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, templateId); + if (template == null) { + throw new CloudRuntimeException("the temporary template is not created, server error, contact your sys admin"); + } + List networkIds = _networkModel.listNetworksUsedByVm(curVm.getId()); + String group = null; + InstanceGroupVO groupVo = getGroupForVm(cmd.getId()); + if (groupVo != null) { + group = groupVo.getName(); + } + UserVm vmResult = null; + List affinityGroupIdList = _affinityGroupDao.findByAccountAndNames(curAccount.getId(), curAccount.getAccountName()) + .stream(). + mapToLong(AffinityGroupVO::getId). + boxed(). + collect(Collectors.toList()); + try { + Map detailMap = new HashMap<>(); + Map> dhcpMap = new HashMap<>(); + Map emptyUserOfVmProperties = new HashMap<>(); + if (dataCenter.getNetworkType() == NetworkType.Basic) { + vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, + size, group, hypervisorType, cmd.getHttpMethod(), userData, null, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, + curVm.getDetails() == null ? detailMap : curVm.getDetails(), null, new HashMap<>(), + null, new HashMap<>(), dynamicScalingEnabled, diskOfferingId); + } else { + vmResult = createAdvancedVirtualMachine(dataCenter, serviceOffering, template, networkIds, curAccount, hostName, displayName, diskOfferingId, size, group, + hypervisorType, cmd.getHttpMethod(), userData, null, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, curVm.getDetails() == null ? detailMap : curVm.getDetails(), + null, new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled, null, diskOfferingId); + } + } catch (CloudRuntimeException e) { + _templateMgr.delete(curAccount.getId(), template.getId(), zoneId); + throw new CloudRuntimeException("Unable to create the VM record"); + } + return vmResult; + } + /** * Persist extra configuration data in the user_vm_details table as key/value pair * @param decodedUrl String consisting of the extra config data to appended onto the vmx file for VMware instances @@ -7149,7 +7456,7 @@ public UserVm moveVMToUser(final AssignVMCmd cmd) throws ResourceAllocationExcep // VV 2: check if account/domain is with in resource limits to create a new vm if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceLimitCheck(newAccount, vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceLimitCheck(newAccount, vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); } // VV 3: check if volumes and primary storage space are with in resource limits @@ -7182,7 +7489,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm.getHypervisorType().toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); // update resource counts for old account - resourceCountDecrement(oldAccount.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountDecrement(oldAccount.getAccountId(), vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); // OWNERSHIP STEP 1: update the vm owner vm.setAccountId(newAccount.getAccountId()); @@ -7194,12 +7501,12 @@ public void doInTransactionWithoutResult(TransactionStatus status) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.volume); - _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.primary_storage, volume.getSize()); volume.setAccountId(newAccount.getAccountId()); volume.setDomainId(newAccount.getDomainId()); _volsDao.persist(volume); _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.volume); - _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.primary_storage, volume.getSize()); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); @@ -7207,7 +7514,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { //update resource count of new account if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceCountIncrement(newAccount.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountIncrement(newAccount.getAccountId(), vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); } //generate usage events to account for this change @@ -7747,7 +8054,7 @@ public UserVm restoreVirtualMachine(final Account caller, final long vmId, final // 1. Save usage event and update resource count for user vm volumes _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.volume, newVol.isDisplay()); - _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.primary_storage, newVol.isDisplay(), new Long(newVol.getSize())); + _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.primary_storage, newVol.isDisplay(), newVol.getSize()); // 2. Create Usage event for the newly created volume UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VOLUME_CREATE, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(), newVol.getDiskOfferingId(), template.getId(), newVol.getSize()); _usageEventDao.persist(usageEvent); @@ -8274,7 +8581,7 @@ private void postProcessingUnmanageVMVolumes(List volumes, UserVmVO vm Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); } _resourceLimitMgr.decrementResourceCount(vm.getAccountId(), ResourceType.volume); - _resourceLimitMgr.decrementResourceCount(vm.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(vm.getAccountId(), ResourceType.primary_storage, volume.getSize()); } } diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 06a56e45f12d..f725b0899763 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -16,56 +16,13 @@ // under the License. package com.cloud.vm; -import com.cloud.storage.Volume; -import com.cloud.storage.dao.VolumeDao; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.cloud.template.VirtualMachineTemplate; -import com.cloud.user.UserData; -import com.cloud.user.UserDataVO; -import com.cloud.user.dao.UserDataDao; -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.cloudstack.api.BaseCmd.HTTPMethod; -import org.apache.cloudstack.api.command.user.vm.ResetVMUserDataCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; -import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.Spy; -import org.mockito.junit.MockitoJUnitRunner; -import org.powermock.core.classloader.annotations.PrepareForTest; - import com.cloud.configuration.Resource; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.InsufficientAddressCapacityException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.Hypervisor; import com.cloud.network.NetworkModel; @@ -78,21 +35,81 @@ import com.cloud.storage.GuestOSVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeApiService; +import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; +import com.cloud.user.UserData; +import com.cloud.user.UserDataVO; import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserDataDao; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; +import org.apache.cloudstack.api.BaseCmd.HTTPMethod; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; +import org.apache.cloudstack.api.command.user.vm.ResetVMUserDataCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; +import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.powermock.core.classloader.annotations.PrepareForTest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.powermock.core.classloader.annotations.PrepareForTest; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + @RunWith(MockitoJUnitRunner.class) public class UserVmManagerImplTest { @@ -156,6 +173,9 @@ public class UserVmManagerImplTest { @Mock private VMTemplateDao templateDao; + @Mock + private CloneVMCmd cloneVMCommand; + @Mock private AccountDao accountDao; @@ -624,6 +644,20 @@ private DiskOfferingVO prepareDiskOffering(long rootSize, long diskOfferingId, l return newRootDiskOffering; } + @Test + public void validateCloneCondition() { + Mockito.when(cloneVMCommand.getTargetVM()).thenReturn(null); + Mockito.when(cloneVMCommand.getAccountName()).thenReturn(null); + Mockito.when(cloneVMCommand.getDomainId()).thenReturn(null); + Exception err = null; + try { + userVmManagerImpl.validateCloneCondition(cloneVMCommand); + } catch (CloudRuntimeException | ResourceUnavailableException | ResourceAllocationException e) { + err = e; + } + assertTrue(err instanceof CloudRuntimeException); + } + private ServiceOfferingVO prepareOfferingsForEncryptionValidation(long diskOfferingId, boolean encryption) { ServiceOfferingVO svcOffering = Mockito.mock(ServiceOfferingVO.class); DiskOfferingVO diskOffering = Mockito.mock(DiskOfferingVO.class); diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index d9878b097659..6ff92f7984a7 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -873,7 +873,6 @@ def test_11_destroy_vm_and_volumes(self): self.assertEqual(Volume.list(self.apiclient, id=vol1.id), None, "List response contains records when it should not") - class TestSecuredVmMigration(cloudstackTestCase): @classmethod @@ -1839,3 +1838,139 @@ def test_01_vapps_vm_cycle(self): cmd = destroyVirtualMachine.destroyVirtualMachineCmd() cmd.id = vm.id self.apiclient.destroyVirtualMachine(cmd) + +class TestCloneVM(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestCloneVM, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + + # Get Zone, Domain and templates + domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + + # if local storage is enabled, alter the offerings to use localstorage + # this step is needed for devcloud + if cls.zone.localstorageenabled == True: + cls.services["service_offerings"]["tiny"]["storagetype"] = 'local' + cls.services["service_offerings"]["small"]["storagetype"] = 'local' + cls.services["service_offerings"]["medium"]["storagetype"] = 'local' + + template = get_suitable_test_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"], + cls.hypervisor + ) + if template == FAILED: + assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"] + + # Set Zones and disk offerings + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + + cls.services["iso1"]["zoneid"] = cls.zone.id + + # Create VMs, NAT Rules etc + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=domain.id + ) + cls._cleanup = [] + cls._cleanup.append(cls.account) + cls.small_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["small"] + ) + cls._cleanup.append(cls.small_offering) + + cls.medium_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["medium"] + ) + cls._cleanup.append(cls.medium_offering) + # create small and large virtual machines + cls.small_virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.small_offering.id, + mode=cls.services["mode"] + ) + cls._cleanup.append(cls.small_virtual_machine) + cls.medium_virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.medium_offering.id, + mode=cls.services["mode"] + ) + cls._cleanup.append(cls.medium_virtual_machine) + cls.virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.small_offering.id, + mode=cls.services["mode"] + ) + cls._cleanup.append(cls.virtual_machine) + + @classmethod + def tearDownClass(cls): + super(TestCloneVM, cls).tearDownClass() + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") + def test_clone_vm_and_volumes(self): + small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; + config = Configurations.list(self.apiclient, + name="kvm.snapshot.enabled" + ) + if config is None: + self.skipTest("Please enable kvm.snapshot.enable global config") + if len(config) == 0 or config[0].value != "true": + self.skipTest("Please enable kvm.snapshot.enable global config") + if self.hypervisor.lower() in ["kvm", "simulator"]: + small_virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.small_offering.id,) + self.cleanup.append(small_virtual_machine) + vol1 = Volume.create( + self.apiclient, + self.services, + account=self.account.name, + diskofferingid=small_disk_offering.id, + domainid=self.account.domainid, + zoneid=self.zone.id + ) + self.cleanup.append(vol1) + small_virtual_machine.attach_volume(self.apiclient, vol1) + self.debug("Clone VM - ID: %s" % small_virtual_machine.id) + try: + clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) + self.cleanup.append(clone_response) + except Exception as e: + self.debug("Clone --" + str(e)) + raise e + self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") \ No newline at end of file diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index e22bfafa4df7..484ef95893ab 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -753,6 +753,21 @@ def reboot(self, apiclient, forced=None): if response[0] == FAIL: raise Exception(response[1]) + def clone(self, apiclient, vm): + """"Clone the instance""" + cmd = cloneVirtualMachine.cloneVirtualMachineCmd() + cmd.virtualmachineid = vm.id + if vm.id is None: + cmd.virtualmachineid = self.id + response = apiclient.cloneVirtualMachine(cmd) + temp = self.id + self.id = response.id + state = self.getState(apiclient, VirtualMachine.RUNNING) + self.id = temp + if (state[0] == FAIL): + raise Exception(state[1]) + return response + def recover(self, apiclient): """Recover the instance""" cmd = recoverVirtualMachine.recoverVirtualMachineCmd() diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 25f86c33b9fc..bce615fb6d29 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -58,6 +58,7 @@ "label.action.configure.stickiness": "Stickiness", "label.action.copy.iso": "Copy ISO", "label.action.copy.template": "Copy template", +"label.action.clone.vm": "Clone VM", "label.action.create.snapshot.from.vmsnapshot": "Create snapshot from VM snapshot", "label.action.create.template.from.volume": "Create template from volume", "label.action.create.volume": "Create volume", @@ -2034,6 +2035,7 @@ "message.action.patch.systemvm": "Please confirm that you want to patch the System VM.", "message.action.primarystorage.enable.maintenance.mode": "Warning: placing the primary storage into maintenance mode will cause all VMs using volumes from it to be stopped. Do you want to continue?", "message.action.reboot.instance": "Please confirm that you want to reboot this instance.", + "message.action.clone.instance": "Please confirm that you want to clone this instance", "message.action.reboot.router": "All services provided by this virtual router will be interrupted. Please confirm that you want to reboot this router.", "message.action.reboot.systemvm": "Please confirm that you want to reboot this system VM.", "message.action.recover.volume": "Please confirm that you would like to recover this volume.", diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 476b055e7fd1..9462454f14cf 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -157,6 +157,21 @@ export default { popup: true, groupMap: (selection, values) => { return selection.map(x => { return { id: x, forced: values.forced } }) } }, + { + api: 'cloneVirtualMachine', + icon: 'camera', + label: 'label.action.clone.vm', + message: 'message.action.clone.instance', + docHelp: 'adminguide/virtual_machines.html#cloning-vms', + dataView: true, + show: (record) => { return true }, + args: ['name', 'virtualmachineid'], + mapping: { + virtualmachineid: { + value: (record, params) => { return record.id } + } + } + }, { api: 'restoreVirtualMachine', icon: 'sync-outlined', diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 9cf1e30865c9..6efe599747dd 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -33,6 +33,9 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.dao.SnapshotDao; import org.apache.cloudstack.quota.QuotaAlertManager; import org.apache.cloudstack.quota.QuotaManager; import org.apache.cloudstack.quota.QuotaStatement; @@ -162,6 +165,8 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna private QuotaAlertManager _alertManager; @Inject private QuotaStatement _quotaStatement; + @Inject + private SnapshotDao _snapshotDao; private String _version = null; private final Calendar _jobExecTime = Calendar.getInstance(); @@ -1627,6 +1632,13 @@ private void createSnapshotHelperEvent(UsageEventVO event) { long zoneId = -1L; long snapId = event.getResourceId(); + + SnapshotVO snapshotInstance = _snapshotDao.findById(snapId); + + if (snapshotInstance != null && snapshotInstance.getSnapshotType() == Snapshot.Type.INTERNAL.ordinal()) { + return; + } + if (EventTypes.EVENT_SNAPSHOT_CREATE.equals(event.getType())) { if (usageSnapshotSelection){ snapSize = event.getVirtualSize();