From d3b8e748a6ccc0d1be13c4d696d08060022e15c5 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 7 Jun 2021 02:58:34 -0400 Subject: [PATCH 001/114] API command initialization --- .../api/command/user/vm/CloneVMCmd.java | 39 +++++++++++++++++++ .../cloud/server/ManagementServerImpl.java | 22 +---------- 2 files changed, 41 insertions(+), 20 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java new file mode 100644 index 000000000000..b02e6f9f3749 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -0,0 +1,39 @@ +package org.apache.cloudstack.api.command.user.vm; + +import com.cloud.exception.*; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.UserVmResponse; + +@APICommand(name = "cloneVirtualMachine", responseObject = UserVmResponse.class, description = "clone a virtual VM in full clone mode", + responseView = ResponseObject.ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) +public class CloneVMCmd extends BaseAsyncCmd implements UserCmd { + + @Override + public String getEventType() { + return null; + } + + @Override + public String getEventDescription() { + return null; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + + } + + @Override + public String getCommandName() { + return null; + } + + @Override + public long getEntityOwnerId() { + return 0; + } +} diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 18afeb42a929..f4f67c89f3db 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -465,26 +465,7 @@ import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; -import org.apache.cloudstack.api.command.user.vm.AddIpToVmNicCmd; -import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; -import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; -import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; -import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd; -import org.apache.cloudstack.api.command.user.vm.ListNicsCmd; -import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; -import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; -import org.apache.cloudstack.api.command.user.vm.RemoveIpFromVmNicCmd; -import org.apache.cloudstack.api.command.user.vm.RemoveNicFromVMCmd; -import org.apache.cloudstack.api.command.user.vm.ResetVMPasswordCmd; -import org.apache.cloudstack.api.command.user.vm.ResetVMSSHKeyCmd; -import org.apache.cloudstack.api.command.user.vm.RestoreVMCmd; -import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd; -import org.apache.cloudstack.api.command.user.vm.StartVMCmd; -import org.apache.cloudstack.api.command.user.vm.StopVMCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateDefaultNicForVMCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateVmNicIpCmd; -import org.apache.cloudstack.api.command.user.vm.UpgradeVMCmd; +import org.apache.cloudstack.api.command.user.vm.*; import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.ListVMGroupsCmd; @@ -2797,6 +2778,7 @@ public long getMemoryOrCpuCapacityByHost(final Long hostId, final short capacity @Override public List> getCommands() { final List> cmdList = new ArrayList>(); + cmdList.add(CloneVMCmd.class); cmdList.add(CreateAccountCmd.class); cmdList.add(DeleteAccountCmd.class); cmdList.add(DisableAccountCmd.class); From e2f72280eb8d56ea01dd4c7e7c14a117897147aa Mon Sep 17 00:00:00 2001 From: junxuan Date: Thu, 10 Jun 2021 19:32:25 -0400 Subject: [PATCH 002/114] first step to create the clone api execution --- .../main/java/com/cloud/event/EventTypes.java | 1 + .../api/command/user/vm/CloneVMCmd.java | 52 ++++++++++++++++--- .../cloud/server/ManagementServerImpl.java | 24 ++++++++- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index ebe7590fd5f6..3eb0b8dda86c 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -105,6 +105,7 @@ public class EventTypes { public static final String EVENT_VM_IMPORT = "VM.IMPORT"; public static final String EVENT_VM_UNMANAGE = "VM.UNMANAGE"; public static final String EVENT_VM_RECOVER = "VM.RECOVER"; + public static final String EVENT_VM_CLONE = "VM.CLONE"; // Domain Router public static final String EVENT_ROUTER_CREATE = "ROUTER.CREATE"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index b02e6f9f3749..de241b2c8cae 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -1,39 +1,77 @@ package org.apache.cloudstack.api.command.user.vm; -import com.cloud.exception.*; +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import com.cloud.uservm.UserVm; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.log4j.Logger; @APICommand(name = "cloneVirtualMachine", responseObject = UserVmResponse.class, description = "clone a virtual VM in full clone mode", - responseView = ResponseObject.ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) + responseView = ResponseObject.ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, entityType = {VirtualMachine.class}) public class CloneVMCmd extends BaseAsyncCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(CloneVMCmd.class.getName()); + private static final String s_name = "clonevirtualmachineresponse"; + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @ACL(accessType = AccessType.OperateEntry) + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType=UserVmResponse.class, + required = true, description = "The ID of the virtual machine") + private Long id; + + public Long getId() { + return this.id; + } @Override public String getEventType() { - return null; + return EventTypes.EVENT_VM_CLONE; + } + + @Override + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.VirtualMachine; } @Override public String getEventDescription() { - return null; + return "Cloning user VM: " + this._uuidMgr.getUuid(VirtualMachine.class, getId()); } @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - + UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", null).get(0); + response.setResponseName("test_clone"); + setResponseObject(response); } @Override public String getCommandName() { - return null; + return s_name; } @Override public long getEntityOwnerId() { - return 0; + UserVm vm = this._responseGenerator.findUserVmById(getId()); + if (vm != null) { + return vm.getAccountId(); + } + return Account.ACCOUNT_ID_SYSTEM; } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index f4f67c89f3db..dde9906073d4 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -465,7 +465,27 @@ import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; -import org.apache.cloudstack.api.command.user.vm.*; +import org.apache.cloudstack.api.command.user.vm.AddIpToVmNicCmd; +import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; +import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; +import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; +import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd; +import org.apache.cloudstack.api.command.user.vm.ListNicsCmd; +import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; +import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; +import org.apache.cloudstack.api.command.user.vm.RemoveIpFromVmNicCmd; +import org.apache.cloudstack.api.command.user.vm.RemoveNicFromVMCmd; +import org.apache.cloudstack.api.command.user.vm.ResetVMPasswordCmd; +import org.apache.cloudstack.api.command.user.vm.ResetVMSSHKeyCmd; +import org.apache.cloudstack.api.command.user.vm.RestoreVMCmd; +import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.cloudstack.api.command.user.vm.StopVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateDefaultNicForVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateVmNicIpCmd; +import org.apache.cloudstack.api.command.user.vm.UpgradeVMCmd; import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.ListVMGroupsCmd; @@ -2778,7 +2798,6 @@ public long getMemoryOrCpuCapacityByHost(final Long hostId, final short capacity @Override public List> getCommands() { final List> cmdList = new ArrayList>(); - cmdList.add(CloneVMCmd.class); cmdList.add(CreateAccountCmd.class); cmdList.add(DeleteAccountCmd.class); cmdList.add(DisableAccountCmd.class); @@ -3064,6 +3083,7 @@ public List> getCommands() { cmdList.add(ExpungeVMCmd.class); cmdList.add(GetVMPasswordCmd.class); cmdList.add(ListVMsCmd.class); + cmdList.add(CloneVMCmd.class); cmdList.add(ScaleVMCmd.class); cmdList.add(RebootVMCmd.class); cmdList.add(RemoveNicFromVMCmd.class); From 6b2604d1ba86f9dcecbdd5d7ca89a6dbb877ad23 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 18 Jun 2021 03:40:07 -0400 Subject: [PATCH 003/114] add findISO and findVO steps --- .../main/java/com/cloud/vm/UserVmService.java | 25 ++++++------- .../api/command/user/vm/CloneVMCmd.java | 35 ++++++++++++------- .../java/com/cloud/vm/UserVmManagerImpl.java | 35 +++++-------------- 3 files changed, 43 insertions(+), 52 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 0b48a4867c00..c3d0a59f0b7f 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -18,24 +18,12 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; -import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; -import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; -import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; -import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; -import org.apache.cloudstack.api.command.user.vm.RemoveNicFromVMCmd; -import org.apache.cloudstack.api.command.user.vm.ResetVMPasswordCmd; -import org.apache.cloudstack.api.command.user.vm.ResetVMSSHKeyCmd; -import org.apache.cloudstack.api.command.user.vm.RestoreVMCmd; -import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd; -import org.apache.cloudstack.api.command.user.vm.StartVMCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateDefaultNicForVMCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateVmNicIpCmd; -import org.apache.cloudstack.api.command.user.vm.UpgradeVMCmd; +import org.apache.cloudstack.api.command.user.vm.*; import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; @@ -85,6 +73,15 @@ public interface UserVmService { */ UserVm destroyVm(long vmId, boolean expunge) throws ResourceUnavailableException, ConcurrentOperationException; + /** + * Clone a specific VM (full clone) + * + * @param cmd + * - the command specifying vmId to be cloned + * @return the VM if cloneVM operation is successful + * */ + Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException; + /** * Resets the password of a virtual machine. * diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index de241b2c8cae..6efb3502aa0e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -10,18 +10,14 @@ import com.cloud.uservm.UserVm; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.ACL; -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiCommandJobType; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseAsyncCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ResponseObject; -import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.*; import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; +import java.util.Optional; + @APICommand(name = "cloneVirtualMachine", responseObject = UserVmResponse.class, description = "clone a virtual VM in full clone mode", responseView = ResponseObject.ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, entityType = {VirtualMachine.class}) public class CloneVMCmd extends BaseAsyncCmd implements UserCmd { @@ -55,10 +51,25 @@ public String getEventDescription() { } @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", null).get(0); - response.setResponseName("test_clone"); - setResponseObject(response); + public void execute() { + Optional result; + try { + CallContext.current().setEventDetails("Vm Id for full clone: " + getId()); + result = _userVmService.cloneVirtualMachine(this); + } catch (ResourceUnavailableException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); + } catch (ConcurrentOperationException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + result.ifPresentOrElse((userVm)-> { + UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", userVm).get(0); + response.setResponseName("test_clone"); + setResponseObject(response); + }, ()-> { + throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, "failed to clone VM: " + getId()); + }); } @Override diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 14cdae66d28e..11f1a2776406 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -22,18 +22,8 @@ import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -63,21 +53,7 @@ import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; -import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; -import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; -import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; -import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; -import org.apache.cloudstack.api.command.user.vm.RemoveNicFromVMCmd; -import org.apache.cloudstack.api.command.user.vm.ResetVMPasswordCmd; -import org.apache.cloudstack.api.command.user.vm.ResetVMSSHKeyCmd; -import org.apache.cloudstack.api.command.user.vm.RestoreVMCmd; -import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd; -import org.apache.cloudstack.api.command.user.vm.SecurityGroupAction; -import org.apache.cloudstack.api.command.user.vm.StartVMCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateDefaultNicForVMCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; -import org.apache.cloudstack.api.command.user.vm.UpdateVmNicIpCmd; -import org.apache.cloudstack.api.command.user.vm.UpgradeVMCmd; +import org.apache.cloudstack.api.command.user.vm.*; import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; @@ -4499,6 +4475,13 @@ protected String validateUserData(String userData, HTTPMethod httpmethod) { return null; } + @Override + @ActionEvent(eventType = EventTypes.EVENT_VM_CLONE, eventDescription = "clone vm", async = true) + public Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException { + VolumeVO volumeInformation = _volsDao.findByUuid(""); + return Optional.ofNullable(null); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CREATE, eventDescription = "starting Vm", async = true) public UserVm startVirtualMachine(DeployVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException { From 8c815aba944108470221e82081b4a70edd9f97dd Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 21 Jun 2021 04:03:04 -0400 Subject: [PATCH 004/114] add creation of root-disk -> template -> vm step --- .../cloud/template/TemplateApiService.java | 9 ++++ .../main/java/com/cloud/vm/UserVmService.java | 18 ++++++- .../api/command/user/vm/CloneVMCmd.java | 35 ++++++++++++-- .../cloud/template/TemplateManagerImpl.java | 11 +++++ .../java/com/cloud/vm/UserVmManagerImpl.java | 48 +++++++++++++++++-- 5 files changed, 112 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index ea818a55a0cc..bcda2bc2f2dc 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -20,6 +20,7 @@ import java.net.URISyntaxException; import java.util.List; +import com.cloud.exception.ResourceUnavailableException; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; @@ -40,6 +41,7 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.response.GetUploadParamsResponse; public interface TemplateApiService { @@ -99,6 +101,13 @@ public interface TemplateApiService { boolean updateTemplateOrIsoPermissions(BaseUpdateTemplateOrIsoPermissionsCmd cmd); + /** + * create a template record for later usage of creating a real template by createPrivateTemplate + * */ + VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner) throws ResourceAllocationException; + + VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd) throws CloudRuntimeException; + VirtualMachineTemplate createPrivateTemplateRecord(CreateTemplateCmd cmd, Account templateOwner) throws ResourceAllocationException; VirtualMachineTemplate createPrivateTemplate(CreateTemplateCmd command) throws CloudRuntimeException; diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index c3d0a59f0b7f..02b67c8e7695 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -23,7 +23,21 @@ import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; -import org.apache.cloudstack.api.command.user.vm.*; +import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; +import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; +import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; +import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; +import org.apache.cloudstack.api.command.user.vm.RemoveNicFromVMCmd; +import org.apache.cloudstack.api.command.user.vm.ResetVMPasswordCmd; +import org.apache.cloudstack.api.command.user.vm.ResetVMSSHKeyCmd; +import org.apache.cloudstack.api.command.user.vm.RestoreVMCmd; +import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateDefaultNicForVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateVmNicIpCmd; +import org.apache.cloudstack.api.command.user.vm.UpgradeVMCmd; import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; @@ -82,6 +96,8 @@ public interface UserVmService { * */ Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException; + void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException; + /** * Resets the password of a virtual machine. * diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 6efb3502aa0e..a740c6031baf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -2,12 +2,12 @@ import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.*; @@ -20,7 +20,7 @@ @APICommand(name = "cloneVirtualMachine", responseObject = UserVmResponse.class, description = "clone a virtual VM in full clone mode", responseView = ResponseObject.ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, entityType = {VirtualMachine.class}) -public class CloneVMCmd extends BaseAsyncCmd implements UserCmd { +public class CloneVMCmd extends BaseAsyncCreateCustomIdCmd implements UserCmd { public static final Logger s_logger = Logger.getLogger(CloneVMCmd.class.getName()); private static final String s_name = "clonevirtualmachineresponse"; @@ -28,7 +28,7 @@ public class CloneVMCmd extends BaseAsyncCmd implements UserCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// @ACL(accessType = AccessType.OperateEntry) - @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType=UserVmResponse.class, + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType=UserVmResponse.class, required = true, description = "The ID of the virtual machine") private Long id; @@ -50,6 +50,29 @@ public String getEventDescription() { return "Cloning user VM: " + this._uuidMgr.getUuid(VirtualMachine.class, getId()); } + @Override + public void create() throws ResourceAllocationException { + VirtualMachineTemplate template = null; + try { + _userVmService.checkCloneCondition(this); + } catch (ResourceUnavailableException e) { + s_logger.warn("Exception: ", e); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); + } + template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId())); + if (template == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to generate a template during clone!"); + } + } + + public String getVMName() { + return getTargetVM().getInstanceName(); + } + + public String getTemplateName() { + return getVMName() + "-QA"; + } + @Override public void execute() { Optional result; @@ -85,4 +108,8 @@ public long getEntityOwnerId() { } return Account.ACCOUNT_ID_SYSTEM; } + + public UserVm getTargetVM() { + return this._userVmService.getUserVm(getId()); + } } diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 3ca0807e6228..698e23119d2a 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -54,6 +54,7 @@ import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; @@ -1781,6 +1782,16 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } } + @Override + @DB + @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template from clone", create = true) + public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner) throws ResourceAllocationException { + Account caller = CallContext.current().getCallingAccount(); + boolean isAdmin = (_accountMgr.isAdmin(caller.getId())); + _accountMgr.checkAccess(caller, null, true, templateOwner); + String name = cmd.getTemplateName(); + return null; + } @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template", create = true) public VMTemplateVO createPrivateTemplateRecord(CreateTemplateCmd cmd, Account templateOwner) throws ResourceAllocationException { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 11f1a2776406..4fb7b83f93ad 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -22,8 +22,19 @@ import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -53,7 +64,22 @@ import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; -import org.apache.cloudstack.api.command.user.vm.*; +import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; +import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; +import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; +import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; +import org.apache.cloudstack.api.command.user.vm.RemoveNicFromVMCmd; +import org.apache.cloudstack.api.command.user.vm.ResetVMPasswordCmd; +import org.apache.cloudstack.api.command.user.vm.ResetVMSSHKeyCmd; +import org.apache.cloudstack.api.command.user.vm.RestoreVMCmd; +import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd; +import org.apache.cloudstack.api.command.user.vm.SecurityGroupAction; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateDefaultNicForVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateVmNicIpCmd; +import org.apache.cloudstack.api.command.user.vm.UpgradeVMCmd; import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; @@ -4475,10 +4501,24 @@ protected String validateUserData(String userData, HTTPMethod httpmethod) { return null; } + @Override + public void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, CloudRuntimeException, ConcurrentOperationException { + UserVm curVm = cmd.getTargetVM(); + if (curVm == null) { + throw new CloudRuntimeException("the VM doesn't exist or not registered in management server!"); + } + UserVmVO vmStatus = _vmDao.findById(cmd.getId()); + if (vmStatus.state != State.Shutdown) { + throw new CloudRuntimeException("You should clone an instance that's shutdown!"); + } + List volumeInformation = _volsDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); + if (CollectionUtils.isEmpty(volumeInformation)) { + throw new CloudRuntimeException("The VM to copy does not have a Volume attached!"); + } + } @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CLONE, eventDescription = "clone vm", async = true) - public Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException { - VolumeVO volumeInformation = _volsDao.findByUuid(""); + public Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException { return Optional.ofNullable(null); } From fb6cb0acb6c96d6ce2da46c742e0af4bdb03e59f Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 23 Jun 2021 02:21:01 -0400 Subject: [PATCH 005/114] prepare the template impl --- .../api/command/user/vm/CloneVMCmd.java | 42 ++++++- .../cloud/template/TemplateManagerImpl.java | 114 ++++++++++++++++++ .../java/com/cloud/vm/UserVmManagerImpl.java | 1 + 3 files changed, 151 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index a740c6031baf..0bc6163e5756 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -7,10 +7,17 @@ import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.uservm.UserVm; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.*; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCustomIdCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.context.CallContext; @@ -32,6 +39,26 @@ public class CloneVMCmd extends BaseAsyncCreateCustomIdCmd implements UserCmd { required = true, description = "The ID of the virtual machine") private Long id; + public Long getVolumeId() { + return volumeId; + } + + public void setVolumeId(Long volumeId) { + this.volumeId = volumeId; + } + + private Long volumeId; + + private VirtualMachineTemplate createdTemplate; + + public void setCreatedTemplate(VirtualMachineTemplate template) { + this.createdTemplate = template; + } + + public VirtualMachineTemplate getCreatedTemplate() { + return this.createdTemplate; + } + public Long getId() { return this.id; } @@ -55,14 +82,17 @@ public void create() throws ResourceAllocationException { VirtualMachineTemplate template = null; try { _userVmService.checkCloneCondition(this); + template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId())); + setCreatedTemplate(template); +// _userVmService.createBasicSecurityGroupVirtualMachine(); // disabled since it crashes } catch (ResourceUnavailableException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); } - template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId())); - if (template == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to generate a template during clone!"); - } + } + + public boolean isPublic() { + return false; } public String getVMName() { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 698e23119d2a..e972748ff5b8 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1790,8 +1790,122 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template boolean isAdmin = (_accountMgr.isAdmin(caller.getId())); _accountMgr.checkAccess(caller, null, true, templateOwner); String name = cmd.getTemplateName(); + if (name.length() > 32) { + name = name.substring(5) + "-QA"; + } + + int bits = 64; // where to specify + boolean requireHVM = true, sshKeyEnabled = true, featured = false; + boolean isPublic = cmd.isPublic(); + Long volumeId = cmd.getVolumeId(); + HypervisorType hyperType = null; + VolumeVO volume = _volumeDao.findById(volumeId); + if (volume == null) { + throw new InvalidParameterValueException("Failed to create private template record, unable to find volume " + volumeId); + } + // check permissions + _accountMgr.checkAccess(caller, null, true, volume); + + // If private template is created from Volume, check that the volume + // will not be active when the private template is + // created + if (!_volumeMgr.volumeInactive(volume)) { + String msg = "Unable to create private template for volume: " + volume.getName() + "; volume is attached to a non-stopped VM, please stop the VM first"; + if (s_logger.isInfoEnabled()) { + s_logger.info(msg); + } + throw new CloudRuntimeException(msg); + } + + hyperType = _volumeDao.getHypervisorType(volumeId); + if (HypervisorType.LXC.equals(hyperType)) { + throw new InvalidParameterValueException("Template creation is not supported for LXC volume: " + volumeId); + } + + _resourceLimitMgr.checkResourceLimit(templateOwner, ResourceType.template); + _resourceLimitMgr.checkResourceLimit(templateOwner, ResourceType.secondary_storage, volume.getSize()); + + Long guestOSId = cmd.getTargetVM().getGuestOSId(); + GuestOSVO guestOS = _guestOSDao.findById(guestOSId); + if (guestOS == null) { + throw new InvalidParameterValueException("GuestOS with ID: " + guestOSId + " does not exist."); + } + + Long nextTemplateId = _tmpltDao.getNextInSequence(Long.class, "id"); + String description = ""; // TODO: add this to clone parameter in the future + boolean isExtractable = false; + Long sourceTemplateId = null; + if (volume != null) { + VMTemplateVO template = ApiDBUtils.findTemplateById(volume.getTemplateId()); + isExtractable = template != null && template.isExtractable() && template.getTemplateType() != Storage.TemplateType.SYSTEM; + if (volume.getIsoId() != null && volume.getIsoId() != 0) { + sourceTemplateId = volume.getIsoId(); + } else if (volume.getTemplateId() != null) { + sourceTemplateId = volume.getTemplateId(); + } + } + + VMTemplateVO privateTemplate = null; + privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable, + TemplateType.USER, null, true, 64, templateOwner.getId(), null, description, + true, guestOS.getId(), true, hyperType, null, new HashMap<>(){{put("template to be cleared", "yes");}}, false, false, false, false); + List stores = _imgStoreDao.findRegionImageStores(); + if (!CollectionUtils.isEmpty(stores)) { + privateTemplate.setCrossZones(true); + } + + privateTemplate.setSourceTemplateId(sourceTemplateId); + + VMTemplateVO template = _tmpltDao.persist(privateTemplate); + // Increment the number of templates + if (template != null) { + Map details = new HashMap(); + + if (sourceTemplateId != null) { + VMTemplateVO sourceTemplate = _tmpltDao.findById(sourceTemplateId); + if (sourceTemplate != null && sourceTemplate.getDetails() != null) { + details.putAll(sourceTemplate.getDetails()); + } + } + + if (volume != null) { + Long vmId = volume.getInstanceId(); + if (vmId != null) { + UserVmVO userVm = _userVmDao.findById(vmId); + if (userVm != null) { + _userVmDao.loadDetails(userVm); + Map vmDetails = userVm.getDetails(); + vmDetails = vmDetails.entrySet() + .stream() + .filter(map -> map.getValue() != null) + .collect(Collectors.toMap(map -> map.getKey(), map -> map.getValue())); + details.putAll(vmDetails); + } + } + } + + if (!details.isEmpty()) { + privateTemplate.setDetails(details); + _tmpltDao.saveDetails(privateTemplate); + } + + _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.template); + _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.secondary_storage, + volume.getSize()); + } + + if (template != null) { + return template; + } else { + throw new CloudRuntimeException("Failed to create a template"); + } + } + + @Override + public VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd) throws CloudRuntimeException { return null; } + @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template", create = true) public VMTemplateVO createPrivateTemplateRecord(CreateTemplateCmd cmd, Account templateOwner) throws ResourceAllocationException { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 4fb7b83f93ad..60607b99a845 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4515,6 +4515,7 @@ public void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableExcept if (CollectionUtils.isEmpty(volumeInformation)) { throw new CloudRuntimeException("The VM to copy does not have a Volume attached!"); } + cmd.setVolumeId(volumeInformation.get(0).getId()); } @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CLONE, eventDescription = "clone vm", async = true) From 77c39c21bf5bcaa0888f411eba8e4002088e338c Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 23 Jun 2021 02:23:28 -0400 Subject: [PATCH 006/114] clear the dot record for later test --- api/src/main/java/com/cloud/template/TemplateApiService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index bcda2bc2f2dc..f0e6f0b264a4 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -20,7 +20,6 @@ import java.net.URISyntaxException; import java.util.List; -import com.cloud.exception.ResourceUnavailableException; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; From 1971a39f4bdb9ac2d09cfd3334dbaf28a7c6a51b Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 25 Jun 2021 11:49:00 -0400 Subject: [PATCH 007/114] checks for VM are implemented --- .../main/java/com/cloud/vm/UserVmService.java | 5 +- .../api/command/user/vm/CloneVMCmd.java | 49 ++++++----- .../cloud/template/TemplateManagerImpl.java | 2 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 83 +++++++++++++++---- 4 files changed, 101 insertions(+), 38 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 02b67c8e7695..a7fe2824763c 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -96,7 +96,7 @@ public interface UserVmService { * */ Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException; - void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException; + void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, ResourceAllocationException; /** * Resets the password of a virtual machine. @@ -442,6 +442,9 @@ UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serviceOffe UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, StorageUnavailableException, ResourceAllocationException; + UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, + StorageUnavailableException, ResourceAllocationException; + UserVm getUserVm(long vmId); /** diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 0bc6163e5756..b834c95dc55f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -1,9 +1,7 @@ package org.apache.cloudstack.api.command.user.vm; import com.cloud.event.EventTypes; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.*; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.uservm.UserVm; @@ -19,6 +17,7 @@ import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; @@ -39,24 +38,19 @@ public class CloneVMCmd extends BaseAsyncCreateCustomIdCmd implements UserCmd { required = true, description = "The ID of the virtual machine") private Long id; - public Long getVolumeId() { - return volumeId; - } - - public void setVolumeId(Long volumeId) { - this.volumeId = volumeId; - } + //Owner information + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") + private String accountName; - private Long volumeId; + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used.") + private Long domainId; - private VirtualMachineTemplate createdTemplate; - - public void setCreatedTemplate(VirtualMachineTemplate template) { - this.createdTemplate = template; + public String getAccountName() { + return accountName; } - public VirtualMachineTemplate getCreatedTemplate() { - return this.createdTemplate; + public Long getDomainId() { + return domainId; } public Long getId() { @@ -79,15 +73,21 @@ public String getEventDescription() { @Override public void create() throws ResourceAllocationException { - VirtualMachineTemplate template = null; try { _userVmService.checkCloneCondition(this); - template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId())); - setCreatedTemplate(template); + UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); + if (vmRecord == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "unable to record a new VM to db!"); + } + setEntityId(vmRecord.getId()); + setEntityUuid(vmRecord.getUuid()); // _userVmService.createBasicSecurityGroupVirtualMachine(); // disabled since it crashes - } catch (ResourceUnavailableException e) { + } catch (ResourceUnavailableException | InsufficientCapacityException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); + } catch (InvalidParameterValueException e) { + s_logger.warn("Exception: ", e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } } @@ -107,7 +107,8 @@ public String getTemplateName() { public void execute() { Optional result; try { - CallContext.current().setEventDetails("Vm Id for full clone: " + getId()); +// CallContext.current().setEventDetails("Vm Id for full clone: " + getId()); +// VirtualMachineTemplate template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId())); result = _userVmService.cloneVirtualMachine(this); } catch (ResourceUnavailableException ex) { s_logger.warn("Exception: ", ex); @@ -116,6 +117,10 @@ public void execute() { s_logger.warn("Exception: ", ex); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } +// catch (ResourceAllocationException ex) { +// s_logger.warn("Exception: ", ex); +// throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage()); +// } result.ifPresentOrElse((userVm)-> { UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", userVm).get(0); response.setResponseName("test_clone"); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index e972748ff5b8..fc91584b2d44 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1797,7 +1797,7 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template int bits = 64; // where to specify boolean requireHVM = true, sshKeyEnabled = true, featured = false; boolean isPublic = cmd.isPublic(); - Long volumeId = cmd.getVolumeId(); + Long volumeId = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT).get(0).getId(); HypervisorType hyperType = null; VolumeVO volume = _volumeDao.findById(volumeId); if (volume == null) { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 60607b99a845..f6973b940dfa 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -51,6 +51,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import com.cloud.user.*; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -292,16 +293,6 @@ import com.cloud.template.TemplateApiService; import com.cloud.template.TemplateManager; import com.cloud.template.VirtualMachineTemplate; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountService; -import com.cloud.user.ResourceLimitService; -import com.cloud.user.SSHKeyPair; -import com.cloud.user.SSHKeyPairVO; -import com.cloud.user.User; -import com.cloud.user.UserStatisticsVO; -import com.cloud.user.UserVO; -import com.cloud.user.VmDiskStatisticsVO; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.user.dao.UserDao; @@ -4502,7 +4493,17 @@ protected String validateUserData(String userData, HTTPMethod httpmethod) { } @Override - public void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, CloudRuntimeException, ConcurrentOperationException { + public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueException, ResourceUnavailableException, CloudRuntimeException, ResourceAllocationException { + final DomainVO domain = _domainDao.findById(cmd.getDomainId()); + final Account account = _accountService.getActiveAccountByName(cmd.getAccountName(), cmd.getDomainId()); + if (domain != null && account != null) { + if (account.getType() == Account.ACCOUNT_TYPE_PROJECT) { + throw new InvalidParameterValueException("Invalid user type: project to clone the VM"); + } + if (account.getState() != Account.State.enabled) { + throw new InvalidParameterValueException("User is not enabled to clone this VM"); + } + } UserVm curVm = cmd.getTargetVM(); if (curVm == null) { throw new CloudRuntimeException("the VM doesn't exist or not registered in management server!"); @@ -4511,11 +4512,59 @@ public void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableExcept if (vmStatus.state != State.Shutdown) { throw new CloudRuntimeException("You should clone an instance that's shutdown!"); } - List volumeInformation = _volsDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); - if (CollectionUtils.isEmpty(volumeInformation)) { + List volumes = _volsDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); + if (CollectionUtils.isEmpty(volumes)) { throw new CloudRuntimeException("The VM to copy does not have a Volume attached!"); } - cmd.setVolumeId(volumeInformation.get(0).getId()); + // verify that the VM doesn't expire + Map details = curVm.getDetails(); + verifyDetails(details); +// Account activeOwner = _accountDao.findById(cmd.getEntityOwnerId()); + long zoneId = curVm.getDataCenterId(); + DataCenter zone = _entityMgr.findById(DataCenter.class, zoneId); + if (zone == null) { + throw new InvalidParameterValueException("Unable to find a zone in the current VM by zone id=" + zoneId); + } + // service offering check + long serviceOfferingId = curVm.getServiceOfferingId(); + ServiceOffering serviceOffering = _entityMgr.findById(ServiceOffering.class, serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Service offering Id for this VM: " + serviceOfferingId + " doesn't exist now"); + } + if (!serviceOffering.isDynamic()) { + for(String detail: details.keySet()) { + if(detail.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || detail.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || detail.equalsIgnoreCase(VmDetailConstants.MEMORY)) { + throw new InvalidParameterValueException("cpuNumber or cpuSpeed or memory should not be specified for static service offering"); + } + } + } + // disk offering check + VolumeVO rootDisk = volumes.get(0); + Long diskOfferingID = rootDisk.getDiskOfferingId(); + DiskOfferingVO diskOffering =null; + if (diskOfferingID != null) { + diskOffering = _diskOfferingDao.findById(diskOfferingID); + if (diskOffering == null) { + throw new CloudRuntimeException("Unable to find disk offering " + diskOfferingID); + } + } + if (!zone.isLocalStorageEnabled()) { + if (serviceOffering.isUseLocalStorage()) { + throw new CloudRuntimeException("Zone is not configured to use local storage now but this service offering " + serviceOffering.getName() + " uses it"); + } + if (diskOffering != null && diskOffering.isUseLocalStorage()) { + throw new CloudRuntimeException("Zone is not configured to use local storage but disk offering " + diskOffering.getName() + " uses it"); + } + } + // resource limit checks & account check + AccountVO activeOwner = _accountDao.findById(cmd.getEntityOwnerId()); + List totalVolumes = _volsDao.findByInstance(cmd.getId()); + _resourceLimitMgr.checkResourceLimit(activeOwner, ResourceType.volume, totalVolumes.size()); + Long totalSize = 0L; + for (VolumeVO volumeToCheck : totalVolumes) { + totalSize += volumeToCheck.getSize(); + } + _resourceLimitMgr.checkResourceLimit(activeOwner, ResourceType.primary_storage, totalSize); } @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CLONE, eventDescription = "clone vm", async = true) @@ -5508,6 +5557,12 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE return vm; } + @Override + public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperationException { + //network configurations and check, then create the template + return null; + } + /** * Persist extra configuration data in the user_vm_details table as key/value pair * @param decodedUrl String consisting of the extra config data to appended onto the vmx file for VMware instances From 368711043ee79256466d61a17ee1066fca985764 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 25 Jun 2021 11:52:02 -0400 Subject: [PATCH 008/114] fix checkstyle --- .../cloudstack/api/command/user/vm/CloneVMCmd.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index b834c95dc55f..83a95b46ba3e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -1,8 +1,11 @@ package org.apache.cloudstack.api.command.user.vm; import com.cloud.event.EventTypes; -import com.cloud.exception.*; -import com.cloud.template.VirtualMachineTemplate; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.user.Account; import com.cloud.uservm.UserVm; import com.cloud.vm.VirtualMachine; @@ -19,7 +22,7 @@ import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.context.CallContext; +//import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; import java.util.Optional; From fbb886cfb0af4fd87f58040635f8ae7c5dba636c Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 25 Jun 2021 11:53:58 -0400 Subject: [PATCH 009/114] compilation fixed --- .../main/java/com/cloud/vm/UserVmManagerImpl.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index f6973b940dfa..a5dbc3cc8a9e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -51,7 +51,17 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.user.*; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountService; +import com.cloud.user.AccountVO; +import com.cloud.user.ResourceLimitService; +import com.cloud.user.SSHKeyPair; +import com.cloud.user.SSHKeyPairVO; +import com.cloud.user.User; +import com.cloud.user.UserStatisticsVO; +import com.cloud.user.UserVO; +import com.cloud.user.VmDiskStatisticsVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; From 16d75daafe4f4a56af95328158314905efc6bf0d Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 25 Jun 2021 17:26:26 -0400 Subject: [PATCH 010/114] checkClone debug --- .../main/java/com/cloud/vm/UserVmManagerImpl.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index a5dbc3cc8a9e..e9281f9a416d 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4504,8 +4504,11 @@ protected String validateUserData(String userData, HTTPMethod httpmethod) { @Override public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueException, ResourceUnavailableException, CloudRuntimeException, ResourceAllocationException { - final DomainVO domain = _domainDao.findById(cmd.getDomainId()); - final Account account = _accountService.getActiveAccountByName(cmd.getAccountName(), cmd.getDomainId()); + if (cmd.getAccountName() != null && cmd.getDomainId() == null) { + throw new InvalidParameterValueException("You must input the domainId together with the account name"); + } + final DomainVO domain = cmd.getDomainId() == null ? null : _domainDao.findById(cmd.getDomainId()); + final Account account = cmd.getAccountName() == null ? null : _accountService.getActiveAccountByName(cmd.getAccountName(), cmd.getDomainId()); if (domain != null && account != null) { if (account.getType() == Account.ACCOUNT_TYPE_PROJECT) { throw new InvalidParameterValueException("Invalid user type: project to clone the VM"); @@ -5570,6 +5573,12 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE @Override public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperationException { //network configurations and check, then create the template + UserVm curVm = cmd.getTargetVM(); + // check if host is available + Long hostId = curVm.getHostId(); + getDestinationHost(hostId, true); + + return null; } From 779681cf2ab17880efa5340a8cf5c2e9224a6358 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 25 Jun 2021 19:14:00 -0400 Subject: [PATCH 011/114] server state update --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index e9281f9a416d..3722e75faf5a 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4522,7 +4522,7 @@ public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueExce throw new CloudRuntimeException("the VM doesn't exist or not registered in management server!"); } UserVmVO vmStatus = _vmDao.findById(cmd.getId()); - if (vmStatus.state != State.Shutdown) { + if (vmStatus.state != State.Shutdown || vmStatus.state != State.Stopped) { throw new CloudRuntimeException("You should clone an instance that's shutdown!"); } List volumes = _volsDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); From 5a81d2b7e81e88acb0b783313b5e48ccec2a2744 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 25 Jun 2021 19:20:26 -0400 Subject: [PATCH 012/114] add and statement --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 3722e75faf5a..354f849cab1e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4522,7 +4522,7 @@ public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueExce throw new CloudRuntimeException("the VM doesn't exist or not registered in management server!"); } UserVmVO vmStatus = _vmDao.findById(cmd.getId()); - if (vmStatus.state != State.Shutdown || vmStatus.state != State.Stopped) { + if (vmStatus.state != State.Shutdown && vmStatus.state != State.Stopped) { throw new CloudRuntimeException("You should clone an instance that's shutdown!"); } List volumes = _volsDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); From 6f4fca9a9a5793a211be41190b1e87107d4b84d5 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 25 Jun 2021 19:35:44 -0400 Subject: [PATCH 013/114] fix null error --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 354f849cab1e..6b0e739dfed1 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4544,7 +4544,7 @@ public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueExce if (serviceOffering == null) { throw new InvalidParameterValueException("Service offering Id for this VM: " + serviceOfferingId + " doesn't exist now"); } - if (!serviceOffering.isDynamic()) { + if (!serviceOffering.isDynamic() && details != null) { for(String detail: details.keySet()) { if(detail.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || detail.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || detail.equalsIgnoreCase(VmDetailConstants.MEMORY)) { throw new InvalidParameterValueException("cpuNumber or cpuSpeed or memory should not be specified for static service offering"); From 896b0331d7750a88aa4c4aa656f99b0a69a597b4 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 28 Jun 2021 09:17:36 -0400 Subject: [PATCH 014/114] test hit the current vm information --- .../api/command/user/vm/CloneVMCmd.java | 10 ++++ .../cloud/template/TemplateManagerImpl.java | 1 - .../java/com/cloud/vm/UserVmManagerImpl.java | 46 ++++++++++++++++--- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 83a95b46ba3e..0479a7d3fb36 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -48,6 +48,8 @@ public class CloneVMCmd extends BaseAsyncCreateCustomIdCmd implements UserCmd { @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used.") private Long domainId; + private Long temporaryTemlateId; + public String getAccountName() { return accountName; } @@ -74,6 +76,14 @@ public String getEventDescription() { return "Cloning user VM: " + this._uuidMgr.getUuid(VirtualMachine.class, getId()); } + public Long getTemporaryTemlateId() { + return this.temporaryTemlateId; + } + + public void setTemporaryTemlateId(Long tempId) { + this.temporaryTemlateId = tempId; + } + @Override public void create() throws ResourceAllocationException { try { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index fc91584b2d44..b96ab18841f0 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1783,7 +1783,6 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } @Override - @DB @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template from clone", create = true) public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 6b0e739dfed1..bc260b2a506a 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -51,6 +51,8 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import com.cloud.network.*; +import com.cloud.network.security.SecurityGroupVO; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -226,14 +228,10 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; -import com.cloud.network.IpAddressManager; -import com.cloud.network.Network; import com.cloud.network.Network.IpAddresses; import com.cloud.network.Network.Provider; import com.cloud.network.Network.Service; -import com.cloud.network.NetworkModel; import com.cloud.network.Networks.TrafficType; -import com.cloud.network.PhysicalNetwork; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; @@ -4525,6 +4523,14 @@ public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueExce if (vmStatus.state != State.Shutdown && vmStatus.state != State.Stopped) { throw new CloudRuntimeException("You should clone an instance that's shutdown!"); } + if (vmStatus.getHypervisorType() != HypervisorType.KVM && vmStatus.getHypervisorType() != HypervisorType.Simulator) { + throw new CloudRuntimeException("The clone operation is only supported on KVM and Simulator!"); + } + Long accountId = curVm.getAccountId(); + Account vmOwner = _accountDao.findById(accountId); + if (vmOwner == null) { + throw new CloudRuntimeException("This VM doesn't have an owner account, please assign one to it"); + } List volumes = _volsDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); if (CollectionUtils.isEmpty(volumes)) { throw new CloudRuntimeException("The VM to copy does not have a Volume attached!"); @@ -5571,14 +5577,40 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } @Override - public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperationException { + public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperationException, ResourceAllocationException, InsufficientAddressCapacityException { //network configurations and check, then create the template UserVm curVm = cmd.getTargetVM(); // check if host is available Long hostId = curVm.getHostId(); getDestinationHost(hostId, true); - - + Long zoneId = curVm.getDataCenterId(); + DataCenter dataCenter = _entityMgr.findById(DataCenter.class, zoneId); + Map vmProperties = curVm.getDetails(); + String keyboard = vmProperties.get(VmDetailConstants.KEYBOARD); + HypervisorType hypervisorType = curVm.getHypervisorType(); + Account curAccount = _accountDao.findById(curVm.getAccountId()); + long callingUserId = CallContext.current().getCallingUserId(); + Account callerAccount = CallContext.current().getCallingAccount(); +// IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, curAccount.getId() == Account.ACCOUNT_ID_SYSTEM, callerAccount, callingUserId, dataCenter, null, null); + long serviceOfferingId = curVm.getServiceOfferingId(); + ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); + List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); + List securityGroupIdList = securityGroupList.stream().map(SecurityGroupVO::getId).collect(Collectors.toList()); + String uuidName = _uuidMgr.generateUuid(UserVm.class, null); + String hostName = generateHostName(uuidName); + String displayName = hostName + "-Clone"; + Long diskOfferingId = curVm.getDiskOfferingId(); + Long size = null; // mutual exclusive with disk offering id + HTTPMethod httpMethod = cmd.getHttpMethod(); + String userData = curVm.getUserData(); + String sshKeyPair = null; + Map ipToNetoworkMap = null; // Since we've specified Ip + boolean isDisplayVM = curVm.isDisplayVm(); + boolean dynamicScalingEnabled = curVm.isDynamicallyScalable(); + +// if (dataCenter.getNetworkType() == NetworkType.Basic) { +// +// } return null; } From cfd131c3c83b9421660fa03aae76e0b4e364fa31 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 28 Jun 2021 09:34:31 -0400 Subject: [PATCH 015/114] fix typo of star import --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index bc260b2a506a..c22ea5c83907 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -51,7 +51,11 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.network.*; +import com.cloud.network.IpAddress; +import com.cloud.network.IpAddressManager; +import com.cloud.network.Network; +import com.cloud.network.NetworkModel; +import com.cloud.network.PhysicalNetwork; import com.cloud.network.security.SecurityGroupVO; import com.cloud.user.Account; import com.cloud.user.AccountManager; From 9677858d48b56ba4903bac5dcc63f413966d95fc Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 28 Jun 2021 09:41:19 -0400 Subject: [PATCH 016/114] fix the null pt except --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index c22ea5c83907..2dac8084911d 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5589,7 +5589,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio getDestinationHost(hostId, true); Long zoneId = curVm.getDataCenterId(); DataCenter dataCenter = _entityMgr.findById(DataCenter.class, zoneId); - Map vmProperties = curVm.getDetails(); + Map vmProperties = curVm.getDetails() != null ? curVm.getDetails() : new HashMap<>(); String keyboard = vmProperties.get(VmDetailConstants.KEYBOARD); HypervisorType hypervisorType = curVm.getHypervisorType(); Account curAccount = _accountDao.findById(curVm.getAccountId()); From 85129eea652cb6c483946a26e0f2795e53a47c00 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 28 Jun 2021 10:21:29 -0400 Subject: [PATCH 017/114] add vm record creation code --- .../apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 5 +++++ .../main/java/com/cloud/template/TemplateManagerImpl.java | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 0479a7d3fb36..ac8f3eebbca6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -6,6 +6,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.uservm.UserVm; import com.cloud.vm.VirtualMachine; @@ -88,6 +89,10 @@ public void setTemporaryTemlateId(Long tempId) { public void create() throws ResourceAllocationException { try { _userVmService.checkCloneCondition(this); + VirtualMachineTemplate template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId())); + if (template == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "failed to create a template to db"); + } UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); if (vmRecord == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "unable to record a new VM to db!"); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index b96ab18841f0..6d7fc7b2ca23 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1786,7 +1786,6 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template from clone", create = true) public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); - boolean isAdmin = (_accountMgr.isAdmin(caller.getId())); _accountMgr.checkAccess(caller, null, true, templateOwner); String name = cmd.getTemplateName(); if (name.length() > 32) { @@ -1794,7 +1793,7 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template } int bits = 64; // where to specify - boolean requireHVM = true, sshKeyEnabled = true, featured = false; + boolean featured = false; boolean isPublic = cmd.isPublic(); Long volumeId = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT).get(0).getId(); HypervisorType hyperType = null; From b6607f4f270411f9d0a6bbeff27fd781f6d3b734 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 28 Jun 2021 12:00:25 -0400 Subject: [PATCH 018/114] add the user vm db creation code --- .../api/command/user/vm/CloneVMCmd.java | 1 + .../java/com/cloud/vm/UserVmManagerImpl.java | 32 ++++++++++++++----- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index ac8f3eebbca6..1879cd27b3eb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -93,6 +93,7 @@ public void create() throws ResourceAllocationException { if (template == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "failed to create a template to db"); } + setTemporaryTemlateId(template.getId()); UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); if (vmRecord == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "unable to record a new VM to db!"); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 2dac8084911d..8cc906273ebf 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -68,6 +68,8 @@ import com.cloud.user.UserStatisticsVO; import com.cloud.user.UserVO; import com.cloud.user.VmDiskStatisticsVO; +import com.cloud.utils.net.MacAddress; +import com.googlecode.ipv6.IPv6Address; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -5581,7 +5583,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } @Override - public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperationException, ResourceAllocationException, InsufficientAddressCapacityException { + public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperationException, ResourceAllocationException, InsufficientCapacityException, ResourceUnavailableException { //network configurations and check, then create the template UserVm curVm = cmd.getTargetVM(); // check if host is available @@ -5595,7 +5597,10 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio Account curAccount = _accountDao.findById(curVm.getAccountId()); long callingUserId = CallContext.current().getCallingUserId(); Account callerAccount = CallContext.current().getCallingAccount(); -// IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, curAccount.getId() == Account.ACCOUNT_ID_SYSTEM, callerAccount, callingUserId, dataCenter, null, null); + IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, curAccount.getId() == Account.ACCOUNT_ID_SYSTEM, callerAccount, callingUserId, dataCenter, null, null); + String ipv6Address = null; + String macAddress = null; + IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), null, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); @@ -5608,14 +5613,25 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio HTTPMethod httpMethod = cmd.getHttpMethod(); String userData = curVm.getUserData(); String sshKeyPair = null; - Map ipToNetoworkMap = null; // Since we've specified Ip + Map ipToNetoworkMap = null; // Since we've specified Ip boolean isDisplayVM = curVm.isDisplayVm(); boolean dynamicScalingEnabled = curVm.isDynamicallyScalable(); - -// if (dataCenter.getNetworkType() == NetworkType.Basic) { -// -// } - return null; + Long templateId = cmd.getTemporaryTemlateId(); + VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, templateId); + if (template == null) { + throw new CloudRuntimeException("the temporary template is not created, server error, contact your sys admin"); + } + List networkIds = null; + String group = null; + InstanceGroupVO groupVo = getGroupForVm(cmd.getId()); + if (groupVo != null) { + group = groupVo.getName(); + } + UserVm vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, + size , group , hypervisorType, cmd.getHttpMethod(), userData , sshKeyPair , ipToNetoworkMap, addr, isDisplayVM , keyboard , null, + curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), + null, new HashMap<>(), dynamicScalingEnabled); + return vmResult; } /** From afae56078259caabf3d271cf3f9f51c89c997cd6 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 28 Jun 2021 12:03:01 -0400 Subject: [PATCH 019/114] add vm creation typo fix --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 8cc906273ebf..16d540034dc7 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -68,8 +68,6 @@ import com.cloud.user.UserStatisticsVO; import com.cloud.user.UserVO; import com.cloud.user.VmDiskStatisticsVO; -import com.cloud.utils.net.MacAddress; -import com.googlecode.ipv6.IPv6Address; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; From 0a1405d0790e3e29fb805b020b3ec1d5ef322bcf Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 28 Jun 2021 12:19:19 -0400 Subject: [PATCH 020/114] change ip to available --- .../apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 4 ++++ server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 1879cd27b3eb..a7de9630736d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -107,6 +107,10 @@ public void create() throws ResourceAllocationException { } catch (InvalidParameterValueException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } finally { + if (getTemporaryTemlateId() != null) { + // TODO: delete template in the service + } } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 16d540034dc7..2bfa26ee5b28 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5595,10 +5595,11 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio Account curAccount = _accountDao.findById(curVm.getAccountId()); long callingUserId = CallContext.current().getCallingUserId(); Account callerAccount = CallContext.current().getCallingAccount(); - IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, curAccount.getId() == Account.ACCOUNT_ID_SYSTEM, callerAccount, callingUserId, dataCenter, null, null); +// IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, curAccount.getId() == Account.ACCOUNT_ID_SYSTEM, callerAccount, callingUserId, dataCenter, null, null); String ipv6Address = null; String macAddress = null; - IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), null, macAddress); +// IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), null, macAddress); + IpAddresses addr = new IpAddresses("127.20.0.183", ipv6Address, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); From 64c0fd51ec6eef65bc62b9c09051da729181a5ef Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 28 Jun 2021 12:21:41 -0400 Subject: [PATCH 021/114] fix ip issue --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 1 + server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index a7de9630736d..074e47b1671a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -110,6 +110,7 @@ public void create() throws ResourceAllocationException { } finally { if (getTemporaryTemlateId() != null) { // TODO: delete template in the service + s_logger.warn("clearing the temporary template: " + getTemporaryTemlateId()); } } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 2bfa26ee5b28..9f24fd8b3620 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -51,7 +51,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.network.IpAddress; +//import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; import com.cloud.network.Network; import com.cloud.network.NetworkModel; From e04dc22896699b24ac3a64e8e5efcf6d6087862c Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 28 Jun 2021 12:31:36 -0400 Subject: [PATCH 022/114] fix the vm creation detail --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 9f24fd8b3620..224f9f6ed38e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5628,7 +5628,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio } UserVm vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, size , group , hypervisorType, cmd.getHttpMethod(), userData , sshKeyPair , ipToNetoworkMap, addr, isDisplayVM , keyboard , null, - curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), + curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); return vmResult; } From 7bef87f11dac1585b5d208801305e5494009416d Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 2 Jul 2021 02:22:53 -0400 Subject: [PATCH 023/114] template zone addition code --- .../java/com/cloud/template/TemplateManagerImpl.java | 9 ++++++--- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 6d7fc7b2ca23..639f8468aaaa 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1789,12 +1789,13 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template _accountMgr.checkAccess(caller, null, true, templateOwner); String name = cmd.getTemplateName(); if (name.length() > 32) { - name = name.substring(5) + "-QA"; + name = name.substring(5) + "-QA-Clone"; } - int bits = 64; // where to specify boolean featured = false; boolean isPublic = cmd.isPublic(); + UserVm curVm = cmd.getTargetVM(); + long zoneId = curVm.getDataCenterId(); Long volumeId = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT).get(0).getId(); HypervisorType hyperType = null; VolumeVO volume = _volumeDao.findById(volumeId); @@ -1853,8 +1854,10 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template } privateTemplate.setSourceTemplateId(sourceTemplateId); - VMTemplateVO template = _tmpltDao.persist(privateTemplate); + // persist this to the template zone area and remember to remove the resource count in the execute phase once in failure or clean up phase + VMTemplateZoneVO templateZone = new VMTemplateZoneVO(zoneId, template.getId(), new Date()); + _tmpltZoneDao.persist(templateZone); // Increment the number of templates if (template != null) { Map details = new HashMap(); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 224f9f6ed38e..037b8461f5a7 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5626,6 +5626,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio if (groupVo != null) { group = groupVo.getName(); } + UserVm vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, size , group , hypervisorType, cmd.getHttpMethod(), userData , sshKeyPair , ipToNetoworkMap, addr, isDisplayVM , keyboard , null, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), From d15f6207fa64d120a39b50b6c2d4e1b1c6d7eeb1 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 2 Jul 2021 02:25:54 -0400 Subject: [PATCH 024/114] trail-issue-1 --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 037b8461f5a7..224f9f6ed38e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5626,7 +5626,6 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio if (groupVo != null) { group = groupVo.getName(); } - UserVm vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, size , group , hypervisorType, cmd.getHttpMethod(), userData , sshKeyPair , ipToNetoworkMap, addr, isDisplayVM , keyboard , null, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), From fee4a95a016a1b3fe410f103afdc41405375e4fb Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 2 Jul 2021 03:08:22 -0400 Subject: [PATCH 025/114] change test ip address --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 224f9f6ed38e..06399bd130bd 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5599,7 +5599,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio String ipv6Address = null; String macAddress = null; // IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), null, macAddress); - IpAddresses addr = new IpAddresses("127.20.0.183", ipv6Address, macAddress); + IpAddresses addr = new IpAddresses("60.147.41.98", ipv6Address, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); From eecacc08c6883ef7591a8701c1ee2d16f7c9aea2 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 2 Jul 2021 10:18:20 -0400 Subject: [PATCH 026/114] add template download record --- .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 639f8468aaaa..8ad16a024b19 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1858,6 +1858,8 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template // persist this to the template zone area and remember to remove the resource count in the execute phase once in failure or clean up phase VMTemplateZoneVO templateZone = new VMTemplateZoneVO(zoneId, template.getId(), new Date()); _tmpltZoneDao.persist(templateZone); + TemplateDataStoreVO voRecord = _tmplStoreDao.createTemplateDirectDownloadEntry(template.getId(), template.getSize()); + _tmplStoreDao.persist(voRecord); // Increment the number of templates if (template != null) { Map details = new HashMap(); From 31824f2b6a8abb43423a93ecaef2c5191bb472bb Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 15:46:18 -0400 Subject: [PATCH 027/114] get the template created and prepare for the vm start --- .../java/com/cloud/network/NetworkModel.java | 4 + .../cloud/template/TemplateApiService.java | 2 +- .../api/command/user/vm/CloneVMCmd.java | 5 +- .../com/cloud/network/NetworkModelImpl.java | 10 +++ .../cloud/template/TemplateManagerImpl.java | 84 +++++++++++++++++-- .../java/com/cloud/vm/UserVmManagerImpl.java | 32 +++++-- 6 files changed, 123 insertions(+), 14 deletions(-) diff --git a/api/src/main/java/com/cloud/network/NetworkModel.java b/api/src/main/java/com/cloud/network/NetworkModel.java index e933a1cc7bd3..0f05b1afa13f 100644 --- a/api/src/main/java/com/cloud/network/NetworkModel.java +++ b/api/src/main/java/com/cloud/network/NetworkModel.java @@ -107,6 +107,10 @@ public interface NetworkModel { List listNetworksUsedByVm(long vmId, boolean isSystem); + default List listNetworksUsedByVm(long vmId) { + throw new UnsupportedOperationException(); + } + Nic getNicInNetwork(long vmId, long networkId); List getNicsForTraffic(long vmId, TrafficType type); diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index f0e6f0b264a4..0e647d67f986 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -105,7 +105,7 @@ public interface TemplateApiService { * */ VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner) throws ResourceAllocationException; - VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd) throws CloudRuntimeException; + VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws CloudRuntimeException; VirtualMachineTemplate createPrivateTemplateRecord(CreateTemplateCmd cmd, Account templateOwner) throws ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 074e47b1671a..e92cf27bc10f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.UserVmResponse; //import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; import java.util.Optional; @@ -131,8 +132,8 @@ public String getTemplateName() { public void execute() { Optional result; try { -// CallContext.current().setEventDetails("Vm Id for full clone: " + getId()); -// VirtualMachineTemplate template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId())); + CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); +// _templateService.createPrivateTemplate(this); result = _userVmService.cloneVirtualMachine(this); } catch (ResourceUnavailableException ex) { s_logger.warn("Exception: ", ex); diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index 4322478d93e1..32643ca2a48f 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -831,6 +832,15 @@ public List listNetworksUsedByVm(long vmId, boolean isSystem) { return networks; } + @Override + public List listNetworksUsedByVm(long vmId) { + return listNetworksUsedByVm(vmId, false). + stream(). + mapToLong(NetworkVO::getId). + boxed(). + collect(Collectors.toList()); + } + @Override public Nic getNicInNetwork(long vmId, long networkId) { return _nicDao.findByNtwkIdAndInstanceId(networkId, vmId); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 8ad16a024b19..19e6b25ba3f0 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1782,6 +1782,85 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } } + @Override + @DB + @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating actual private template", create = true) + public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws CloudRuntimeException { + UserVm curVm = cmd.getTargetVM(); + long templateId = cmd.getTemporaryTemlateId(); + final Long accountId = curVm.getAccountId(); + Account caller = CallContext.current().getCallingAccount(); + List volumes = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); + VolumeVO targetVolume = volumes.get(0); + long volumeId = targetVolume.getId(); + VMTemplateVO finalTmpProduct = null; + try { + TemplateInfo cloneTempalateInfp = _tmplFactory.getTemplate(templateId, DataStoreRole.Image); + long zoneId = curVm.getDataCenterId(); + AsyncCallFuture future = null; + VolumeInfo vInfo = _volFactory.getVolume(volumeId); + DataStore store = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + future = _tmpltSvr.createTemplateFromVolumeAsync(vInfo, cloneTempalateInfp, store); + CommandResult result = null; + try { + result = future.get(); + + if (result.isFailed()) { + finalTmpProduct = null; + s_logger.debug("Failed to create template: " + result.getResult()); + throw new CloudRuntimeException("Failed to create template: " + result.getResult()); + } + if (_dataStoreMgr.isRegionStore(store)) { + _tmpltSvr.associateTemplateToZone(templateId, null); + } else { + // Already done in the record to db step + } + finalTmpProduct = _tmpltDao.findById(templateId); + TemplateDataStoreVO srcTmpltStore = _tmplStoreDao.findByStoreTemplate(store.getId(), templateId); + UsageEventVO usageEvent = + new UsageEventVO(EventTypes.EVENT_TEMPLATE_CREATE, finalTmpProduct.getAccountId(), zoneId, finalTmpProduct.getId(), privateTemplate.getName(), null, + finalTmpProduct.getSourceTemplateId(), srcTmpltStore.getPhysicalSize(), finalTmpProduct.getSize()); + _usageEventDao.persist(usageEvent); + } catch (InterruptedException e) { + s_logger.debug("Failed to create template for id: " + templateId, e); + throw new CloudRuntimeException("Failed to create template" , e); + } catch (ExecutionException e) { + s_logger.debug("Failed to create template for id: " + templateId, e); + throw new CloudRuntimeException("Failed to create template ", e); + } + + } finally { + if (finalTmpProduct == null) { + final VolumeVO volumeFinal = targetVolume; + final SnapshotVO snapshotFinal = null; + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + // template_store_ref entries should have been removed using our + // DataObject.processEvent command in case of failure, but clean + // it up here to avoid + // some leftovers which will cause removing template from + // vm_template table fail. + _tmplStoreDao.deletePrimaryRecordsForTemplate(templateId); + // Remove the template_zone_ref record + _tmpltZoneDao.deletePrimaryRecordsForTemplate(templateId); + // Remove the template record + _tmpltDao.expunge(templateId); + + // decrement resource count + if (accountId != null) { + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template); + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, new Long(volumeFinal != null ? volumeFinal.getSize() + : snapshotFinal.getSize())); + } + } + }); + + } + } + return null; + } + @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template from clone", create = true) public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner) throws ResourceAllocationException { @@ -1904,11 +1983,6 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template } } - @Override - public VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd) throws CloudRuntimeException { - return null; - } - @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template", create = true) public VMTemplateVO createPrivateTemplateRecord(CreateTemplateCmd cmd, Account templateOwner) throws ResourceAllocationException { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 06399bd130bd..78d229988799 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5599,7 +5599,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio String ipv6Address = null; String macAddress = null; // IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), null, macAddress); - IpAddresses addr = new IpAddresses("60.147.41.98", ipv6Address, macAddress); + IpAddresses addr = new IpAddresses("60.147.41.99", ipv6Address, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); @@ -5620,16 +5620,36 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio if (template == null) { throw new CloudRuntimeException("the temporary template is not created, server error, contact your sys admin"); } - List networkIds = null; + List networkIds = _networkModel.listNetworksUsedByVm(curVm.getId()); String group = null; InstanceGroupVO groupVo = getGroupForVm(cmd.getId()); if (groupVo != null) { group = groupVo.getName(); } - UserVm vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, - size , group , hypervisorType, cmd.getHttpMethod(), userData , sshKeyPair , ipToNetoworkMap, addr, isDisplayVM , keyboard , null, - curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), - null, new HashMap<>(), dynamicScalingEnabled); + UserVm vmResult = null; + List affinityGroupIdList = _affinityGroupDao.findByAccountAndNames(curAccount.getId(), curAccount.getAccountName()) + .stream(). + mapToLong(AffinityGroupVO::getId). + boxed(). + collect(Collectors.toList()); + if (dataCenter.getNetworkType() == NetworkType.Basic) { + vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, + size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, + curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), + null, new HashMap<>(), dynamicScalingEnabled); + } else { + if (dataCenter.isSecurityGroupEnabled()) { + vmResult = createAdvancedSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, networkIds, securityGroupIdList, curAccount, hostName, + displayName, diskOfferingId, size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, + affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), + null, new HashMap<>(), dynamicScalingEnabled); + } else { + vmResult = createAdvancedVirtualMachine(dataCenter, serviceOffering, template, networkIds, curAccount, hostName, displayName, diskOfferingId, size, group, + hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), + cmd.getCustomId(), new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); + } + } + return vmResult; } From e3bf89459584e511ca05c806d0f59300d4102089 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 16:51:46 -0400 Subject: [PATCH 028/114] fix unknown symbol --- .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 19e6b25ba3f0..a1ac73bb4998 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1818,7 +1818,7 @@ public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws Cloud finalTmpProduct = _tmpltDao.findById(templateId); TemplateDataStoreVO srcTmpltStore = _tmplStoreDao.findByStoreTemplate(store.getId(), templateId); UsageEventVO usageEvent = - new UsageEventVO(EventTypes.EVENT_TEMPLATE_CREATE, finalTmpProduct.getAccountId(), zoneId, finalTmpProduct.getId(), privateTemplate.getName(), null, + new UsageEventVO(EventTypes.EVENT_TEMPLATE_CREATE, finalTmpProduct.getAccountId(), zoneId, finalTmpProduct.getId(), finalTmpProduct.getName(), null, finalTmpProduct.getSourceTemplateId(), srcTmpltStore.getPhysicalSize(), finalTmpProduct.getSize()); _usageEventDao.persist(usageEvent); } catch (InterruptedException e) { From 99e50434144102c27f2387a23dde0ee46e3035e3 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 17:27:32 -0400 Subject: [PATCH 029/114] finish the final private creation --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index e92cf27bc10f..9cf9007a6943 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -133,7 +133,7 @@ public void execute() { Optional result; try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); -// _templateService.createPrivateTemplate(this); + _templateService.createPrivateTemplate(this); result = _userVmService.cloneVirtualMachine(this); } catch (ResourceUnavailableException ex) { s_logger.warn("Exception: ", ex); From 40abfac2c349e19ea9f58c0c156003cd49474f64 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 17:49:26 -0400 Subject: [PATCH 030/114] add logger information --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 9cf9007a6943..f71c8a18d2ae 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -101,7 +101,6 @@ public void create() throws ResourceAllocationException { } setEntityId(vmRecord.getId()); setEntityUuid(vmRecord.getUuid()); -// _userVmService.createBasicSecurityGroupVirtualMachine(); // disabled since it crashes } catch (ResourceUnavailableException | InsufficientCapacityException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); @@ -133,6 +132,8 @@ public void execute() { Optional result; try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); + s_logger.info("creating actual template id: " + getTemporaryTemlateId()); + s_logger.info("starting actual VM id: " + getEntityId()); _templateService.createPrivateTemplate(this); result = _userVmService.cloneVirtualMachine(this); } catch (ResourceUnavailableException ex) { From d235477b349d7682c3e5e9d563bcdb81fb4aae31 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 18:04:40 -0400 Subject: [PATCH 031/114] add autobox statement --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index f71c8a18d2ae..270699c6ae08 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -82,7 +82,7 @@ public Long getTemporaryTemlateId() { return this.temporaryTemlateId; } - public void setTemporaryTemlateId(Long tempId) { + public void setTemporaryTemlateId(long tempId) { this.temporaryTemlateId = tempId; } @@ -94,6 +94,7 @@ public void create() throws ResourceAllocationException { if (template == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "failed to create a template to db"); } + s_logger.info("The template id recorded is: " + template.getId()); setTemporaryTemlateId(template.getId()); UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); if (vmRecord == null) { From a48d7fa367fe8ae16091bf6a348cda59d300f9fa Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 21:46:45 -0400 Subject: [PATCH 032/114] add template Id setting --- .../org/apache/cloudstack/api/BaseAsyncCreateCmd.java | 9 +++++++++ .../cloudstack/api/command/user/vm/CloneVMCmd.java | 3 ++- .../java/com/cloud/template/TemplateManagerImpl.java | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java index 60c2a183ad33..869431b8b03a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java @@ -23,6 +23,8 @@ public abstract class BaseAsyncCreateCmd extends BaseAsyncCmd { private String uuid; + private Long templateId; + public abstract void create() throws ResourceAllocationException; public Long getEntityId() { @@ -49,4 +51,11 @@ public String getCreateEventDescription() { return null; } + public Long getTemplateId() { + return templateId; + } + + public void setTemplateId(Long id) { + this.templateId = id; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 270699c6ae08..0cadbbba1dc8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -95,6 +95,7 @@ public void create() throws ResourceAllocationException { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "failed to create a template to db"); } s_logger.info("The template id recorded is: " + template.getId()); + setTemplateId(template.getId()); setTemporaryTemlateId(template.getId()); UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); if (vmRecord == null) { @@ -133,7 +134,7 @@ public void execute() { Optional result; try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); - s_logger.info("creating actual template id: " + getTemporaryTemlateId()); + s_logger.info("creating actual template id: " + getTemplateId()); s_logger.info("starting actual VM id: " + getEntityId()); _templateService.createPrivateTemplate(this); result = _userVmService.cloneVirtualMachine(this); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index a1ac73bb4998..5546a6a07e23 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1787,7 +1787,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating actual private template", create = true) public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws CloudRuntimeException { UserVm curVm = cmd.getTargetVM(); - long templateId = cmd.getTemporaryTemlateId(); + long templateId = cmd.getTemplateId(); final Long accountId = curVm.getAccountId(); Account caller = CallContext.current().getCallingAccount(); List volumes = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); From 7fb099482d1415bd6758920c2cda86319ca5319c Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 22:19:39 -0400 Subject: [PATCH 033/114] use uuid as replacement of template --- .../org/apache/cloudstack/api/BaseAsyncCreateCmd.java | 10 ---------- .../cloudstack/api/command/user/vm/CloneVMCmd.java | 7 +++---- .../java/com/cloud/template/TemplateManagerImpl.java | 2 +- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java index 869431b8b03a..10fee857dfcf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCmd.java @@ -23,8 +23,6 @@ public abstract class BaseAsyncCreateCmd extends BaseAsyncCmd { private String uuid; - private Long templateId; - public abstract void create() throws ResourceAllocationException; public Long getEntityId() { @@ -50,12 +48,4 @@ public String getCreateEventType() { public String getCreateEventDescription() { return null; } - - public Long getTemplateId() { - return templateId; - } - - public void setTemplateId(Long id) { - this.templateId = id; - } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 0cadbbba1dc8..83ae60a6c3ef 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -95,14 +95,13 @@ public void create() throws ResourceAllocationException { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "failed to create a template to db"); } s_logger.info("The template id recorded is: " + template.getId()); - setTemplateId(template.getId()); - setTemporaryTemlateId(template.getId()); UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); if (vmRecord == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "unable to record a new VM to db!"); } + setTemporaryTemlateId(template.getId()); setEntityId(vmRecord.getId()); - setEntityUuid(vmRecord.getUuid()); + setEntityUuid(String.valueOf(template.getId())); } catch (ResourceUnavailableException | InsufficientCapacityException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); @@ -134,7 +133,7 @@ public void execute() { Optional result; try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); - s_logger.info("creating actual template id: " + getTemplateId()); + s_logger.info("creating actual template id: " + getEntityUuid()); s_logger.info("starting actual VM id: " + getEntityId()); _templateService.createPrivateTemplate(this); result = _userVmService.cloneVirtualMachine(this); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 5546a6a07e23..f5494b7c8444 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1787,7 +1787,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating actual private template", create = true) public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws CloudRuntimeException { UserVm curVm = cmd.getTargetVM(); - long templateId = cmd.getTemplateId(); + long templateId = Long.parseLong(cmd.getEntityUuid()); final Long accountId = curVm.getAccountId(); Account caller = CallContext.current().getCallingAccount(); List volumes = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); From ed9ac4e2fc02ff171497ca3b750452cc518badb9 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 22:27:23 -0400 Subject: [PATCH 034/114] fix the template recording bug --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 83ae60a6c3ef..179d5efd32f1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -95,11 +95,11 @@ public void create() throws ResourceAllocationException { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "failed to create a template to db"); } s_logger.info("The template id recorded is: " + template.getId()); + setTemporaryTemlateId(template.getId()); UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); if (vmRecord == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "unable to record a new VM to db!"); } - setTemporaryTemlateId(template.getId()); setEntityId(vmRecord.getId()); setEntityUuid(String.valueOf(template.getId())); } catch (ResourceUnavailableException | InsufficientCapacityException e) { From 23a53cb22ecbdeaf3e67c6295fbb8c696b1442e0 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 22:53:31 -0400 Subject: [PATCH 035/114] fix instance creation null --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 2 +- .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 179d5efd32f1..b724dc21f000 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -133,7 +133,7 @@ public void execute() { Optional result; try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); - s_logger.info("creating actual template id: " + getEntityUuid()); + s_logger.info("creating actual template id: " + Long.parseLong(getEntityUuid())); s_logger.info("starting actual VM id: " + getEntityId()); _templateService.createPrivateTemplate(this); result = _userVmService.cloneVirtualMachine(this); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index f5494b7c8444..0b0309f9187e 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1807,7 +1807,7 @@ public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws Cloud if (result.isFailed()) { finalTmpProduct = null; - s_logger.debug("Failed to create template: " + result.getResult()); + s_logger.warn("Failed to create template: " + result.getResult()); throw new CloudRuntimeException("Failed to create template: " + result.getResult()); } if (_dataStoreMgr.isRegionStore(store)) { @@ -1815,6 +1815,7 @@ public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws Cloud } else { // Already done in the record to db step } + s_logger.info("successfully created the template with Id: " + templateId); finalTmpProduct = _tmpltDao.findById(templateId); TemplateDataStoreVO srcTmpltStore = _tmplStoreDao.findByStoreTemplate(store.getId(), templateId); UsageEventVO usageEvent = @@ -1830,6 +1831,7 @@ public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws Cloud } } finally { + finalTmpProduct = _tmpltDao.findById(templateId); if (finalTmpProduct == null) { final VolumeVO volumeFinal = targetVolume; final SnapshotVO snapshotFinal = null; From 53785ed1c657e74eb0ebd312a97f9469a92e46e0 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 4 Jul 2021 23:52:35 -0400 Subject: [PATCH 036/114] fix the template creation sequence --- .../cloudstack/api/command/user/vm/CloneVMCmd.java | 5 ++--- .../com/cloud/template/TemplateManagerImpl.java | 13 ++++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index b724dc21f000..91144ab72651 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -96,12 +96,13 @@ public void create() throws ResourceAllocationException { } s_logger.info("The template id recorded is: " + template.getId()); setTemporaryTemlateId(template.getId()); + _templateService.createPrivateTemplate(this); UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); if (vmRecord == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "unable to record a new VM to db!"); } setEntityId(vmRecord.getId()); - setEntityUuid(String.valueOf(template.getId())); + setEntityUuid(vmRecord.getUuid()); } catch (ResourceUnavailableException | InsufficientCapacityException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); @@ -133,9 +134,7 @@ public void execute() { Optional result; try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); - s_logger.info("creating actual template id: " + Long.parseLong(getEntityUuid())); s_logger.info("starting actual VM id: " + getEntityId()); - _templateService.createPrivateTemplate(this); result = _userVmService.cloneVirtualMachine(this); } catch (ResourceUnavailableException ex) { s_logger.warn("Exception: ", ex); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 0b0309f9187e..1955aa96f625 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1787,7 +1787,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating actual private template", create = true) public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws CloudRuntimeException { UserVm curVm = cmd.getTargetVM(); - long templateId = Long.parseLong(cmd.getEntityUuid()); + long templateId = cmd.getTemporaryTemlateId(); final Long accountId = curVm.getAccountId(); Account caller = CallContext.current().getCallingAccount(); List volumes = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); @@ -1814,6 +1814,8 @@ public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws Cloud _tmpltSvr.associateTemplateToZone(templateId, null); } else { // Already done in the record to db step + VMTemplateZoneVO templateZone = new VMTemplateZoneVO(zoneId, templateId, new Date()); + _tmpltZoneDao.persist(templateZone); } s_logger.info("successfully created the template with Id: " + templateId); finalTmpProduct = _tmpltDao.findById(templateId); @@ -1937,10 +1939,11 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template privateTemplate.setSourceTemplateId(sourceTemplateId); VMTemplateVO template = _tmpltDao.persist(privateTemplate); // persist this to the template zone area and remember to remove the resource count in the execute phase once in failure or clean up phase - VMTemplateZoneVO templateZone = new VMTemplateZoneVO(zoneId, template.getId(), new Date()); - _tmpltZoneDao.persist(templateZone); - TemplateDataStoreVO voRecord = _tmplStoreDao.createTemplateDirectDownloadEntry(template.getId(), template.getSize()); - _tmplStoreDao.persist(voRecord); +// VMTemplateZoneVO templateZone = new VMTemplateZoneVO(zoneId, template.getId(), new Date()); +// _tmpltZoneDao.persist(templateZone); +// TemplateDataStoreVO voRecord = _tmplStoreDao.createTemplateDirectDownloadEntry(template.getId(), template.getSize()); +// voRecord.setDataStoreId(2); +// _tmplStoreDao.persist(voRecord); // Increment the number of templates if (template != null) { Map details = new HashMap(); From 306a7f42d3aac05ac17ae0ac5d422ca0420c1cbf Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 5 Jul 2021 02:22:23 -0400 Subject: [PATCH 037/114] finish the start vm coding --- .../main/java/com/cloud/vm/UserVmService.java | 2 +- .../api/command/user/vm/CloneVMCmd.java | 10 +++++----- .../java/com/cloud/vm/UserVmManagerImpl.java | 19 +++++++++++++++++-- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index a7fe2824763c..90f41d860ffa 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -94,7 +94,7 @@ public interface UserVmService { * - the command specifying vmId to be cloned * @return the VM if cloneVM operation is successful * */ - Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException; + Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 91144ab72651..438f31854e79 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -143,12 +143,12 @@ public void execute() { s_logger.warn("Exception: ", ex); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } -// catch (ResourceAllocationException ex) { -// s_logger.warn("Exception: ", ex); -// throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage()); -// } + catch (ResourceAllocationException | InsufficientCapacityException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage()); + } result.ifPresentOrElse((userVm)-> { - UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", userVm).get(0); + UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", result.get()).get(0); response.setResponseName("test_clone"); setResponseObject(response); }, ()-> { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 78d229988799..6c423beba7b9 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4589,10 +4589,25 @@ public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueExce } _resourceLimitMgr.checkResourceLimit(activeOwner, ResourceType.primary_storage, totalSize); } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CLONE, eventDescription = "clone vm", async = true) - public Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException { - return Optional.ofNullable(null); + public Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException, InsufficientCapacityException, ResourceAllocationException { + long vmId = cmd.getEntityId(); + UserVmVO curVm = _vmDao.findById(vmId); + Long podId = curVm.getPodIdToDeployIn(); + Long clusterId = null; + Long hostId = curVm.getHostId(); + Map additonalParams = new HashMap<>(); + Map diskOfferingMap = null; + if (MapUtils.isNotEmpty(curVm.getDetails()) && curVm.getDetails().containsKey(ApiConstants.BootType.UEFI.toString())) { + Map map = curVm.getDetails(); + additonalParams.put(VirtualMachineProfile.Param.UefiFlag, "Yes"); + additonalParams.put(VirtualMachineProfile.Param.BootType, ApiConstants.BootType.UEFI.toString()); + additonalParams.put(VirtualMachineProfile.Param.BootMode, map.get(ApiConstants.BootType.UEFI.toString())); + } + + return Optional.of(startVirtualMachine(vmId, podId, clusterId, hostId, diskOfferingMap, additonalParams, null)); } @Override From 8ecb6a587b933b08e514ad67cad8e7bc9c0165b6 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 5 Jul 2021 02:36:48 -0400 Subject: [PATCH 038/114] fix start virtual machine null pointer bug --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 6c423beba7b9..e94225b3e4c1 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4599,7 +4599,7 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnava Long clusterId = null; Long hostId = curVm.getHostId(); Map additonalParams = new HashMap<>(); - Map diskOfferingMap = null; + Map diskOfferingMap = new HashMap<>(); if (MapUtils.isNotEmpty(curVm.getDetails()) && curVm.getDetails().containsKey(ApiConstants.BootType.UEFI.toString())) { Map map = curVm.getDetails(); additonalParams.put(VirtualMachineProfile.Param.UefiFlag, "Yes"); From a1bf574b8e8b28a1cfbc7ed842174bb801b97f30 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 5 Jul 2021 02:49:21 -0400 Subject: [PATCH 039/114] add public ip address for kvm host test --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index e94225b3e4c1..483f58991bba 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5614,7 +5614,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio String ipv6Address = null; String macAddress = null; // IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), null, macAddress); - IpAddresses addr = new IpAddresses("60.147.41.99", ipv6Address, macAddress); + IpAddresses addr = new IpAddresses("172.20.0.97", ipv6Address, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); From 96393890f03fd6c27f9319774947b22f7fcad3fd Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 5 Jul 2021 12:20:28 -0400 Subject: [PATCH 040/114] new clone ip added --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 483f58991bba..c269e2fd71c9 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5614,7 +5614,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio String ipv6Address = null; String macAddress = null; // IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), null, macAddress); - IpAddresses addr = new IpAddresses("172.20.0.97", ipv6Address, macAddress); + IpAddresses addr = new IpAddresses("172.20.0.100", ipv6Address, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); From 0871f74a4e0c6c0eb5c5693affb1f446cb9d070a Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 5 Jul 2021 12:55:53 -0400 Subject: [PATCH 041/114] change test ip to 98 --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index c269e2fd71c9..1ffb479c22ff 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5614,7 +5614,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio String ipv6Address = null; String macAddress = null; // IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), null, macAddress); - IpAddresses addr = new IpAddresses("172.20.0.100", ipv6Address, macAddress); + IpAddresses addr = new IpAddresses("172.20.0.98", ipv6Address, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); From f5b896e5a8291a73bffdf6e715cac2b1fba3a74a Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 02:22:04 -0400 Subject: [PATCH 042/114] fixing the first password issue --- .../cloud/template/TemplateApiService.java | 3 +- .../api/command/user/vm/CloneVMCmd.java | 14 ++- .../cloud/template/TemplateManagerImpl.java | 93 ++++++++++++------- .../java/com/cloud/vm/UserVmManagerImpl.java | 6 +- 4 files changed, 79 insertions(+), 37 deletions(-) diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index 0e647d67f986..fd45499ef6b4 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -20,6 +20,7 @@ import java.net.URISyntaxException; import java.util.List; +import com.cloud.storage.VolumeApiService; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; @@ -103,7 +104,7 @@ public interface TemplateApiService { /** * create a template record for later usage of creating a real template by createPrivateTemplate * */ - VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner) throws ResourceAllocationException; + VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner, VolumeApiService serviceObj) throws ResourceAllocationException; VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws CloudRuntimeException; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 438f31854e79..41837c93ac72 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -6,6 +6,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.storage.Snapshot; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.uservm.UserVm; @@ -52,6 +53,8 @@ public class CloneVMCmd extends BaseAsyncCreateCustomIdCmd implements UserCmd { private Long temporaryTemlateId; + private Long temporarySnapShotId; + public String getAccountName() { return accountName; } @@ -82,6 +85,15 @@ public Long getTemporaryTemlateId() { return this.temporaryTemlateId; } + public void setTemporarySnapShotId(Long snapshotId) { + this.temporarySnapShotId = snapshotId; + } + + public Long getTemporarySnapShotId() { + return temporarySnapShotId; + } + + public void setTemporaryTemlateId(long tempId) { this.temporaryTemlateId = tempId; } @@ -90,7 +102,7 @@ public void setTemporaryTemlateId(long tempId) { public void create() throws ResourceAllocationException { try { _userVmService.checkCloneCondition(this); - VirtualMachineTemplate template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId())); + VirtualMachineTemplate template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId()), _volumeService); if (template == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "failed to create a template to db"); } diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 1955aa96f625..3e68ef0f9605 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -33,6 +33,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.*; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; @@ -143,29 +144,9 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.projects.Project; import com.cloud.projects.ProjectManager; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.GuestOSVO; -import com.cloud.storage.ImageStoreUploadMonitorImpl; -import com.cloud.storage.LaunchPermissionVO; -import com.cloud.storage.Snapshot; -import com.cloud.storage.SnapshotVO; -import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; -import com.cloud.storage.StorageManager; -import com.cloud.storage.StoragePool; -import com.cloud.storage.StoragePoolHostVO; -import com.cloud.storage.StoragePoolStatus; -import com.cloud.storage.TemplateProfile; -import com.cloud.storage.Upload; -import com.cloud.storage.VMTemplateHostVO; -import com.cloud.storage.VMTemplateStoragePoolVO; -import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.VMTemplateZoneVO; -import com.cloud.storage.Volume; -import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.SnapshotDao; @@ -1788,19 +1769,50 @@ public void doInTransactionWithoutResult(TransactionStatus status) { public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws CloudRuntimeException { UserVm curVm = cmd.getTargetVM(); long templateId = cmd.getTemporaryTemlateId(); + long snapshotId = cmd.getTemporarySnapShotId(); final Long accountId = curVm.getAccountId(); Account caller = CallContext.current().getCallingAccount(); List volumes = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); VolumeVO targetVolume = volumes.get(0); long volumeId = targetVolume.getId(); VMTemplateVO finalTmpProduct = null; + SnapshotVO snapshot = null; try { TemplateInfo cloneTempalateInfp = _tmplFactory.getTemplate(templateId, DataStoreRole.Image); long zoneId = curVm.getDataCenterId(); AsyncCallFuture future = null; VolumeInfo vInfo = _volFactory.getVolume(volumeId); DataStore store = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); - future = _tmpltSvr.createTemplateFromVolumeAsync(vInfo, cloneTempalateInfp, store); + snapshot = _snapshotDao.findById(snapshotId); +// future = _tmpltSvr.createTemplateFromVolumeAsync(vInfo, cloneTempalateInfp, store); + // create template from snapshot + DataStoreRole dataStoreRole = ApiResponseHelper.getDataStoreRole(snapshot, _snapshotStoreDao, _dataStoreMgr); + SnapshotInfo snapInfo = _snapshotFactory.getSnapshot(snapshotId, dataStoreRole); + if (dataStoreRole == DataStoreRole.Image) { + if (snapInfo == null) { + snapInfo = _snapshotFactory.getSnapshot(snapshotId, DataStoreRole.Primary); + if(snapInfo == null) { + throw new CloudRuntimeException("Cannot find snapshot "+snapshotId); + } + // We need to copy the snapshot onto secondary. + SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.BACKUP); + snapshotStrategy.backupSnapshot(snapInfo); + + // Attempt to grab it again. + snapInfo = _snapshotFactory.getSnapshot(snapshotId, dataStoreRole); + if(snapInfo == null) { + throw new CloudRuntimeException("Cannot find snapshot " + snapshotId + " on secondary and could not create backup"); + } + } + _accountMgr.checkAccess(caller, null, true, snapInfo); + DataStore snapStore = snapInfo.getDataStore(); + + if (snapStore != null) { + store = snapStore; // pick snapshot image store to create template + } + } + future = _tmpltSvr.createTemplateFromSnapshotAsync(snapInfo, cloneTempalateInfp, store); + // wait for the result to converge CommandResult result = null; try { result = future.get(); @@ -1836,7 +1848,7 @@ public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws Cloud finalTmpProduct = _tmpltDao.findById(templateId); if (finalTmpProduct == null) { final VolumeVO volumeFinal = targetVolume; - final SnapshotVO snapshotFinal = null; + final SnapshotVO snapshotFinal = snapshot; Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { @@ -1867,7 +1879,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template from clone", create = true) - public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner) throws ResourceAllocationException { + public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner, VolumeApiService volumeService) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); _accountMgr.checkAccess(caller, null, true, templateOwner); String name = cmd.getTemplateName(); @@ -1883,21 +1895,22 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template HypervisorType hyperType = null; VolumeVO volume = _volumeDao.findById(volumeId); if (volume == null) { - throw new InvalidParameterValueException("Failed to create private template record, unable to find volume " + volumeId); + throw new InvalidParameterValueException("Failed to create private template record, unable to find root volume " + volumeId); } + // check permissions _accountMgr.checkAccess(caller, null, true, volume); // If private template is created from Volume, check that the volume // will not be active when the private template is // created - if (!_volumeMgr.volumeInactive(volume)) { - String msg = "Unable to create private template for volume: " + volume.getName() + "; volume is attached to a non-stopped VM, please stop the VM first"; - if (s_logger.isInfoEnabled()) { - s_logger.info(msg); - } - throw new CloudRuntimeException(msg); - } +// if (!_volumeMgr.volumeInactive(volume)) { +// String msg = "Unable to create private template for volume: " + volume.getName() + "; volume is attached to a non-stopped VM, please stop the VM first"; +// if (s_logger.isInfoEnabled()) { +// s_logger.info(msg); +// } +// throw new CloudRuntimeException(msg); +// } hyperType = _volumeDao.getHypervisorType(volumeId); if (HypervisorType.LXC.equals(hyperType)) { @@ -1913,7 +1926,23 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template throw new InvalidParameterValueException("GuestOS with ID: " + guestOSId + " does not exist."); } + // get snapshot from this step + + Long nextTemplateId = _tmpltDao.getNextInSequence(Long.class, "id"); + s_logger.info("Creating snapshot for the tempalte creation"); + SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, null, curVm.getDisplayName() + "-Clone-" + nextTemplateId, Snapshot.LocationType.PRIMARY); + if (snapshot == null) { + throw new CloudRuntimeException("Unable to create a snapshot during the template creation recording"); + } + Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, null, snapshot.getId(), caller, false, Snapshot.LocationType.PRIMARY, false, new HashMap<>()); + if (snapshotEntity == null) { + throw new CloudRuntimeException("Error when creating the snapshot entity"); + } + if (snapshotEntity.getState() != Snapshot.State.BackedUp) { + throw new CloudRuntimeException("Async backup of snapshot happens during the clone for snapshot id: " + snapshot.getId()); + } + cmd.setTemporarySnapShotId(snapshot.getId()); String description = ""; // TODO: add this to clone parameter in the future boolean isExtractable = false; Long sourceTemplateId = null; @@ -1978,7 +2007,7 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.template); _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.secondary_storage, - volume.getSize()); + snapshot.getSize()); } if (template != null) { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 1ffb479c22ff..a2955491d742 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4524,9 +4524,9 @@ public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueExce throw new CloudRuntimeException("the VM doesn't exist or not registered in management server!"); } UserVmVO vmStatus = _vmDao.findById(cmd.getId()); - if (vmStatus.state != State.Shutdown && vmStatus.state != State.Stopped) { - throw new CloudRuntimeException("You should clone an instance that's shutdown!"); - } +// if (vmStatus.state != State.Shutdown && vmStatus.state != State.Stopped) { +// throw new CloudRuntimeException("You should clone an instance that's shutdown!"); +// } if (vmStatus.getHypervisorType() != HypervisorType.KVM && vmStatus.getHypervisorType() != HypervisorType.Simulator) { throw new CloudRuntimeException("The clone operation is only supported on KVM and Simulator!"); } From 8bebd3b6dd781dfa3d1aa50eeccd4fa61bdbdc86 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 02:23:40 -0400 Subject: [PATCH 043/114] fix the style code --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 41837c93ac72..cdd5cb342c4d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -6,7 +6,6 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.storage.Snapshot; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.uservm.UserVm; From 21ac7a77aa1ccaa92af6eb98ca62dd5afd7355e5 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 02:26:07 -0400 Subject: [PATCH 044/114] audit the star --- .../cloud/template/TemplateManagerImpl.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 3e68ef0f9605..e00c5b8650ab 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -33,7 +33,27 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.storage.*; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.ImageStoreUploadMonitorImpl; +import com.cloud.storage.LaunchPermissionVO; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.TemplateProfile; +import com.cloud.storage.Upload; +import com.cloud.storage.VMTemplateHostVO; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; +import com.cloud.storage.VolumeVO; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; From 385df67c0d69535c119b525c2b402bec7dfcc9dc Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 02:57:36 -0400 Subject: [PATCH 045/114] location type changed --- .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index e00c5b8650ab..15f982f5e430 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1951,11 +1951,11 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template Long nextTemplateId = _tmpltDao.getNextInSequence(Long.class, "id"); s_logger.info("Creating snapshot for the tempalte creation"); - SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, null, curVm.getDisplayName() + "-Clone-" + nextTemplateId, Snapshot.LocationType.PRIMARY); + SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, null, curVm.getDisplayName() + "-Clone-" + nextTemplateId, null); if (snapshot == null) { throw new CloudRuntimeException("Unable to create a snapshot during the template creation recording"); } - Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, null, snapshot.getId(), caller, false, Snapshot.LocationType.PRIMARY, false, new HashMap<>()); + Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, null, snapshot.getId(), caller, false, null, false, new HashMap<>()); if (snapshotEntity == null) { throw new CloudRuntimeException("Error when creating the snapshot entity"); } From cec1ee9cfd6d7b027e5f52021e451804fce96fd5 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 03:09:30 -0400 Subject: [PATCH 046/114] fix the snapshot policy --- .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 15f982f5e430..48f5407f6398 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1951,11 +1951,11 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template Long nextTemplateId = _tmpltDao.getNextInSequence(Long.class, "id"); s_logger.info("Creating snapshot for the tempalte creation"); - SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, null, curVm.getDisplayName() + "-Clone-" + nextTemplateId, null); + SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, Snapshot.MANUAL_POLICY_ID, curVm.getDisplayName() + "-Clone-" + nextTemplateId, null); if (snapshot == null) { throw new CloudRuntimeException("Unable to create a snapshot during the template creation recording"); } - Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, null, snapshot.getId(), caller, false, null, false, new HashMap<>()); + Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, Snapshot.MANUAL_POLICY_ID, snapshot.getId(), caller, false, null, false, new HashMap<>()); if (snapshotEntity == null) { throw new CloudRuntimeException("Error when creating the snapshot entity"); } From d4d7570ad46b24669f7c542c3dc2f2872d8a6552 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 20:54:26 -0400 Subject: [PATCH 047/114] add automatic ip assignment- --- .../main/java/com/cloud/vm/UserVmManagerImpl.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index a2955491d742..1e07f2fbf8d3 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -52,10 +52,7 @@ import javax.xml.parsers.ParserConfigurationException; //import com.cloud.network.IpAddress; -import com.cloud.network.IpAddressManager; -import com.cloud.network.Network; -import com.cloud.network.NetworkModel; -import com.cloud.network.PhysicalNetwork; +import com.cloud.network.*; import com.cloud.network.security.SecurityGroupVO; import com.cloud.user.Account; import com.cloud.user.AccountManager; @@ -5610,11 +5607,12 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio Account curAccount = _accountDao.findById(curVm.getAccountId()); long callingUserId = CallContext.current().getCallingUserId(); Account callerAccount = CallContext.current().getCallingAccount(); -// IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, curAccount.getId() == Account.ACCOUNT_ID_SYSTEM, callerAccount, callingUserId, dataCenter, null, null); +// IpAddress ipAddress = _ipAddrMgr.assignPublicIpAddress(zoneId, curVm.getPodIdToDeployIn(), callerAccount, VlanType.DirectAttached, ) + IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, false, callerAccount, callingUserId, dataCenter, null, null); String ipv6Address = null; String macAddress = null; -// IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), null, macAddress); - IpAddresses addr = new IpAddresses("172.20.0.98", ipv6Address, macAddress); + IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), ipv6Address, macAddress); +// IpAddresses addr = new IpAddresses("172.20.0.98", ipv6Address, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); From 52f281035c13a311dc91692dda66c9a6055b0148 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 21:01:17 -0400 Subject: [PATCH 048/114] add ip allocation --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 1e07f2fbf8d3..9ff1814ef36e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5608,7 +5608,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio long callingUserId = CallContext.current().getCallingUserId(); Account callerAccount = CallContext.current().getCallingAccount(); // IpAddress ipAddress = _ipAddrMgr.assignPublicIpAddress(zoneId, curVm.getPodIdToDeployIn(), callerAccount, VlanType.DirectAttached, ) - IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, false, callerAccount, callingUserId, dataCenter, null, null); + IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, false, callerAccount, callingUserId, dataCenter, true, null); String ipv6Address = null; String macAddress = null; IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), ipv6Address, macAddress); From 1a475b3916d8549352f48be954957ace63feb113 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 21:02:49 -0400 Subject: [PATCH 049/114] fix the check-style typo --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 9ff1814ef36e..8ecd7c1ca91e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -52,7 +52,11 @@ import javax.xml.parsers.ParserConfigurationException; //import com.cloud.network.IpAddress; -import com.cloud.network.*; +import com.cloud.network.IpAddress; +import com.cloud.network.IpAddressManager; +import com.cloud.network.Network; +import com.cloud.network.NetworkModel; +import com.cloud.network.PhysicalNetwork; import com.cloud.network.security.SecurityGroupVO; import com.cloud.user.Account; import com.cloud.user.AccountManager; From b64c48bf3a8e7ab0e9c7ada9a72024a6ff8fb1ce Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 23:28:31 -0400 Subject: [PATCH 050/114] try null address for vm creation --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 8ecd7c1ca91e..636c9ddbf905 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5612,10 +5612,10 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio long callingUserId = CallContext.current().getCallingUserId(); Account callerAccount = CallContext.current().getCallingAccount(); // IpAddress ipAddress = _ipAddrMgr.assignPublicIpAddress(zoneId, curVm.getPodIdToDeployIn(), callerAccount, VlanType.DirectAttached, ) - IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, false, callerAccount, callingUserId, dataCenter, true, null); +// IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, false, callerAccount, callingUserId, dataCenter, true, null); String ipv6Address = null; String macAddress = null; - IpAddresses addr = new IpAddresses(ipAddress.getVmIp(), ipv6Address, macAddress); + IpAddresses addr = new IpAddresses(null, ipv6Address, macAddress); // IpAddresses addr = new IpAddresses("172.20.0.98", ipv6Address, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); From 6b145d17494d6467c71d7caab7db56195fbec832 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 7 Jul 2021 23:31:19 -0400 Subject: [PATCH 051/114] ignore unused ip --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 636c9ddbf905..4d23854009f6 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -50,9 +50,8 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; - //import com.cloud.network.IpAddress; -import com.cloud.network.IpAddress; +//import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; import com.cloud.network.Network; import com.cloud.network.NetworkModel; From fbf37531d0312af3ed9d94f76330f0514625d351 Mon Sep 17 00:00:00 2001 From: junxuan Date: Thu, 8 Jul 2021 00:37:27 -0400 Subject: [PATCH 052/114] finish the network allocation --- .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 2 +- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 48f5407f6398..79a7646b813e 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1979,7 +1979,7 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template VMTemplateVO privateTemplate = null; privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable, TemplateType.USER, null, true, 64, templateOwner.getId(), null, description, - true, guestOS.getId(), true, hyperType, null, new HashMap<>(){{put("template to be cleared", "yes");}}, false, false, false, false); + false, guestOS.getId(), true, hyperType, null, new HashMap<>(){{put("template to be cleared", "yes");}}, false, false, false, false); List stores = _imgStoreDao.findRegionImageStores(); if (!CollectionUtils.isEmpty(stores)) { privateTemplate.setCrossZones(true); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 4d23854009f6..46863728851c 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5665,7 +5665,6 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio cmd.getCustomId(), new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); } } - return vmResult; } From 63c7c5e9b61a0ce39013439e23117d2949018ae7 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 9 Jul 2021 00:50:27 -0400 Subject: [PATCH 053/114] datadisk code added --- .../com/cloud/storage/VolumeApiService.java | 4 + .../main/java/com/cloud/vm/UserVmService.java | 3 +- .../api/command/user/vm/CloneVMCmd.java | 2 +- .../cloud/storage/VolumeApiServiceImpl.java | 7 + .../java/com/cloud/vm/UserVmManagerImpl.java | 140 ++++++++++++------ 5 files changed, 110 insertions(+), 46 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 5c4130158cd8..5ea49650ae37 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -21,6 +21,8 @@ import java.net.MalformedURLException; import java.util.Map; +import com.cloud.exception.StorageUnavailableException; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; @@ -92,6 +94,8 @@ public interface VolumeApiService { Volume detachVolumeViaDestroyVM(long vmId, long volumeId); + Volume cloneDataVolume(CloneVMCmd cmd, long snapshotId, Volume volume) throws StorageUnavailableException; + Volume detachVolumeFromVM(DetachVolumeCmd cmd); Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 90f41d860ffa..74c090eb5f32 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Optional; +import com.cloud.storage.VolumeApiService; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; @@ -94,7 +95,7 @@ public interface UserVmService { * - the command specifying vmId to be cloned * @return the VM if cloneVM operation is successful * */ - Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; + Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService) throws ResourceUnavailableException, ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index cdd5cb342c4d..66414dc28969 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -146,7 +146,7 @@ public void execute() { try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); s_logger.info("starting actual VM id: " + getEntityId()); - result = _userVmService.cloneVirtualMachine(this); + result = _userVmService.cloneVirtualMachine(this, _volumeService); } catch (ResourceUnavailableException ex) { s_logger.warn("Exception: ", ex); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index a2b2a4978c0d..732ece2345db 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -31,6 +31,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; @@ -900,6 +901,12 @@ public VolumeVO createVolume(CreateVolumeCmd cmd) { } } + @Override + public Volume cloneDataVolume(CloneVMCmd cmd, long snapshotId, Volume volume) throws StorageUnavailableException { + long vmId = cmd.getEntityId(); + return createVolumeFromSnapshot((VolumeVO) volume, snapshotId, vmId); + } + protected VolumeVO createVolumeFromSnapshot(VolumeVO volume, long snapshotId, Long vmId) throws StorageUnavailableException { VolumeInfo createdVolume = null; SnapshotVO snapshot = _snapshotDao.findById(snapshotId); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 46863728851c..4be22defc2f6 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -22,19 +22,8 @@ import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -57,6 +46,7 @@ import com.cloud.network.NetworkModel; import com.cloud.network.PhysicalNetwork; import com.cloud.network.security.SecurityGroupVO; +import com.cloud.storage.*; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -68,6 +58,7 @@ import com.cloud.user.UserStatisticsVO; import com.cloud.user.UserVO; import com.cloud.user.VmDiskStatisticsVO; +import com.cloud.utils.db.*; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -125,15 +116,13 @@ import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachCommand; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.*; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; +import org.owasp.esapi.util.CollectionsUtil; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -273,26 +262,9 @@ import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.service.dao.ServiceOfferingDetailsDao; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.DiskOfferingVO; -import com.cloud.storage.GuestOSCategoryVO; -import com.cloud.storage.GuestOSVO; -import com.cloud.storage.ScopeType; -import com.cloud.storage.Snapshot; -import com.cloud.storage.SnapshotVO; -import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Storage.TemplateType; -import com.cloud.storage.StorageManager; -import com.cloud.storage.StoragePool; -import com.cloud.storage.StoragePoolStatus; -import com.cloud.storage.VMTemplateStorageResourceAssoc; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.VMTemplateZoneVO; -import com.cloud.storage.Volume; -import com.cloud.storage.VolumeApiService; -import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; @@ -319,16 +291,6 @@ import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.crypt.RSAHelper; -import com.cloud.utils.db.DB; -import com.cloud.utils.db.EntityManager; -import com.cloud.utils.db.GlobalLock; -import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionCallbackWithException; -import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; -import com.cloud.utils.db.TransactionStatus; -import com.cloud.utils.db.UUIDManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExecutionException; import com.cloud.utils.fsm.NoTransitionException; @@ -4590,11 +4552,101 @@ public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueExce _resourceLimitMgr.checkResourceLimit(activeOwner, ResourceType.primary_storage, totalSize); } + private VolumeVO saveDataDiskVolumeFromSnapShot(final Account owner, final Boolean displayVolume, final Long zoneId, final Long diskOfferingId, + final Storage.ProvisioningType provisioningType, final Long size, final Long minIops, final Long maxIops, final VolumeVO parentVolume, final String volumeName, final String uuid, final Map details) { + return Transaction.execute((TransactionCallback) status -> { + VolumeVO volume = new VolumeVO(volumeName, -1, -1, -1, -1, new Long(-1), null, null, provisioningType, 0, Volume.Type.DATADISK); + volume.setPoolId(null); + volume.setUuid(uuid); + volume.setDataCenterId(zoneId); + volume.setPodId(null); + volume.setAccountId(owner.getId()); + volume.setDomainId(owner.getDomainId()); + volume.setDiskOfferingId(diskOfferingId); + volume.setSize(size); + volume.setMinIops(minIops); + volume.setMaxIops(maxIops); + volume.setInstanceId(null); + volume.setUpdated(new Date()); + volume.setDisplayVolume(displayVolume); + if (parentVolume != null) { + volume.setTemplateId(parentVolume.getTemplateId()); + volume.setFormat(parentVolume.getFormat()); + } else { + volume.setTemplateId(null); + } + + volume = _volsDao.persist(volume); + CallContext.current().setEventDetails("Volume Id: " + volume.getUuid()); + + // Increment resource count during allocation; if actual creation fails, + // decrement it + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume, displayVolume); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, displayVolume, new Long(volume.getSize())); + return volume; + }); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CLONE, eventDescription = "clone vm", async = true) - public Optional cloneVirtualMachine(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException, InsufficientCapacityException, ResourceAllocationException { + public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException, InsufficientCapacityException, ResourceAllocationException { long vmId = cmd.getEntityId(); UserVmVO curVm = _vmDao.findById(vmId); + // create and attach data disk + long targetClonedVmId = cmd.getId(); + Account caller = CallContext.current().getCallingAccount(); + List dataDisks = _volsDao.findByInstanceAndType(targetClonedVmId, Volume.Type.DATADISK); + List createdSnapshots = new ArrayList<>(); + List createdVolumes = new ArrayList<>(); + long zoneId = cmd.getTargetVM().getDataCenterId(); + s_logger.info("Trying to attach data disk before starting the VM..."); + if (dataDisks.size() > 0) { + VolumeVO newDatadisk = null; + try { + for (VolumeVO dataDisk : dataDisks) { + long diskId = dataDisk.getId(); + SnapshotVO dataSnapShot = (SnapshotVO) volumeService.allocSnapshot(diskId, Snapshot.MANUAL_POLICY_ID, "DataDisk-Clone" + dataDisk.getName(), null); + if (dataSnapShot == null) { + throw new CloudRuntimeException("Unable to allocate snapshot of data disk: " + dataDisk.getId() + " name: " + dataDisk.getName()); + } + createdSnapshots.add(dataSnapShot); + SnapshotVO snapshotEntity = (SnapshotVO) volumeService.takeSnapshot(diskId, Snapshot.MANUAL_POLICY_ID, dataSnapShot.getId(), caller, false, null, false, new HashMap<>()); + if (snapshotEntity == null) { + throw new CloudRuntimeException("Error when creating the snapshot entity"); + } + if (snapshotEntity.getState() != Snapshot.State.BackedUp) { + throw new CloudRuntimeException("Async backup of snapshot happens during the clone for snapshot id: " + dataSnapShot.getId()); + } + long diskOfferingId = snapshotEntity.getDiskOfferingId(); + DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); + Long minIops = snapshotEntity.getMinIops(); + Long maxIops = snapshotEntity.getMaxIops(); + Long size = snapshotEntity.getSize(); + Storage.ProvisioningType provisioningType = diskOffering.getProvisioningType(); + DataCenterVO dataCenter = _dcDao.findById(zoneId); + String volumeName = snapshotEntity.getName() + "-DataDisk-Volume"; + VolumeVO parentVolume = _volsDao.findByIdIncludingRemoved(snapshotEntity.getVolumeId()); + newDatadisk = saveDataDiskVolumeFromSnapShot(caller, false, zoneId, + diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, volumeName, _uuidMgr.generateUuid(Volume.class, null), new HashMap<>()); + VolumeVO volumeEntity = (VolumeVO) volumeService.cloneDataVolume(cmd, snapshotEntity.getId(), newDatadisk); + createdVolumes.add(volumeEntity); + } + for (VolumeVO createdVol : createdVolumes) { + ((VolumeApiServiceImpl) volumeService).attachVolumeToVM(vmId, createdVol.getId(), createdVol.getDeviceId()); + } + } catch (CloudRuntimeException e){ + // clear the created disks + s_logger.warn("data disk process failed during clone, clearing the temporary resources..."); + if (newDatadisk != null) { + _resourceLimitMgr.decrementResourceCount(caller.getId(), ResourceType.volume, false); + _resourceLimitMgr.decrementResourceCount(caller.getId(), ResourceType.primary_storage, false, new Long(newDatadisk.getSize())); + } + throw new CloudRuntimeException(e.getMessage()); + } finally { + // clear the temporary data snapshots + } + } + // start the VM if successfull Long podId = curVm.getPodIdToDeployIn(); Long clusterId = null; Long hostId = curVm.getHostId(); From fcb81528deba61fee9abb9e9a0db7e2f8149c0d9 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 9 Jul 2021 14:51:23 -0400 Subject: [PATCH 054/114] fix the import typos --- .../java/com/cloud/vm/UserVmManagerImpl.java | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 4be22defc2f6..f06f8cdad073 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -22,8 +22,19 @@ import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -46,7 +57,24 @@ import com.cloud.network.NetworkModel; import com.cloud.network.PhysicalNetwork; import com.cloud.network.security.SecurityGroupVO; -import com.cloud.storage.*; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.GuestOSCategoryVO; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; +import com.cloud.storage.VolumeApiServiceImpl; +import com.cloud.storage.VolumeVO; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -58,7 +86,17 @@ import com.cloud.user.UserStatisticsVO; import com.cloud.user.UserVO; import com.cloud.user.VmDiskStatisticsVO; -import com.cloud.utils.db.*; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionCallbackWithException; +import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; +import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.db.UUIDManager; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -116,13 +154,15 @@ import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachCommand; -import org.apache.cloudstack.storage.datastore.db.*; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; -import org.owasp.esapi.util.CollectionsUtil; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -4468,9 +4508,11 @@ protected String validateUserData(String userData, HTTPMethod httpmethod) { @Override public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueException, ResourceUnavailableException, CloudRuntimeException, ResourceAllocationException { + if (cmd.getAccountName() != null && cmd.getDomainId() == null) { throw new InvalidParameterValueException("You must input the domainId together with the account name"); } + final DomainVO domain = cmd.getDomainId() == null ? null : _domainDao.findById(cmd.getDomainId()); final Account account = cmd.getAccountName() == null ? null : _accountService.getActiveAccountByName(cmd.getAccountName(), cmd.getDomainId()); if (domain != null && account != null) { @@ -4492,6 +4534,10 @@ public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueExce if (vmStatus.getHypervisorType() != HypervisorType.KVM && vmStatus.getHypervisorType() != HypervisorType.Simulator) { throw new CloudRuntimeException("The clone operation is only supported on KVM and Simulator!"); } + String kvmEnabled = _configDao.getValue("kvm.snapshot.enabled"); + if (kvmEnabled == null || !kvmEnabled.equalsIgnoreCase("true")) { + throw new CloudRuntimeException("Clone VM is not supported, as snapshots are disabled"); + } Long accountId = curVm.getAccountId(); Account vmOwner = _accountDao.findById(accountId); if (vmOwner == null) { From 78d5295b926d0d6607d7e64310b123dd9906af2e Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 11 Jul 2021 03:29:34 -0400 Subject: [PATCH 055/114] fix the service api impl --- api/src/main/java/com/cloud/storage/VolumeApiService.java | 2 ++ .../main/java/com/cloud/storage/VolumeApiServiceImpl.java | 5 +++++ server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 5ea49650ae37..c9a5139043f6 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -105,6 +105,8 @@ Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account acc Volume updateVolume(long volumeId, String path, String state, Long storageId, Boolean displayVolume, String customId, long owner, String chainInfo); + Volume attachVolumeToVm(CloneVMCmd cmd, Long volumeId, Long deviceId); + /** * Extracts the volume to a particular location. * diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 732ece2345db..95072721109a 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -1670,6 +1670,11 @@ private Volume orchestrateAttachVolumeToVM(Long vmId, Long volumeId, Long device return newVol; } + @Override + public Volume attachVolumeToVm(CloneVMCmd cmd, Long volumeId, Long deviceId) { + return attachVolumeToVM(cmd.getEntityId(), volumeId, deviceId); + } + public Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId) { Account caller = CallContext.current().getCallingAccount(); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index f06f8cdad073..471b88c5815d 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4678,7 +4678,8 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol createdVolumes.add(volumeEntity); } for (VolumeVO createdVol : createdVolumes) { - ((VolumeApiServiceImpl) volumeService).attachVolumeToVM(vmId, createdVol.getId(), createdVol.getDeviceId()); +// ((VolumeApiServiceImpl) volumeService).attachVolumeToVM(vmId, createdVol.getId(), createdVol.getDeviceId()); + volumeService.attachVolumeToVm(cmd, createdVol.getId(), createdVol.getDeviceId()); } } catch (CloudRuntimeException e){ // clear the created disks From 1414e3a2621bfd5d9e15b1125835262b7014d4c0 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 11 Jul 2021 14:21:03 -0400 Subject: [PATCH 056/114] clear unused import --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 471b88c5815d..9a940d402d4d 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -73,7 +73,6 @@ import com.cloud.storage.VMTemplateZoneVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; -import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.storage.VolumeVO; import com.cloud.user.Account; import com.cloud.user.AccountManager; From 877172893d2f555b50b5b8a70ca7ae0a383a0839 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 11 Jul 2021 15:48:16 -0400 Subject: [PATCH 057/114] ake the data disk shown --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 9a940d402d4d..7b0a2ff7d8ac 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4671,7 +4671,7 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol DataCenterVO dataCenter = _dcDao.findById(zoneId); String volumeName = snapshotEntity.getName() + "-DataDisk-Volume"; VolumeVO parentVolume = _volsDao.findByIdIncludingRemoved(snapshotEntity.getVolumeId()); - newDatadisk = saveDataDiskVolumeFromSnapShot(caller, false, zoneId, + newDatadisk = saveDataDiskVolumeFromSnapShot(caller, true, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, volumeName, _uuidMgr.generateUuid(Volume.class, null), new HashMap<>()); VolumeVO volumeEntity = (VolumeVO) volumeService.cloneDataVolume(cmd, snapshotEntity.getId(), newDatadisk); createdVolumes.add(volumeEntity); From c6279ce1c70a5a74c877441addb4fa3a6d1bc0a1 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 12 Jul 2021 18:08:53 -0400 Subject: [PATCH 058/114] data disk snapshot clearance and exception handler --- .../main/java/com/cloud/vm/UserVmService.java | 3 +- .../api/command/user/vm/CloneVMCmd.java | 5 ++-- .../java/com/cloud/vm/UserVmManagerImpl.java | 29 ++++++++++++++----- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 74c090eb5f32..da8c2373ca79 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -21,6 +21,7 @@ import java.util.Optional; import com.cloud.storage.VolumeApiService; +import com.cloud.storage.snapshot.SnapshotApiService; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; @@ -95,7 +96,7 @@ public interface UserVmService { * - the command specifying vmId to be cloned * @return the VM if cloneVM operation is successful * */ - Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService) throws ResourceUnavailableException, ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; + Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService, SnapshotApiService snapshotService) throws ResourceUnavailableException, ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 66414dc28969..136f5b9ce087 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -108,6 +108,7 @@ public void create() throws ResourceAllocationException { s_logger.info("The template id recorded is: " + template.getId()); setTemporaryTemlateId(template.getId()); _templateService.createPrivateTemplate(this); + _snapshotService.deleteSnapshot(getTemporarySnapShotId()); UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); if (vmRecord == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "unable to record a new VM to db!"); @@ -146,7 +147,7 @@ public void execute() { try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); s_logger.info("starting actual VM id: " + getEntityId()); - result = _userVmService.cloneVirtualMachine(this, _volumeService); + result = _userVmService.cloneVirtualMachine(this, _volumeService, _snapshotService); } catch (ResourceUnavailableException ex) { s_logger.warn("Exception: ", ex); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); @@ -160,7 +161,7 @@ public void execute() { } result.ifPresentOrElse((userVm)-> { UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", result.get()).get(0); - response.setResponseName("test_clone"); + response.setResponseName("full_clone"); setResponseObject(response); }, ()-> { throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, "failed to clone VM: " + getId()); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 7b0a2ff7d8ac..d5691007469b 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -74,6 +74,7 @@ import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; +import com.cloud.storage.snapshot.SnapshotApiService; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -4634,7 +4635,7 @@ private VolumeVO saveDataDiskVolumeFromSnapShot(final Account owner, final Boole @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CLONE, eventDescription = "clone vm", async = true) - public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException, InsufficientCapacityException, ResourceAllocationException { + public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService, SnapshotApiService snapshotService) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException, InsufficientCapacityException, ResourceAllocationException { long vmId = cmd.getEntityId(); UserVmVO curVm = _vmDao.findById(vmId); // create and attach data disk @@ -4676,22 +4677,34 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol VolumeVO volumeEntity = (VolumeVO) volumeService.cloneDataVolume(cmd, snapshotEntity.getId(), newDatadisk); createdVolumes.add(volumeEntity); } - for (VolumeVO createdVol : createdVolumes) { -// ((VolumeApiServiceImpl) volumeService).attachVolumeToVM(vmId, createdVol.getId(), createdVol.getDeviceId()); - volumeService.attachVolumeToVm(cmd, createdVol.getId(), createdVol.getDeviceId()); - } } catch (CloudRuntimeException e){ - // clear the created disks s_logger.warn("data disk process failed during clone, clearing the temporary resources..."); + for (VolumeVO dataDiskToClear : createdVolumes) { + volumeService.destroyVolume(dataDiskToClear.getId(), caller, true, false); + } + // clear the created disks if (newDatadisk != null) { - _resourceLimitMgr.decrementResourceCount(caller.getId(), ResourceType.volume, false); - _resourceLimitMgr.decrementResourceCount(caller.getId(), ResourceType.primary_storage, false, new Long(newDatadisk.getSize())); + volumeService.destroyVolume(newDatadisk.getId(), caller, true, false); } throw new CloudRuntimeException(e.getMessage()); } finally { // clear the temporary data snapshots + for (Snapshot snapshotLeftOver : createdSnapshots) { + snapshotService.deleteSnapshot(snapshotLeftOver.getId()); + } } } + + for (VolumeVO createdVol : createdVolumes) { + try { + volumeService.attachVolumeToVm(cmd, createdVol.getId(), createdVol.getDeviceId()); + } catch (CloudRuntimeException e) { + s_logger.warn("data disk: " + createdVol.getId() + " attachment to VM " + vmId + " failed due to" + e.getMessage()); + s_logger.info("Clearing the data disk: " + createdVol.getId()); + volumeService.destroyVolume(createdVol.getId(), caller, true, true); + } + } + // start the VM if successfull Long podId = curVm.getPodIdToDeployIn(); Long clusterId = null; From 0415cbc9683a096cc443cfeb4b7e65e5dfea2556 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 16 Jul 2021 00:01:22 -0400 Subject: [PATCH 059/114] error handling complates --- .../main/java/com/cloud/storage/Snapshot.java | 3 +- .../api/command/user/vm/CloneVMCmd.java | 17 ++++--- .../storage/snapshot/SnapshotManagerImpl.java | 12 +++-- .../cloud/template/TemplateManagerImpl.java | 13 ++--- .../java/com/cloud/vm/UserVmManagerImpl.java | 50 +++++++++---------- .../com/cloud/usage/UsageManagerImpl.java | 12 +++++ 6 files changed, 62 insertions(+), 45 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/Snapshot.java b/api/src/main/java/com/cloud/storage/Snapshot.java index 2f3a59541d9a..64917fe64213 100644 --- a/api/src/main/java/com/cloud/storage/Snapshot.java +++ b/api/src/main/java/com/cloud/storage/Snapshot.java @@ -26,7 +26,7 @@ public interface Snapshot extends ControlledEntity, Identity, InternalIdentity, StateObject { public enum Type { - MANUAL, RECURRING, TEMPLATE, HOURLY, DAILY, WEEKLY, MONTHLY; + MANUAL, RECURRING, TEMPLATE, HOURLY, DAILY, WEEKLY, MONTHLY, INTERNAL; private int max = 8; public void setMax(int max) { @@ -71,6 +71,7 @@ enum LocationType { } public static final long MANUAL_POLICY_ID = 0L; + public static final long INTERNAL_POLICY_ID = 7L; @Override long getAccountId(); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 136f5b9ce087..b5e388dbb5e4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -9,6 +9,7 @@ import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; @@ -108,23 +109,27 @@ public void create() throws ResourceAllocationException { s_logger.info("The template id recorded is: " + template.getId()); setTemporaryTemlateId(template.getId()); _templateService.createPrivateTemplate(this); - _snapshotService.deleteSnapshot(getTemporarySnapShotId()); UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); if (vmRecord == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "unable to record a new VM to db!"); } setEntityId(vmRecord.getId()); setEntityUuid(vmRecord.getUuid()); - } catch (ResourceUnavailableException | InsufficientCapacityException e) { + } + catch (ResourceUnavailableException | InsufficientCapacityException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); } catch (InvalidParameterValueException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } catch (ServerApiException e) { + throw new ServerApiException(e.getErrorCode(), e.getDescription()); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } finally { - if (getTemporaryTemlateId() != null) { - // TODO: delete template in the service - s_logger.warn("clearing the temporary template: " + getTemporaryTemlateId()); + if (getTemporarySnapShotId() != null) { + _snapshotService.deleteSnapshot(getTemporarySnapShotId()); + s_logger.warn("clearing the temporary snapshot: " + getTemporarySnapShotId()); } } } @@ -138,7 +143,7 @@ public String getVMName() { } public String getTemplateName() { - return getVMName() + "-QA"; + return (getVMName() + "-Clone-" + _uuidMgr.generateUuid(VirtualMachineTemplate.class, null)).substring(0, 32); } @Override diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 06da5d1f0026..dbd5790a8658 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -665,7 +665,7 @@ public Pair, Integer> listSnapshots(ListSnapshotsCmd cm sb.and("snapshotTypeEQ", sb.entity().getsnapshotType(), SearchCriteria.Op.IN); sb.and("snapshotTypeNEQ", sb.entity().getsnapshotType(), SearchCriteria.Op.NEQ); sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); - + sb.and("snapshotTypeInternal", sb.entity().getsnapshotType(), SearchCriteria.Op.NEQ); if (tags != null && !tags.isEmpty()) { SearchBuilder tagSearch = _resourceTagDao.createSearchBuilder(); for (int count = 0; count < tags.size(); count++) { @@ -737,7 +737,7 @@ public Pair, Integer> listSnapshots(ListSnapshotsCmd cm // Show only MANUAL and RECURRING snapshot types sc.setParameters("snapshotTypeNEQ", Snapshot.Type.TEMPLATE.ordinal()); } - + sc.setParameters("snapshotTypeInternal", Type.INTERNAL.ordinal()); Pair, Integer> result = _snapshotDao.searchAndCount(sc, searchFilter); return new Pair, Integer>(result.first(), result.second()); } @@ -1027,7 +1027,13 @@ public List findRecurringSnapshotSchedule(ListRecurringSnaps private Type getSnapshotType(Long policyId) { if (policyId.equals(Snapshot.MANUAL_POLICY_ID)) { return Type.MANUAL; - } else { + } + + else if (policyId.equals(Snapshot.INTERNAL_POLICY_ID)) { + return Type.INTERNAL; + } + + else { SnapshotPolicyVO spstPolicyVO = _snapshotPolicyDao.findById(policyId); IntervalType intvType = DateUtil.getIntervalType(spstPolicyVO.getInterval()); return getSnapshotType(intvType); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 55399b34ddd2..188a06766b09 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1906,17 +1906,14 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template if (guestOS == null) { throw new InvalidParameterValueException("GuestOS with ID: " + guestOSId + " does not exist."); } - // get snapshot from this step - - Long nextTemplateId = _tmpltDao.getNextInSequence(Long.class, "id"); s_logger.info("Creating snapshot for the tempalte creation"); - SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, Snapshot.MANUAL_POLICY_ID, curVm.getDisplayName() + "-Clone-" + nextTemplateId, null); + SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, Snapshot.INTERNAL_POLICY_ID, curVm.getDisplayName() + "-Clone-" + nextTemplateId, null); if (snapshot == null) { throw new CloudRuntimeException("Unable to create a snapshot during the template creation recording"); } - Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, Snapshot.MANUAL_POLICY_ID, snapshot.getId(), caller, false, null, false, new HashMap<>()); + Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, Snapshot.INTERNAL_POLICY_ID, snapshot.getId(), caller, false, null, false, new HashMap<>()); if (snapshotEntity == null) { throw new CloudRuntimeException("Error when creating the snapshot entity"); } @@ -1991,11 +1988,7 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template snapshot.getSize()); } - if (template != null) { - return template; - } else { - throw new CloudRuntimeException("Failed to create a template"); - } + return template; } @Override diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 89b907a6664e..f0a680827e60 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4648,12 +4648,12 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol try { for (VolumeVO dataDisk : dataDisks) { long diskId = dataDisk.getId(); - SnapshotVO dataSnapShot = (SnapshotVO) volumeService.allocSnapshot(diskId, Snapshot.MANUAL_POLICY_ID, "DataDisk-Clone" + dataDisk.getName(), null); + SnapshotVO dataSnapShot = (SnapshotVO) volumeService.allocSnapshot(diskId, Snapshot.INTERNAL_POLICY_ID, "DataDisk-Clone" + dataDisk.getName(), null); if (dataSnapShot == null) { throw new CloudRuntimeException("Unable to allocate snapshot of data disk: " + dataDisk.getId() + " name: " + dataDisk.getName()); } createdSnapshots.add(dataSnapShot); - SnapshotVO snapshotEntity = (SnapshotVO) volumeService.takeSnapshot(diskId, Snapshot.MANUAL_POLICY_ID, dataSnapShot.getId(), caller, false, null, false, new HashMap<>()); + SnapshotVO snapshotEntity = (SnapshotVO) volumeService.takeSnapshot(diskId, Snapshot.INTERNAL_POLICY_ID, dataSnapShot.getId(), caller, false, null, false, new HashMap<>()); if (snapshotEntity == null) { throw new CloudRuntimeException("Error when creating the snapshot entity"); } @@ -4674,6 +4674,10 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol VolumeVO volumeEntity = (VolumeVO) volumeService.cloneDataVolume(cmd, snapshotEntity.getId(), newDatadisk); createdVolumes.add(volumeEntity); } + + for (VolumeVO createdVol : createdVolumes) { + volumeService.attachVolumeToVm(cmd, createdVol.getId(), createdVol.getDeviceId()); + } } catch (CloudRuntimeException e){ s_logger.warn("data disk process failed during clone, clearing the temporary resources..."); for (VolumeVO dataDiskToClear : createdVolumes) { @@ -4683,6 +4687,7 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol if (newDatadisk != null) { volumeService.destroyVolume(newDatadisk.getId(), caller, true, false); } + destroyVm(vmId, true); throw new CloudRuntimeException(e.getMessage()); } finally { // clear the temporary data snapshots @@ -4692,16 +4697,6 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol } } - for (VolumeVO createdVol : createdVolumes) { - try { - volumeService.attachVolumeToVm(cmd, createdVol.getId(), createdVol.getDeviceId()); - } catch (CloudRuntimeException e) { - s_logger.warn("data disk: " + createdVol.getId() + " attachment to VM " + vmId + " failed due to" + e.getMessage()); - s_logger.info("Clearing the data disk: " + createdVol.getId()); - volumeService.destroyVolume(createdVol.getId(), caller, true, true); - } - } - // start the VM if successfull Long podId = curVm.getPodIdToDeployIn(); Long clusterId = null; @@ -5756,22 +5751,27 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio mapToLong(AffinityGroupVO::getId). boxed(). collect(Collectors.toList()); - if (dataCenter.getNetworkType() == NetworkType.Basic) { - vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, - size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, - curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), - null, new HashMap<>(), dynamicScalingEnabled); - } else { - if (dataCenter.isSecurityGroupEnabled()) { - vmResult = createAdvancedSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, networkIds, securityGroupIdList, curAccount, hostName, - displayName, diskOfferingId, size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, - affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), + try { + if (dataCenter.getNetworkType() == NetworkType.Basic) { + vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, + size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, + curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); } else { - vmResult = createAdvancedVirtualMachine(dataCenter, serviceOffering, template, networkIds, curAccount, hostName, displayName, diskOfferingId, size, group, - hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), - cmd.getCustomId(), new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); + if (dataCenter.isSecurityGroupEnabled()) { + vmResult = createAdvancedSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, networkIds, securityGroupIdList, curAccount, hostName, + displayName, diskOfferingId, size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, + affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), + null, new HashMap<>(), dynamicScalingEnabled); + } else { + vmResult = createAdvancedVirtualMachine(dataCenter, serviceOffering, template, networkIds, curAccount, hostName, displayName, diskOfferingId, size, group, + hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), + cmd.getCustomId(), new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); + } } + } catch (CloudRuntimeException e) { + _templateMgr.delete(curAccount.getId(), template.getId(), zoneId); + throw new CloudRuntimeException("Unable to create the VM record"); } return vmResult; } diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 899f558861ee..90a5bec8b535 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -33,6 +33,9 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.dao.SnapshotDao; import org.apache.cloudstack.quota.QuotaAlertManager; import org.apache.cloudstack.quota.QuotaManager; import org.apache.cloudstack.quota.QuotaStatement; @@ -162,6 +165,8 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna private QuotaAlertManager _alertManager; @Inject private QuotaStatement _quotaStatement; + @Inject + private SnapshotDao _snapshotDao; private String _version = null; private final Calendar _jobExecTime = Calendar.getInstance(); @@ -1633,6 +1638,13 @@ private void createSnapshotHelperEvent(UsageEventVO event) { long zoneId = -1L; long snapId = event.getResourceId(); + + SnapshotVO snapshotInstance = _snapshotDao.findById(snapId); + + if (snapshotInstance != null && snapshotInstance.getsnapshotType() == Snapshot.Type.INTERNAL.ordinal()) { + return; + } + if (EventTypes.EVENT_SNAPSHOT_CREATE.equals(event.getType())) { if (usageSnapshotSelection){ snapSize = event.getVirtualSize(); From 03e380f971d516e420f43f419fdee3dee6b8695b Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 19 Jul 2021 19:35:53 -0400 Subject: [PATCH 060/114] modify usage manager --- usage/src/main/java/com/cloud/usage/UsageManagerImpl.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 90a5bec8b535..6550d0255505 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1641,7 +1641,13 @@ private void createSnapshotHelperEvent(UsageEventVO event) { SnapshotVO snapshotInstance = _snapshotDao.findById(snapId); - if (snapshotInstance != null && snapshotInstance.getsnapshotType() == Snapshot.Type.INTERNAL.ordinal()) { + if (snapshotInstance == null) { + return; + } + + String typeDescription = snapshotInstance.getTypeDescription(); + + if (typeDescription == null || typeDescription.trim().equalsIgnoreCase("INTERNAL")) { return; } From 4d6d6cf604948f71a3379756a1a2d6556ac0749f Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 19 Jul 2021 19:40:21 -0400 Subject: [PATCH 061/114] change the usage server --- usage/src/main/java/com/cloud/usage/UsageManagerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 6550d0255505..57bfff020043 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -33,7 +33,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; import com.cloud.storage.dao.SnapshotDao; import org.apache.cloudstack.quota.QuotaAlertManager; From 678d9d81a0371f250e78c1a43e2c8281e6425446 Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 19 Jul 2021 19:52:13 -0400 Subject: [PATCH 062/114] debug logger enabled --- .../src/main/java/com/cloud/usage/UsageManagerImpl.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 57bfff020043..c4fe1d7920ce 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -33,6 +33,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; import com.cloud.storage.dao.SnapshotDao; import org.apache.cloudstack.quota.QuotaAlertManager; @@ -1639,16 +1640,14 @@ private void createSnapshotHelperEvent(UsageEventVO event) { long snapId = event.getResourceId(); SnapshotVO snapshotInstance = _snapshotDao.findById(snapId); - - if (snapshotInstance == null) { - return; - } - String typeDescription = snapshotInstance.getTypeDescription(); if (typeDescription == null || typeDescription.trim().equalsIgnoreCase("INTERNAL")) { + return; } + s_logger.warn("snapshot type: " + snapshotInstance.getTypeDescription() + " ---" + snapshotInstance.getsnapshotType() + "---name----" + snapshotInstance.getName()); + s_logger.warn("type ordinal: " + Snapshot.Type.INTERNAL.ordinal()); if (EventTypes.EVENT_SNAPSHOT_CREATE.equals(event.getType())) { if (usageSnapshotSelection){ From 8ea6949890faf8732ab3ff4df5320b669955c4dd Mon Sep 17 00:00:00 2001 From: junxuan Date: Mon, 19 Jul 2021 20:28:45 -0400 Subject: [PATCH 063/114] fix the usage event snapshot event recording --- .../storage/snapshot/SnapshotServiceImpl.java | 7 +++++-- .../storage/snapshot/SnapshotManagerImpl.java | 14 ++++++++++---- .../java/com/cloud/usage/UsageManagerImpl.java | 6 +----- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index b8788fbef988..a763a8bf0de1 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -218,8 +218,11 @@ public SnapshotResult takeSnapshot(SnapshotInfo snap) { try { result = future.get(); - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_ON_PRIMARY, snap.getAccountId(), snap.getDataCenterId(), snap.getId(), - snap.getName(), null, null, snapshotOnPrimary.getSize(), snapshotOnPrimary.getSize(), snap.getClass().getName(), snap.getUuid()); + SnapshotVO snapVO = _snapshotDao.findById(snap.getId()); + if (snapVO == null || snapVO.getsnapshotType() != Snapshot.Type.INTERNAL.ordinal()) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_ON_PRIMARY, snap.getAccountId(), snap.getDataCenterId(), snap.getId(), + snap.getName(), null, null, snapshotOnPrimary.getSize(), snapshotOnPrimary.getSize(), snap.getClass().getName(), snap.getUuid()); + } return result; } catch (InterruptedException e) { s_logger.debug("Failed to create snapshot", e); diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index dbd5790a8658..6dcb412355cd 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -588,8 +588,11 @@ public boolean deleteSnapshot(long snapshotId) { if (result) { if (snapshotCheck.getState() == Snapshot.State.BackedUp) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_DELETE, snapshotCheck.getAccountId(), snapshotCheck.getDataCenterId(), snapshotId, - snapshotCheck.getName(), null, null, 0L, snapshotCheck.getClass().getName(), snapshotCheck.getUuid()); + SnapshotVO snapVO = _snapshotDao.findById(snapshotId); + if (snapVO == null || snapVO.getsnapshotType() != Type.INTERNAL.ordinal()) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_DELETE, snapshotCheck.getAccountId(), snapshotCheck.getDataCenterId(), snapshotId, + snapshotCheck.getName(), null, null, 0L, snapshotCheck.getClass().getName(), snapshotCheck.getUuid()); + } } if (snapshotCheck.getState() != Snapshot.State.Error && snapshotCheck.getState() != Snapshot.State.Destroyed) { @@ -1186,8 +1189,11 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc throw new CloudRuntimeException("Could not find snapshot"); } } - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, - snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); + SnapshotVO snapInstance = _snapshotDao.findById(snapshot.getId()); + if (snapInstance == null || snapInstance.getsnapshotType() != Type.INTERNAL.ordinal()) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, + snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); + } // Correct the resource count of snapshot in case of delta snapshots. _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index c4fe1d7920ce..90a5bec8b535 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1640,14 +1640,10 @@ private void createSnapshotHelperEvent(UsageEventVO event) { long snapId = event.getResourceId(); SnapshotVO snapshotInstance = _snapshotDao.findById(snapId); - String typeDescription = snapshotInstance.getTypeDescription(); - - if (typeDescription == null || typeDescription.trim().equalsIgnoreCase("INTERNAL")) { + if (snapshotInstance != null && snapshotInstance.getsnapshotType() == Snapshot.Type.INTERNAL.ordinal()) { return; } - s_logger.warn("snapshot type: " + snapshotInstance.getTypeDescription() + " ---" + snapshotInstance.getsnapshotType() + "---name----" + snapshotInstance.getName()); - s_logger.warn("type ordinal: " + Snapshot.Type.INTERNAL.ordinal()); if (EventTypes.EVENT_SNAPSHOT_CREATE.equals(event.getType())) { if (usageSnapshotSelection){ From d9eac6e6f03dec84a6a46b366af43d65d4a833b7 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 25 Jul 2021 17:55:31 -0400 Subject: [PATCH 064/114] add front end for clone virtuial machine --- ui/public/locales/en.json | 2 ++ ui/src/config/section/compute.js | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 870686b5f387..03d5b0fd25d2 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -82,6 +82,7 @@ "label.action.copy.iso.processing": "Copying ISO....", "label.action.copy.template": "Copy Template", "label.action.copy.template.processing": "Copying Template....", + "label.action.clone.vm": "Clone VM", "label.action.create.template.from.vm": "Create Template from VM", "label.action.create.template.from.volume": "Create Template from Volume", "label.action.create.template.processing": "Creating Template....", @@ -2443,6 +2444,7 @@ "message.action.manage.cluster": "Please confirm that you want to manage the cluster.", "message.action.primarystorage.enable.maintenance.mode": "Warning: placing the primary storage into maintenance mode will cause all VMs using volumes from it to be stopped. Do you want to continue?", "message.action.reboot.instance": "Please confirm that you want to reboot this instance.", + "message.action.clone.instance": "Please confirm that you want to clone this instance", "message.action.reboot.router": "All services provided by this virtual router will be interrupted. Please confirm that you want to reboot this router.", "message.action.reboot.systemvm": "Please confirm that you want to reboot this system VM.", "message.action.recover.volume": "Please confirm that you would like to recover this volume.", diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index d6ecbf53b96a..4fce4d55de47 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -133,6 +133,21 @@ export default { return fields } }, + { + api: 'cloneVirtualMachine', + icon: 'reload', + label: 'label.action.clone.vm', + message: 'message.action.clone.instance', + docHelp: 'adminguide/virtual_machines.html#stopping-and-starting-vms', + dataView: true, + show: (record) => { return ['Running'].includes(record.state) }, + args: ["virtualmachineid"], + mapping: { + virtualmachineid: { + value: (record, params) => {return record.id} + } + } + }, { api: 'restoreVirtualMachine', icon: 'sync', From 14fa30c2d758a78aff0aac7f901e52b1dbc2f6cc Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 25 Jul 2021 18:00:04 -0400 Subject: [PATCH 065/114] fix lint --- ui/src/config/section/compute.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 4fce4d55de47..279db83f4865 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -141,13 +141,13 @@ export default { docHelp: 'adminguide/virtual_machines.html#stopping-and-starting-vms', dataView: true, show: (record) => { return ['Running'].includes(record.state) }, - args: ["virtualmachineid"], + args: ['virtualmachineid'], mapping: { virtualmachineid: { - value: (record, params) => {return record.id} + value: (record, params) => { return record.id } } } - }, + }, { api: 'restoreVirtualMachine', icon: 'sync', From 0d893f0ca7b639cb3e8d9f3a88004defa2a5c9a3 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 25 Jul 2021 18:53:14 -0400 Subject: [PATCH 066/114] modify show condition --- ui/src/config/section/compute.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 279db83f4865..562ded3e5b6d 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -140,7 +140,7 @@ export default { message: 'message.action.clone.instance', docHelp: 'adminguide/virtual_machines.html#stopping-and-starting-vms', dataView: true, - show: (record) => { return ['Running'].includes(record.state) }, + show: (record) => { return true }, args: ['virtualmachineid'], mapping: { virtualmachineid: { From c72654c3f47159873eab4e61818d0c4ae6eda60f Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 25 Jul 2021 19:37:05 -0400 Subject: [PATCH 067/114] change ui icon --- ui/src/config/section/compute.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 562ded3e5b6d..bf9fbc94d6fa 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -135,7 +135,7 @@ export default { }, { api: 'cloneVirtualMachine', - icon: 'reload', + icon: 'camera', label: 'label.action.clone.vm', message: 'message.action.clone.instance', docHelp: 'adminguide/virtual_machines.html#stopping-and-starting-vms', From b0ab18b18b95803b2d5cfed96a90109a8bf9e06a Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 28 Jul 2021 03:10:01 -0400 Subject: [PATCH 068/114] ping first test case --- .../com/cloud/vm/UserVmManagerImplTest.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index ae647f349df7..a0208d62ca82 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -32,13 +32,16 @@ import java.util.List; import java.util.Map; +import com.cloud.exception.*; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; +import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.context.CallContext; @@ -57,10 +60,6 @@ import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.exception.InsufficientAddressCapacityException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.NetworkModel; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; @@ -145,6 +144,9 @@ public class UserVmManagerImplTest { @Mock private VMTemplateDao templateDao; + @Mock + private CloneVMCmd cloneVMCommand; + private long vmId = 1l; private static final long GiB_TO_BYTES = 1024 * 1024 * 1024; @@ -571,4 +573,16 @@ public void validateRemoveEncryptedPasswordFromUserVmVoDetails(){ Mockito.verify(userVmVoMock, Mockito.times(1)).setDetails(detailsMock); Mockito.verify(userVmDao, Mockito.times(1)).saveDetails(userVmVoMock); } + + @Test + public void validateCloneCondition() { + Mockito.when(cloneVMCommand.getTargetVM()).thenReturn(null); + Exception err = null; + try { + userVmManagerImpl.checkCloneCondition(cloneVMCommand); + } catch (CloudRuntimeException | ResourceUnavailableException | ResourceAllocationException e) { + err = e; + } + assertTrue(err instanceof CloudRuntimeException); + } } From a423887573df270bc3d0fd81d6a22c36efd3e20e Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 28 Jul 2021 03:25:31 -0400 Subject: [PATCH 069/114] fix checkstyle --- .../src/test/java/com/cloud/vm/UserVmManagerImplTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index a0208d62ca82..8a0ed28726df 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -32,7 +32,11 @@ import java.util.List; import java.util.Map; -import com.cloud.exception.*; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VMTemplateVO; From 4a3fc6d0c31235da20b34612a1e4308555c5b5c6 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 28 Jul 2021 03:30:38 -0400 Subject: [PATCH 070/114] fix null mackito --- server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 8a0ed28726df..149853a64c5f 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -581,6 +581,8 @@ public void validateRemoveEncryptedPasswordFromUserVmVoDetails(){ @Test public void validateCloneCondition() { Mockito.when(cloneVMCommand.getTargetVM()).thenReturn(null); + Mockito.when(cloneVMCommand.getAccountName()).thenReturn(null); + Mockito.when(cloneVMCommand.getDomainId()).thenReturn(null); Exception err = null; try { userVmManagerImpl.checkCloneCondition(cloneVMCommand); From d925bedb69a2332051ff669b09662e95aa2e1c04 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 6 Aug 2021 03:48:37 -0400 Subject: [PATCH 071/114] add integration test for clone --- test/integration/smoke/test_vm_life_cycle.py | 21 ++++++++++++++++++++ tools/marvin/marvin/lib/base.py | 11 ++++++++++ 2 files changed, 32 insertions(+) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 61b3a22a6c8e..731256946889 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -876,6 +876,27 @@ def test_11_destroy_vm_and_volumes(self): self.assertEqual(Volume.list(self.apiclient, id=vol1.id), None, "List response contains records when it should not") + @attr(tags = ["devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") + def test_12_clone_vm_and_volumes(self): + small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; + small_virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.small_offering.id,) + vol1 = Volume.create( + self.apiclient, + self.services, + account=self.account.name, + diskofferingid=small_disk_offering.id, + domainid=self.account.domainid, + zoneid=self.zone.id + ) + small_virtual_machine.attach_volume(self.apiclient, vol1) + self.debug("Clone VM - ID: %s" % small_virtual_machine.id) + small_virtual_machine.clone(self.apiclient) + self.assertEqual(VirtualMachine.list(self.apiclient, id=small_virtual_machine.id), None, "List response contains records when it should not") class TestSecuredVmMigration(cloudstackTestCase): diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 916af64d96cc..25b460fca9a0 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -744,6 +744,17 @@ def reboot(self, apiclient, forced=None): if response[0] == FAIL: raise Exception(response[1]) + def clone(self, apiclient, forced=None): + """"Clone the instance""" + cmd = cloneVirtualMachine.cloneVirtualMachineCmd() + cmd.id = self.id + if forced: + cmd.forced =forced + apiclient.cloneVirtualMachine(cmd) + response = self.getState(apiclient, VirtualMachine.RUNNING) + if (response[0] == FAIL): + raise Exception(response[1]) + def recover(self, apiclient): """Recover the instance""" cmd = recoverVirtualMachine.recoverVirtualMachineCmd() From cdbbd607c8eb53a72aed12bbb1b583f0e62f27e5 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 6 Aug 2021 12:56:38 -0400 Subject: [PATCH 072/114] add clone tag to test --- test/integration/smoke/test_vm_life_cycle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 731256946889..4e35367314e9 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -876,7 +876,7 @@ def test_11_destroy_vm_and_volumes(self): self.assertEqual(Volume.list(self.apiclient, id=vol1.id), None, "List response contains records when it should not") - @attr(tags = ["devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") + @attr(tags = ["clone", "devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") def test_12_clone_vm_and_volumes(self): small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; small_virtual_machine = VirtualMachine.create( From 6af4d28569796ae14af402b39d2a513572e7dfc4 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 6 Aug 2021 13:07:30 -0400 Subject: [PATCH 073/114] fix the virtualmachine id attribute --- tools/marvin/marvin/lib/base.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 25b460fca9a0..5d97229407cc 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -744,12 +744,10 @@ def reboot(self, apiclient, forced=None): if response[0] == FAIL: raise Exception(response[1]) - def clone(self, apiclient, forced=None): + def clone(self, apiclient): """"Clone the instance""" cmd = cloneVirtualMachine.cloneVirtualMachineCmd() - cmd.id = self.id - if forced: - cmd.forced =forced + cmd.virtualmachineid = self.id apiclient.cloneVirtualMachine(cmd) response = self.getState(apiclient, VirtualMachine.RUNNING) if (response[0] == FAIL): From a743d20820741d7d6c0ac73a14e86f36cd466121 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sat, 7 Aug 2021 03:19:29 -0400 Subject: [PATCH 074/114] fix the integration target --- test/integration/smoke/test_vm_life_cycle.py | 6 +++--- tools/marvin/marvin/lib/base.py | 16 ++++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 4e35367314e9..bc8a1a1d8e24 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -876,7 +876,7 @@ def test_11_destroy_vm_and_volumes(self): self.assertEqual(Volume.list(self.apiclient, id=vol1.id), None, "List response contains records when it should not") - @attr(tags = ["clone", "devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") + @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") def test_12_clone_vm_and_volumes(self): small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; small_virtual_machine = VirtualMachine.create( @@ -895,8 +895,8 @@ def test_12_clone_vm_and_volumes(self): ) small_virtual_machine.attach_volume(self.apiclient, vol1) self.debug("Clone VM - ID: %s" % small_virtual_machine.id) - small_virtual_machine.clone(self.apiclient) - self.assertEqual(VirtualMachine.list(self.apiclient, id=small_virtual_machine.id), None, "List response contains records when it should not") + clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) + self.assertEqual(VirtualMachine.list(self.apiclient, id=clone_response.id), None, "List response contains records when it should not") class TestSecuredVmMigration(cloudstackTestCase): diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 5d97229407cc..401cce825537 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -744,14 +744,18 @@ def reboot(self, apiclient, forced=None): if response[0] == FAIL: raise Exception(response[1]) - def clone(self, apiclient): + def clone(self, apiclient, vm): """"Clone the instance""" cmd = cloneVirtualMachine.cloneVirtualMachineCmd() - cmd.virtualmachineid = self.id - apiclient.cloneVirtualMachine(cmd) - response = self.getState(apiclient, VirtualMachine.RUNNING) - if (response[0] == FAIL): - raise Exception(response[1]) + cmd.virtualmachineid = vm.id + response = apiclient.cloneVirtualMachine(cmd) + temp = self.id + self.id = response.id + state = self.getState(apiclient, VirtualMachine.RUNNING) + self.id = temp + if (state[0] == FAIL): + raise Exception(state[1]) + return response def recover(self, apiclient): """Recover the instance""" From 1194b637235c63b1ac4155d70875f8e593527103 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sat, 7 Aug 2021 21:22:40 -0400 Subject: [PATCH 075/114] debug msg and vm id logged --- test/integration/smoke/test_vm_life_cycle.py | 6 +++++- tools/marvin/marvin/lib/base.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index bc8a1a1d8e24..bd65e0763730 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -895,7 +895,11 @@ def test_12_clone_vm_and_volumes(self): ) small_virtual_machine.attach_volume(self.apiclient, vol1) self.debug("Clone VM - ID: %s" % small_virtual_machine.id) - clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) + try: + clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) + except Exception as e: + self.debug("Clone --" + str(e)) + raise e self.assertEqual(VirtualMachine.list(self.apiclient, id=clone_response.id), None, "List response contains records when it should not") class TestSecuredVmMigration(cloudstackTestCase): diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 401cce825537..9b34cd68ab60 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -748,6 +748,8 @@ def clone(self, apiclient, vm): """"Clone the instance""" cmd = cloneVirtualMachine.cloneVirtualMachineCmd() cmd.virtualmachineid = vm.id + if vm.id is None: + cmd.virtualmachineid = self.id response = apiclient.cloneVirtualMachine(cmd) temp = self.id self.id = response.id From 0521448939196b9cdb0e6cfcb21d8f115e8ad8b9 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sat, 7 Aug 2021 23:31:15 -0400 Subject: [PATCH 076/114] fix the size integration test --- test/integration/smoke/test_vm_life_cycle.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index bd65e0763730..cd0b440e1e03 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -884,7 +884,8 @@ def test_12_clone_vm_and_volumes(self): self.services["small"], accountid=self.account.name, domainid=self.account.domainid, - serviceofferingid=self.small_offering.id,) + serviceofferingid=self.small_offering.id, + rootdisksize=100,) vol1 = Volume.create( self.apiclient, self.services, From de15e9d687f7416997961a7a154c2b97911c017d Mon Sep 17 00:00:00 2001 From: junxuan Date: Sat, 7 Aug 2021 23:38:03 -0400 Subject: [PATCH 077/114] fix the resource limit check --- test/integration/smoke/test_vm_life_cycle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index cd0b440e1e03..217cc2c5076a 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -885,7 +885,7 @@ def test_12_clone_vm_and_volumes(self): accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.small_offering.id, - rootdisksize=100,) + rootdisksize=1,) vol1 = Volume.create( self.apiclient, self.services, From 76f9869177965d1fc71b7c81aa292472bfeb7735 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sat, 7 Aug 2021 23:45:23 -0400 Subject: [PATCH 078/114] fix min root size --- test/integration/smoke/test_vm_life_cycle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 217cc2c5076a..4b8bbaa5c510 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -885,7 +885,7 @@ def test_12_clone_vm_and_volumes(self): accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.small_offering.id, - rootdisksize=1,) + rootdisksize=2,) vol1 = Volume.create( self.apiclient, self.services, From dee440681d16c8c444aa088922438877efd2957a Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 8 Aug 2021 00:27:15 -0400 Subject: [PATCH 079/114] fix size error when the template is not healthy --- .../main/java/com/cloud/template/TemplateManagerImpl.java | 6 ++++++ test/integration/smoke/test_vm_life_cycle.py | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 188a06766b09..2bc2be2dfd48 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1813,6 +1813,12 @@ public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws Cloud s_logger.info("successfully created the template with Id: " + templateId); finalTmpProduct = _tmpltDao.findById(templateId); TemplateDataStoreVO srcTmpltStore = _tmplStoreDao.findByStoreTemplate(store.getId(), templateId); + try { + srcTmpltStore.getSize(); + } catch (NullPointerException e) { + srcTmpltStore.setSize(0L); + _tmplStoreDao.update(srcTmpltStore.getId(), srcTmpltStore); + } UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_TEMPLATE_CREATE, finalTmpProduct.getAccountId(), zoneId, finalTmpProduct.getId(), finalTmpProduct.getName(), null, finalTmpProduct.getSourceTemplateId(), srcTmpltStore.getPhysicalSize(), finalTmpProduct.getSize()); diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 4b8bbaa5c510..bd65e0763730 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -884,8 +884,7 @@ def test_12_clone_vm_and_volumes(self): self.services["small"], accountid=self.account.name, domainid=self.account.domainid, - serviceofferingid=self.small_offering.id, - rootdisksize=2,) + serviceofferingid=self.small_offering.id,) vol1 = Volume.create( self.apiclient, self.services, From b1ad52cec266e15e33e79771f3418985b6fa74c3 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 8 Aug 2021 01:59:33 -0400 Subject: [PATCH 080/114] fix the permission issue --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index f0a680827e60..a5bec03aaf5f 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4635,6 +4635,7 @@ private VolumeVO saveDataDiskVolumeFromSnapShot(final Account owner, final Boole public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService, SnapshotApiService snapshotService) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException, InsufficientCapacityException, ResourceAllocationException { long vmId = cmd.getEntityId(); UserVmVO curVm = _vmDao.findById(vmId); + Account curVmAccount = _accountDao.findById(curVm.getAccountId()); // create and attach data disk long targetClonedVmId = cmd.getId(); Account caller = CallContext.current().getCallingAccount(); @@ -4669,7 +4670,7 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol DataCenterVO dataCenter = _dcDao.findById(zoneId); String volumeName = snapshotEntity.getName() + "-DataDisk-Volume"; VolumeVO parentVolume = _volsDao.findByIdIncludingRemoved(snapshotEntity.getVolumeId()); - newDatadisk = saveDataDiskVolumeFromSnapShot(caller, true, zoneId, + newDatadisk = saveDataDiskVolumeFromSnapShot(curVmAccount, true, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, volumeName, _uuidMgr.generateUuid(Volume.class, null), new HashMap<>()); VolumeVO volumeEntity = (VolumeVO) volumeService.cloneDataVolume(cmd, snapshotEntity.getId(), newDatadisk); createdVolumes.add(volumeEntity); From 94015718b9ecc42eb230799bc84fe4c4078d8342 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 8 Aug 2021 02:18:19 -0400 Subject: [PATCH 081/114] finish the assertion of clone vm integration test --- test/integration/smoke/test_vm_life_cycle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index bd65e0763730..979bb5467c5f 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -900,7 +900,7 @@ def test_12_clone_vm_and_volumes(self): except Exception as e: self.debug("Clone --" + str(e)) raise e - self.assertEqual(VirtualMachine.list(self.apiclient, id=clone_response.id), None, "List response contains records when it should not") + self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") class TestSecuredVmMigration(cloudstackTestCase): From 0cafec91eda99eca74c5ce041adf027c1204466c Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 8 Aug 2021 02:27:22 -0400 Subject: [PATCH 082/114] migrate clone vm to seperate class cycle test --- test/integration/smoke/test_vm_life_cycle.py | 152 +++++++++++++++---- 1 file changed, 126 insertions(+), 26 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 979bb5467c5f..81940aa9e466 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -876,32 +876,6 @@ def test_11_destroy_vm_and_volumes(self): self.assertEqual(Volume.list(self.apiclient, id=vol1.id), None, "List response contains records when it should not") - @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") - def test_12_clone_vm_and_volumes(self): - small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; - small_virtual_machine = VirtualMachine.create( - self.apiclient, - self.services["small"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.small_offering.id,) - vol1 = Volume.create( - self.apiclient, - self.services, - account=self.account.name, - diskofferingid=small_disk_offering.id, - domainid=self.account.domainid, - zoneid=self.zone.id - ) - small_virtual_machine.attach_volume(self.apiclient, vol1) - self.debug("Clone VM - ID: %s" % small_virtual_machine.id) - try: - clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) - except Exception as e: - self.debug("Clone --" + str(e)) - raise e - self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") - class TestSecuredVmMigration(cloudstackTestCase): @classmethod @@ -1968,3 +1942,129 @@ def test_01_vapps_vm_cycle(self): cmd = destroyVirtualMachine.destroyVirtualMachineCmd() cmd.id = vm.id self.apiclient.destroyVirtualMachine(cmd) + +class TestCloneVM(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestVMLifeCycle, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + + # Get Zone, Domain and templates + domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + + # if local storage is enabled, alter the offerings to use localstorage + # this step is needed for devcloud + if cls.zone.localstorageenabled == True: + cls.services["service_offerings"]["tiny"]["storagetype"] = 'local' + cls.services["service_offerings"]["small"]["storagetype"] = 'local' + cls.services["service_offerings"]["medium"]["storagetype"] = 'local' + + template = get_suitable_test_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"], + cls.hypervisor + ) + if template == FAILED: + assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"] + + # Set Zones and disk offerings + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + + cls.services["iso1"]["zoneid"] = cls.zone.id + + # Create VMs, NAT Rules etc + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=domain.id + ) + + cls.small_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["small"] + ) + + cls.medium_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["medium"] + ) + # create small and large virtual machines + cls.small_virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.small_offering.id, + mode=cls.services["mode"] + ) + cls.medium_virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.medium_offering.id, + mode=cls.services["mode"] + ) + cls.virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.small_offering.id, + mode=cls.services["mode"] + ) + cls._cleanup = [ + cls.small_offering, + cls.medium_offering, + cls.account + ] + + @classmethod + def tearDownClass(cls): + super(TestVMLifeCycle, cls).tearDownClass() + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + + def tearDown(self): + try: + # Clean up, terminate the created ISOs + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") + def test_clone_vm_and_volumes(self): + small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; + small_virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.small_offering.id,) + vol1 = Volume.create( + self.apiclient, + self.services, + account=self.account.name, + diskofferingid=small_disk_offering.id, + domainid=self.account.domainid, + zoneid=self.zone.id + ) + small_virtual_machine.attach_volume(self.apiclient, vol1) + self.debug("Clone VM - ID: %s" % small_virtual_machine.id) + try: + clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) + except Exception as e: + self.debug("Clone --" + str(e)) + raise e + self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") \ No newline at end of file From 43d00b306cb26e8c6d75dbbd9ac26480ea5aacec Mon Sep 17 00:00:00 2001 From: junxuan Date: Sun, 8 Aug 2021 02:30:26 -0400 Subject: [PATCH 083/114] fix typo --- test/integration/smoke/test_vm_life_cycle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 81940aa9e466..d853324c1ef0 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -1947,7 +1947,7 @@ class TestCloneVM(cloudstackTestCase): @classmethod def setUpClass(cls): - testClient = super(TestVMLifeCycle, cls).getClsTestClient() + testClient = super(TestCloneVM, cls).getClsTestClient() cls.apiclient = testClient.getApiClient() cls.services = testClient.getParsedTestDataConfig() cls.hypervisor = testClient.getHypervisorInfo() @@ -2028,7 +2028,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - super(TestVMLifeCycle, cls).tearDownClass() + super(TestCloneVM, cls).tearDownClass() def setUp(self): self.apiclient = self.testClient.getApiClient() From fa552ceda2c00215fd9d90e7e35f34330c8bbbcf Mon Sep 17 00:00:00 2001 From: Junxuan Wu <23434323+atrocitytheme@users.noreply.github.com> Date: Mon, 9 Aug 2021 15:03:59 -0400 Subject: [PATCH 084/114] Test (#8) * add integration test and fix owner privilege --- .../cloud/template/TemplateManagerImpl.java | 6 + .../java/com/cloud/vm/UserVmManagerImpl.java | 3 +- test/integration/smoke/test_vm_life_cycle.py | 127 +++++++++++++++++- tools/marvin/marvin/lib/base.py | 15 +++ 4 files changed, 149 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 188a06766b09..2bc2be2dfd48 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1813,6 +1813,12 @@ public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws Cloud s_logger.info("successfully created the template with Id: " + templateId); finalTmpProduct = _tmpltDao.findById(templateId); TemplateDataStoreVO srcTmpltStore = _tmplStoreDao.findByStoreTemplate(store.getId(), templateId); + try { + srcTmpltStore.getSize(); + } catch (NullPointerException e) { + srcTmpltStore.setSize(0L); + _tmplStoreDao.update(srcTmpltStore.getId(), srcTmpltStore); + } UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_TEMPLATE_CREATE, finalTmpProduct.getAccountId(), zoneId, finalTmpProduct.getId(), finalTmpProduct.getName(), null, finalTmpProduct.getSourceTemplateId(), srcTmpltStore.getPhysicalSize(), finalTmpProduct.getSize()); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ead8768dd949..cf5586b885a7 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4638,6 +4638,7 @@ private VolumeVO saveDataDiskVolumeFromSnapShot(final Account owner, final Boole public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService, SnapshotApiService snapshotService) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException, InsufficientCapacityException, ResourceAllocationException { long vmId = cmd.getEntityId(); UserVmVO curVm = _vmDao.findById(vmId); + Account curVmAccount = _accountDao.findById(curVm.getAccountId()); // create and attach data disk long targetClonedVmId = cmd.getId(); Account caller = CallContext.current().getCallingAccount(); @@ -4672,7 +4673,7 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol DataCenterVO dataCenter = _dcDao.findById(zoneId); String volumeName = snapshotEntity.getName() + "-DataDisk-Volume"; VolumeVO parentVolume = _volsDao.findByIdIncludingRemoved(snapshotEntity.getVolumeId()); - newDatadisk = saveDataDiskVolumeFromSnapShot(caller, true, zoneId, + newDatadisk = saveDataDiskVolumeFromSnapShot(curVmAccount, true, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, volumeName, _uuidMgr.generateUuid(Volume.class, null), new HashMap<>()); VolumeVO volumeEntity = (VolumeVO) volumeService.cloneDataVolume(cmd, snapshotEntity.getId(), newDatadisk); createdVolumes.add(volumeEntity); diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 61b3a22a6c8e..d853324c1ef0 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -876,7 +876,6 @@ def test_11_destroy_vm_and_volumes(self): self.assertEqual(Volume.list(self.apiclient, id=vol1.id), None, "List response contains records when it should not") - class TestSecuredVmMigration(cloudstackTestCase): @classmethod @@ -1943,3 +1942,129 @@ def test_01_vapps_vm_cycle(self): cmd = destroyVirtualMachine.destroyVirtualMachineCmd() cmd.id = vm.id self.apiclient.destroyVirtualMachine(cmd) + +class TestCloneVM(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestCloneVM, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + + # Get Zone, Domain and templates + domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + + # if local storage is enabled, alter the offerings to use localstorage + # this step is needed for devcloud + if cls.zone.localstorageenabled == True: + cls.services["service_offerings"]["tiny"]["storagetype"] = 'local' + cls.services["service_offerings"]["small"]["storagetype"] = 'local' + cls.services["service_offerings"]["medium"]["storagetype"] = 'local' + + template = get_suitable_test_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"], + cls.hypervisor + ) + if template == FAILED: + assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"] + + # Set Zones and disk offerings + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + + cls.services["iso1"]["zoneid"] = cls.zone.id + + # Create VMs, NAT Rules etc + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=domain.id + ) + + cls.small_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["small"] + ) + + cls.medium_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["medium"] + ) + # create small and large virtual machines + cls.small_virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.small_offering.id, + mode=cls.services["mode"] + ) + cls.medium_virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.medium_offering.id, + mode=cls.services["mode"] + ) + cls.virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.small_offering.id, + mode=cls.services["mode"] + ) + cls._cleanup = [ + cls.small_offering, + cls.medium_offering, + cls.account + ] + + @classmethod + def tearDownClass(cls): + super(TestCloneVM, cls).tearDownClass() + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + + def tearDown(self): + try: + # Clean up, terminate the created ISOs + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") + def test_clone_vm_and_volumes(self): + small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; + small_virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.small_offering.id,) + vol1 = Volume.create( + self.apiclient, + self.services, + account=self.account.name, + diskofferingid=small_disk_offering.id, + domainid=self.account.domainid, + zoneid=self.zone.id + ) + small_virtual_machine.attach_volume(self.apiclient, vol1) + self.debug("Clone VM - ID: %s" % small_virtual_machine.id) + try: + clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) + except Exception as e: + self.debug("Clone --" + str(e)) + raise e + self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") \ No newline at end of file diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 916af64d96cc..9b34cd68ab60 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -744,6 +744,21 @@ def reboot(self, apiclient, forced=None): if response[0] == FAIL: raise Exception(response[1]) + def clone(self, apiclient, vm): + """"Clone the instance""" + cmd = cloneVirtualMachine.cloneVirtualMachineCmd() + cmd.virtualmachineid = vm.id + if vm.id is None: + cmd.virtualmachineid = self.id + response = apiclient.cloneVirtualMachine(cmd) + temp = self.id + self.id = response.id + state = self.getState(apiclient, VirtualMachine.RUNNING) + self.id = temp + if (state[0] == FAIL): + raise Exception(state[1]) + return response + def recover(self, apiclient): """Recover the instance""" cmd = recoverVirtualMachine.recoverVirtualMachineCmd() From 13bb0a590a2b9b67b7125cefe635ee7de2a3ab68 Mon Sep 17 00:00:00 2001 From: junxuan Date: Thu, 12 Aug 2021 23:52:10 -0400 Subject: [PATCH 085/114] refactor the clone to service layer --- .../main/java/com/cloud/storage/Snapshot.java | 1 + .../com/cloud/storage/VolumeApiService.java | 4 +-- .../main/java/com/cloud/vm/UserVmService.java | 1 + .../api/command/user/vm/CloneVMCmd.java | 36 +++---------------- .../cloud/storage/VolumeApiServiceImpl.java | 7 +--- .../java/com/cloud/vm/UserVmManagerImpl.java | 31 ++++++++++++++-- 6 files changed, 39 insertions(+), 41 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/Snapshot.java b/api/src/main/java/com/cloud/storage/Snapshot.java index 64917fe64213..9dc7d45b036e 100644 --- a/api/src/main/java/com/cloud/storage/Snapshot.java +++ b/api/src/main/java/com/cloud/storage/Snapshot.java @@ -27,6 +27,7 @@ public interface Snapshot extends ControlledEntity, Identity, InternalIdentity, StateObject { public enum Type { MANUAL, RECURRING, TEMPLATE, HOURLY, DAILY, WEEKLY, MONTHLY, INTERNAL; + // New types should be defined after INTERNAL, and change the max value private int max = 8; public void setMax(int max) { diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index c9a5139043f6..f8ff9c885f95 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -94,7 +94,7 @@ public interface VolumeApiService { Volume detachVolumeViaDestroyVM(long vmId, long volumeId); - Volume cloneDataVolume(CloneVMCmd cmd, long snapshotId, Volume volume) throws StorageUnavailableException; + Volume cloneDataVolume(long vmId, long snapshotId, Volume volume) throws StorageUnavailableException; Volume detachVolumeFromVM(DetachVolumeCmd cmd); @@ -105,7 +105,7 @@ Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account acc Volume updateVolume(long volumeId, String path, String state, Long storageId, Boolean displayVolume, String customId, long owner, String chainInfo); - Volume attachVolumeToVm(CloneVMCmd cmd, Long volumeId, Long deviceId); + Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId); /** * Extracts the volume to a particular location. diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index da8c2373ca79..4eb006c90e04 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -100,6 +100,7 @@ public interface UserVmService { void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, ResourceAllocationException; + void prepareCloneVirtualMachine(CloneVMCmd cmd) throws ResourceAllocationException, InsufficientCapacityException, ResourceUnavailableException; /** * Resets the password of a virtual machine. * diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index b5e388dbb5e4..3ea51897cfb4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -12,27 +12,18 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.ACL; -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiCommandJobType; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseAsyncCreateCustomIdCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ResponseObject; -import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.*; import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.UserVmResponse; -//import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; import java.util.Optional; -@APICommand(name = "cloneVirtualMachine", responseObject = UserVmResponse.class, description = "clone a virtual VM in full clone mode", - responseView = ResponseObject.ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, entityType = {VirtualMachine.class}) -public class CloneVMCmd extends BaseAsyncCreateCustomIdCmd implements UserCmd { +@APICommand(name = "cloneVirtualMachine", responseObject = UserVmResponse.class, description = "clone a virtual VM", + responseView = ResponseObject.ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, entityType = {VirtualMachine.class}, since="4.16.0") +public class CloneVMCmd extends BaseAsyncCreateCmd implements UserCmd { public static final Logger s_logger = Logger.getLogger(CloneVMCmd.class.getName()); private static final String s_name = "clonevirtualmachineresponse"; @@ -102,19 +93,7 @@ public void setTemporaryTemlateId(long tempId) { public void create() throws ResourceAllocationException { try { _userVmService.checkCloneCondition(this); - VirtualMachineTemplate template = _templateService.createPrivateTemplateRecord(this, _accountService.getAccount(getEntityOwnerId()), _volumeService); - if (template == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "failed to create a template to db"); - } - s_logger.info("The template id recorded is: " + template.getId()); - setTemporaryTemlateId(template.getId()); - _templateService.createPrivateTemplate(this); - UserVm vmRecord = _userVmService.recordVirtualMachineToDB(this); - if (vmRecord == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "unable to record a new VM to db!"); - } - setEntityId(vmRecord.getId()); - setEntityUuid(vmRecord.getUuid()); + _userVmService.prepareCloneVirtualMachine(this); } catch (ResourceUnavailableException | InsufficientCapacityException e) { s_logger.warn("Exception: ", e); @@ -126,11 +105,6 @@ public void create() throws ResourceAllocationException { throw new ServerApiException(e.getErrorCode(), e.getDescription()); } catch (CloudRuntimeException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); - } finally { - if (getTemporarySnapShotId() != null) { - _snapshotService.deleteSnapshot(getTemporarySnapShotId()); - s_logger.warn("clearing the temporary snapshot: " + getTemporarySnapShotId()); - } } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 59d0e3aed920..f692a9bf03d9 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -910,8 +910,7 @@ public VolumeVO createVolume(CreateVolumeCmd cmd) { } @Override - public Volume cloneDataVolume(CloneVMCmd cmd, long snapshotId, Volume volume) throws StorageUnavailableException { - long vmId = cmd.getEntityId(); + public Volume cloneDataVolume(long vmId, long snapshotId, Volume volume) throws StorageUnavailableException { return createVolumeFromSnapshot((VolumeVO) volume, snapshotId, vmId); } @@ -1706,10 +1705,6 @@ private Volume orchestrateAttachVolumeToVM(Long vmId, Long volumeId, Long device } @Override - public Volume attachVolumeToVm(CloneVMCmd cmd, Long volumeId, Long deviceId) { - return attachVolumeToVM(cmd.getEntityId(), volumeId, deviceId); - } - public Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId) { Account caller = CallContext.current().getCallingAccount(); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index a5bec03aaf5f..7382c894a95e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -532,6 +532,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private BackupDao backupDao; @Inject private BackupManager backupManager; + @Inject + private SnapshotApiService _snapshotService; private ScheduledExecutorService _executor = null; private ScheduledExecutorService _vmIpFetchExecutor = null; @@ -4630,6 +4632,30 @@ private VolumeVO saveDataDiskVolumeFromSnapShot(final Account owner, final Boole }); } + @Override + public void prepareCloneVirtualMachine(CloneVMCmd cmd) throws ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + try { + VirtualMachineTemplate template = _tmplService.createPrivateTemplateRecord(cmd, _accountService.getAccount(cmd.getEntityOwnerId()), _volumeService); + if (template == null) { + throw new CloudRuntimeException("failed to create a template to db"); + } + s_logger.info("The template id recorded is: " + template.getId()); + cmd.setTemporaryTemlateId(template.getId()); + _tmplService.createPrivateTemplate(cmd); + UserVm vmRecord = recordVirtualMachineToDB(cmd); + if (vmRecord == null) { + throw new CloudRuntimeException("Unable to record the VM to DB!"); + } + cmd.setEntityUuid(vmRecord.getUuid()); + cmd.setEntityId(vmRecord.getId()); + } finally { + if (cmd.getTemporarySnapShotId() != null) { + _snapshotService.deleteSnapshot(cmd.getTemporarySnapShotId()); + s_logger.warn("clearing the temporary snapshot: " + cmd.getTemporarySnapShotId()); + } + } + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CLONE, eventDescription = "clone vm", async = true) public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService, SnapshotApiService snapshotService) throws ResourceUnavailableException, ConcurrentOperationException, CloudRuntimeException, InsufficientCapacityException, ResourceAllocationException { @@ -4672,12 +4698,13 @@ public Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService vol VolumeVO parentVolume = _volsDao.findByIdIncludingRemoved(snapshotEntity.getVolumeId()); newDatadisk = saveDataDiskVolumeFromSnapShot(curVmAccount, true, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, volumeName, _uuidMgr.generateUuid(Volume.class, null), new HashMap<>()); - VolumeVO volumeEntity = (VolumeVO) volumeService.cloneDataVolume(cmd, snapshotEntity.getId(), newDatadisk); + VolumeVO volumeEntity = (VolumeVO) volumeService.cloneDataVolume(cmd.getEntityId(), snapshotEntity.getId(), newDatadisk); createdVolumes.add(volumeEntity); } for (VolumeVO createdVol : createdVolumes) { - volumeService.attachVolumeToVm(cmd, createdVol.getId(), createdVol.getDeviceId()); +// volumeService.attachVolumeToVm(cmd, createdVol.getId(), createdVol.getDeviceId()); + volumeService.attachVolumeToVM(cmd.getEntityId(), createdVol.getId(), createdVol.getDeviceId()); } } catch (CloudRuntimeException e){ s_logger.warn("data disk process failed during clone, clearing the temporary resources..."); From 8b552628e5d5f8622aa52f39a3b5e9926a9d5005 Mon Sep 17 00:00:00 2001 From: junxuan Date: Thu, 12 Aug 2021 23:56:01 -0400 Subject: [PATCH 086/114] fix checkstyle --- .../main/java/com/cloud/storage/VolumeApiService.java | 1 - .../cloudstack/api/command/user/vm/CloneVMCmd.java | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index f8ff9c885f95..5eb93ca9556c 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -22,7 +22,6 @@ import java.util.Map; import com.cloud.exception.StorageUnavailableException; -import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 3ea51897cfb4..af45c6ac810d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -12,7 +12,15 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.*; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.UserVmResponse; From 56f4d1ca8f6f19cd435deb3af04042e5f0a7c540 Mon Sep 17 00:00:00 2001 From: junxuan Date: Thu, 12 Aug 2021 23:59:01 -0400 Subject: [PATCH 087/114] server style fix: --- .../src/main/java/com/cloud/storage/VolumeApiServiceImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index f692a9bf03d9..ed978fc57649 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -32,8 +32,6 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; - -import org.apache.cloudstack.api.command.user.vm.CloneVMCmd; import com.cloud.api.query.dao.ServiceOfferingJoinDao; import com.cloud.api.query.vo.ServiceOfferingJoinVO; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; From 4efd6b639f34eeffb67256739e58cd92c712c35f Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 13 Aug 2021 00:06:19 -0400 Subject: [PATCH 088/114] fix custom id option --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 7382c894a95e..1d74bd3dee50 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5783,18 +5783,18 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio if (dataCenter.getNetworkType() == NetworkType.Basic) { vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, - curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), + curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), null, new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); } else { if (dataCenter.isSecurityGroupEnabled()) { vmResult = createAdvancedSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, networkIds, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, - affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), cmd.getCustomId(), new HashMap<>(), + affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), null, new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); } else { vmResult = createAdvancedVirtualMachine(dataCenter, serviceOffering, template, networkIds, curAccount, hostName, displayName, diskOfferingId, size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), - cmd.getCustomId(), new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); + null, new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); } } } catch (CloudRuntimeException e) { From aa02972b1383c76598a553081817ff67b2f38945 Mon Sep 17 00:00:00 2001 From: junxuan Date: Fri, 13 Aug 2021 00:33:04 -0400 Subject: [PATCH 089/114] fix virtualmachine id variable naming --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index af45c6ac810d..a6be10e73ef0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -41,7 +41,7 @@ public class CloneVMCmd extends BaseAsyncCreateCmd implements UserCmd { @ACL(accessType = AccessType.OperateEntry) @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType=UserVmResponse.class, required = true, description = "The ID of the virtual machine") - private Long id; + private Long virtualmachineid; //Owner information @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") @@ -63,7 +63,7 @@ public Long getDomainId() { } public Long getId() { - return this.id; + return this.virtualmachineid; } @Override public String getEventType() { From 5fa517eaffbc2a0fbdb82ffa404af0b7c1901168 Mon Sep 17 00:00:00 2001 From: junxuan Date: Tue, 17 Aug 2021 18:40:06 -0400 Subject: [PATCH 090/114] fix remaining bugs and refactor --- api/src/main/java/com/cloud/vm/UserVmService.java | 2 +- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 2 +- .../cloudstack/engine/orchestration/VolumeOrchestrator.java | 3 +++ .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 3 ++- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 4eb006c90e04..9348349f9922 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -98,7 +98,7 @@ public interface UserVmService { * */ Optional cloneVirtualMachine(CloneVMCmd cmd, VolumeApiService volumeService, SnapshotApiService snapshotService) throws ResourceUnavailableException, ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; - void checkCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, ResourceAllocationException; + void validateCloneCondition(CloneVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException, ResourceAllocationException; void prepareCloneVirtualMachine(CloneVMCmd cmd) throws ResourceAllocationException, InsufficientCapacityException, ResourceUnavailableException; /** diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index a6be10e73ef0..1bedab703920 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -100,7 +100,7 @@ public void setTemporaryTemlateId(long tempId) { @Override public void create() throws ResourceAllocationException { try { - _userVmService.checkCloneCondition(this); + _userVmService.validateCloneCondition(this); _userVmService.prepareCloneVirtualMachine(this); } catch (ResourceUnavailableException | InsufficientCapacityException e) { diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 9322481f211f..d51c6992ebf3 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -349,6 +349,9 @@ public StoragePool findStoragePool(DiskProfile dskCh, DataCenter dc, Pod pod, Lo final List poolList = allocator.allocateToPool(dskCh, profile, plan, avoidList, StoragePoolAllocator.RETURN_UPTO_ALL); if (poolList != null && !poolList.isEmpty()) { + if (vm == null) { + return (StoragePool) dataStoreMgr.getDataStore(poolList.get(0).getId(), DataStoreRole.Primary); + } // Check if the preferred storage pool can be used. If yes, use it. Optional storagePool = getPreferredStoragePool(poolList, vm); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 45e97d2a0525..5831488e7fb4 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1871,6 +1871,7 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template _accountMgr.checkAccess(caller, null, true, templateOwner); String name = cmd.getTemplateName(); if (name.length() > 32) { + name = name.substring(5) + "-QA-Clone"; } @@ -1896,7 +1897,7 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template // if (s_logger.isInfoEnabled()) { // s_logger.info(msg); // } -// throw new CloudRuntimeException(msg); +// / throw new CloudRuntimeException(msg); // } hyperType = _volumeDao.getHypervisorType(volumeId); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ec805c640791..32ef1e077685 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4519,7 +4519,7 @@ protected String validateUserData(String userData, HTTPMethod httpmethod) { } @Override - public void checkCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueException, ResourceUnavailableException, CloudRuntimeException, ResourceAllocationException { + public void validateCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueException, ResourceUnavailableException, CloudRuntimeException, ResourceAllocationException { if (cmd.getAccountName() != null && cmd.getDomainId() == null) { throw new InvalidParameterValueException("You must input the domainId together with the account name"); diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 149853a64c5f..038768c5b9c1 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -585,7 +585,7 @@ public void validateCloneCondition() { Mockito.when(cloneVMCommand.getDomainId()).thenReturn(null); Exception err = null; try { - userVmManagerImpl.checkCloneCondition(cloneVMCommand); + userVmManagerImpl.validateCloneCondition(cloneVMCommand); } catch (CloudRuntimeException | ResourceUnavailableException | ResourceAllocationException e) { err = e; } From 8abf1419d89d3fd33c4c0ebc35fc01f59a0c01a8 Mon Sep 17 00:00:00 2001 From: junxuan Date: Tue, 17 Aug 2021 20:24:47 -0400 Subject: [PATCH 091/114] remove getter and setter of cloneVmCmd --- .../cloud/template/TemplateApiService.java | 8 ++- .../main/java/com/cloud/vm/UserVmService.java | 2 +- .../api/command/user/vm/CloneVMCmd.java | 21 -------- .../cloud/template/TemplateManagerImpl.java | 51 ++++++++++++------- .../java/com/cloud/vm/UserVmManagerImpl.java | 21 ++++---- 5 files changed, 51 insertions(+), 52 deletions(-) diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index fd45499ef6b4..60b30745a8b0 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -20,7 +20,9 @@ import java.net.URISyntaxException; import java.util.List; +import com.cloud.storage.Snapshot; import com.cloud.storage.VolumeApiService; +import com.cloud.uservm.UserVm; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; @@ -101,12 +103,14 @@ public interface TemplateApiService { boolean updateTemplateOrIsoPermissions(BaseUpdateTemplateOrIsoPermissionsCmd cmd); + Snapshot createSnapshotFromTemplateOwner(long vmId, UserVm curVm, Account templateOwner, VolumeApiService volumeService) throws ResourceAllocationException; + /** * create a template record for later usage of creating a real template by createPrivateTemplate * */ - VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner, VolumeApiService serviceObj) throws ResourceAllocationException; + VirtualMachineTemplate createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner, VolumeApiService serviceObj, Snapshot snapshot) throws ResourceAllocationException; - VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws CloudRuntimeException; + VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd, long snapshotId, long templateId) throws CloudRuntimeException; VirtualMachineTemplate createPrivateTemplateRecord(CreateTemplateCmd cmd, Account templateOwner) throws ResourceAllocationException; diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 9348349f9922..7bdbdfc802ef 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -445,7 +445,7 @@ UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serviceOffe UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, StorageUnavailableException, ResourceAllocationException; - UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, + UserVm recordVirtualMachineToDB(CloneVMCmd cmd, long templateId) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, StorageUnavailableException, ResourceAllocationException; UserVm getUserVm(long vmId); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 1bedab703920..6e3443f23cd4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -50,10 +50,6 @@ public class CloneVMCmd extends BaseAsyncCreateCmd implements UserCmd { @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used.") private Long domainId; - private Long temporaryTemlateId; - - private Long temporarySnapShotId; - public String getAccountName() { return accountName; } @@ -80,23 +76,6 @@ public String getEventDescription() { return "Cloning user VM: " + this._uuidMgr.getUuid(VirtualMachine.class, getId()); } - public Long getTemporaryTemlateId() { - return this.temporaryTemlateId; - } - - public void setTemporarySnapShotId(Long snapshotId) { - this.temporarySnapShotId = snapshotId; - } - - public Long getTemporarySnapShotId() { - return temporarySnapShotId; - } - - - public void setTemporaryTemlateId(long tempId) { - this.temporaryTemlateId = tempId; - } - @Override public void create() throws ResourceAllocationException { try { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 5831488e7fb4..b07858b128ac 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1747,10 +1747,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @Override @DB @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating actual private template", create = true) - public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd) throws CloudRuntimeException { + public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd, long snapshotId, long templateId) throws CloudRuntimeException { UserVm curVm = cmd.getTargetVM(); - long templateId = cmd.getTemporaryTemlateId(); - long snapshotId = cmd.getTemporarySnapShotId(); final Long accountId = curVm.getAccountId(); Account caller = CallContext.current().getCallingAccount(); List volumes = _volumeDao.findByInstanceAndType(cmd.getId(), Volume.Type.ROOT); @@ -1864,9 +1862,39 @@ public void doInTransactionWithoutResult(TransactionStatus status) { return null; } + @Override + public Snapshot createSnapshotFromTemplateOwner(long vmId, UserVm curVm, Account templateOwner, VolumeApiService volumeService) throws ResourceAllocationException { + Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, templateOwner); +// UserVm curVm = cmd.getTargetVM(); + Long nextSnapId = _tmpltDao.getNextInSequence(Long.class, "id"); + Long volumeId = _volumeDao.findByInstanceAndType(vmId, Volume.Type.ROOT).get(0).getId(); + VolumeVO volume = _volumeDao.findById(volumeId); + if (volume == null) { + throw new InvalidParameterValueException("Failed to create private template record, unable to find root volume " + volumeId); + } + + // check permissions + _accountMgr.checkAccess(caller, null, true, volume); + s_logger.info("Creating snapshot for the tempalte creation"); + SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, Snapshot.INTERNAL_POLICY_ID, curVm.getDisplayName() + "-Clone-" + nextSnapId, null); + if (snapshot == null) { + throw new CloudRuntimeException("Unable to create a snapshot during the template creation recording"); + } + Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, Snapshot.INTERNAL_POLICY_ID, snapshot.getId(), caller, false, null, false, new HashMap<>()); + if (snapshotEntity == null) { + throw new CloudRuntimeException("Error when creating the snapshot entity"); + } + if (snapshotEntity.getState() != Snapshot.State.BackedUp) { + throw new CloudRuntimeException("Async backup of snapshot happens during the clone for snapshot id: " + snapshot.getId()); + } + + return snapshot; + } + @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template from clone", create = true) - public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner, VolumeApiService volumeService) throws ResourceAllocationException { + public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner, VolumeApiService volumeService, Snapshot snapshot) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); _accountMgr.checkAccess(caller, null, true, templateOwner); String name = cmd.getTemplateName(); @@ -1915,19 +1943,6 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template } // get snapshot from this step Long nextTemplateId = _tmpltDao.getNextInSequence(Long.class, "id"); - s_logger.info("Creating snapshot for the tempalte creation"); - SnapshotVO snapshot = (SnapshotVO) volumeService.allocSnapshot(volumeId, Snapshot.INTERNAL_POLICY_ID, curVm.getDisplayName() + "-Clone-" + nextTemplateId, null); - if (snapshot == null) { - throw new CloudRuntimeException("Unable to create a snapshot during the template creation recording"); - } - Snapshot snapshotEntity = volumeService.takeSnapshot(volumeId, Snapshot.INTERNAL_POLICY_ID, snapshot.getId(), caller, false, null, false, new HashMap<>()); - if (snapshotEntity == null) { - throw new CloudRuntimeException("Error when creating the snapshot entity"); - } - if (snapshotEntity.getState() != Snapshot.State.BackedUp) { - throw new CloudRuntimeException("Async backup of snapshot happens during the clone for snapshot id: " + snapshot.getId()); - } - cmd.setTemporarySnapShotId(snapshot.getId()); String description = ""; // TODO: add this to clone parameter in the future boolean isExtractable = false; Long sourceTemplateId = null; @@ -1992,7 +2007,7 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.template); _resourceLimitMgr.incrementResourceCount(templateOwner.getId(), ResourceType.secondary_storage, - snapshot.getSize()); + ((SnapshotVO) snapshot).getSize()); } return template; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 32ef1e077685..07eda4911863 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4647,24 +4647,27 @@ private VolumeVO saveDataDiskVolumeFromSnapShot(final Account owner, final Boole @Override public void prepareCloneVirtualMachine(CloneVMCmd cmd) throws ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + Long temporarySnapshotId = null; try { - VirtualMachineTemplate template = _tmplService.createPrivateTemplateRecord(cmd, _accountService.getAccount(cmd.getEntityOwnerId()), _volumeService); + Account owner = _accountService.getAccount(cmd.getEntityOwnerId()); + Snapshot snapshot = _tmplService.createSnapshotFromTemplateOwner(cmd.getId(), cmd.getTargetVM(), owner, _volumeService); + temporarySnapshotId = snapshot.getId(); + VirtualMachineTemplate template = _tmplService.createPrivateTemplateRecord(cmd, owner, _volumeService, snapshot); if (template == null) { throw new CloudRuntimeException("failed to create a template to db"); } s_logger.info("The template id recorded is: " + template.getId()); - cmd.setTemporaryTemlateId(template.getId()); - _tmplService.createPrivateTemplate(cmd); - UserVm vmRecord = recordVirtualMachineToDB(cmd); + _tmplService.createPrivateTemplate(cmd, snapshot.getId(), template.getId()); + UserVm vmRecord = recordVirtualMachineToDB(cmd, template.getId()); if (vmRecord == null) { throw new CloudRuntimeException("Unable to record the VM to DB!"); } cmd.setEntityUuid(vmRecord.getUuid()); cmd.setEntityId(vmRecord.getId()); } finally { - if (cmd.getTemporarySnapShotId() != null) { - _snapshotService.deleteSnapshot(cmd.getTemporarySnapShotId()); - s_logger.warn("clearing the temporary snapshot: " + cmd.getTemporarySnapShotId()); + if (temporarySnapshotId != null) { + _snapshotService.deleteSnapshot(temporarySnapshotId); + s_logger.warn("clearing the temporary snapshot: " + temporarySnapshotId); } } } @@ -5740,7 +5743,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } @Override - public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperationException, ResourceAllocationException, InsufficientCapacityException, ResourceUnavailableException { + public UserVm recordVirtualMachineToDB(CloneVMCmd cmd, long templateId) throws ConcurrentOperationException, ResourceAllocationException, InsufficientCapacityException, ResourceUnavailableException { //network configurations and check, then create the template UserVm curVm = cmd.getTargetVM(); // check if host is available @@ -5769,13 +5772,11 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd) throws ConcurrentOperatio String displayName = hostName + "-Clone"; Long diskOfferingId = curVm.getDiskOfferingId(); Long size = null; // mutual exclusive with disk offering id - HTTPMethod httpMethod = cmd.getHttpMethod(); String userData = curVm.getUserData(); String sshKeyPair = null; Map ipToNetoworkMap = null; // Since we've specified Ip boolean isDisplayVM = curVm.isDisplayVm(); boolean dynamicScalingEnabled = curVm.isDynamicallyScalable(); - Long templateId = cmd.getTemporaryTemlateId(); VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, templateId); if (template == null) { throw new CloudRuntimeException("the temporary template is not created, server error, contact your sys admin"); From 50d250e129a2e5ab26bfd1614896b9f462b797b4 Mon Sep 17 00:00:00 2001 From: junxuan Date: Tue, 17 Aug 2021 20:29:22 -0400 Subject: [PATCH 092/114] fix checkstyle:trailingspace --- .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index b07858b128ac..8171082976ef 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1888,10 +1888,8 @@ public Snapshot createSnapshotFromTemplateOwner(long vmId, UserVm curVm, Account if (snapshotEntity.getState() != Snapshot.State.BackedUp) { throw new CloudRuntimeException("Async backup of snapshot happens during the clone for snapshot id: " + snapshot.getId()); } - return snapshot; } - @Override @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template from clone", create = true) public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account templateOwner, VolumeApiService volumeService, Snapshot snapshot) throws ResourceAllocationException { @@ -1913,7 +1911,6 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template if (volume == null) { throw new InvalidParameterValueException("Failed to create private template record, unable to find root volume " + volumeId); } - // check permissions _accountMgr.checkAccess(caller, null, true, volume); From ee6ce66f8cdd960a6887dd1cdcb4686ae2164952 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 18 Aug 2021 16:11:03 -0400 Subject: [PATCH 093/114] change execute sequence and apply new tests --- .../api/command/user/vm/CloneVMCmd.java | 23 +++++++++++++++---- test/integration/smoke/test_vm_life_cycle.py | 10 +++++++- ui/src/config/section/compute.js | 4 ++-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 6e3443f23cd4..92a86f3af706 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -34,6 +34,7 @@ public class CloneVMCmd extends BaseAsyncCreateCmd implements UserCmd { public static final Logger s_logger = Logger.getLogger(CloneVMCmd.class.getName()); private static final String s_name = "clonevirtualmachineresponse"; + private static final String CLONE_IDENTIFIER = "Clone"; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -43,6 +44,9 @@ public class CloneVMCmd extends BaseAsyncCreateCmd implements UserCmd { required = true, description = "The ID of the virtual machine") private Long virtualmachineid; + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "name of the cloned virtual machine") + private String name; + //Owner information @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") private String accountName; @@ -58,6 +62,14 @@ public Long getDomainId() { return domainId; } + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + public Long getId() { return this.virtualmachineid; } @@ -80,9 +92,8 @@ public String getEventDescription() { public void create() throws ResourceAllocationException { try { _userVmService.validateCloneCondition(this); - _userVmService.prepareCloneVirtualMachine(this); } - catch (ResourceUnavailableException | InsufficientCapacityException e) { + catch (ResourceUnavailableException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); } catch (InvalidParameterValueException e) { @@ -100,11 +111,14 @@ public boolean isPublic() { } public String getVMName() { - return getTargetVM().getInstanceName(); + if (getName() == null) { + return getTargetVM().getInstanceName() + "-" + CLONE_IDENTIFIER; + } + return getName(); } public String getTemplateName() { - return (getVMName() + "-Clone-" + _uuidMgr.generateUuid(VirtualMachineTemplate.class, null)).substring(0, 32); + return (getVMName() + "-" + _uuidMgr.generateUuid(VirtualMachineTemplate.class, null)).substring(0, 32); } @Override @@ -112,6 +126,7 @@ public void execute() { Optional result; try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); + _userVmService.prepareCloneVirtualMachine(this); s_logger.info("starting actual VM id: " + getEntityId()); result = _userVmService.cloneVirtualMachine(this, _volumeService, _snapshotService); } catch (ResourceUnavailableException ex) { diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 24c69d2a841c..7ea8a03b3086 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -2094,6 +2094,10 @@ def tearDown(self): @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") def test_clone_vm_and_volumes(self): small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; + Configurations.update(self.api_client, + name="kvm.snapshot.enabled", + value="true" + ) small_virtual_machine = VirtualMachine.create( self.apiclient, self.services["small"], @@ -2115,4 +2119,8 @@ def test_clone_vm_and_volumes(self): except Exception as e: self.debug("Clone --" + str(e)) raise e - self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") \ No newline at end of file + self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") + Configurations.update(self.api_client, + name="kvm.snapshot.enabled", + value="false" + ) \ No newline at end of file diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 264d99562d20..e9a2e9b492db 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -141,10 +141,10 @@ export default { icon: 'camera', label: 'label.action.clone.vm', message: 'message.action.clone.instance', - docHelp: 'adminguide/virtual_machines.html#stopping-and-starting-vms', + docHelp: 'adminguide/virtual_machines.html#cloning-vms', dataView: true, show: (record) => { return true }, - args: ['virtualmachineid'], + args: ['name', 'virtualmachineid'], mapping: { virtualmachineid: { value: (record, params) => { return record.id } From 9ffd3359d65bc53e1842a7e2519b62c50e8abc90 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 18 Aug 2021 17:02:07 -0400 Subject: [PATCH 094/114] fix null name --- ui/src/config/section/compute.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index e9a2e9b492db..89fda017142d 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -148,6 +148,9 @@ export default { mapping: { virtualmachineid: { value: (record, params) => { return record.id } + }, + name: { + value: (record, params) => {return record.name} } } }, From 862f9017ee31305b920da3ece4950231b2d0d73c Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 18 Aug 2021 17:03:28 -0400 Subject: [PATCH 095/114] fix ui --- ui/src/config/section/compute.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 89fda017142d..64166041c816 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -150,8 +150,8 @@ export default { value: (record, params) => { return record.id } }, name: { - value: (record, params) => {return record.name} - } + value: (record, params) => { return record.name } + }, } }, { From 28e417cf5f6758457df0366ea42bbf18d3d22eee Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 18 Aug 2021 17:04:29 -0400 Subject: [PATCH 096/114] typo check --- ui/src/config/section/compute.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 64166041c816..49c2d242f5ec 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -151,7 +151,7 @@ export default { }, name: { value: (record, params) => { return record.name } - }, + } } }, { From 4e7f40f3378c06a6c78967f1efb1b9794080a810 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 18 Aug 2021 17:12:23 -0400 Subject: [PATCH 097/114] fix null pointer --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 3 ++- ui/src/config/section/compute.js | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index 92a86f3af706..bd6dadcf12f2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -92,8 +92,9 @@ public String getEventDescription() { public void create() throws ResourceAllocationException { try { _userVmService.validateCloneCondition(this); + _userVmService.prepareCloneVirtualMachine(this); } - catch (ResourceUnavailableException e) { + catch (ResourceUnavailableException | InsufficientCapacityException e) { s_logger.warn("Exception: ", e); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, e.getMessage()); } catch (InvalidParameterValueException e) { diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 49c2d242f5ec..e9a2e9b492db 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -148,9 +148,6 @@ export default { mapping: { virtualmachineid: { value: (record, params) => { return record.id } - }, - name: { - value: (record, params) => { return record.name } } } }, From a44bba93d1c3746c41c655e874e82479e0b27ef6 Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 18 Aug 2021 19:13:18 -0400 Subject: [PATCH 098/114] add new test --- test/integration/smoke/test_vm_life_cycle.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 7ea8a03b3086..bdfb7ab73faf 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -2094,10 +2094,10 @@ def tearDown(self): @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") def test_clone_vm_and_volumes(self): small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; - Configurations.update(self.api_client, - name="kvm.snapshot.enabled", - value="true" - ) +# Configurations.update(self.api_client, +# name="kvm.snapshot.enabled", +# value="true" +# ) small_virtual_machine = VirtualMachine.create( self.apiclient, self.services["small"], @@ -2120,7 +2120,7 @@ def test_clone_vm_and_volumes(self): self.debug("Clone --" + str(e)) raise e self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") - Configurations.update(self.api_client, - name="kvm.snapshot.enabled", - value="false" - ) \ No newline at end of file +# Configurations.update(self.api_client, +# name="kvm.snapshot.enabled", +# value="false" +# ) \ No newline at end of file From 826ea4deab865492329fda7e5154791fdfec570c Mon Sep 17 00:00:00 2001 From: junxuan Date: Wed, 18 Aug 2021 19:30:33 -0400 Subject: [PATCH 099/114] finish the test --- test/integration/smoke/test_vm_life_cycle.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index bdfb7ab73faf..df17b4af0a8f 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -2094,10 +2094,10 @@ def tearDown(self): @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") def test_clone_vm_and_volumes(self): small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; -# Configurations.update(self.api_client, -# name="kvm.snapshot.enabled", -# value="true" -# ) + Configurations.update(self.apiclient, + name="kvm.snapshot.enabled", + value="true" + ) small_virtual_machine = VirtualMachine.create( self.apiclient, self.services["small"], @@ -2120,7 +2120,7 @@ def test_clone_vm_and_volumes(self): self.debug("Clone --" + str(e)) raise e self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") -# Configurations.update(self.api_client, -# name="kvm.snapshot.enabled", -# value="false" -# ) \ No newline at end of file + Configurations.update(self.apiclient, + name="kvm.snapshot.enabled", + value="false" + ) \ No newline at end of file From 2ab24721a61f76226c66792bbd0116b6ec11597b Mon Sep 17 00:00:00 2001 From: junxuan Date: Thu, 19 Aug 2021 13:43:42 -0400 Subject: [PATCH 100/114] update the smoke test to ignore specific vm --- test/integration/smoke/test_vm_life_cycle.py | 58 ++++++++++---------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index df17b4af0a8f..3ae40bf7433b 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -631,7 +631,7 @@ def test_08_migrate_vm(self): if self.hypervisor.lower() in ["lxc"]: self.skipTest("Migration is not supported on LXC") - # For KVM, two hosts used for migration should be present in same cluster + # For KVM, two hosts used fo r migration should be present in same cluster # For XenServer and VMware, migration is possible between hosts belonging to different clusters # with the help of XenMotion and Vmotion respectively. @@ -2094,33 +2094,31 @@ def tearDown(self): @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") def test_clone_vm_and_volumes(self): small_disk_offering = DiskOffering.list(self.apiclient, name='Small')[0]; - Configurations.update(self.apiclient, - name="kvm.snapshot.enabled", - value="true" + config = Configurations.list(self.apiclient, + name="kvm.snapshot.enabled" ) - small_virtual_machine = VirtualMachine.create( - self.apiclient, - self.services["small"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.small_offering.id,) - vol1 = Volume.create( - self.apiclient, - self.services, - account=self.account.name, - diskofferingid=small_disk_offering.id, - domainid=self.account.domainid, - zoneid=self.zone.id - ) - small_virtual_machine.attach_volume(self.apiclient, vol1) - self.debug("Clone VM - ID: %s" % small_virtual_machine.id) - try: - clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) - except Exception as e: - self.debug("Clone --" + str(e)) - raise e - self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") - Configurations.update(self.apiclient, - name="kvm.snapshot.enabled", - value="false" - ) \ No newline at end of file + if len(config) == 0 or config[0].value != "true": + self.skipTest("Please enable kvm.snapshot.enable global config") + if self.hypervisor.lower() in ["kvm", "simulator"]: + small_virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.small_offering.id,) + vol1 = Volume.create( + self.apiclient, + self.services, + account=self.account.name, + diskofferingid=small_disk_offering.id, + domainid=self.account.domainid, + zoneid=self.zone.id + ) + small_virtual_machine.attach_volume(self.apiclient, vol1) + self.debug("Clone VM - ID: %s" % small_virtual_machine.id) + try: + clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) + except Exception as e: + self.debug("Clone --" + str(e)) + raise e + self.assertTrue(VirtualMachine.list(self.apiclient, id=clone_response.id) is not None, "vm id should be populated") \ No newline at end of file From c3e836a2cf207268f5bdea5d5bdc4cfd27e5b20b Mon Sep 17 00:00:00 2001 From: junxuan Date: Thu, 19 Aug 2021 14:44:11 -0400 Subject: [PATCH 101/114] fix restore vm --- .../org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index bd6dadcf12f2..a2925cd0614d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -127,7 +127,6 @@ public void execute() { Optional result; try { CallContext.current().setEventDetails("Vm Id for full clone: " + getEntityId()); - _userVmService.prepareCloneVirtualMachine(this); s_logger.info("starting actual VM id: " + getEntityId()); result = _userVmService.cloneVirtualMachine(this, _volumeService, _snapshotService); } catch (ResourceUnavailableException ex) { From 2fd4ca096d3045377907bc637ece49e0835c39d7 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sat, 21 Aug 2021 22:40:02 -0400 Subject: [PATCH 102/114] fix typo and delete trivial comments --- .../api/command/user/vm/CloneVMCmd.java | 16 +++++++++++++++ .../cloud/template/TemplateManagerImpl.java | 20 ------------------- .../java/com/cloud/vm/UserVmManagerImpl.java | 9 --------- test/integration/smoke/test_vm_life_cycle.py | 2 +- 4 files changed, 17 insertions(+), 30 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java index a2925cd0614d..33314ac7ac89 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CloneVMCmd.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package org.apache.cloudstack.api.command.user.vm; import com.cloud.event.EventTypes; diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 8171082976ef..7380186017c9 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1763,7 +1763,6 @@ public VirtualMachineTemplate createPrivateTemplate(CloneVMCmd cmd, long snapsho VolumeInfo vInfo = _volFactory.getVolume(volumeId); DataStore store = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); snapshot = _snapshotDao.findById(snapshotId); -// future = _tmpltSvr.createTemplateFromVolumeAsync(vInfo, cloneTempalateInfp, store); // create template from snapshot DataStoreRole dataStoreRole = ApiResponseHelper.getDataStoreRole(snapshot, _snapshotStoreDao, _dataStoreMgr); SnapshotInfo snapInfo = _snapshotFactory.getSnapshot(snapshotId, dataStoreRole); @@ -1913,18 +1912,6 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template } // check permissions _accountMgr.checkAccess(caller, null, true, volume); - - // If private template is created from Volume, check that the volume - // will not be active when the private template is - // created -// if (!_volumeMgr.volumeInactive(volume)) { -// String msg = "Unable to create private template for volume: " + volume.getName() + "; volume is attached to a non-stopped VM, please stop the VM first"; -// if (s_logger.isInfoEnabled()) { -// s_logger.info(msg); -// } -// / throw new CloudRuntimeException(msg); -// } - hyperType = _volumeDao.getHypervisorType(volumeId); if (HypervisorType.LXC.equals(hyperType)) { throw new InvalidParameterValueException("Template creation is not supported for LXC volume: " + volumeId); @@ -1964,13 +1951,6 @@ public VMTemplateVO createPrivateTemplateRecord(CloneVMCmd cmd, Account template privateTemplate.setSourceTemplateId(sourceTemplateId); VMTemplateVO template = _tmpltDao.persist(privateTemplate); - // persist this to the template zone area and remember to remove the resource count in the execute phase once in failure or clean up phase -// VMTemplateZoneVO templateZone = new VMTemplateZoneVO(zoneId, template.getId(), new Date()); -// _tmpltZoneDao.persist(templateZone); -// TemplateDataStoreVO voRecord = _tmplStoreDao.createTemplateDirectDownloadEntry(template.getId(), template.getSize()); -// voRecord.setDataStoreId(2); -// _tmplStoreDao.persist(voRecord); - // Increment the number of templates if (template != null) { Map details = new HashMap(); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 07eda4911863..c11aa0f54f7b 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4540,9 +4540,6 @@ public void validateCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueE throw new CloudRuntimeException("the VM doesn't exist or not registered in management server!"); } UserVmVO vmStatus = _vmDao.findById(cmd.getId()); -// if (vmStatus.state != State.Shutdown && vmStatus.state != State.Stopped) { -// throw new CloudRuntimeException("You should clone an instance that's shutdown!"); -// } if (vmStatus.getHypervisorType() != HypervisorType.KVM && vmStatus.getHypervisorType() != HypervisorType.Simulator) { throw new CloudRuntimeException("The clone operation is only supported on KVM and Simulator!"); } @@ -4562,7 +4559,6 @@ public void validateCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueE // verify that the VM doesn't expire Map details = curVm.getDetails(); verifyDetails(details); -// Account activeOwner = _accountDao.findById(cmd.getEntityOwnerId()); long zoneId = curVm.getDataCenterId(); DataCenter zone = _entityMgr.findById(DataCenter.class, zoneId); if (zone == null) { @@ -5755,14 +5751,9 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd, long templateId) throws C String keyboard = vmProperties.get(VmDetailConstants.KEYBOARD); HypervisorType hypervisorType = curVm.getHypervisorType(); Account curAccount = _accountDao.findById(curVm.getAccountId()); - long callingUserId = CallContext.current().getCallingUserId(); - Account callerAccount = CallContext.current().getCallingAccount(); -// IpAddress ipAddress = _ipAddrMgr.assignPublicIpAddress(zoneId, curVm.getPodIdToDeployIn(), callerAccount, VlanType.DirectAttached, ) -// IpAddress ipAddress = _ipAddrMgr.allocateIp(curAccount, false, callerAccount, callingUserId, dataCenter, true, null); String ipv6Address = null; String macAddress = null; IpAddresses addr = new IpAddresses(null, ipv6Address, macAddress); -// IpAddresses addr = new IpAddresses("172.20.0.98", ipv6Address, macAddress); long serviceOfferingId = curVm.getServiceOfferingId(); ServiceOffering serviceOffering = _serviceOfferingDao.findById(curVm.getId(), serviceOfferingId); List securityGroupList = _securityGroupMgr.getSecurityGroupsForVm(curVm.getId()); diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 3ae40bf7433b..93702647a7f8 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -631,7 +631,7 @@ def test_08_migrate_vm(self): if self.hypervisor.lower() in ["lxc"]: self.skipTest("Migration is not supported on LXC") - # For KVM, two hosts used fo r migration should be present in same cluster + # For KVM, two hosts used for migration should be present in same cluster # For XenServer and VMware, migration is possible between hosts belonging to different clusters # with the help of XenMotion and Vmotion respectively. From 6ff0ce4ff982bd8b00319fdd4273c264a97c18e4 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sat, 21 Aug 2021 23:17:40 -0400 Subject: [PATCH 103/114] optimize test implementation --- test/integration/smoke/test_vm_life_cycle.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 93702647a7f8..b556952ca8dc 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -2033,16 +2033,19 @@ def setUpClass(cls): cls.services["account"], domainid=domain.id ) - + cls._cleanup = [] + cls._cleanup.append(cls.account) cls.small_offering = ServiceOffering.create( cls.apiclient, cls.services["service_offerings"]["small"] ) + cls._cleanup.append(cls.small_offering) cls.medium_offering = ServiceOffering.create( cls.apiclient, cls.services["service_offerings"]["medium"] ) + cls._cleanup.append(cls.medium_offering) # create small and large virtual machines cls.small_virtual_machine = VirtualMachine.create( cls.apiclient, @@ -2052,6 +2055,7 @@ def setUpClass(cls): serviceofferingid=cls.small_offering.id, mode=cls.services["mode"] ) + cls._cleanup.append(cls.small_virtual_machine) cls.medium_virtual_machine = VirtualMachine.create( cls.apiclient, cls.services["small"], @@ -2060,6 +2064,7 @@ def setUpClass(cls): serviceofferingid=cls.medium_offering.id, mode=cls.services["mode"] ) + cls._cleanup.append(cls.medium_virtual_machine) cls.virtual_machine = VirtualMachine.create( cls.apiclient, cls.services["small"], @@ -2068,11 +2073,7 @@ def setUpClass(cls): serviceofferingid=cls.small_offering.id, mode=cls.services["mode"] ) - cls._cleanup = [ - cls.small_offering, - cls.medium_offering, - cls.account - ] + cls._cleanup.append(cls.virtual_machine) @classmethod def tearDownClass(cls): @@ -2084,12 +2085,7 @@ def setUp(self): self.cleanup = [] def tearDown(self): - try: - # Clean up, terminate the created ISOs - cleanup_resources(self.apiclient, self.cleanup) - except Exception as e: - raise Exception("Warning: Exception during cleanup : %s" % e) - return + super(TestCloneVM, self).tearDown() @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") def test_clone_vm_and_volumes(self): From 520e9515a109d990ac18fa001aa934a041f110f3 Mon Sep 17 00:00:00 2001 From: junxuan Date: Sat, 21 Aug 2021 23:42:37 -0400 Subject: [PATCH 104/114] add clean up resources --- test/integration/smoke/test_vm_life_cycle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index b556952ca8dc..c6bcbd028cb3 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -2102,6 +2102,7 @@ def test_clone_vm_and_volumes(self): accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.small_offering.id,) + self.cleanup.append(small_virtual_machine) vol1 = Volume.create( self.apiclient, self.services, @@ -2110,10 +2111,12 @@ def test_clone_vm_and_volumes(self): domainid=self.account.domainid, zoneid=self.zone.id ) + self.cleanup.append(vol1) small_virtual_machine.attach_volume(self.apiclient, vol1) self.debug("Clone VM - ID: %s" % small_virtual_machine.id) try: clone_response = small_virtual_machine.clone(self.apiclient, small_virtual_machine) + self.cleanup.append(clone_response) except Exception as e: self.debug("Clone --" + str(e)) raise e From 5072889653454664e8f5efc23657b738eea7aaf8 Mon Sep 17 00:00:00 2001 From: junxuan Date: Tue, 24 Aug 2021 00:53:36 -0400 Subject: [PATCH 105/114] fix the orchestrator --- .../engine/orchestration/VolumeOrchestrator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index d51c6992ebf3..e1c6bf9753d8 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -311,7 +311,10 @@ private Optional getMatchingStoragePool(String preferredPoolId, Lis } private Optional getPreferredStoragePool(List poolList, VirtualMachine vm) { - String accountStoragePoolUuid = StorageManager.PreferredStoragePool.valueIn(vm.getAccountId()); + String accountStoragePoolUuid = null; + if (vm != null) { + accountStoragePoolUuid = StorageManager.PreferredStoragePool.valueIn(vm.getAccountId()); + } Optional storagePool = getMatchingStoragePool(accountStoragePoolUuid, poolList); if (storagePool.isPresent()) { @@ -349,9 +352,6 @@ public StoragePool findStoragePool(DiskProfile dskCh, DataCenter dc, Pod pod, Lo final List poolList = allocator.allocateToPool(dskCh, profile, plan, avoidList, StoragePoolAllocator.RETURN_UPTO_ALL); if (poolList != null && !poolList.isEmpty()) { - if (vm == null) { - return (StoragePool) dataStoreMgr.getDataStore(poolList.get(0).getId(), DataStoreRole.Primary); - } // Check if the preferred storage pool can be used. If yes, use it. Optional storagePool = getPreferredStoragePool(poolList, vm); From a1e22b784d4930d257ce28b03e61b8fc5473e293 Mon Sep 17 00:00:00 2001 From: junxuan Date: Tue, 24 Aug 2021 01:27:30 -0400 Subject: [PATCH 106/114] add None checker if kvm enabled var doesn't exist --- test/integration/smoke/test_vm_life_cycle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index c6bcbd028cb3..fe3bfd5e37c4 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -2093,6 +2093,8 @@ def test_clone_vm_and_volumes(self): config = Configurations.list(self.apiclient, name="kvm.snapshot.enabled" ) + if config is None: + self.skipTest("Please enable kvm.snapshot.enable global config") if len(config) == 0 or config[0].value != "true": self.skipTest("Please enable kvm.snapshot.enable global config") if self.hypervisor.lower() in ["kvm", "simulator"]: From ae25c4a25bf6fd84769ac885bae8bea6763067cd Mon Sep 17 00:00:00 2001 From: junxuan Date: Tue, 24 Aug 2021 02:09:50 -0400 Subject: [PATCH 107/114] add cleanup code --- test/integration/smoke/test_vm_life_cycle.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index fe3bfd5e37c4..896985dcefae 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -2085,7 +2085,10 @@ def setUp(self): self.cleanup = [] def tearDown(self): - super(TestCloneVM, self).tearDown() + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) @attr(tags = ["clone","devcloud", "advanced", "smoke", "basic", "sg"], required_hardware="false") def test_clone_vm_and_volumes(self): From b89f7888b5bb8405908c8d5812686ec71273c38f Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 8 Nov 2021 19:23:16 +0530 Subject: [PATCH 108/114] Fixed issues with merge conflicts --- .../cloud/storage/VolumeApiServiceImpl.java | 2 - .../java/com/cloud/vm/UserVmManagerImpl.java | 59 ++++++++++++++++--- .../com/cloud/vm/UserVmManagerImplTest.java | 2 +- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 6cad61feb3cf..9f475105af2e 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -33,8 +33,6 @@ import javax.inject.Inject; -import com.cloud.api.query.dao.ServiceOfferingJoinDao; -import com.cloud.api.query.vo.ServiceOfferingJoinVO; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index b70232557775..0fbd5cabd7dd 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.vm; +import static com.cloud.configuration.ConfigurationManagerImpl.VM_USERDATA_MAX_LENGTH; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import java.io.IOException; @@ -26,6 +27,7 @@ import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -51,9 +53,6 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.network.router.CommandSetupHelper; -import com.cloud.network.router.NetworkHelper; -import com.cloud.network.security.SecurityGroupVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -120,6 +119,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -230,10 +230,14 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; +import com.cloud.network.IpAddressManager; +import com.cloud.network.Network; import com.cloud.network.Network.IpAddresses; import com.cloud.network.Network.Provider; import com.cloud.network.Network.Service; +import com.cloud.network.NetworkModel; import com.cloud.network.Networks.TrafficType; +import com.cloud.network.PhysicalNetwork; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; @@ -246,6 +250,8 @@ import com.cloud.network.element.UserDataServiceProvider; import com.cloud.network.guru.NetworkGuru; import com.cloud.network.lb.LoadBalancingRulesManager; +import com.cloud.network.router.CommandSetupHelper; +import com.cloud.network.router.NetworkHelper; import com.cloud.network.router.VpcVirtualNetworkApplianceManager; import com.cloud.network.rules.FirewallManager; import com.cloud.network.rules.FirewallRuleVO; @@ -254,6 +260,7 @@ import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.security.SecurityGroup; import com.cloud.network.security.SecurityGroupManager; +import com.cloud.network.security.SecurityGroupVO; import com.cloud.network.security.dao.SecurityGroupDao; import com.cloud.network.vpc.VpcManager; import com.cloud.offering.DiskOffering; @@ -271,9 +278,26 @@ import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.GuestOSCategoryVO; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolStatus; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; @@ -281,11 +305,23 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.snapshot.SnapshotApiService; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.TemplateApiService; import com.cloud.template.TemplateManager; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountService; +import com.cloud.user.AccountVO; +import com.cloud.user.ResourceLimitService; +import com.cloud.user.SSHKeyPair; +import com.cloud.user.SSHKeyPairVO; +import com.cloud.user.User; +import com.cloud.user.UserStatisticsVO; +import com.cloud.user.UserVO; +import com.cloud.user.VmDiskStatisticsVO; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.user.dao.UserDao; @@ -300,6 +336,17 @@ import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.crypt.RSAHelper; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionCallbackWithException; +import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; +import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.db.UUIDManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExecutionException; import com.cloud.utils.fsm.NoTransitionException; @@ -316,10 +363,6 @@ import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; -import java.util.HashSet; -import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; - -import static com.cloud.configuration.ConfigurationManagerImpl.VM_USERDATA_MAX_LENGTH; public class UserVmManagerImpl extends ManagerBase implements UserVmManager, VirtualMachineGuru, UserVmService, Configurable { private static final Logger s_logger = Logger.getLogger(UserVmManagerImpl.class); @@ -5952,7 +5995,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd, long templateId) throws C } else { vmResult = createAdvancedVirtualMachine(dataCenter, serviceOffering, template, networkIds, curAccount, hostName, displayName, diskOfferingId, size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), - null, new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled); + null, new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled, null); } } } catch (CloudRuntimeException e) { diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index ef9cedc79887..cbc1426f21cf 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -156,7 +156,7 @@ public class UserVmManagerImplTest { @Mock private CloneVMCmd cloneVMCommand; - @Mock + @Mock private AccountDao accountDao; @Mock From 7b048c4fb53d79c306c4381f7fac68fb7ffc4022 Mon Sep 17 00:00:00 2001 From: atrocitytheme Date: Fri, 4 Mar 2022 21:58:49 -0800 Subject: [PATCH 109/114] resolve the compatability issue --- usage/src/main/java/com/cloud/usage/UsageManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 905a7d5c98a7..b18ff3d49229 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1635,7 +1635,7 @@ private void createSnapshotHelperEvent(UsageEventVO event) { SnapshotVO snapshotInstance = _snapshotDao.findById(snapId); - if (snapshotInstance != null && snapshotInstance.getsnapshotType() == Snapshot.Type.INTERNAL.ordinal()) { + if (snapshotInstance != null && snapshotInstance.getSnapshotType() == Snapshot.Type.INTERNAL.ordinal()) { return; } From 40788cf926cf8f07c75cd14da0e88a2a5529df7d Mon Sep 17 00:00:00 2001 From: atrocitytheme Date: Fri, 4 Mar 2022 22:03:02 -0800 Subject: [PATCH 110/114] resolve the checkstyle imports --- .../com/cloud/vm/UserVmManagerImplTest.java | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 85339f46abc1..b27535fb5cd1 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -19,7 +19,11 @@ import com.cloud.configuration.Resource; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.exception.*; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.Hypervisor; import com.cloud.network.NetworkModel; import com.cloud.network.dao.NetworkDao; @@ -27,11 +31,19 @@ import com.cloud.offering.ServiceOffering; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; -import com.cloud.storage.*; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeApiService; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.user.*; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.ResourceLimitService; +import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; import com.cloud.uservm.UserVm; import com.cloud.utils.exception.CloudRuntimeException; @@ -61,9 +73,17 @@ import java.util.List; import java.util.Map; -import static org.junit.Assert.*; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class UserVmManagerImplTest { From 8da90f70d10f0420416c35cdf6e6e5534132c18a Mon Sep 17 00:00:00 2001 From: atrocitytheme Date: Fri, 4 Mar 2022 22:59:47 -0800 Subject: [PATCH 111/114] Implementation for compatability --- .../java/com/cloud/vm/UserVmManagerImpl.java | 97 ++++++++----------- 1 file changed, 38 insertions(+), 59 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 5115311398da..608b49854a8c 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -23,20 +23,8 @@ import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -46,13 +34,11 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; - import javax.inject.Inject; import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; - import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -134,7 +120,6 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; - import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -1244,14 +1229,14 @@ private UserVm upgradeStoppedVirtualMachine(Long vmId, Long svcOffId, Map currentCpu) { - _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); + _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.cpu, (long) (newCpu - currentCpu)); } else if (currentCpu > newCpu) { - _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.cpu, new Long(currentCpu - newCpu)); + _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.cpu, (long) (currentCpu - newCpu)); } if (newMemory > currentMemory) { - _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.memory, new Long(newMemory - currentMemory)); + _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.memory, (long) (newMemory - currentMemory)); } else if (currentMemory > newMemory) { - _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.memory, new Long(currentMemory - newMemory)); + _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.memory, (long) (currentMemory - newMemory)); } } @@ -1979,11 +1964,11 @@ private boolean upgradeRunningVirtualMachine(Long vmId, Long newServiceOfferingI // Increment CPU and Memory count accordingly. if (newCpu > currentCpu) { - _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); + _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.cpu, (long) (newCpu - currentCpu)); } if (memoryDiff > 0) { - _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(memoryDiff)); + _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.memory, (long) memoryDiff); } // #1 Check existing host has capacity @@ -2015,11 +2000,11 @@ private boolean upgradeRunningVirtualMachine(Long vmId, Long newServiceOfferingI if (!success) { // Decrement CPU and Memory count accordingly. if (newCpu > currentCpu) { - _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); + _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.cpu, (long) (newCpu - currentCpu)); } if (memoryDiff > 0) { - _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(memoryDiff)); + _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, (long) memoryDiff); } } } @@ -2224,7 +2209,7 @@ public UserVm recoverVirtualMachine(RecoverVMCmd cmd) throws ResourceAllocationE // First check that the maximum number of UserVMs, CPU and Memory limit for the given // accountId will not be exceeded if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceLimitCheck(account, vm.isDisplayVm(), new Long(serviceOffering.getCpu()), new Long(serviceOffering.getRamSize())); + resourceLimitCheck(account, vm.isDisplayVm(), serviceOffering.getCpu().longValue(), serviceOffering.getRamSize().longValue()); } _haMgr.cancelDestroy(vm, vm.getHostId()); @@ -2259,7 +2244,7 @@ public UserVm recoverVirtualMachine(RecoverVMCmd cmd) throws ResourceAllocationE } //Update Resource Count for the given account - resourceCountIncrement(account.getId(), vm.isDisplayVm(), new Long(serviceOffering.getCpu()), new Long(serviceOffering.getRamSize())); + resourceCountIncrement(account.getId(), vm.isDisplayVm(), serviceOffering.getCpu().longValue(), serviceOffering.getRamSize().longValue()); } }); @@ -2537,7 +2522,7 @@ private void updateVmStateForFailedVmCreation(Long vmId, Long hostId) { ServiceOfferingVO offering = _serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); // Update Resource Count for the given account - resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); } } } @@ -3852,7 +3837,7 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe volumesSize += verifyAndGetDiskSize(diskOffering, diskSize); } if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceLimitCheck(owner, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceLimitCheck(owner, isDisplayVm, offering.getCpu().longValue(), offering.getRamSize().longValue()); } _resourceLimitMgr.checkResourceLimit(owner, ResourceType.volume, (isIso || diskOfferingId == null ? 1 : 2)); @@ -4436,7 +4421,7 @@ public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCap } //Update Resource Count for the given account - resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountIncrement(accountId, isDisplayVm, offering.getCpu().longValue(), offering.getRamSize().longValue()); } return vm; } @@ -4797,9 +4782,6 @@ public void validateCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueE } } if (!zone.isLocalStorageEnabled()) { - if (serviceOffering.isUseLocalStorage()) { - throw new CloudRuntimeException("Zone is not configured to use local storage now but this service offering " + serviceOffering.getName() + " uses it"); - } if (diskOffering != null && diskOffering.isUseLocalStorage()) { throw new CloudRuntimeException("Zone is not configured to use local storage but disk offering " + diskOffering.getName() + " uses it"); } @@ -4818,7 +4800,7 @@ public void validateCloneCondition(CloneVMCmd cmd) throws InvalidParameterValueE private VolumeVO saveDataDiskVolumeFromSnapShot(final Account owner, final Boolean displayVolume, final Long zoneId, final Long diskOfferingId, final Storage.ProvisioningType provisioningType, final Long size, final Long minIops, final Long maxIops, final VolumeVO parentVolume, final String volumeName, final String uuid, final Map details) { return Transaction.execute((TransactionCallback) status -> { - VolumeVO volume = new VolumeVO(volumeName, -1, -1, -1, -1, new Long(-1), null, null, provisioningType, 0, Volume.Type.DATADISK); + VolumeVO volume = new VolumeVO(volumeName, -1, -1, -1, -1, Long.valueOf(-1), null, null, provisioningType, 0, Volume.Type.DATADISK); volume.setPoolId(null); volume.setUuid(uuid); volume.setDataCenterId(zoneId); @@ -4845,7 +4827,7 @@ private VolumeVO saveDataDiskVolumeFromSnapShot(final Account owner, final Boole // Increment resource count during allocation; if actual creation fails, // decrement it _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume, displayVolume); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, displayVolume, new Long(volume.getSize())); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, displayVolume, volume.getSize()); return volume; }); } @@ -5649,7 +5631,7 @@ public UserVm destroyVm(long vmId, boolean expunge) throws ResourceUnavailableEx ServiceOfferingVO offering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()); //Update Resource Count for the given account - resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); } return _vmDao.findById(vmId); } else { @@ -6023,7 +6005,7 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd, long templateId) throws C UserVm curVm = cmd.getTargetVM(); // check if host is available Long hostId = curVm.getHostId(); - getDestinationHost(hostId, true); + getDestinationHost(hostId, true, false); Long zoneId = curVm.getDataCenterId(); DataCenter dataCenter = _entityMgr.findById(DataCenter.class, zoneId); Map vmProperties = curVm.getDetails() != null ? curVm.getDetails() : new HashMap<>(); @@ -6040,7 +6022,8 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd, long templateId) throws C String uuidName = _uuidMgr.generateUuid(UserVm.class, null); String hostName = generateHostName(uuidName); String displayName = hostName + "-Clone"; - Long diskOfferingId = curVm.getDiskOfferingId(); + VolumeVO curVolume = _volsDao.findByInstance(curVm.getId()).get(0); + Long diskOfferingId = curVolume.getDiskOfferingId(); Long size = null; // mutual exclusive with disk offering id String userData = curVm.getUserData(); String sshKeyPair = null; @@ -6064,22 +6047,18 @@ public UserVm recordVirtualMachineToDB(CloneVMCmd cmd, long templateId) throws C boxed(). collect(Collectors.toList()); try { + Map detailMap = new HashMap<>(); + Map> dhcpMap = new HashMap<>(); + Map emptyUserOfVmProperties = new HashMap<>(); if (dataCenter.getNetworkType() == NetworkType.Basic) { vmResult = createBasicSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, securityGroupIdList, curAccount, hostName, displayName, diskOfferingId, - size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, - curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), null, new HashMap<>(), - null, new HashMap<>(), dynamicScalingEnabled); + size, group, hypervisorType, cmd.getHttpMethod(), userData, null, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, + curVm.getDetails() == null ? detailMap : curVm.getDetails(), null, new HashMap<>(), + null, new HashMap<>(), dynamicScalingEnabled, diskOfferingId); } else { - if (dataCenter.isSecurityGroupEnabled()) { - vmResult = createAdvancedSecurityGroupVirtualMachine(dataCenter, serviceOffering, template, networkIds, securityGroupIdList, curAccount, hostName, - displayName, diskOfferingId, size, group, hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, - affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), null, new HashMap<>(), - null, new HashMap<>(), dynamicScalingEnabled); - } else { - vmResult = createAdvancedVirtualMachine(dataCenter, serviceOffering, template, networkIds, curAccount, hostName, displayName, diskOfferingId, size, group, - hypervisorType, cmd.getHttpMethod(), userData, sshKeyPair, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, curVm.getDetails() == null ? new HashMap<>() : curVm.getDetails(), - null, new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled, null); - } + vmResult = createAdvancedVirtualMachine(dataCenter, serviceOffering, template, networkIds, curAccount, hostName, displayName, diskOfferingId, size, group, + hypervisorType, cmd.getHttpMethod(), userData, null, ipToNetoworkMap, addr, isDisplayVM, keyboard, affinityGroupIdList, curVm.getDetails() == null ? detailMap : curVm.getDetails(), + null, new HashMap<>(), null, new HashMap<>(), dynamicScalingEnabled, null, diskOfferingId); } } catch (CloudRuntimeException e) { _templateMgr.delete(curAccount.getId(), template.getId(), zoneId); @@ -7189,7 +7168,7 @@ public UserVm moveVMToUser(final AssignVMCmd cmd) throws ResourceAllocationExcep // VV 2: check if account/domain is with in resource limits to create a new vm if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceLimitCheck(newAccount, vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceLimitCheck(newAccount, vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); } // VV 3: check if volumes and primary storage space are with in resource limits @@ -7222,7 +7201,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm.getHypervisorType().toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); // update resource counts for old account - resourceCountDecrement(oldAccount.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountDecrement(oldAccount.getAccountId(), vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); // OWNERSHIP STEP 1: update the vm owner vm.setAccountId(newAccount.getAccountId()); @@ -7234,12 +7213,12 @@ public void doInTransactionWithoutResult(TransactionStatus status) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.volume); - _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.primary_storage, volume.getSize()); volume.setAccountId(newAccount.getAccountId()); volume.setDomainId(newAccount.getDomainId()); _volsDao.persist(volume); _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.volume); - _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.primary_storage, volume.getSize()); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); @@ -7247,7 +7226,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { //update resource count of new account if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceCountIncrement(newAccount.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountIncrement(newAccount.getAccountId(), vm.isDisplayVm(), offering.getCpu().longValue(), offering.getRamSize().longValue()); } //generate usage events to account for this change @@ -7787,7 +7766,7 @@ public UserVm restoreVirtualMachine(final Account caller, final long vmId, final // 1. Save usage event and update resource count for user vm volumes _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.volume, newVol.isDisplay()); - _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.primary_storage, newVol.isDisplay(), new Long(newVol.getSize())); + _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.primary_storage, newVol.isDisplay(), newVol.getSize()); // 2. Create Usage event for the newly created volume UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VOLUME_CREATE, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(), newVol.getDiskOfferingId(), template.getId(), newVol.getSize()); _usageEventDao.persist(usageEvent); @@ -8273,8 +8252,8 @@ private void removeVMFromAffinityGroups(long vmId) { */ private void postProcessingUnmanageVM(UserVmVO vm) { ServiceOfferingVO offering = _serviceOfferingDao.findById(vm.getServiceOfferingId()); - Long cpu = offering.getCpu() != null ? new Long(offering.getCpu()) : 0L; - Long ram = offering.getRamSize() != null ? new Long(offering.getRamSize()) : 0L; + Long cpu = offering.getCpu() != null ? offering.getCpu().longValue() : 0L; + Long ram = offering.getRamSize() != null ? offering.getRamSize().longValue() : 0L; // First generate a VM stop event if the VM was not stopped already if (vm.getState() != State.Stopped) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_STOP, vm.getAccountId(), vm.getDataCenterId(), @@ -8302,7 +8281,7 @@ private void postProcessingUnmanageVMVolumes(List volumes, UserVmVO vm Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); } _resourceLimitMgr.decrementResourceCount(vm.getAccountId(), ResourceType.volume); - _resourceLimitMgr.decrementResourceCount(vm.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(vm.getAccountId(), ResourceType.primary_storage, volume.getSize()); } } From 04a6f05cc02528c578b5ddc3ad5aa09e7a96f191 Mon Sep 17 00:00:00 2001 From: atrocitytheme Date: Fri, 4 Mar 2022 23:00:17 -0800 Subject: [PATCH 112/114] fix checkstyle --- .../main/java/com/cloud/vm/UserVmManagerImpl.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 608b49854a8c..5034f340e12c 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -23,8 +23,20 @@ import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; From fb99861d6fb2ab2cb07b6c1210ca14aac3d3aa58 Mon Sep 17 00:00:00 2001 From: atrocitytheme Date: Fri, 4 Mar 2022 23:24:39 -0800 Subject: [PATCH 113/114] fix snapshot method compatability --- .../storage/snapshot/SnapshotManagerImpl.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 6b1ebe982d75..d281102d1a75 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -327,7 +327,7 @@ public Snapshot revertSnapshot(Long snapshotId) { boolean result = snapshotStrategy.revertSnapshot(snapshotInfo); if (result) { // update volume size and primary storage count - _resourceLimitMgr.decrementResourceCount(snapshot.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize() - snapshot.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshot.getAccountId(), ResourceType.primary_storage, volume.getSize() - snapshot.getSize()); volume.setSize(snapshot.getSize()); _volsDao.update(volume.getId(), volume); return snapshotInfo; @@ -493,7 +493,7 @@ public Snapshot backupSnapshotFromVmSnapshot(Long snapshotId, Long vmId, Long vo } catch (Exception e) { s_logger.debug("Failed to backup snapshot from vm snapshot", e); _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.secondary_storage, volume.getSize()); throw new CloudRuntimeException("Failed to backup snapshot from vm snapshot", e); } return snapshotInfo; @@ -600,7 +600,7 @@ public boolean deleteSnapshot(long snapshotId) { if (snapshotCheck.getState() == Snapshot.State.BackedUp) { SnapshotVO snapVO = _snapshotDao.findById(snapshotId); - if (snapVO == null || snapVO.getsnapshotType() != Type.INTERNAL.ordinal()) { + if (snapVO == null || snapVO.getSnapshotType() != Type.INTERNAL.ordinal()) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_DELETE, snapshotCheck.getAccountId(), snapshotCheck.getDataCenterId(), snapshotId, snapshotCheck.getName(), null, null, 0L, snapshotCheck.getClass().getName(), snapshotCheck.getUuid()); } @@ -612,7 +612,7 @@ public boolean deleteSnapshot(long snapshotId) { if (snapshotCheck.getState() == Snapshot.State.BackedUp) { if (snapshotStoreRef != null) { - _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.secondary_storage, new Long(snapshotStoreRef.getPhysicalSize())); + _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.secondary_storage, snapshotStoreRef.getPhysicalSize()); } } } @@ -679,7 +679,7 @@ public Pair, Integer> listSnapshots(ListSnapshotsCmd cm sb.and("snapshotTypeEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.IN); sb.and("snapshotTypeNEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.NEQ); sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); - sb.and("snapshotTypeInternal", sb.entity().getsnapshotType(), SearchCriteria.Op.NEQ); + sb.and("snapshotTypeInternal", sb.entity().getSnapshotType(), SearchCriteria.Op.NEQ); if (tags != null && !tags.isEmpty()) { SearchBuilder tagSearch = _resourceTagDao.createSearchBuilder(); for (int count = 0; count < tags.size(); count++) { @@ -812,7 +812,7 @@ public boolean deleteSnapshotDirsForAccount(long accountId) { if (Type.MANUAL == snapshot.getRecurringType()) { _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.snapshot); if (snapshotStoreRef != null) { - _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, new Long(snapshotStoreRef.getPhysicalSize())); + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, snapshotStoreRef.getPhysicalSize()); } } @@ -1255,13 +1255,13 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc throw new CloudRuntimeException(String.format("Could not find snapshot %s [%s] on [%s]", snapshot.getName(), snapshot.getUuid(), snapshot.getLocationType())); } SnapshotVO snapInstance = _snapshotDao.findById(snapshot.getId()); - if (snapInstance == null || snapInstance.getsnapshotType() != Type.INTERNAL.ordinal()) { + if (snapInstance == null || snapInstance.getSnapshotType() != Type.INTERNAL.ordinal()) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); } // Correct the resource count of snapshot in case of delta snapshots. - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, volume.getSize() - snapshotStoreRef.getPhysicalSize()); } catch (Exception e) { s_logger.debug("post process snapshot failed", e); } @@ -1270,14 +1270,14 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc s_logger.debug("Failed to create snapshot" + cre.getLocalizedMessage()); } _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, volume.getSize()); throw cre; } catch (Exception e) { if (s_logger.isDebugEnabled()) { s_logger.debug("Failed to create snapshot", e); } _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, volume.getSize()); throw new CloudRuntimeException("Failed to create snapshot", e); } return snapshot; @@ -1502,7 +1502,7 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, try { _resourceLimitMgr.checkResourceLimit(owner, ResourceType.snapshot); - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.secondary_storage, new Long(volume.getSize()).longValue()); + _resourceLimitMgr.checkResourceLimit(owner, ResourceType.secondary_storage, volume.getSize().longValue()); } catch (ResourceAllocationException e) { if (snapshotType != Type.MANUAL) { String msg = "Snapshot resource limit exceeded for account id : " + owner.getId() + ". Failed to create recurring snapshots"; @@ -1552,7 +1552,7 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, throw new CloudRuntimeException("Failed to create snapshot for volume: " + volume.getId()); } _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.snapshot); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, volume.getSize()); return snapshot; } } From 63d3da1bd8e5a35f0a9bc89ce707f7194df4e368 Mon Sep 17 00:00:00 2001 From: atrocitytheme Date: Fri, 4 Mar 2022 23:42:06 -0800 Subject: [PATCH 114/114] fix legacy snapshot methods --- .../apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index a763a8bf0de1..3aaf063b860c 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -219,7 +219,7 @@ public SnapshotResult takeSnapshot(SnapshotInfo snap) { try { result = future.get(); SnapshotVO snapVO = _snapshotDao.findById(snap.getId()); - if (snapVO == null || snapVO.getsnapshotType() != Snapshot.Type.INTERNAL.ordinal()) { + if (snapVO == null || snapVO.getSnapshotType() != Snapshot.Type.INTERNAL.ordinal()) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_ON_PRIMARY, snap.getAccountId(), snap.getDataCenterId(), snap.getId(), snap.getName(), null, null, snapshotOnPrimary.getSize(), snapshotOnPrimary.getSize(), snap.getClass().getName(), snap.getUuid()); }