Merge remote-tracking branch 'apache/4.18' into main

Author: DaanHoogland

agent/src/main/java/com/cloud/agent/Agent.java

@@ -42,6 +42,7 @@
 
 import com.cloud.resource.AgentStatusUpdater;
 import com.cloud.resource.ResourceStatusUpdater;
+import com.cloud.agent.api.PingAnswer;
 import com.cloud.utils.NumbersUtil;
 import org.apache.cloudstack.agent.lb.SetupMSListAnswer;
 import org.apache.cloudstack.agent.lb.SetupMSListCommand;
@@ -842,6 +843,9 @@ public void processResponse(final Response response, final Link link) {
                     listener.processControlResponse(response, (AgentControlAnswer)answer);
                 }
             }
+        } else if (answer instanceof PingAnswer && (((PingAnswer) answer).isSendStartup()) && _reconnectAllowed) {
+            s_logger.info("Management server requested startup command to reinitialize the agent");
+            sendStartup(link);
         } else {
             setLastPingResponseTime();
         }

core/src/main/java/com/cloud/agent/api/PingAnswer.java

@@ -22,15 +22,26 @@
 public class PingAnswer extends Answer {
     private PingCommand _command = null;
 
+    private boolean sendStartup = false;
+
     protected PingAnswer() {
     }
 
-    public PingAnswer(PingCommand cmd) {
+    public PingAnswer(PingCommand cmd, boolean sendStartup) {
         super(cmd);
         _command = cmd;
+        this.sendStartup = sendStartup;
     }
 
     public PingCommand getCommand() {
         return _command;
     }
+
+    public boolean isSendStartup() {
+        return sendStartup;
+    }
+
+    public void setSendStartup(boolean sendStartup) {
+        this.sendStartup = sendStartup;
+    }
 }

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -40,6 +40,7 @@
 
 import com.cloud.configuration.Config;
 import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.db.GlobalLock;
 import org.apache.cloudstack.agent.lb.IndirectAgentLB;
 import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
@@ -799,49 +800,65 @@ public boolean stop() {
         return true;
     }
 
+    protected Status getNextStatusOnDisconnection(Host host, final Status.Event event) {
+        final Status currentStatus = host.getStatus();
+        Status nextStatus;
+        if (currentStatus == Status.Down || currentStatus == Status.Alert || currentStatus == Status.Removed) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug(String.format("Host %s is already %s", host.getUuid(), currentStatus));
+            }
+            nextStatus = currentStatus;
+        } else {
+            try {
+                nextStatus = currentStatus.getNextStatus(event);
+            } catch (final NoTransitionException e) {
+                final String err = String.format("Cannot find next status for %s as current status is %s for agent %s", event, currentStatus, host.getUuid());
+                s_logger.debug(err);
+                throw new CloudRuntimeException(err);
+            }
+
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug(String.format("The next status of agent %s is %s, current status is %s", host.getUuid(), nextStatus, currentStatus));
+            }
+        }
+        return nextStatus;
+    }
+
     protected boolean handleDisconnectWithoutInvestigation(final AgentAttache attache, final Status.Event event, final boolean transitState, final boolean removeAgent) {
         final long hostId = attache.getId();
 
-        s_logger.info("Host " + hostId + " is disconnecting with event " + event);
-        Status nextStatus = null;
-        final HostVO host = _hostDao.findById(hostId);
-        if (host == null) {
-            s_logger.warn("Can't find host with " + hostId);
-            nextStatus = Status.Removed;
-        } else {
-            final Status currentStatus = host.getStatus();
-            if (currentStatus == Status.Down || currentStatus == Status.Alert || currentStatus == Status.Removed) {
-                if (s_logger.isDebugEnabled()) {
-                    s_logger.debug("Host " + hostId + " is already " + currentStatus);
-                }
-                nextStatus = currentStatus;
-            } else {
-                try {
-                    nextStatus = currentStatus.getNextStatus(event);
-                } catch (final NoTransitionException e) {
-                    final String err = "Cannot find next status for " + event + " as current status is " + currentStatus + " for agent " + hostId;
-                    s_logger.debug(err);
-                    throw new CloudRuntimeException(err);
+        boolean result = false;
+        GlobalLock joinLock = getHostJoinLock(hostId);
+        if (joinLock.lock(60)) {
+            try {
+                s_logger.info(String.format("Host %d is disconnecting with event %s", hostId, event));
+                Status nextStatus = null;
+                final HostVO host = _hostDao.findById(hostId);
+                if (host == null) {
+                    s_logger.warn(String.format("Can't find host with %d", hostId));
+                    nextStatus = Status.Removed;
+                } else {
+                    nextStatus = getNextStatusOnDisconnection(host, event);
+                    caService.purgeHostCertificate(host);
                 }
 
                 if (s_logger.isDebugEnabled()) {
-                    s_logger.debug("The next status of agent " + hostId + "is " + nextStatus + ", current status is " + currentStatus);
+                    s_logger.debug(String.format("Deregistering link for %d with state %s", hostId, nextStatus));
                 }
-            }
-            caService.purgeHostCertificate(host);
-        }
 
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Deregistering link for " + hostId + " with state " + nextStatus);
-        }
+                removeAgent(attache, nextStatus);
 
-        removeAgent(attache, nextStatus);
-        // update the DB
-        if (host != null && transitState) {
-            disconnectAgent(host, event, _nodeId);
+                if (host != null && transitState) {
+                    // update the state for host in DB as per the event
+                    disconnectAgent(host, event, _nodeId);
+                }
+            } finally {
+                joinLock.unlock();
+            }
+            result = true;
         }
-
-        return true;
+        joinLock.releaseRef();
+        return result;
     }
 
     protected boolean handleDisconnectWithInvestigation(final AgentAttache attache, Status.Event event) {
@@ -1102,26 +1119,23 @@ protected AgentAttache createAttacheForConnect(final HostVO host, final Link lin
         return attache;
     }
 
-    private AgentAttache handleConnectedAgent(final Link link, final StartupCommand[] startup, final Request request) {
-        AgentAttache attache = null;
-        ReadyCommand ready = null;
-        try {
-            final List<String> agentMSHostList = new ArrayList<>();
-            String lbAlgorithm = null;
-            if (startup != null && startup.length > 0) {
-                final String agentMSHosts = startup[0].getMsHostList();
-                if (StringUtils.isNotEmpty(agentMSHosts)) {
-                    String[] msHosts = agentMSHosts.split("@");
-                    if (msHosts.length > 1) {
-                        lbAlgorithm = msHosts[1];
-                    }
-                    agentMSHostList.addAll(Arrays.asList(msHosts[0].split(",")));
+    private AgentAttache sendReadyAndGetAttache(HostVO host, ReadyCommand ready, Link link, StartupCommand[] startup) throws ConnectionException {
+        final List<String> agentMSHostList = new ArrayList<>();
+        String lbAlgorithm = null;
+        if (startup != null && startup.length > 0) {
+            final String agentMSHosts = startup[0].getMsHostList();
+            if (StringUtils.isNotEmpty(agentMSHosts)) {
+                String[] msHosts = agentMSHosts.split("@");
+                if (msHosts.length > 1) {
+                    lbAlgorithm = msHosts[1];
                 }
+                agentMSHostList.addAll(Arrays.asList(msHosts[0].split(",")));
             }
-
-            final HostVO host = _resourceMgr.createHostVOForConnectedAgent(startup);
-            if (host != null) {
-                ready = new ReadyCommand(host.getDataCenterId(), host.getId(), NumbersUtil.enableHumanReadableSizes);
+        }
+        AgentAttache attache = null;
+        GlobalLock joinLock = getHostJoinLock(host.getId());
+        if (joinLock.lock(60)) {
+            try {
 
                 if (!indirectAgentLB.compareManagementServerList(host.getId(), host.getDataCenterId(), agentMSHostList, lbAlgorithm)) {
                     final List<String> newMSList = indirectAgentLB.getManagementServerList(host.getId(), host.getDataCenterId(), null);
@@ -1133,6 +1147,24 @@ private AgentAttache handleConnectedAgent(final Link link, final StartupCommand[
 
                 attache = createAttacheForConnect(host, link);
                 attache = notifyMonitorsOfConnection(attache, startup, false);
+            } finally {
+                joinLock.unlock();
+            }
+        } else {
+            throw new ConnectionException(true, "Unable to acquire lock on host " + host.getUuid());
+        }
+        joinLock.releaseRef();
+        return attache;
+    }
+
+    private AgentAttache handleConnectedAgent(final Link link, final StartupCommand[] startup, final Request request) {
+        AgentAttache attache = null;
+        ReadyCommand ready = null;
+        try {
+            final HostVO host = _resourceMgr.createHostVOForConnectedAgent(startup);
+            if (host != null) {
+                ready = new ReadyCommand(host.getDataCenterId(), host.getId(), NumbersUtil.enableHumanReadableSizes);
+                attache = sendReadyAndGetAttache(host, ready, link, startup);
             }
         } catch (final Exception e) {
             s_logger.debug("Failed to handle host connection: ", e);
@@ -1312,6 +1344,8 @@ protected void processRequest(final Link link, final Request request) {
                     connectAgent(link, cmds, request);
                 }
                 return;
+            } else if (cmd instanceof StartupCommand) {
+                connectAgent(link, cmds, request);
             }
 
             final long hostId = attache.getId();
@@ -1366,7 +1400,10 @@ protected void processRequest(final Link link, final Request request) {
                         handleCommands(attache, request.getSequence(), new Command[] {cmd});
                         if (cmd instanceof PingCommand) {
                             final long cmdHostId = ((PingCommand)cmd).getHostId();
+                            boolean requestStartupCommand = false;
 
+                            final HostVO host = _hostDao.findById(Long.valueOf(cmdHostId));
+                            boolean gatewayAccessible = true;
                             // if the router is sending a ping, verify the
                             // gateway was pingable
                             if (cmd instanceof PingRoutingCommand) {
@@ -1391,7 +1428,10 @@ protected void processRequest(final Link link, final Request request) {
                                     s_logger.debug("Not processing " + PingRoutingCommand.class.getSimpleName() + " for agent id=" + cmdHostId + "; can't find the host in the DB");
                                 }
                             }
-                            answer = new PingAnswer((PingCommand)cmd);
+                            if (host!= null && host.getStatus() != Status.Up && gatewayAccessible) {
+                                requestStartupCommand = true;
+                            }
+                            answer = new PingAnswer((PingCommand)cmd, requestStartupCommand);
                         } else if (cmd instanceof ReadyAnswer) {
                             final HostVO host = _hostDao.findById(attache.getId());
                             if (host == null) {
@@ -1913,4 +1953,8 @@ public void propagateChangeToAgents(Map<String, String> params) {
             sendCommandToAgents(hostsPerZone, params);
         }
     }
+
+    private GlobalLock getHostJoinLock(Long hostId) {
+        return GlobalLock.getInternLock(String.format("%s-%s", "Host-Join", hostId));
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -1343,8 +1343,8 @@ protected void setupMemoryBalloonStatsPeriod(Connect conn) {
                 }
                 s_logger.debug(String.format("The memory balloon stats period [%s] has been set successfully for the VM (Libvirt Domain) with ID [%s] and name [%s].",
                         currentVmBalloonStatsPeriod, vmId, dm.getName()));
-            } catch (final LibvirtException e) {
-                s_logger.warn("Failed to set up memory balloon stats period." + e.getMessage());
+            } catch (final Exception e) {
+                s_logger.warn(String.format("Failed to set up memory balloon stats period for the VM %s with exception %s", parser.getName(), e.getMessage()));
             }
         }
     }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java

@@ -59,6 +59,8 @@ public class LibvirtDomainXMLParser {
     private Integer vncPort;
     private String desc;
 
+    private String name;
+
     public boolean parseDomainXML(String domXML) {
         DocumentBuilder builder;
         try {
@@ -71,6 +73,7 @@ public boolean parseDomainXML(String domXML) {
             Element rootElement = doc.getDocumentElement();
 
             desc = getTagValue("description", rootElement);
+            name = getTagValue("name", rootElement);
 
             Element devices = (Element)rootElement.getElementsByTagName("devices").item(0);
             NodeList disks = devices.getElementsByTagName("disk");
@@ -312,15 +315,19 @@ public boolean parseDomainXML(String domXML) {
                 String path = getTagValue("backend", rng);
                 String bytes = getAttrValue("rate", "bytes", rng);
                 String period = getAttrValue("rate", "period", rng);
-
-                if (StringUtils.isEmpty(backendModel)) {
-                    def = new RngDef(path, Integer.parseInt(bytes), Integer.parseInt(period));
+                if (StringUtils.isAnyEmpty(bytes, period)) {
+                    s_logger.debug(String.format("Bytes and period in the rng section should not be null, please check the VM %s", name));
                 } else {
-                    def = new RngDef(path, RngBackendModel.valueOf(backendModel.toUpperCase()),
-                                     Integer.parseInt(bytes), Integer.parseInt(period));
+                    if (StringUtils.isEmpty(backendModel)) {
+                        def = new RngDef(path, Integer.parseInt(bytes), Integer.parseInt(period));
+                    } else {
+                        def = new RngDef(path, RngBackendModel.valueOf(backendModel.toUpperCase()),
+                                Integer.parseInt(bytes), Integer.parseInt(period));
+                    }
+                }
+                if (def != null) {
+                    rngDefs.add(def);
                 }
-
-                rngDefs.add(def);
             }
 
             NodeList watchDogs = devices.getElementsByTagName("watchdog");
@@ -427,4 +434,8 @@ public List<WatchDogDef> getWatchDogs() {
     public String getDescription() {
         return desc;
     }
+
+    public String getName() {
+        return name;
+    }
 }