Reconnect offline agents (#359)

Author: timja

src/main/java/com/microsoft/azure/vmagent/AzureVMCloudOnceRetentionStrategy.java

@@ -38,6 +38,11 @@ public long check(final AzureVMComputer agentComputer) {
                 done(agentComputer);
             }
         }
+
+        if (agentComputer.isOffline() && !agentComputer.isConnecting() && agentComputer.isLaunchSupported()) {
+            agentComputer.tryReconnect();
+        }
+
         return 1;
     }
 

src/main/java/com/microsoft/azure/vmagent/AzureVMCloudPoolRetentionStrategy.java

@@ -98,12 +98,11 @@ public void run() {
         final int currentPoolSize
                 = ((AzureVMCloudPoolRetentionStrategy) currentTemplate.getRetentionStrategy()).getPoolSize();
 
-        Computer.threadPoolForRemoting.submit(new Runnable() {
-            @Override
-            public void run() {
-                checkPoolSizeAndDelete(agentComputer, currentPoolSize);
-            }
-        });
+        Computer.threadPoolForRemoting.submit(() -> checkPoolSizeAndDelete(agentComputer, currentPoolSize));
+
+        if (agentComputer.isOffline() && !agentComputer.isConnecting() && agentComputer.isLaunchSupported()) {
+            agentComputer.tryReconnect();
+        }
 
         return 1;
     }

src/main/java/com/microsoft/azure/vmagent/AzureVMCloudRetensionStrategy.java

@@ -61,39 +61,38 @@ public long check(@NonNull AzureVMComputer agentNode) {
         return check(agentNode, new ExecutionEngine());
     }
 
-    protected long check(final AzureVMComputer agentNode, ExecutionEngine executionEngine) {
+    protected long check(final AzureVMComputer agentComputer, ExecutionEngine executionEngine) {
         // Determine whether we can recycle this machine.
         // The CRS is the way that nodes that are currently operating "correctly"
         // can be retained/reclaimed.  Any failure modes need to be dealt with through
-        // the clean up task.
+        // the cleanup task.
 
-        boolean canRecycle = true;
         // Node must be idle
-        canRecycle &= agentNode.isIdle();
+        boolean canRecycle = agentComputer.isIdle();
         // The node must also be online.  This also implies not temporarily disconnected
         // (like by a user).
-        canRecycle &= agentNode.isOnline();
+        canRecycle &= agentComputer.isOnline();
         // The configured idle time must be > 0 (which means leave forever)
         canRecycle &= idleTerminationMillis > 0;
         // The number of ms it's been idle must be greater than the current idle time.
-        canRecycle &= idleTerminationMillis < (System.currentTimeMillis() - agentNode.getIdleStartMilliseconds());
+        canRecycle &= idleTerminationMillis < (System.currentTimeMillis() - agentComputer.getIdleStartMilliseconds());
 
-        if (agentNode.getNode() == null) {
+        if (agentComputer.getNode() == null) {
             return 1;
         }
 
-        final AzureVMAgent agent = agentNode.getNode();
+        final AzureVMAgent agent = agentComputer.getNode();
 
         if (canRecycle) {
             LOGGER.log(Level.INFO, "Idle timeout reached for agent: {0}, action: {1}",
-                    new Object[]{agentNode.getName(), agent.isShutdownOnIdle() ? "shutdown" : "delete"});
+                    new Object[]{agentComputer.getName(), agent.isShutdownOnIdle() ? "shutdown" : "delete"});
 
             Callable<Void> task = () -> {
                 // Block cleanup while we execute so the cleanup task doesn't try to take it
                 // away (node will go offline).  Also blocks cleanup in case of shutdown.
                 agent.blockCleanUpAction();
                 if (agent.isShutdownOnIdle()) {
-                    LOGGER.log(Level.INFO, "Going to idleTimeout agent: {0}", agentNode.getName());
+                    LOGGER.log(Level.INFO, "Going to idleTimeout agent: {0}", agentComputer.getName());
                     agent.shutdown(Messages._Idle_Timeout_Shutdown());
                 } else {
                     agent.deprovision(Messages._Idle_Timeout_Delete());
@@ -112,24 +111,31 @@ protected long check(final AzureVMComputer agentNode, ExecutionEngine executionE
                                 defaultTimeoutInSeconds
                         ));
             } catch (AzureCloudException ae) {
-                LOGGER.log(Level.WARNING, String.format("Could not terminate or shutdown %s", agentNode.getName()), ae);
+                LOGGER.log(Level.WARNING, String.format("Could not terminate or shutdown %s",
+                        agentComputer.getName()), ae);
                 // If we have an exception, set the agent for deletion.
                 // It's unlikely we'll be able to shut it down properly ever.
-                AzureVMAgent node = agentNode.getNode();
+                AzureVMAgent node = agentComputer.getNode();
                 if (node != null) {
                     node.setCleanUpAction(CleanUpAction.DELETE, Messages._Failed_Initial_Shutdown_Or_Delete());
                 }
             } catch (Exception e) {
                 LOGGER.log(Level.WARNING,
-                        String.format("Exception occurred while calling timeout on node %s", agentNode.getName()), e);
+                        String.format("Exception occurred while calling timeout on node %s",
+                                agentComputer.getName()), e);
                 // If we have an exception, set the agent for deletion.
                 // It's unlikely we'll be able to shut it down properly ever.
-                AzureVMAgent node = agentNode.getNode();
+                AzureVMAgent node = agentComputer.getNode();
                 if (node != null) {
                     node.setCleanUpAction(CleanUpAction.DELETE, Messages._Failed_Initial_Shutdown_Or_Delete());
                 }
             }
         }
+
+        if (agentComputer.isOffline() && !agentComputer.isConnecting() && agentComputer.isLaunchSupported()) {
+            agentComputer.tryReconnect();
+        }
+
         return 1;
     }
 

src/main/resources/scripts/ubuntuInstallMavenScript.sh

@@ -1,6 +1,6 @@
 # Install Maven
 
-MAVEN_VERSION=3.8.4
+MAVEN_VERSION=3.8.5
 
 sudo curl -O https://downloads.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz
 sudo tar zxvf apache-maven-${MAVEN_VERSION}-bin.tar.gz -C /opt/

src/main/webapp/help-initScript.html

@@ -17,7 +17,7 @@
 sudo apt-get install -y git
 
 # Install Maven
-MAVEN_VERSION=3.8.4
+MAVEN_VERSION=3.8.5
 
 sudo curl -O https://downloads.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz
 sudo tar zxvf apache-maven-${MAVEN_VERSION}-bin.tar.gz -C /opt/