exec session PipedOutputStream passed to Dockerutil for handling exit properly, updatecontainer logic fixed for minimize downtime ,agent container image layers optimized

Author: emirhandurmus

Dockerfile

@@ -106,73 +106,76 @@ RUN true && \
     usermod -aG root,wheel iofog-agent && \
     true
 
+# Intermediate stage to collect all ubi-dep files
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest AS ubi-dep-staging
+COPY --from=ubi-dep /usr/share/zoneinfo /staging/usr/share/zoneinfo
+COPY --from=ubi-dep /usr/bin/curl /staging/usr/bin/curl
+COPY --from=ubi-dep /usr/bin/grep /staging/usr/bin/grep
+COPY --from=ubi-dep /usr/bin/gzip /staging/usr/bin/gzip
+COPY --from=ubi-dep /usr/bin/pgrep /staging/usr/bin/pgrep
+COPY --from=ubi-dep /usr/bin/awk /staging/usr/bin/awk
+COPY --from=ubi-dep /etc/ssl/certs/ca-bundle.crt /staging/etc/ssl/certs/ca-bundle.crt
+COPY --from=ubi-dep /etc/pki/tls/certs/ca-bundle.crt /staging/etc/pki/tls/certs/ca-bundle.crt
+COPY --from=ubi-dep /usr/lib64/libc.so.6 /staging/usr/lib64/libc.so.6
+COPY --from=ubi-dep /usr/lib64/libcom_err.so.2 /staging/usr/lib64/libcom_err.so.2
+COPY --from=ubi-dep /usr/lib64/libcrypto.so.3 /staging/usr/lib64/libcrypto.so.3
+COPY --from=ubi-dep /usr/lib64/libcurl.so.4 /staging/usr/lib64/libcurl.so.4
+COPY --from=ubi-dep /usr/lib64/libffi.so.8 /staging/usr/lib64/libffi.so.8
+COPY --from=ubi-dep /usr/lib64/libgmp.so.10 /staging/usr/lib64/libgmp.so.10
+COPY --from=ubi-dep /usr/lib64/libgnutls.so.30 /staging/usr/lib64/libgnutls.so.30
+COPY --from=ubi-dep /usr/lib64/libgssapi_krb5.so.2 /staging/usr/lib64/libgssapi_krb5.so.2
+COPY --from=ubi-dep /usr/lib64/libpcre.so.1 /staging/usr/lib64/libpcre.so.1
+COPY --from=ubi-dep /usr/lib64/libhogweed.so.6 /staging/usr/lib64/libhogweed.so.6
+COPY --from=ubi-dep /usr/lib64/libidn2.so.0 /staging/usr/lib64/libidn2.so.0
+COPY --from=ubi-dep /usr/lib64/libk5crypto.so.3 /staging/usr/lib64/libk5crypto.so.3
+COPY --from=ubi-dep /usr/lib64/libkeyutils.so.1 /staging/usr/lib64/libkeyutils.so.1
+COPY --from=ubi-dep /usr/lib64/libkrb5.so.3 /staging/usr/lib64/libkrb5.so.3
+COPY --from=ubi-dep /usr/lib64/libkrb5support.so.0 /staging/usr/lib64/libkrb5support.so.0
+COPY --from=ubi-dep /usr/lib64/libnettle.so.8 /staging/usr/lib64/libnettle.so.8
+COPY --from=ubi-dep /usr/lib64/libnghttp2.so.14 /staging/usr/lib64/libnghttp2.so.14
+COPY --from=ubi-dep /usr/lib64/libp11-kit.so.0 /staging/usr/lib64/libp11-kit.so.0
+COPY --from=ubi-dep /usr/lib64/libresolv.so.2 /staging/usr/lib64/libresolv.so.2
+COPY --from=ubi-dep /usr/lib64/libssl.so.3 /staging/usr/lib64/libssl.so.3
+COPY --from=ubi-dep /usr/lib64/libtasn1.so.6 /staging/usr/lib64/libtasn1.so.6
+COPY --from=ubi-dep /usr/lib64/libunistring.so.2 /staging/usr/lib64/libunistring.so.2
+COPY --from=ubi-dep /usr/lib64/libz.so.1 /staging/usr/lib64/libz.so.1
+COPY --from=ubi-dep /usr/lib64/libzstd.so.1 /staging/usr/lib64/libzstd.so.1
+COPY --from=ubi-dep /usr/lib64/libm.so.6 /staging/usr/lib64/libm.so.6
+COPY --from=ubi-dep /usr/lib64/libmpfr.so.6 /staging/usr/lib64/libmpfr.so.6
+COPY --from=ubi-dep /usr/lib64/libreadline.so.8 /staging/usr/lib64/libreadline.so.8
+COPY --from=ubi-dep /usr/lib64/libsigsegv.so.2 /staging/usr/lib64/libsigsegv.so.2
+COPY --from=ubi-dep /usr/lib64/libtinfo.so.6 /staging/usr/lib64/libtinfo.so.6
+COPY --from=ubi-dep /usr/lib64/libprocps.so.8 /staging/usr/lib64/libprocps.so.8
+COPY --from=ubi-dep /usr/lib64/libsystemd.so.0 /staging/usr/lib64/libsystemd.so.0
+COPY --from=ubi-dep /usr/lib64/liblz4.so.1 /staging/usr/lib64/liblz4.so.1
+COPY --from=ubi-dep /usr/lib64/libcap.so.2 /staging/usr/lib64/libcap.so.2
+COPY --from=ubi-dep /usr/lib64/libgcrypt.so.20 /staging/usr/lib64/libgcrypt.so.20
+COPY --from=ubi-dep /usr/lib64/libgpg-error.so.0 /staging/usr/lib64/libgpg-error.so.0
+COPY --from=ubi-dep /usr/lib64/liblzma.so.5 /staging/usr/lib64/liblzma.so.5
+COPY --from=ubi-dep /etc/passwd /staging/etc/passwd
+COPY --from=ubi-dep /etc/group /staging/etc/group
+COPY --from=ubi-dep /etc/shadow /staging/etc/shadow
+
+# Intermediate stage to collect all builder files
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest AS builder-staging
+COPY --from=builder packaging/iofog-agent/usr /staging/usr
+COPY --from=builder packaging/iofog-agent/etc/systemd/system/iofog-agent.service /staging/etc/systemd/system/iofog-agent.service
+COPY --from=builder packaging/iofog-agent/etc/bash_completion.d /staging/etc/bash_completion.d
+COPY --from=builder packaging/iofog-agent/etc/iofog-agent /staging/etc/iofog-agent
+
 # Final stage using UBI Micro
 FROM registry.access.redhat.com/ubi9/ubi-micro:latest
 
-# Copy dependencies from the ubi-dep stage
-COPY --from=ubi-dep /usr/share/zoneinfo /usr/share/zoneinfo
-COPY --from=ubi-dep /usr/bin/curl /usr/bin/
-COPY --from=ubi-dep /usr/bin/grep /usr/bin/
-COPY --from=ubi-dep /usr/bin/gzip /usr/bin/
-COPY --from=ubi-dep /usr/bin/pgrep /usr/bin/
-COPY --from=ubi-dep /usr/bin/awk /usr/bin/
-COPY --from=ubi-dep /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/
-COPY --from=ubi-dep /etc/pki/tls/certs/ca-bundle.crt /etc/pki/tls/certs/
-
-# Copy required shared libraries for curl grep awk
-COPY --from=ubi-dep /usr/lib64/libc.so.6 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libcom_err.so.2 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libcrypto.so.3 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libcurl.so.4 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libffi.so.8 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libgmp.so.10 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libgnutls.so.30 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libgssapi_krb5.so.2 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libpcre.so.1 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libhogweed.so.6 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libidn2.so.0 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libk5crypto.so.3 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libkeyutils.so.1 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libkrb5.so.3 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libkrb5support.so.0 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libnettle.so.8 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libnghttp2.so.14 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libp11-kit.so.0 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libresolv.so.2 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libssl.so.3 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libtasn1.so.6 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libunistring.so.2 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libz.so.1 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libzstd.so.1 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libm.so.6 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libmpfr.so.6 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libreadline.so.8 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libsigsegv.so.2 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libtinfo.so.6 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libprocps.so.8 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libsystemd.so.0 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/liblz4.so.1 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libcap.so.2  /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libgcrypt.so.20  /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/libgpg-error.so.0 /usr/lib64/
-COPY --from=ubi-dep /usr/lib64/liblzma.so.5 /usr/lib64/
-# COPY --from=ubi-dep /usr/lib64/libblkid.so.1 /usr/lib64/
-
-
-# Copy the iofog-agent user and related configurations
-COPY --from=ubi-dep /etc/passwd /etc/passwd
-COPY --from=ubi-dep /etc/group /etc/group
-COPY --from=ubi-dep /etc/shadow /etc/shadow
+# Copy all dependencies from the staging stage in a single layer
+COPY --from=ubi-dep-staging /staging/ /
 
 
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH="${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
-COPY --from=builder packaging/iofog-agent/usr ./usr
-COPY --from=builder packaging/iofog-agent/etc/systemd/system/iofog-agent.service /etc/systemd/system/iofog-agent.service
-COPY --from=builder packaging/iofog-agent/etc/bash_completion.d /etc/bash_completion.d/
-COPY --from=builder packaging/iofog-agent/etc/iofog-agent /etc/iofog-agent/
+# Copy all files from builder staging stage in a single layer
+COPY --from=builder-staging /staging/ /
 
 RUN true && \
     mv /etc/iofog-agent/config_new.xml /etc/iofog-agent/config.xml && \

build.gradle

@@ -5,7 +5,7 @@ plugins {
 
 allprojects {
     group = 'org.eclipse'
-    version = '3.5.5'
+    version = '3.5.6'
 }
 
 subprojects {

iofog-agent-daemon/src/main/java/org/eclipse/iofog/process_manager/ContainerManager.java

@@ -29,6 +29,7 @@
 import org.eclipse.iofog.process_manager.ExecSessionCallback;
 import java.util.concurrent.CompletableFuture;
 import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
 
 /**
  * provides methods to manage Docker containers
@@ -71,15 +72,51 @@ private Registry getRegistry(Microservice microservice) throws AgentSystemExcept
 
 	/**
 	 * removes an existing {@link Container} and creates a new one
+	 * Improved flow: Pull image first while old container is still running (minimizes downtime),
+	 * then stop old container (releases ports), then create and start new container.
 	 *
 	 * @param withCleanUp if true then removes old image and volumes
 	 * @throws Exception exception
 	 */
 	private void updateContainer(Microservice microservice, boolean withCleanUp) throws Exception {
 		LoggingService.logInfo(MODULE_NAME, "Start update container for microservice : " + microservice.getImageName());
 		microservice.setUpdating(true);
+		docker = DockerUtil.getInstance();
+		
+		// Step 1: Pull new image while old container is still running
+		// This keeps the service available during the slow image pull operation
+		setMicroserviceStatus(microservice.getMicroserviceUuid(), MicroserviceState.PULLING);
+		Registry registry = getRegistry(microservice);
+		if (!registry.getUrl().equals("from_cache")) {
+			try {
+				docker.pullImage(microservice.getImageName(), microservice.getMicroserviceUuid(), 
+						microservice.getPlatform(), registry);
+				StatusReporter.setProcessManagerStatus().setMicroservicesStatePercentage(
+						microservice.getMicroserviceUuid(), Constants.PERCENTAGE_COMPLETION);
+				LoggingService.logInfo(MODULE_NAME, "Successfully pulled image \"" + microservice.getImageName() + "\" while old container was running");
+			} catch (Exception e) {
+				LoggingService.logError(MODULE_NAME, 
+						"unable to pull \"" + microservice.getImageName() + "\" from registry. trying local cache",
+						new AgentSystemException(e.getMessage(), e));
+				// Continue with local cache if pull fails
+			}
+		}
+		
+		// Verify image exists (either pulled or in cache)
+		if (!docker.findLocalImage(microservice.getImageName())) {
+			microservice.setUpdating(false);
+			throw new NotFoundException("Image not found: " + microservice.getImageName() + 
+					". Pull failed and image not in local cache.");
+		}
+		
+		// Step 2: Now stop and remove old container (releases ports)
+		// Downtime starts here, but it's brief compared to pull time
 		removeContainerByMicroserviceUuid(microservice.getMicroserviceUuid(), withCleanUp);
-		createContainer(microservice);
+		
+		// Step 3: Create and start new container (can use same ports now)
+		// Pass false to createContainer to skip pulling since we already pulled
+		createContainer(microservice, false);
+		
 		microservice.setUpdating(false);
 		LoggingService.logDebug(MODULE_NAME, "Finished update container for microservice : " + microservice.getImageName());
 	}
@@ -260,17 +297,19 @@ public void execute(ContainerTask task) throws Exception {
 				case CREATE_EXEC:
 					if (microserviceOptional.isPresent()) {
 						ExecSessionCallback pmCallback = task.getCallback();
-						// Get the stdin pipe from ProcessManager.ExecSessionCallback
+						// Get both pipes from ProcessManager.ExecSessionCallback
 						PipedInputStream stdinPipe = pmCallback.getStdinPipe();
-						if (stdinPipe == null) {
-							throw new AgentSystemException("Failed to get stdin pipe from callback", null);
+						PipedOutputStream stdinOutputStream = pmCallback.getStdin();
+						if (stdinPipe == null || stdinOutputStream == null) {
+							throw new AgentSystemException("Failed to get stdin pipes from callback", null);
 						}
 
 						// Create a new DockerUtil.ExecSessionCallback that forwards to the ProcessManager callback
 						DockerUtil.ExecSessionCallback dockerCallback = docker.new ExecSessionCallback(
 							"iofog_" + task.getMicroserviceUuid(), // Use a unique ID for the exec session
 							30, // 30 minutes timeout
-							stdinPipe
+							stdinPipe,
+							stdinOutputStream  // Pass the output stream
 						) {
 							@Override
 							public void onNext(Frame frame) {

iofog-agent-daemon/src/main/java/org/eclipse/iofog/process_manager/DockerUtil.java

@@ -1390,24 +1390,20 @@ public class ExecSessionCallback extends ResultCallback.Adapter<Frame> {
         private PipedInputStream ptyStdinPipe;
         private long lastActivityTime;
 
-        public ExecSessionCallback(String execId, long inactivityTimeoutMinutes, PipedInputStream stdinPipe) {
+        public ExecSessionCallback(String execId, long inactivityTimeoutMinutes, 
+                                  PipedInputStream stdinPipe, PipedOutputStream stdinOutputStream) {
             this.execId = execId;
             this.startTime = System.currentTimeMillis();
             this.inactivityTimeoutMinutes = inactivityTimeoutMinutes;
             this.lastActivityTime = startTime;
 
-            // Use the provided pipe instead of creating a new one
+            // Use the provided pipes instead of creating new ones
             this.ptyStdinPipe = stdinPipe;
+            this.ptyStdin = stdinOutputStream;  // Use the provided output stream
 
-            // Create the output stream connected to the input pipe
-            try {
-                this.ptyStdin = new PipedOutputStream(ptyStdinPipe);
-                LoggingService.logDebug(MODULE_NAME, "Created output stream for exec session: " + execId + 
-                    ", ptyStdin=" + (ptyStdin != null) + 
-                    ", ptyStdinPipe=" + (ptyStdinPipe != null));
-            } catch (IOException e) {
-                LoggingService.logError(MODULE_NAME, "Failed to create output stream for exec session: " + execId, e);
-            }
+            LoggingService.logDebug(MODULE_NAME, "Created exec session callback: " + execId + 
+                ", ptyStdin=" + (ptyStdin != null) + 
+                ", ptyStdinPipe=" + (ptyStdinPipe != null));
         }
 
         private void resetInactivityTimer() {
@@ -1450,8 +1446,8 @@ public void onComplete() {
 
         @Override
         public void close() throws IOException {
-            if (ptyStdinPipe != null) ptyStdinPipe.close();
-            if (ptyStdin != null) ptyStdin.close();
+            // Don't close pipes we don't own - ProcessManager owns them
+            // Only call super.close() to clean up the callback itself
             super.close();
         }
 

iofog-agent-daemon/src/main/java/org/eclipse/iofog/process_manager/ExecSessionCallback.java

@@ -334,4 +334,9 @@ public PipedInputStream getStdinPipe() {
         LoggingService.logDebug(MODULE_NAME, "Getting stdin pipe: " + (ptyStdinPipe != null));
         return ptyStdinPipe;
     }
+
+    public PipedOutputStream getStdin() {
+        LoggingService.logDebug(MODULE_NAME, "Getting stdin output stream: " + (stdin != null));
+        return (PipedOutputStream) stdin;
+    }
 } 

iofog-agent-daemon/src/test/java/org/eclipse/iofog/command_line/CommandLineActionTest.java

@@ -93,7 +93,7 @@ public void setup() {
                 .thenReturn(new HashMap<>())
                 .thenThrow(new Exception("item not found or defined more than once"));
 
-        Mockito.when(CmdProperties.getVersion()).thenReturn("3.5.5");
+        Mockito.when(CmdProperties.getVersion()).thenReturn("3.5.6");
         Mockito.when(CmdProperties.getVersionMessage()).thenReturn(version);
         Mockito.when(CmdProperties.getDeprovisionMessage()).thenReturn("Deprovisioning from controller ... %s");
         Mockito.when(CmdProperties.getProvisionMessage()).thenReturn("Provisioning with key \"%s\" ... Result: %s");
@@ -364,7 +364,7 @@ private static boolean isEqual(List list1, List list2) {
             "0.00 MB\\nSystem Available Memory     : " +
             "0.00 MB\\nSystem Total CPU            : 0.00 %";
 
-    private String version = "ioFog Agent 3.5.5 \n" +
+    private String version = "ioFog Agent 3.5.6 \n" +
             "Copyright (c) 2023 Datasance Teknoloji A.S. \n" +
             "Eclipse ioFog is provided under the Eclipse Public License 2.0 (EPL-2.0) \n" +
             "https://www.eclipse.org/legal/epl-v20.html";

iofog-agent-daemon/src/test/java/org/eclipse/iofog/utils/CmdPropertiesTest.java

@@ -46,7 +46,7 @@ public void tearDown() throws Exception {
 
     //@Test
     //public void getVersionMessage() {
-    //    assertEquals("ioFog Agent 3.5.5 \nCopyright (c) 2023 Datasance Teknoloji A.S. \nEclipse ioFog is provided under the Eclipse Public License 2.0 (EPL-2.0) \nhttps://www.eclipse.org/legal/epl-v20.html",
+    //    assertEquals("ioFog Agent 3.5.6 \nCopyright (c) 2023 Datasance Teknoloji A.S. \nEclipse ioFog is provided under the Eclipse Public License 2.0 (EPL-2.0) \nhttps://www.eclipse.org/legal/epl-v20.html",
     //            CmdProperties.getVersionMessage());
     //}