Add JVM memory expiration checks, use a specific timeout for checks

common/src/main/java/org/jboss/fuse/mvnd/common/Environment.java

@@ -65,12 +65,19 @@ public enum Environment {
     /**
      * Internal option to specify the list of maven extension to register
      */
-    DAEMON_CORE_EXTENSIONS("daemon.core.extensions", null);
+    DAEMON_CORE_EXTENSIONS("daemon.core.extensions", null),
+    /**
+     * Interval to check if the daemon should expire
+     */
+    EXPIRATION_CHECK_DELAY_MS("daemon.expirationCheckDelayMs", null),
+    ;
 
     public static final int DEFAULT_IDLE_TIMEOUT = (int) TimeUnit.HOURS.toMillis(3);
 
     public static final int DEFAULT_KEEP_ALIVE = (int) TimeUnit.SECONDS.toMillis(1);
 
+    public static final int DEFAULT_EXPIRATION_CHECK_DELAY = (int) TimeUnit.SECONDS.toMillis(10);
+
     public static final int DEFAULT_MAX_LOST_KEEP_ALIVE = 3;
 
     public static final int DEFAULT_MIN_THREADS = 1;

daemon/src/main/java/org/jboss/fuse/mvnd/daemon/DaemonExpiration.java

@@ -31,6 +31,7 @@ import org.jboss.fuse.mvnd.common.DaemonState;
 
 import static org.jboss.fuse.mvnd.common.DaemonExpirationStatus.DO_NOT_EXPIRE;
 import static org.jboss.fuse.mvnd.common.DaemonExpirationStatus.GRACEFUL_EXPIRE;
+import static org.jboss.fuse.mvnd.common.DaemonExpirationStatus.IMMEDIATE_EXPIRE;
 import static org.jboss.fuse.mvnd.common.DaemonExpirationStatus.QUIET_EXPIRE;
 import static org.jboss.fuse.mvnd.daemon.DaemonExpiration.DaemonExpirationResult.NOT_TRIGGERED;
 
@@ -58,18 +59,21 @@ public class DaemonExpiration {
     }
 
     static DaemonExpirationStrategy gcTrashing() {
-        // TODO
+        return daemon -> daemon.getMemoryStatus().isTrashing()
-        return daemon -> NOT_TRIGGERED;
+                ? new DaemonExpirationResult(IMMEDIATE_EXPIRE, "JVM garbage collector thrashing")
+                : NOT_TRIGGERED;
     }
 
     static DaemonExpirationStrategy lowHeapSpace() {
-        // TODO
+        return daemon -> daemon.getMemoryStatus().isHeapSpaceExhausted()
-        return daemon -> NOT_TRIGGERED;
+                ? new DaemonExpirationResult(GRACEFUL_EXPIRE, "after running out of JVM memory")
+                : NOT_TRIGGERED;
     }
 
     static DaemonExpirationStrategy lowNonHeap() {
-        // TODO
+        return daemon -> daemon.getMemoryStatus().isNonHeapSpaceExhausted()
-        return daemon -> NOT_TRIGGERED;
+                ? new DaemonExpirationResult(GRACEFUL_EXPIRE, "after running out of JVM memory")
+                : NOT_TRIGGERED;
     }
 
     static DaemonExpirationStrategy lowMemory(double minFreeMemoryPercentage) {

daemon/src/main/java/org/jboss/fuse/mvnd/daemon/DaemonMemoryStatus.java

@@ -0,0 +1,223 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jboss.fuse.mvnd.daemon;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryPoolMXBean;
+import java.lang.management.MemoryUsage;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.List;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class DaemonMemoryStatus {
+
+    static final int MAX_EVENTS = 20;
+
+    final GcStrategy strategy;
+
+    final GarbageCollectorMXBean garbageCollectorMXBean;
+    final MemoryPoolMXBean heapMemoryPoolMXBean;
+    final MemoryPoolMXBean nonHeapMemoryPoolMXBean;
+    final Clock clock;
+    final Deque<GcEvent> heapEvents = new ConcurrentLinkedDeque<>();
+    final Deque<GcEvent> nonHeapEvents = new ConcurrentLinkedDeque<>();
+
+    public enum GcStrategy {
+        ORACLE_PARALLEL_CMS("PS Old Gen", "Metaspace", "PS MarkSweep", 1.2, 80, 80, 5.0),
+        ORACLE_6_CMS("CMS Old Gen", "Metaspace", "ConcurrentMarkSweep", 1.2, 80, 80, 5.0),
+        ORACLE_SERIAL("Tenured Gen", "Metaspace", "MarkSweepCompact", 1.2, 80, 80, 5.0),
+        ORACLE_G1("G1 Old Gen", "Metaspace", "G1 Old Generation", 0.4, 75, 80, 2.0),
+        IBM_ALL("Java heap", "Not Used", "MarkSweepCompact", 0.8, 70, -1, 6.0);
+
+        final String garbageCollector;
+        final String heapMemoryPool;
+        final String nonHeapMemoryPool;
+        final int heapUsageThreshold;
+        final double heapRateThreshold;
+        final int nonHeapUsageThreshold;
+        final double thrashingThreshold;
+
+        GcStrategy(String heapMemoryPool, String nonHeapMemoryPool, String garbageCollector,
+                double heapRateThreshold, int heapUsageThreshold,
+                int nonHeapUsageThreshold, double thrashingThreshold) {
+            this.garbageCollector = garbageCollector;
+            this.heapMemoryPool = heapMemoryPool;
+            this.nonHeapMemoryPool = nonHeapMemoryPool;
+            this.heapUsageThreshold = heapUsageThreshold;
+            this.heapRateThreshold = heapRateThreshold;
+            this.nonHeapUsageThreshold = nonHeapUsageThreshold;
+            this.thrashingThreshold = thrashingThreshold;
+        }
+    }
+
+    static class GcEvent {
+        final Instant timestamp;
+        final MemoryUsage usage;
+        final long count;
+
+        public GcEvent(Instant timestamp, MemoryUsage usage, long count) {
+            this.timestamp = timestamp;
+            this.usage = usage;
+            this.count = count;
+        }
+    }
+
+    static class GcStats {
+        final double gcRate;
+        final int usedPercent;
+
+        public GcStats(double gcRate, int usedPercent) {
+            this.gcRate = gcRate;
+            this.usedPercent = usedPercent;
+        }
+    }
+
+    public DaemonMemoryStatus(ScheduledExecutorService executor) {
+        List<GarbageCollectorMXBean> garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans();
+        List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
+        GcStrategy strategy = null;
+        GarbageCollectorMXBean garbageCollector = null;
+        MemoryPoolMXBean heapMemoryPoolMXBean = null;
+        MemoryPoolMXBean nonHeapMemoryPoolMXBean = null;
+        for (GcStrategy testStrategy : GcStrategy.values()) {
+            garbageCollector = garbageCollectors.stream()
+                    .filter(gc -> gc.getName().equals(testStrategy.garbageCollector))
+                    .findFirst().orElse(null);
+            heapMemoryPoolMXBean = memoryPoolMXBeans.stream()
+                    .filter(mp -> mp.getName().equals(testStrategy.heapMemoryPool))
+                    .findFirst().orElse(null);
+            nonHeapMemoryPoolMXBean = memoryPoolMXBeans.stream()
+                    .filter(mp -> mp.getName().equals(testStrategy.nonHeapMemoryPool))
+                    .findFirst().orElse(null);
+            if (garbageCollector != null && heapMemoryPoolMXBean != null && nonHeapMemoryPoolMXBean != null) {
+                strategy = testStrategy;
+                break;
+            }
+        }
+        if (strategy != null) {
+            this.strategy = strategy;
+            this.garbageCollectorMXBean = garbageCollector;
+            this.heapMemoryPoolMXBean = heapMemoryPoolMXBean;
+            this.nonHeapMemoryPoolMXBean = nonHeapMemoryPoolMXBean;
+            this.clock = Clock.systemUTC();
+            executor.scheduleAtFixedRate(this::gatherData, 1, 1, TimeUnit.SECONDS);
+        } else {
+            this.strategy = null;
+            this.garbageCollectorMXBean = null;
+            this.heapMemoryPoolMXBean = null;
+            this.nonHeapMemoryPoolMXBean = null;
+            this.clock = null;
+        }
+    }
+
+    protected void gatherData() {
+        GcEvent latest = heapEvents.peekLast();
+        long currentCount = garbageCollectorMXBean.getCollectionCount();
+        // There has been a GC event
+        if (latest == null || latest.count != currentCount) {
+            slideAndInsert(heapEvents, new GcEvent(clock.instant(), heapMemoryPoolMXBean.getCollectionUsage(), currentCount));
+        }
+        slideAndInsert(nonHeapEvents, new GcEvent(clock.instant(), nonHeapMemoryPoolMXBean.getUsage(), -1));
+    }
+
+    private void slideAndInsert(Deque<GcEvent> events, GcEvent event) {
+        events.addLast(event);
+        while (events.size() > MAX_EVENTS) {
+            events.pollFirst();
+        }
+    }
+
+    public boolean isTrashing() {
+        if (strategy.heapUsageThreshold != 0 && strategy.thrashingThreshold != 0) {
+            GcStats stats = heapStats();
+            return stats != null
+                    && stats.usedPercent >= strategy.heapUsageThreshold
+                    && stats.gcRate >= strategy.thrashingThreshold;
+        } else {
+            return false;
+        }
+    }
+
+    public boolean isHeapSpaceExhausted() {
+        if (strategy.heapUsageThreshold != 0 && strategy.heapRateThreshold != 0) {
+            GcStats stats = heapStats();
+            return stats != null
+                    && stats.usedPercent >= strategy.heapUsageThreshold
+                    && stats.gcRate >= strategy.heapRateThreshold;
+        } else {
+            return false;
+        }
+    }
+
+    public boolean isNonHeapSpaceExhausted() {
+        if (strategy.nonHeapUsageThreshold != 0) {
+            GcStats stats = nonHeapStats();
+            return stats != null
+                    && stats.usedPercent >= strategy.nonHeapUsageThreshold;
+        } else {
+            return false;
+        }
+    }
+
+    private GcStats heapStats() {
+        if (heapEvents.size() >= 5) {
+            // Maximum pool size is fixed, so we should only need to get it from the first event
+            GcEvent first = heapEvents.iterator().next();
+            long maxSizeInBytes = first.usage.getMax();
+            if (maxSizeInBytes > 0) {
+                double gcRate = gcRate(heapEvents);
+                int usagePercent = (int) (averageUsage(heapEvents) * 100.0f / maxSizeInBytes);
+                return new GcStats(gcRate, usagePercent);
+            }
+        }
+        return null;
+    }
+
+    private GcStats nonHeapStats() {
+        if (nonHeapEvents.size() >= 5) {
+            // Maximum pool size is fixed, so we should only need to get it from the first event
+            GcEvent first = heapEvents.iterator().next();
+            long maxSizeInBytes = first.usage.getMax();
+            if (maxSizeInBytes > 0) {
+                int usagePercent = (int) (averageUsage(nonHeapEvents) * 100.0f / maxSizeInBytes);
+                return new GcStats(0, usagePercent);
+            }
+        }
+        return null;
+    }
+
+    private double gcRate(Deque<GcEvent> events) {
+        GcEvent first = events.peekFirst();
+        GcEvent last = events.peekLast();
+        // Total number of garbage collection events observed in the window
+        double gcCountDelta = last.count - first.count;
+        // Time interval between the first event in the window and the last
+        double timeDelta = Duration.between(first.timestamp, last.timestamp).toMillis();
+        return gcCountDelta / timeDelta;
+    }
+
+    private double averageUsage(Collection<GcEvent> events) {
+        return events.stream().mapToLong(e -> e.usage.getUsed()).average().getAsDouble();
+    }
+
+}

daemon/src/main/java/org/jboss/fuse/mvnd/daemon/Server.java

@@ -83,6 +83,7 @@ public class Server implements AutoCloseable, Runnable {
     private final Lock expirationLock = new ReentrantLock();
     private final Lock stateLock = new ReentrantLock();
     private final Condition condition = stateLock.newCondition();
+    private final DaemonMemoryStatus memoryStatus;
 
     public Server(String uid) throws IOException {
         this.uid = uid;
@@ -99,6 +100,7 @@ public class Server implements AutoCloseable, Runnable {
                     .asInt();
             executor = Executors.newScheduledThreadPool(1);
             strategy = DaemonExpiration.master();
+            memoryStatus = new DaemonMemoryStatus(executor);
 
             List<String> opts = new ArrayList<>();
             Environment.DAEMON_EXT_CLASSPATH.systemProperty().asOptional()
@@ -117,6 +119,10 @@ public class Server implements AutoCloseable, Runnable {
         }
     }
 
+    public DaemonMemoryStatus getMemoryStatus() {
+        return memoryStatus;
+    }
+
     public void close() {
         try {
             try {
@@ -156,8 +162,12 @@ public class Server implements AutoCloseable, Runnable {
 
     public void run() {
         try {
+            int expirationCheckDelayMs = Environment.EXPIRATION_CHECK_DELAY_MS
+                    .systemProperty()
+                    .orDefault(() -> String.valueOf(Environment.DEFAULT_EXPIRATION_CHECK_DELAY))
+                    .asInt();
             executor.scheduleAtFixedRate(this::expirationCheck,
-                    info.getIdleTimeout(), info.getIdleTimeout(), TimeUnit.MILLISECONDS);
+                    expirationCheckDelayMs, expirationCheckDelayMs, TimeUnit.MILLISECONDS);
             LOGGER.info("Daemon started");
             new DaemonThread(this::accept).start();
             awaitStop();