diff --git a/common/src/main/java/org/jboss/fuse/mvnd/common/Environment.java b/common/src/main/java/org/jboss/fuse/mvnd/common/Environment.java index 2f558459..f1a096c8 100644 --- a/common/src/main/java/org/jboss/fuse/mvnd/common/Environment.java +++ b/common/src/main/java/org/jboss/fuse/mvnd/common/Environment.java @@ -65,12 +65,19 @@ public enum Environment { /** * Internal option to specify the list of maven extension to register */ - DAEMON_CORE_EXTENSIONS("daemon.core.extensions", null); + DAEMON_CORE_EXTENSIONS("daemon.core.extensions", null), + /** + * Interval to check if the daemon should expire + */ + EXPIRATION_CHECK_DELAY_MS("daemon.expirationCheckDelayMs", null), + ; public static final int DEFAULT_IDLE_TIMEOUT = (int) TimeUnit.HOURS.toMillis(3); public static final int DEFAULT_KEEP_ALIVE = (int) TimeUnit.SECONDS.toMillis(1); + public static final int DEFAULT_EXPIRATION_CHECK_DELAY = (int) TimeUnit.SECONDS.toMillis(10); + public static final int DEFAULT_MAX_LOST_KEEP_ALIVE = 3; public static final int DEFAULT_MIN_THREADS = 1; diff --git a/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/DaemonExpiration.java b/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/DaemonExpiration.java index 242f7d0c..fc998500 100644 --- a/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/DaemonExpiration.java +++ b/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/DaemonExpiration.java @@ -31,6 +31,7 @@ import org.jboss.fuse.mvnd.common.DaemonState; import static org.jboss.fuse.mvnd.common.DaemonExpirationStatus.DO_NOT_EXPIRE; import static org.jboss.fuse.mvnd.common.DaemonExpirationStatus.GRACEFUL_EXPIRE; +import static org.jboss.fuse.mvnd.common.DaemonExpirationStatus.IMMEDIATE_EXPIRE; import static org.jboss.fuse.mvnd.common.DaemonExpirationStatus.QUIET_EXPIRE; import static org.jboss.fuse.mvnd.daemon.DaemonExpiration.DaemonExpirationResult.NOT_TRIGGERED; @@ -58,18 +59,21 @@ public class DaemonExpiration { } static DaemonExpirationStrategy gcTrashing() { - // TODO - return daemon -> NOT_TRIGGERED; + return daemon -> daemon.getMemoryStatus().isTrashing() + ? new DaemonExpirationResult(IMMEDIATE_EXPIRE, "JVM garbage collector thrashing") + : NOT_TRIGGERED; } static DaemonExpirationStrategy lowHeapSpace() { - // TODO - return daemon -> NOT_TRIGGERED; + return daemon -> daemon.getMemoryStatus().isHeapSpaceExhausted() + ? new DaemonExpirationResult(GRACEFUL_EXPIRE, "after running out of JVM memory") + : NOT_TRIGGERED; } static DaemonExpirationStrategy lowNonHeap() { - // TODO - return daemon -> NOT_TRIGGERED; + return daemon -> daemon.getMemoryStatus().isNonHeapSpaceExhausted() + ? new DaemonExpirationResult(GRACEFUL_EXPIRE, "after running out of JVM memory") + : NOT_TRIGGERED; } static DaemonExpirationStrategy lowMemory(double minFreeMemoryPercentage) { diff --git a/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/DaemonMemoryStatus.java b/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/DaemonMemoryStatus.java new file mode 100644 index 00000000..c6e3b967 --- /dev/null +++ b/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/DaemonMemoryStatus.java @@ -0,0 +1,223 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.fuse.mvnd.daemon; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryUsage; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.Deque; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class DaemonMemoryStatus { + + static final int MAX_EVENTS = 20; + + final GcStrategy strategy; + + final GarbageCollectorMXBean garbageCollectorMXBean; + final MemoryPoolMXBean heapMemoryPoolMXBean; + final MemoryPoolMXBean nonHeapMemoryPoolMXBean; + final Clock clock; + final Deque heapEvents = new ConcurrentLinkedDeque<>(); + final Deque nonHeapEvents = new ConcurrentLinkedDeque<>(); + + public enum GcStrategy { + ORACLE_PARALLEL_CMS("PS Old Gen", "Metaspace", "PS MarkSweep", 1.2, 80, 80, 5.0), + ORACLE_6_CMS("CMS Old Gen", "Metaspace", "ConcurrentMarkSweep", 1.2, 80, 80, 5.0), + ORACLE_SERIAL("Tenured Gen", "Metaspace", "MarkSweepCompact", 1.2, 80, 80, 5.0), + ORACLE_G1("G1 Old Gen", "Metaspace", "G1 Old Generation", 0.4, 75, 80, 2.0), + IBM_ALL("Java heap", "Not Used", "MarkSweepCompact", 0.8, 70, -1, 6.0); + + final String garbageCollector; + final String heapMemoryPool; + final String nonHeapMemoryPool; + final int heapUsageThreshold; + final double heapRateThreshold; + final int nonHeapUsageThreshold; + final double thrashingThreshold; + + GcStrategy(String heapMemoryPool, String nonHeapMemoryPool, String garbageCollector, + double heapRateThreshold, int heapUsageThreshold, + int nonHeapUsageThreshold, double thrashingThreshold) { + this.garbageCollector = garbageCollector; + this.heapMemoryPool = heapMemoryPool; + this.nonHeapMemoryPool = nonHeapMemoryPool; + this.heapUsageThreshold = heapUsageThreshold; + this.heapRateThreshold = heapRateThreshold; + this.nonHeapUsageThreshold = nonHeapUsageThreshold; + this.thrashingThreshold = thrashingThreshold; + } + } + + static class GcEvent { + final Instant timestamp; + final MemoryUsage usage; + final long count; + + public GcEvent(Instant timestamp, MemoryUsage usage, long count) { + this.timestamp = timestamp; + this.usage = usage; + this.count = count; + } + } + + static class GcStats { + final double gcRate; + final int usedPercent; + + public GcStats(double gcRate, int usedPercent) { + this.gcRate = gcRate; + this.usedPercent = usedPercent; + } + } + + public DaemonMemoryStatus(ScheduledExecutorService executor) { + List garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans(); + List memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans(); + GcStrategy strategy = null; + GarbageCollectorMXBean garbageCollector = null; + MemoryPoolMXBean heapMemoryPoolMXBean = null; + MemoryPoolMXBean nonHeapMemoryPoolMXBean = null; + for (GcStrategy testStrategy : GcStrategy.values()) { + garbageCollector = garbageCollectors.stream() + .filter(gc -> gc.getName().equals(testStrategy.garbageCollector)) + .findFirst().orElse(null); + heapMemoryPoolMXBean = memoryPoolMXBeans.stream() + .filter(mp -> mp.getName().equals(testStrategy.heapMemoryPool)) + .findFirst().orElse(null); + nonHeapMemoryPoolMXBean = memoryPoolMXBeans.stream() + .filter(mp -> mp.getName().equals(testStrategy.nonHeapMemoryPool)) + .findFirst().orElse(null); + if (garbageCollector != null && heapMemoryPoolMXBean != null && nonHeapMemoryPoolMXBean != null) { + strategy = testStrategy; + break; + } + } + if (strategy != null) { + this.strategy = strategy; + this.garbageCollectorMXBean = garbageCollector; + this.heapMemoryPoolMXBean = heapMemoryPoolMXBean; + this.nonHeapMemoryPoolMXBean = nonHeapMemoryPoolMXBean; + this.clock = Clock.systemUTC(); + executor.scheduleAtFixedRate(this::gatherData, 1, 1, TimeUnit.SECONDS); + } else { + this.strategy = null; + this.garbageCollectorMXBean = null; + this.heapMemoryPoolMXBean = null; + this.nonHeapMemoryPoolMXBean = null; + this.clock = null; + } + } + + protected void gatherData() { + GcEvent latest = heapEvents.peekLast(); + long currentCount = garbageCollectorMXBean.getCollectionCount(); + // There has been a GC event + if (latest == null || latest.count != currentCount) { + slideAndInsert(heapEvents, new GcEvent(clock.instant(), heapMemoryPoolMXBean.getCollectionUsage(), currentCount)); + } + slideAndInsert(nonHeapEvents, new GcEvent(clock.instant(), nonHeapMemoryPoolMXBean.getUsage(), -1)); + } + + private void slideAndInsert(Deque events, GcEvent event) { + events.addLast(event); + while (events.size() > MAX_EVENTS) { + events.pollFirst(); + } + } + + public boolean isTrashing() { + if (strategy.heapUsageThreshold != 0 && strategy.thrashingThreshold != 0) { + GcStats stats = heapStats(); + return stats != null + && stats.usedPercent >= strategy.heapUsageThreshold + && stats.gcRate >= strategy.thrashingThreshold; + } else { + return false; + } + } + + public boolean isHeapSpaceExhausted() { + if (strategy.heapUsageThreshold != 0 && strategy.heapRateThreshold != 0) { + GcStats stats = heapStats(); + return stats != null + && stats.usedPercent >= strategy.heapUsageThreshold + && stats.gcRate >= strategy.heapRateThreshold; + } else { + return false; + } + } + + public boolean isNonHeapSpaceExhausted() { + if (strategy.nonHeapUsageThreshold != 0) { + GcStats stats = nonHeapStats(); + return stats != null + && stats.usedPercent >= strategy.nonHeapUsageThreshold; + } else { + return false; + } + } + + private GcStats heapStats() { + if (heapEvents.size() >= 5) { + // Maximum pool size is fixed, so we should only need to get it from the first event + GcEvent first = heapEvents.iterator().next(); + long maxSizeInBytes = first.usage.getMax(); + if (maxSizeInBytes > 0) { + double gcRate = gcRate(heapEvents); + int usagePercent = (int) (averageUsage(heapEvents) * 100.0f / maxSizeInBytes); + return new GcStats(gcRate, usagePercent); + } + } + return null; + } + + private GcStats nonHeapStats() { + if (nonHeapEvents.size() >= 5) { + // Maximum pool size is fixed, so we should only need to get it from the first event + GcEvent first = heapEvents.iterator().next(); + long maxSizeInBytes = first.usage.getMax(); + if (maxSizeInBytes > 0) { + int usagePercent = (int) (averageUsage(nonHeapEvents) * 100.0f / maxSizeInBytes); + return new GcStats(0, usagePercent); + } + } + return null; + } + + private double gcRate(Deque events) { + GcEvent first = events.peekFirst(); + GcEvent last = events.peekLast(); + // Total number of garbage collection events observed in the window + double gcCountDelta = last.count - first.count; + // Time interval between the first event in the window and the last + double timeDelta = Duration.between(first.timestamp, last.timestamp).toMillis(); + return gcCountDelta / timeDelta; + } + + private double averageUsage(Collection events) { + return events.stream().mapToLong(e -> e.usage.getUsed()).average().getAsDouble(); + } + +} diff --git a/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/Server.java b/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/Server.java index d66efbe0..1b0a2abe 100644 --- a/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/Server.java +++ b/daemon/src/main/java/org/jboss/fuse/mvnd/daemon/Server.java @@ -83,6 +83,7 @@ public class Server implements AutoCloseable, Runnable { private final Lock expirationLock = new ReentrantLock(); private final Lock stateLock = new ReentrantLock(); private final Condition condition = stateLock.newCondition(); + private final DaemonMemoryStatus memoryStatus; public Server(String uid) throws IOException { this.uid = uid; @@ -99,6 +100,7 @@ public class Server implements AutoCloseable, Runnable { .asInt(); executor = Executors.newScheduledThreadPool(1); strategy = DaemonExpiration.master(); + memoryStatus = new DaemonMemoryStatus(executor); List opts = new ArrayList<>(); Environment.DAEMON_EXT_CLASSPATH.systemProperty().asOptional() @@ -117,6 +119,10 @@ public class Server implements AutoCloseable, Runnable { } } + public DaemonMemoryStatus getMemoryStatus() { + return memoryStatus; + } + public void close() { try { try { @@ -156,8 +162,12 @@ public class Server implements AutoCloseable, Runnable { public void run() { try { + int expirationCheckDelayMs = Environment.EXPIRATION_CHECK_DELAY_MS + .systemProperty() + .orDefault(() -> String.valueOf(Environment.DEFAULT_EXPIRATION_CHECK_DELAY)) + .asInt(); executor.scheduleAtFixedRate(this::expirationCheck, - info.getIdleTimeout(), info.getIdleTimeout(), TimeUnit.MILLISECONDS); + expirationCheckDelayMs, expirationCheckDelayMs, TimeUnit.MILLISECONDS); LOGGER.info("Daemon started"); new DaemonThread(this::accept).start(); awaitStop();