apache/maven-mvnd
1
0
Fork 0
mirror of https://github.com/apache/maven-mvnd.git synced 2025-10-15 06:40:54 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Switch back to takari smart builder (#805)

Browse Source
This commit is contained in:
Guillaume Nodet
2023-03-10 08:48:08 +01:00
committed by GitHub
parent 6e4e9df4a9
commit 65da451ecd
20 changed files with 22 additions and 2059 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
daemon/pom.xml
View File

@@ -54,6 +54,12 @@
<artifactId>plexus-interactivity-api</artifactId> <artifactId>plexus-interactivity-api</artifactId>
</dependency> </dependency>
<dependency>
<groupId>io.takari.maven</groupId>
<artifactId>takari-smart-builder</artifactId>
<version>0.6.2</version>
</dependency>
<!-- Logging --> <!-- Logging -->
<dependency> <dependency>
<groupId>ch.qos.logback</groupId> <groupId>ch.qos.logback</groupId>

335
daemon/src/main/java/org/mvndaemon/mvnd/builder/DependencyGraph.java
View File

@@ -1,335 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.io.IOException;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.execution.ProjectDependencyGraph;
import org.apache.maven.project.MavenProject;
/**
* File origin:
* https://github.com/takari/takari-smart-builder/blob/takari-smart-builder-0.6.1/src/main/java/io/takari/maven/builder/smart/DependencyGraph.java
*/
public class DependencyGraph<K> {
private final List<K> projects;
private final Map<K, List<K>> upstreams;
private final Map<K, Set<K>> transitiveUpstreams;
private final Map<K, List<K>> downstreams;
@SuppressWarnings("unchecked")
public static DependencyGraph<MavenProject> fromMaven(MavenSession session) {
Map<String, Object> data = session.getRequest().getData();
DependencyGraph<MavenProject> graph = (DependencyGraph<MavenProject>) data.get(DependencyGraph.class.getName());
if (graph == null) {
graph = fromMaven(session.getProjectDependencyGraph());
data.put(DependencyGraph.class.getName(), graph);
}
return graph;
}
static DependencyGraph<MavenProject> fromMaven(ProjectDependencyGraph graph) {
final List<MavenProject> projects = graph.getSortedProjects();
Map<MavenProject, List<MavenProject>> upstreams =
projects.stream().collect(Collectors.toMap(p -> p, p -> graph.getUpstreamProjects(p, false)));
Map<MavenProject, List<MavenProject>> downstreams =
projects.stream().collect(Collectors.toMap(p -> p, p -> graph.getDownstreamProjects(p, false)));
return new DependencyGraph<>(Collections.unmodifiableList(projects), upstreams, downstreams);
}
public DependencyGraph(List<K> projects, Map<K, List<K>> upstreams, Map<K, List<K>> downstreams) {
this.projects = projects;
this.upstreams = upstreams;
this.downstreams = downstreams;
this.transitiveUpstreams = new HashMap<>();
projects.forEach(this::transitiveUpstreams); // topological ordering of projects matters
}
DependencyGraph(
List<K> projects,
Map<K, List<K>> upstreams,
Map<K, List<K>> downstreams,
Map<K, Set<K>> transitiveUpstreams) {
this.projects = projects;
this.upstreams = upstreams;
this.downstreams = downstreams;
this.transitiveUpstreams = transitiveUpstreams;
}
public Stream<K> getDownstreamProjects(K project) {
return downstreams.get(project).stream();
}
public Stream<K> getUpstreamProjects(K project) {
return upstreams.get(project).stream();
}
public boolean isRoot(K project) {
return upstreams.get(project).isEmpty();
}
public Stream<K> getProjects() {
return projects.stream();
}
public int computeMaxWidth(int max, long maxTimeMillis) {
return new DagWidth<>(this).getMaxWidth(max, maxTimeMillis);
}
public void store(Function<K, String> toString, Path path) {
try (Writer w = Files.newBufferedWriter(path)) {
store(toString, w);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public void store(Function<K, String> toString, Appendable w) {
getProjects().forEach(k -> {
try {
w.append(toString.apply(k));
w.append(" = ");
w.append(getUpstreamProjects(k).map(toString).collect(Collectors.joining(",")));
w.append(System.lineSeparator());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
store(k -> k.toString(), sb);
return sb.toString();
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((downstreams == null) ? 0 : downstreams.hashCode());
result = prime * result + ((projects == null) ? 0 : projects.hashCode());
result = prime * result + ((upstreams == null) ? 0 : upstreams.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
@SuppressWarnings("unchecked")
DependencyGraph<K> other = (DependencyGraph<K>) obj;
if (downstreams == null) {
if (other.downstreams != null) return false;
} else if (!downstreams.equals(other.downstreams)) return false;
if (projects == null) {
if (other.projects != null) return false;
} else if (!projects.equals(other.projects)) return false;
if (upstreams == null) {
if (other.upstreams != null) return false;
} else if (!upstreams.equals(other.upstreams)) return false;
return true;
}
/**
* Creates a new {@link DependencyGraph} which is a <a href="https://en.wikipedia.org/wiki/Transitive_reduction">
* transitive reduction</a> of this {@link DependencyGraph}. The reduction operation keeps the set of graph nodes
* unchanged and it reduces the set of edges in the following way: An edge {@code C -> A} is removed if an edge
* {@code C -> B} exists such that {@code A != B} and the set of nodes reachable from {@code B} contains {@code A};
* otherwise the edge {@code C -> A} is kept in the reduced graph.
* <p>
* Examples:
*
* <pre>
* Original Reduced
*
* A A
* /| /
* B | B
* \| \
* C C
*
*
* A A
* /|\ /
* B | | B
* \| | \
* C | C
* \| \
* D D
*
* </pre>
*
*
* @return a transitive reduction of this {@link DependencyGraph}
*/
DependencyGraph<K> reduce() {
final Map<K, List<K>> newUpstreams = new HashMap<>();
final Map<K, List<K>> newDownstreams = new HashMap<>();
for (K node : projects) {
final List<K> oldNodeUpstreams = upstreams.get(node);
final List<K> newNodeUpstreams;
newDownstreams.computeIfAbsent(node, k -> new ArrayList<>());
if (oldNodeUpstreams.size() == 0) {
newNodeUpstreams = new ArrayList<>(oldNodeUpstreams);
} else if (oldNodeUpstreams.size() == 1) {
newNodeUpstreams = new ArrayList<>(oldNodeUpstreams);
newDownstreams
.computeIfAbsent(newNodeUpstreams.get(0), k -> new ArrayList<>())
.add(node);
} else {
newNodeUpstreams = new ArrayList<>(oldNodeUpstreams.size());
for (K leftNode : oldNodeUpstreams) {
if (oldNodeUpstreams.stream()
.filter(rightNode -> leftNode != rightNode)
.noneMatch(rightNode ->
transitiveUpstreams.get(rightNode).contains(leftNode))) {
newNodeUpstreams.add(leftNode);
newDownstreams
.computeIfAbsent(leftNode, k -> new ArrayList<>())
.add(node);
}
}
}
newUpstreams.put(node, newNodeUpstreams);
}
return new DependencyGraph<K>(projects, newUpstreams, newDownstreams, transitiveUpstreams);
}
/**
* Compute the set of nodes reachable from the given {@code node} through the {@code is upstream of} relation. The
* {@code node} itself is not a part of the returned set.
*
* @param node the node for which the transitive upstream should be computed
* @return the set of transitive upstreams
*/
Set<K> transitiveUpstreams(K node) {
Set<K> result = transitiveUpstreams.get(node);
if (result == null) {
final List<K> firstOrderUpstreams = this.upstreams.get(node);
result = new HashSet<>(firstOrderUpstreams);
firstOrderUpstreams.stream().map(this::transitiveUpstreams).forEach(result::addAll);
transitiveUpstreams.put(node, result);
}
return result;
}
static class DagWidth<K> {
private final DependencyGraph<K> graph;
public DagWidth(DependencyGraph<K> graph) {
this.graph = graph.reduce();
}
public int getMaxWidth() {
return getMaxWidth(Integer.MAX_VALUE);
}
public int getMaxWidth(int maxmax) {
return getMaxWidth(maxmax, Long.MAX_VALUE);
}
public int getMaxWidth(int maxmax, long maxTimeMillis) {
int max = 0;
if (maxmax < graph.transitiveUpstreams.size()) {
// try inverted upstream bound
Map<Set<K>, Set<K>> mapByUpstreams = new HashMap<>();
graph.transitiveUpstreams.forEach((k, ups) -> {
mapByUpstreams.computeIfAbsent(ups, n -> new HashSet<>()).add(k);
});
max = mapByUpstreams.values().stream().mapToInt(Set::size).max().orElse(0);
if (max >= maxmax) {
return maxmax;
}
}
long tmax = System.currentTimeMillis() + maxTimeMillis;
int tries = 0;
SubsetIterator iterator = new SubsetIterator(getRoots());
while (max < maxmax && iterator.hasNext()) {
if (++tries % 100 == 0 && System.currentTimeMillis() < tmax) {
return maxmax;
}
List<K> l = iterator.next();
max = Math.max(max, l.size());
}
return Math.min(max, maxmax);
}
private class SubsetIterator implements Iterator<List<K>> {
final List<List<K>> nexts = new ArrayList<>();
final Set<List<K>> visited = new HashSet<>();
public SubsetIterator(List<K> roots) {
nexts.add(roots);
}
@Override
public boolean hasNext() {
return !nexts.isEmpty();
}
@Override
public List<K> next() {
List<K> list = nexts.remove(0);
list.stream()
.map(node -> ensembleWithChildrenOf(list, node))
.filter(visited::add)
.forEach(nexts::add);
return list;
}
}
private List<K> getRoots() {
return graph.getProjects().filter(graph::isRoot).collect(Collectors.toList());
}
List<K> ensembleWithChildrenOf(List<K> list, K node) {
final List<K> result = Stream.concat(
list.stream().filter(k -> !Objects.equals(k, node)),
graph.getDownstreamProjects(node).filter(k -> graph.transitiveUpstreams.get(k).stream()
.noneMatch(k2 -> !Objects.equals(k2, node) && list.contains(k2))))
.distinct()
.collect(Collectors.toList());
return result;
}
}
}

133
daemon/src/main/java/org/mvndaemon/mvnd/builder/ProjectComparator.java
View File

@@ -1,133 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.function.ToLongFunction;
import org.apache.maven.project.MavenProject;
/**
* Project comparator (factory) that uses project build time to establish build order.
* <p>
* Internally, each project is assigned a weight, which is calculated as sum of project build time
* and maximum weight of any of the project's downstream dependencies. The project weights are
* calculated by recursively traversing project dependency graph starting from build root projects,
* i.e. projects that do not have any upstream dependencies.
* <p>
* Project build times are estimated based on values persisted during a previous build. Average
* build time is used for projects that do not have persisted build time.
* <p>
* If there are no persisted build times, all projects build times are assumed the same (arbitrary)
* value of 1. This means that the project with the longest downstream dependency trail will be
* built first.
* <p>
* Currently, historical build times are stored in
* <code>${session.request/baseDirectory}/.mvn/timing.properties</code> file. The timings file is
* written only if <code>${session.request/baseDirectory}/.mvn</code> directory is already present.
*
* File origin:
* https://github.com/takari/takari-smart-builder/blob/takari-smart-builder-0.6.1/src/main/java/io/takari/maven/builder/smart/ProjectComparator.java
*/
class ProjectComparator {
public static Comparator<MavenProject> create(DependencyGraph<MavenProject> graph) {
return create0(graph, Collections.emptyMap(), ProjectComparator::id);
}
static <K> Comparator<K> create0(
DependencyGraph<K> dependencyGraph,
Map<String, AtomicLong> historicalServiceTimes,
Function<K, String> toKey) {
final long defaultServiceTime = average(historicalServiceTimes.values());
final Map<K, Long> serviceTimes = new HashMap<>();
final Set<K> rootProjects = new HashSet<>();
dependencyGraph.getProjects().forEach(project -> {
long serviceTime = getServiceTime(historicalServiceTimes, project, defaultServiceTime, toKey);
serviceTimes.put(project, serviceTime);
if (dependencyGraph.isRoot(project)) {
rootProjects.add(project);
}
});
final Map<K, Long> projectWeights = calculateWeights(dependencyGraph, serviceTimes, rootProjects);
return Comparator.comparingLong((ToLongFunction<K>) projectWeights::get)
.thenComparing(toKey, String::compareTo)
.reversed();
}
private static long average(Collection<AtomicLong> values) {
return (long)
(values.stream().mapToLong(AtomicLong::longValue).average().orElse(1.0d));
}
private static <K> long getServiceTime(
Map<String, AtomicLong> serviceTimes, K project, long defaultServiceTime, Function<K, String> toKey) {
AtomicLong serviceTime = serviceTimes.get(toKey.apply(project));
return serviceTime != null ? serviceTime.longValue() : defaultServiceTime;
}
private static <K> Map<K, Long> calculateWeights(
DependencyGraph<K> dependencyGraph, Map<K, Long> serviceTimes, Collection<K> rootProjects) {
Map<K, Long> weights = new HashMap<>();
for (K rootProject : rootProjects) {
calculateWeights(dependencyGraph, serviceTimes, rootProject, weights);
}
return weights;
}
/**
* Returns the maximum sum of build time along a path from the project to an exit project. An
* "exit project" is a project without downstream dependencies.
*/
private static <K> long calculateWeights(
DependencyGraph<K> dependencyGraph, Map<K, Long> serviceTimes, K project, Map<K, Long> weights) {
long weight = serviceTimes.get(project)
+ dependencyGraph
.getDownstreamProjects(project)
.mapToLong(successor -> {
long successorWeight;
if (weights.containsKey(successor)) {
successorWeight = weights.get(successor);
} else {
successorWeight = calculateWeights(dependencyGraph, serviceTimes, successor, weights);
}
return successorWeight;
})
.max()
.orElse(0);
weights.put(project, weight);
return weight;
}
static String id(MavenProject project) {
return project.getGroupId() + ':' + project.getArtifactId() + ':' + project.getVersion();
}
}

130
daemon/src/main/java/org/mvndaemon/mvnd/builder/ProjectExecutorService.java
View File

@@ -1,130 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.util.Collection;
import java.util.Comparator;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.maven.lifecycle.internal.BuildThreadFactory;
import org.apache.maven.project.MavenProject;
/**
* {@link ThreadPoolExecutor} wrapper.
* <p>
* Uses {@link PriorityBlockingQueue} and provided {@link Comparator} to order queue
* {@link ProjectRunnable} tasks.
*
* File origin:
* https://github.com/takari/takari-smart-builder/blob/takari-smart-builder-0.6.1/src/main/java/io/takari/maven/builder/smart/ProjectExecutorService.java
*/
class ProjectExecutorService {
private final ExecutorService executor;
private final BlockingQueue<Future<MavenProject>> completion = new LinkedBlockingQueue<>();
private final Comparator<Runnable> taskComparator;
public ProjectExecutorService(final int degreeOfConcurrency, final Comparator<MavenProject> projectComparator) {
this.taskComparator = Comparator.comparing(r -> ((ProjectRunnable) r).getProject(), projectComparator);
final BlockingQueue<Runnable> executorWorkQueue =
new PriorityBlockingQueue<>(degreeOfConcurrency, taskComparator);
executor =
new ThreadPoolExecutor(
degreeOfConcurrency, // corePoolSize
degreeOfConcurrency, // maximumPoolSize
0L,
TimeUnit.MILLISECONDS, // keepAliveTime, unit
executorWorkQueue, // workQueue
new BuildThreadFactory() // threadFactory
) {
@Override
protected void beforeExecute(Thread t, Runnable r) {
ProjectExecutorService.this.beforeExecute(t, r);
}
};
}
public void submitAll(final Collection<? extends ProjectRunnable> tasks) {
// when there are available worker threads, tasks are immediately executed, i.e. bypassed the
// ordered queued. need to sort tasks, such that submission order matches desired execution
// order
tasks.stream().sorted(taskComparator).map(ProjectFutureTask::new).forEach(executor::execute);
}
/**
* Returns {@link MavenProject} corresponding to the next completed task, waiting if none are yet
* present.
*/
public MavenProject take() throws InterruptedException, ExecutionException {
return completion.take().get();
}
public void shutdown() {
executor.shutdown();
}
public void cancel() {
executor.shutdownNow();
}
// hook to allow pausing executor during unit tests
protected void beforeExecute(Thread t, Runnable r) {}
// for testing purposes only
public void awaitShutdown() throws InterruptedException {
executor.shutdown();
while (!executor.awaitTermination(5, TimeUnit.SECONDS))
;
}
static interface ProjectRunnable extends Runnable {
public MavenProject getProject();
}
private class ProjectFutureTask extends FutureTask<MavenProject> implements ProjectRunnable {
private ProjectRunnable task;
public ProjectFutureTask(ProjectRunnable task) {
super(task, task.getProject());
this.task = task;
}
@Override
protected void done() {
completion.add(this);
}
@Override
public MavenProject getProject() {
return task.getProject();
}
}
}

122
daemon/src/main/java/org/mvndaemon/mvnd/builder/ReactorBuildQueue.java
View File

@@ -1,122 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Stream;
import org.apache.maven.project.MavenProject;
/**
* Reactor build queue manages reactor modules that are waiting for their upstream dependencies
* build to finish.
*
* File origin:
* https://github.com/takari/takari-smart-builder/blob/takari-smart-builder-0.6.1/src/main/java/io/takari/maven/builder/smart/ReactorBuildQueue.java
*/
class ReactorBuildQueue {
private final DependencyGraph<MavenProject> graph;
private final Set<MavenProject> rootProjects;
private final Set<MavenProject> projects;
/**
* Projects waiting for other projects to finish
*/
private final Set<MavenProject> blockedProjects;
private final Set<MavenProject> finishedProjects;
public ReactorBuildQueue(Collection<MavenProject> projects, DependencyGraph<MavenProject> graph) {
this.graph = graph;
this.projects = new HashSet<>();
this.rootProjects = new HashSet<>();
this.blockedProjects = new HashSet<>();
this.finishedProjects = new HashSet<>();
projects.forEach(project -> {
this.projects.add(project);
if (this.graph.isRoot(project)) {
this.rootProjects.add(project);
} else {
this.blockedProjects.add(project);
}
});
}
/**
* Marks specified project as finished building. Returns, possible empty, set of project's
* downstream dependencies that become ready to build.
*/
public Set<MavenProject> onProjectFinish(MavenProject project) {
finishedProjects.add(project);
Set<MavenProject> downstreamProjects = new HashSet<>();
getDownstreamProjects(project)
.filter(successor -> blockedProjects.contains(successor) && isProjectReady(successor))
.forEach(successor -> {
blockedProjects.remove(successor);
downstreamProjects.add(successor);
});
return downstreamProjects;
}
public Stream<MavenProject> getDownstreamProjects(MavenProject project) {
return graph.getDownstreamProjects(project);
}
private boolean isProjectReady(MavenProject project) {
return graph.getUpstreamProjects(project).allMatch(finishedProjects::contains);
}
/**
* Returns {@code true} when no more projects are left to schedule.
*/
public boolean isEmpty() {
return blockedProjects.isEmpty();
}
/**
* Returns reactor build root projects, that is, projects that do not have upstream dependencies.
*/
public Set<MavenProject> getRootProjects() {
return rootProjects;
}
public int getBlockedCount() {
return blockedProjects.size();
}
public int getFinishedCount() {
return finishedProjects.size();
}
public int getReadyCount() {
return projects.size() - blockedProjects.size() - finishedProjects.size();
}
public Set<MavenProject> getReadyProjects() {
Set<MavenProject> projects = new HashSet<>(this.projects);
projects.removeAll(blockedProjects);
projects.removeAll(finishedProjects);
return projects;
}
}

199
daemon/src/main/java/org/mvndaemon/mvnd/builder/ReactorBuildStats.java
View File

@@ -1,199 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.collect.ImmutableMap;
import org.apache.maven.project.MavenProject;
/**
* File origin:
* https://github.com/takari/takari-smart-builder/blob/takari-smart-builder-0.6.1/src/main/java/io/takari/maven/builder/smart/ReactorBuildStats.java
*/
class ReactorBuildStats {
/**
* Time, in nanoseconds, a worker thread was executing the project build lifecycle. In addition to
* Maven plugin goals execution includes any "overhead" time Maven spends resolving project
* dependencies, calculating build time and perform any post-execution cleanup and maintenance.
*/
private final Map<String, AtomicLong> serviceTimes;
/**
* Time, in nanoseconds, when the project was a bottleneck of entire build, i.e. when not all
* available CPU cores were utilized, presumably because the project build time and dependency
* structure prevented higher degree of parallelism.
*/
private final Map<String, AtomicLong> bottleneckTimes;
private long startTime;
private long stopTime;
private ReactorBuildStats(Map<String, AtomicLong> serviceTimes, Map<String, AtomicLong> bottleneckTimes) {
this.serviceTimes = ImmutableMap.copyOf(serviceTimes);
this.bottleneckTimes = ImmutableMap.copyOf(bottleneckTimes);
}
private static String projectGAV(MavenProject project) {
return project.getGroupId() + ":" + project.getArtifactId() + ":" + project.getVersion();
}
public static ReactorBuildStats create(Collection<MavenProject> projects) {
ImmutableMap.Builder<String, AtomicLong> serviceTimes = ImmutableMap.builder();
ImmutableMap.Builder<String, AtomicLong> bottleneckTimes = ImmutableMap.builder();
projects.stream().map(ReactorBuildStats::projectGAV).forEach(key -> {
serviceTimes.put(key, new AtomicLong());
bottleneckTimes.put(key, new AtomicLong());
});
return new ReactorBuildStats(serviceTimes.build(), bottleneckTimes.build());
}
public void recordStart() {
this.startTime = System.nanoTime();
}
public void recordStop() {
this.stopTime = System.nanoTime();
}
public void recordServiceTime(MavenProject project, long durationNanos) {
AtomicLong serviceTime = serviceTimes.get(projectGAV(project));
if (serviceTime == null) {
throw new IllegalStateException(
"Unknown project " + projectGAV(project) + ", found " + serviceTimes.keySet());
}
serviceTime.addAndGet(durationNanos);
}
public void recordBottlenecks(Set<MavenProject> projects, int degreeOfConcurrency, long durationNanos) {
// only projects that result in single-threaded builds
if (projects.size() == 1) {
projects.forEach(p -> bottleneckTimes.get(projectGAV(p)).addAndGet(durationNanos));
}
}
//
// Reporting
//
public long totalServiceTime(TimeUnit unit) {
long nanos =
serviceTimes.values().stream().mapToLong(AtomicLong::longValue).sum();
return unit.convert(nanos, TimeUnit.NANOSECONDS);
}
public long walltimeTime(TimeUnit unit) {
return unit.convert(stopTime - startTime, TimeUnit.NANOSECONDS);
}
public String renderCriticalPath(DependencyGraph<MavenProject> graph) {
return renderCriticalPath(graph, ReactorBuildStats::projectGAV);
}
public <K> String renderCriticalPath(DependencyGraph<K> graph, Function<K, String> toKey) {
StringBuilder result = new StringBuilder();
// render critical path
long criticalPathServiceTime = 0;
result.append("Build critical path service times (and bottleneck** times):");
for (K project : calculateCriticalPath(graph, toKey)) {
result.append('\n');
String key = toKey.apply(project);
criticalPathServiceTime += serviceTimes.get(key).get();
appendProjectTimes(result, key);
}
result.append(
String.format("\nBuild critical path total service time %s", formatDuration(criticalPathServiceTime)));
// render bottleneck projects
List<String> bottleneckProjects = getBottleneckProjects();
if (!bottleneckProjects.isEmpty()) {
long bottleneckTotalTime = 0;
result.append("\nBuild bottleneck projects service times (and bottleneck** times):");
for (String bottleneck : bottleneckProjects) {
result.append('\n');
bottleneckTotalTime += bottleneckTimes.get(bottleneck).get();
appendProjectTimes(result, bottleneck);
}
result.append(String.format("\nBuild bottlenecks total time %s", formatDuration(bottleneckTotalTime)));
}
result.append("\n** Bottlenecks are projects that limit build concurrency");
result.append("\n removing bottlenecks improves overall build time");
return result.toString();
}
private void appendProjectTimes(StringBuilder result, String project) {
final long serviceTime = serviceTimes.get(project).get();
final long bottleneckTime = bottleneckTimes.get(project).get();
result.append(String.format(" %-60s %s", project, formatDuration(serviceTime)));
if (bottleneckTime > 0) {
result.append(String.format(" (%s)", formatDuration(bottleneckTime)));
}
}
private List<String> getBottleneckProjects() {
Comparator<String> comparator = (a, b) -> {
long ta = bottleneckTimes.get(a).longValue();
long tb = bottleneckTimes.get(b).longValue();
if (tb > ta) {
return 1;
} else if (tb < ta) {
return -1;
}
return 0;
};
return bottleneckTimes.keySet().stream() //
.sorted(comparator) //
.filter(project -> bottleneckTimes.get(project).get() > 0) //
.collect(Collectors.toList());
}
private String formatDuration(long nanos) {
long secs = TimeUnit.NANOSECONDS.toSeconds(nanos);
return String.format("%5d s", secs);
}
private <K> List<K> calculateCriticalPath(DependencyGraph<K> graph, Function<K, String> toKey) {
Comparator<K> comparator = ProjectComparator.create0(graph, serviceTimes, toKey);
Stream<K> rootProjects = graph.getProjects().filter(graph::isRoot);
List<K> criticalPath = new ArrayList<>();
K project = getCriticalProject(rootProjects, comparator);
do {
criticalPath.add(project);
} while ((project = getCriticalProject(graph.getDownstreamProjects(project), comparator)) != null);
return criticalPath;
}
private <K> K getCriticalProject(Stream<K> projects, Comparator<K> comparator) {
return projects.min(comparator).orElse(null);
}
}

167
daemon/src/main/java/org/mvndaemon/mvnd/builder/SmartBuilder.java
View File

@@ -1,167 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.lifecycle.internal.LifecycleModuleBuilder;
import org.apache.maven.lifecycle.internal.ProjectBuildList;
import org.apache.maven.lifecycle.internal.ReactorBuildStatus;
import org.apache.maven.lifecycle.internal.ReactorContext;
import org.apache.maven.lifecycle.internal.TaskSegment;
import org.apache.maven.lifecycle.internal.builder.Builder;
import org.apache.maven.project.MavenProject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Trivial Maven {@link Builder} implementation. All interesting stuff happens in
* {@link SmartBuilderImpl} .
*
* File origin:
* https://github.com/takari/takari-smart-builder/blob/takari-smart-builder-0.6.1/src/main/java/io/takari/maven/builder/smart/SmartBuilder.java
*/
@Singleton
@Named("smart")
public class SmartBuilder implements Builder {
public static final String PROP_PROFILING = "smartbuilder.profiling";
private final Logger logger = LoggerFactory.getLogger(getClass());
private final LifecycleModuleBuilder moduleBuilder;
private volatile SmartBuilderImpl builder;
private volatile boolean canceled;
private static SmartBuilder INSTANCE;
public static SmartBuilder cancel() {
SmartBuilder builder = INSTANCE;
if (builder != null) {
builder.doCancel();
}
return builder;
}
@Inject
public SmartBuilder(LifecycleModuleBuilder moduleBuilder) {
this.moduleBuilder = moduleBuilder;
INSTANCE = this;
}
void doCancel() {
canceled = true;
SmartBuilderImpl b = builder;
if (b != null) {
b.cancel();
}
}
public void doneCancel() {
canceled = false;
}
@Override
public synchronized void build(
final MavenSession session,
final ReactorContext reactorContext,
ProjectBuildList projectBuilds,
final List<TaskSegment> taskSegments,
ReactorBuildStatus reactorBuildStatus)
throws ExecutionException, InterruptedException {
session.getRepositorySession().getData().set(ReactorBuildStatus.class, reactorBuildStatus);
DependencyGraph<MavenProject> graph = DependencyGraph.fromMaven(session);
// log overall build info
final int degreeOfConcurrency = session.getRequest().getDegreeOfConcurrency();
logger.info(
"Task segments : " + taskSegments.stream().map(Object::toString).collect(Collectors.joining(" ")));
logger.info("Build maximum degree of concurrency is " + degreeOfConcurrency);
logger.info("Total number of projects is " + session.getProjects().size());
// the actual build execution
List<Map.Entry<TaskSegment, ReactorBuildStats>> allstats = new ArrayList<>();
for (TaskSegment taskSegment : taskSegments) {
Set<MavenProject> projects =
projectBuilds.getByTaskSegment(taskSegment).getProjects();
if (canceled) {
return;
}
builder = new SmartBuilderImpl(moduleBuilder, session, reactorContext, taskSegment, projects, graph);
try {
ReactorBuildStats stats = builder.build();
allstats.add(new AbstractMap.SimpleEntry<>(taskSegment, stats));
} finally {
builder = null;
}
}
if (session.getResult().hasExceptions()) {
// don't report stats of failed builds
return;
}
// log stats of each task segment
for (Map.Entry<TaskSegment, ReactorBuildStats> entry : allstats) {
TaskSegment taskSegment = entry.getKey();
ReactorBuildStats stats = entry.getValue();
Set<MavenProject> projects =
projectBuilds.getByTaskSegment(taskSegment).getProjects();
logger.debug("Task segment {}, number of projects {}", taskSegment, projects.size());
final long walltimeReactor = stats.walltimeTime(TimeUnit.NANOSECONDS);
final long walltimeService = stats.totalServiceTime(TimeUnit.NANOSECONDS);
final String effectiveConcurrency = String.format("%2.2f", ((double) walltimeService) / walltimeReactor);
logger.info(
"Segment walltime {} s, segment projects service time {} s, effective/maximum degree of concurrency {}/{}",
TimeUnit.NANOSECONDS.toSeconds(walltimeReactor),
TimeUnit.NANOSECONDS.toSeconds(walltimeService),
effectiveConcurrency,
degreeOfConcurrency);
if (projects.size() > 1 && isProfiling(session)) {
logger.info(stats.renderCriticalPath(graph));
}
}
}
private boolean isProfiling(MavenSession session) {
String value = session.getUserProperties().getProperty(PROP_PROFILING);
if (value == null) {
value = session.getSystemProperties().getProperty(PROP_PROFILING);
}
return Boolean.parseBoolean(value);
}
}

219
daemon/src/main/java/org/mvndaemon/mvnd/builder/SmartBuilderImpl.java
View File

@@ -1,219 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.apache.maven.execution.BuildFailure;
import org.apache.maven.execution.BuildSuccess;
import org.apache.maven.execution.BuildSummary;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.lifecycle.internal.LifecycleModuleBuilder;
import org.apache.maven.lifecycle.internal.ReactorContext;
import org.apache.maven.lifecycle.internal.TaskSegment;
import org.apache.maven.lifecycle.internal.builder.Builder;
import org.apache.maven.project.MavenProject;
import org.mvndaemon.mvnd.builder.ProjectExecutorService.ProjectRunnable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Maven {@link Builder} implementation that schedules execution of the reactor modules on the build
* critical path first. Build critical path is estimated based on module build times collected
* during a previous build, or based on module's downstream dependency trail length, if no prior
* build time information is available.
*
* @author Brian Toal
*/
class SmartBuilderImpl {
private final Logger logger = LoggerFactory.getLogger(SmartBuilder.class);
// global components
private final LifecycleModuleBuilder lifecycleModuleBuilder;
// session-level components
private final MavenSession rootSession;
private final ReactorContext reactorContext;
private final TaskSegment taskSegment;
//
private final ReactorBuildQueue reactorBuildQueue;
private final ProjectExecutorService executor;
private final int degreeOfConcurrency;
//
private final ReactorBuildStats stats;
SmartBuilderImpl(
LifecycleModuleBuilder lifecycleModuleBuilder,
MavenSession session,
ReactorContext reactorContext,
TaskSegment taskSegment,
Set<MavenProject> projects,
DependencyGraph<MavenProject> graph) {
this.lifecycleModuleBuilder = lifecycleModuleBuilder;
this.rootSession = session;
this.reactorContext = reactorContext;
this.taskSegment = taskSegment;
this.degreeOfConcurrency = session.getRequest().getDegreeOfConcurrency();
final Comparator<MavenProject> projectComparator = ProjectComparator.create(graph);
this.reactorBuildQueue = new ReactorBuildQueue(projects, graph);
this.executor = new ProjectExecutorService(degreeOfConcurrency, projectComparator);
this.stats = ReactorBuildStats.create(projects);
}
private static String projectGA(MavenProject project) {
return project.getGroupId() + ":" + project.getArtifactId();
}
public ReactorBuildStats build() throws ExecutionException, InterruptedException {
stats.recordStart();
Set<MavenProject> rootProjects = reactorBuildQueue.getRootProjects();
// this is the main build loop
submitAll(rootProjects);
long timstampSubmit = System.nanoTime();
int submittedCount = rootProjects.size();
while (submittedCount > 0) {
Set<MavenProject> bottlenecks = null;
if (submittedCount < degreeOfConcurrency) {
bottlenecks = reactorBuildQueue.getReadyProjects();
}
try {
MavenProject completedProject = executor.take();
if (bottlenecks != null) {
stats.recordBottlenecks(bottlenecks, degreeOfConcurrency, System.nanoTime() - timstampSubmit);
}
logCompleted(completedProject);
Set<MavenProject> readyProjects = reactorBuildQueue.onProjectFinish(completedProject);
submitAll(readyProjects);
timstampSubmit = System.nanoTime();
submittedCount += (readyProjects.size() - 1);
logBuildQueueStatus();
} catch (ExecutionException e) {
// we get here when unhandled exception or error occurred on the worker thread
// this can be low-level system problem, like OOME, or runtime exception in maven code
// there is no meaningful recovery, so we shutdown and rethrow the exception
shutdown();
throw e;
}
}
shutdown();
stats.recordStop();
return stats;
}
private void logBuildQueueStatus() {
int blockedCount = reactorBuildQueue.getBlockedCount();
int finishedCount = reactorBuildQueue.getFinishedCount();
int readyCount = reactorBuildQueue.getReadyCount();
String runningProjects = "";
if (readyCount < degreeOfConcurrency && blockedCount > 0) {
runningProjects = reactorBuildQueue.getReadyProjects().stream()
.map(SmartBuilderImpl::projectGA)
.collect(Collectors.joining(" ", "[", "]"));
}
logger.debug(
"Builder state: blocked={} finished={} ready-or-running={} {}",
blockedCount,
finishedCount,
readyCount,
runningProjects);
}
private void logCompleted(MavenProject project) {
BuildSummary buildSummary = rootSession.getResult().getBuildSummary(project);
String message = "SKIPPED";
if (buildSummary instanceof BuildSuccess) {
message = "SUCCESS";
} else if (buildSummary instanceof BuildFailure) {
message = "FAILURE";
} else if (buildSummary != null) {
logger.warn("Unexpected project build summary class {}", buildSummary.getClass());
message = "UNKNOWN";
}
logger.debug("{} build of project {}:{}", message, project.getGroupId(), project.getArtifactId());
}
private void shutdown() {
executor.shutdown();
}
public void cancel() {
executor.cancel();
}
private void submitAll(Set<MavenProject> readyProjects) {
List<ProjectBuildTask> tasks = new ArrayList<>();
for (MavenProject project : readyProjects) {
tasks.add(new ProjectBuildTask(project));
logger.debug("Ready {}:{}", project.getGroupId(), project.getArtifactId());
}
executor.submitAll(tasks);
}
/* package */ void buildProject(MavenProject project) {
logger.debug("STARTED build of project {}:{}", project.getGroupId(), project.getArtifactId());
try {
MavenSession copiedSession = rootSession.clone();
lifecycleModuleBuilder.buildProject(copiedSession, rootSession, reactorContext, project, taskSegment);
} catch (RuntimeException ex) {
// preserve the xml stack trace, and the java cause chain
rootSession.getResult().addException(new RuntimeException(project.getName() + ": " + ex.getMessage(), ex));
}
}
class ProjectBuildTask implements ProjectRunnable {
private final MavenProject project;
ProjectBuildTask(MavenProject project) {
this.project = project;
}
@Override
public void run() {
final long start = System.nanoTime();
try {
buildProject(project);
} finally {
stats.recordServiceTime(project, System.nanoTime() - start);
}
}
@Override
public MavenProject getProject() {
return project;
}
}
}

2
daemon/src/main/java/org/mvndaemon/mvnd/daemon/ClientDispatcher.java
View File

@@ -26,6 +26,7 @@ import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import io.takari.maven.builder.smart.DependencyGraph;
import org.apache.maven.execution.ExecutionEvent; import org.apache.maven.execution.ExecutionEvent;
import org.apache.maven.execution.MavenSession; import org.apache.maven.execution.MavenSession;
import org.apache.maven.plugin.MojoExecution; import org.apache.maven.plugin.MojoExecution;
@@ -33,7 +34,6 @@ import org.apache.maven.project.MavenProject;
import org.eclipse.aether.transfer.TransferEvent; import org.eclipse.aether.transfer.TransferEvent;
import org.eclipse.aether.transfer.TransferEvent.EventType; import org.eclipse.aether.transfer.TransferEvent.EventType;
import org.eclipse.aether.transfer.TransferEvent.RequestType; import org.eclipse.aether.transfer.TransferEvent.RequestType;
import org.mvndaemon.mvnd.builder.DependencyGraph;
import org.mvndaemon.mvnd.common.Message; import org.mvndaemon.mvnd.common.Message;
import org.mvndaemon.mvnd.common.Message.BuildException; import org.mvndaemon.mvnd.common.Message.BuildException;
import org.mvndaemon.mvnd.common.Message.BuildStarted; import org.mvndaemon.mvnd.common.Message.BuildStarted;

2
daemon/src/main/java/org/mvndaemon/mvnd/daemon/Server.java
View File

@@ -50,8 +50,8 @@ import java.util.function.Consumer;
import java.util.function.Predicate; import java.util.function.Predicate;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import io.takari.maven.builder.smart.SmartBuilder;
import org.apache.maven.cli.DaemonCli; import org.apache.maven.cli.DaemonCli;
import org.mvndaemon.mvnd.builder.SmartBuilder;
import org.mvndaemon.mvnd.common.DaemonConnection; import org.mvndaemon.mvnd.common.DaemonConnection;
import org.mvndaemon.mvnd.common.DaemonException; import org.mvndaemon.mvnd.common.DaemonException;
import org.mvndaemon.mvnd.common.DaemonExpirationStatus; import org.mvndaemon.mvnd.common.DaemonExpirationStatus;

40
daemon/src/test/java/org/mvndaemon/mvnd/builder/AbstractSmartBuilderTest.java
View File

@@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import org.apache.maven.project.MavenProject;
import org.junit.jupiter.api.Assertions;
abstract class AbstractSmartBuilderTest {
protected void assertProjects(Collection<MavenProject> actual, MavenProject... expected) {
Assertions.assertEquals(new HashSet<MavenProject>(Arrays.asList(expected)), new HashSet<>(actual));
}
protected MavenProject newProject(String artifactId) {
MavenProject project = new MavenProject();
project.setGroupId("test");
project.setArtifactId(artifactId);
project.setVersion("1");
return project;
}
}

346
daemon/src/test/java/org/mvndaemon/mvnd/builder/DagWidthTest.java
View File

@@ -1,346 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.mvndaemon.mvnd.builder.DependencyGraph.DagWidth;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class DagWidthTest {
@Test
void testSimpleGraph() {
DependencyGraph<String> graph = newSimpleGraph();
assertEquals(4, new DagWidth<>(graph).getMaxWidth(12));
}
/**
* <pre>
* A B
* /|\ / \
* C D E F
* \|
* G
* </pre>
*/
private DependencyGraph<String> newSimpleGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.emptyList(),
"C", Collections.singletonList("A"),
"D", Collections.singletonList("A"),
"E", Arrays.asList("A", "B"),
"F", Collections.singletonList("B"),
"G", Arrays.asList("D", "E"));
}
@Test
void tripleLinearGraph() {
DependencyGraph<String> graph = newTripleLinearGraph();
assertEquals(1, new DagWidth<>(graph).getMaxWidth());
}
/**
* <pre>
* A
* /|
* B |
* \|
* C
* </pre>
*/
private DependencyGraph<String> newTripleLinearGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Arrays.asList("A", "B"));
}
@Test
void quadrupleLinearGraph() {
DependencyGraph<String> graph = newQuadrupleLinearGraph();
assertEquals(1, new DagWidth<>(graph).getMaxWidth());
}
/**
* <pre>
* A
* /|\
* B | |
* \| |
* C |
* \|
* D
* </pre>
*/
private DependencyGraph<String> newQuadrupleLinearGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Arrays.asList("B", "A"),
"D", Arrays.asList("C", "A"));
}
@Test
void quadrupleLinearGraph2() {
DependencyGraph<String> graph = newQuadrupleLinearGraph2();
assertEquals(1, new DagWidth<>(graph).getMaxWidth());
}
/**
* <pre>
* A
* /|\
* B | |
* |\| |
* | C |
* \|/
* D
* </pre>
*/
private DependencyGraph<String> newQuadrupleLinearGraph2() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Arrays.asList("B", "A"),
"D", Arrays.asList("B", "C", "A"));
}
@Test
void multilevelSum() {
DependencyGraph<String> graph = newMultilevelSumGraph();
assertEquals(5, new DagWidth<>(graph).getMaxWidth());
}
/**
* <pre>
* A
* /|\
* B C D
* /|\ \|
* E F G H
* </pre>
*/
private DependencyGraph<String> newMultilevelSumGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Collections.singletonList("A"),
"D", Collections.singletonList("A"),
"E", Collections.singletonList("B"),
"F", Collections.singletonList("B"),
"G", Collections.singletonList("B"),
"H", Arrays.asList("C", "D"));
}
@Test
void wideGraph() {
DependencyGraph<String> graph = newWideGraph();
assertEquals(3, new DagWidth<>(graph).getMaxWidth());
}
/**
* <pre>
* A
* /|\
* B C D
* |
* E
* </pre>
*/
private DependencyGraph<String> newWideGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Collections.singletonList("A"),
"D", Collections.singletonList("A"),
"E", Collections.singletonList("D"));
}
@Test
void testSingle() {
DependencyGraph<String> graph = newSingleGraph();
assertEquals(1, new DagWidth<>(graph).getMaxWidth(12));
}
/**
* <pre>
* A
* </pre>
*/
private DependencyGraph<String> newSingleGraph() {
return newGraph("A", Collections.emptyList());
}
@Test
void testLinear() {
DependencyGraph<String> graph = newLinearGraph();
assertEquals(1, new DagWidth<>(graph).getMaxWidth(12));
}
/**
* <pre>
* A
* |
* B
* |
* C
* |
* D
* </pre>
*/
private DependencyGraph<String> newLinearGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Collections.singletonList("B"),
"D", Collections.singletonList("C"));
}
@Test
public void testHugeGraph() {
DependencyGraph<String> graph = newHugeGraph();
DagWidth<String> w = new DagWidth<>(graph);
List<String> d = w.ensembleWithChildrenOf(
graph.getDownstreamProjects("org.apache.camel:camel").collect(Collectors.toList()),
"org.apache.camel:camel-parent");
assertEquals(12, w.getMaxWidth(12));
}
private DependencyGraph<String> newHugeGraph() {
Map<String, List<String>> upstreams = new HashMap<>();
try (BufferedReader r =
new BufferedReader(new InputStreamReader(getClass().getResourceAsStream("huge-graph.properties")))) {
r.lines().forEach(l -> {
int idxEq = l.indexOf(" = ");
if (!l.startsWith("#") && idxEq > 0) {
String k = l.substring(0, idxEq).trim();
String[] ups = l.substring(idxEq + 3).trim().split(",");
List<String> list = Stream.of(ups)
.map(String::trim)
.filter(s -> !s.isEmpty())
.collect(Collectors.toList());
upstreams.put(k, list);
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
return newGraph(upstreams);
}
@Test
public void reduce() {
assertSameReduced(newSimpleGraph());
assertReduced(
newTripleLinearGraph(),
"A",
Collections.emptyList(),
"B",
Collections.singletonList("A"),
"C",
Arrays.asList("B"));
assertReduced(
newQuadrupleLinearGraph(),
"A",
Collections.emptyList(),
"B",
Collections.singletonList("A"),
"C",
Arrays.asList("B"),
"D",
Arrays.asList("C"));
assertReduced(
newQuadrupleLinearGraph2(),
"A",
Collections.emptyList(),
"B",
Collections.singletonList("A"),
"C",
Arrays.asList("B"),
"D",
Arrays.asList("C"));
assertSameReduced(newMultilevelSumGraph());
assertSameReduced(newWideGraph());
assertSameReduced(newSingleGraph());
assertSameReduced(newLinearGraph());
}
@Test
public void testToString() {
DependencyGraph<String> graph = newSingleGraph();
assertEquals("A = " + System.lineSeparator(), graph.toString());
}
@SuppressWarnings("unchecked")
static DependencyGraph<String> newGraph(Object... upstreams) {
final Map<String, List<String>> upstreamsMap = new HashMap<>();
for (int i = 0; i < upstreams.length; i++) {
upstreamsMap.put((String) upstreams[i++], (List<String>) upstreams[i]);
}
return newGraph(upstreamsMap);
}
static <K> DependencyGraph<K> newGraph(Map<K, List<K>> upstreams) {
List<K> nodes = Stream.concat(
upstreams.keySet().stream(), upstreams.values().stream().flatMap(List::stream))
.distinct()
.sorted()
.collect(Collectors.toList());
Map<K, List<K>> downstreams = nodes.stream().collect(Collectors.toMap(k -> k, k -> new ArrayList<>()));
upstreams.forEach((k, ups) -> {
ups.forEach(up -> downstreams.get(up).add(k));
});
return new DependencyGraph<>(nodes, upstreams, downstreams);
}
static void assertReduced(DependencyGraph<String> graph, Object... expectedUpstreams) {
final DependencyGraph<String> reduced = graph.reduce();
final DependencyGraph<String> expectedGraph = newGraph(expectedUpstreams);
assertEquals(expectedGraph, reduced);
}
static void assertSameReduced(DependencyGraph<String> graph) {
final DependencyGraph<String> reduced = graph.reduce();
assertEquals(graph, reduced);
}
}

77
daemon/src/test/java/org/mvndaemon/mvnd/builder/ProjectComparatorTest.java
View File

@@ -1,77 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.util.Comparator;
import java.util.HashMap;
import java.util.PriorityQueue;
import java.util.Queue;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.maven.project.MavenProject;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static org.mvndaemon.mvnd.builder.ProjectComparator.id;
public class ProjectComparatorTest extends AbstractSmartBuilderTest {
@Test
public void testPriorityQueueOrder() {
MavenProject a = newProject("a"), b = newProject("b"), c = newProject("c");
TestProjectDependencyGraph graph = new TestProjectDependencyGraph(a, b, c);
graph.addDependency(b, a);
DependencyGraph<MavenProject> dp = DependencyGraph.fromMaven(graph);
Comparator<MavenProject> cmp = ProjectComparator.create0(dp, new HashMap<>(), ProjectComparator::id);
Queue<MavenProject> queue = new PriorityQueue<>(3, cmp);
queue.add(a);
queue.add(b);
queue.add(c);
Assertions.assertEquals(a, queue.poll());
Assertions.assertEquals(c, queue.poll());
Assertions.assertEquals(b, queue.poll());
}
@Test
public void testPriorityQueueOrder_historicalServiceTimes() {
MavenProject a = newProject("a"), b = newProject("b"), c = newProject("c");
TestProjectDependencyGraph graph = new TestProjectDependencyGraph(a, b, c);
graph.addDependency(b, a);
DependencyGraph<MavenProject> dp = DependencyGraph.fromMaven(graph);
HashMap<String, AtomicLong> serviceTimes = new HashMap<>();
serviceTimes.put(id(a), new AtomicLong(1L));
serviceTimes.put(id(b), new AtomicLong(1L));
serviceTimes.put(id(c), new AtomicLong(3L));
Comparator<MavenProject> cmp = ProjectComparator.create0(dp, serviceTimes, ProjectComparator::id);
Queue<MavenProject> queue = new PriorityQueue<>(3, cmp);
queue.add(a);
queue.add(b);
queue.add(c);
Assertions.assertEquals(c, queue.poll());
Assertions.assertEquals(a, queue.poll());
Assertions.assertEquals(b, queue.poll());
}
}

136
daemon/src/test/java/org/mvndaemon/mvnd/builder/ProjectExecutorServiceTest.java
View File

@@ -1,136 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.util.concurrent.Monitor;
import org.apache.maven.project.MavenProject;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mvndaemon.mvnd.builder.ProjectExecutorService.ProjectRunnable;
import static org.mvndaemon.mvnd.builder.ProjectComparator.id;
public class ProjectExecutorServiceTest extends AbstractSmartBuilderTest {
@Test
public void testBuildOrder() throws Exception {
final MavenProject a = newProject("a");
final MavenProject b = newProject("b");
final MavenProject c = newProject("c");
TestProjectDependencyGraph graph = new TestProjectDependencyGraph(a, b, c);
graph.addDependency(b, a);
DependencyGraph<MavenProject> dp = DependencyGraph.fromMaven(graph);
HashMap<String, AtomicLong> serviceTimes = new HashMap<>();
serviceTimes.put(id(a), new AtomicLong(1L));
serviceTimes.put(id(b), new AtomicLong(1L));
serviceTimes.put(id(c), new AtomicLong(3L));
Comparator<MavenProject> cmp = ProjectComparator.create0(dp, serviceTimes, ProjectComparator::id);
PausibleProjectExecutorService executor = new PausibleProjectExecutorService(1, cmp);
final List<MavenProject> executed = new ArrayList<>();
class TestProjectRunnable implements ProjectRunnable {
private final MavenProject project;
TestProjectRunnable(MavenProject project) {
this.project = project;
}
@Override
public void run() {
executed.add(project);
}
@Override
public MavenProject getProject() {
return project;
}
}
// the executor has single work thread and is paused
// first task execution is blocked because the executor is paused
// the subsequent tasks are queued and thus queue order can be asserted
// this one gets stuck on the worker thread
executor.submitAll(Collections.singleton(new TestProjectRunnable(a)));
// these are queued and ordered
executor.submitAll(
Arrays.asList(new TestProjectRunnable(a), new TestProjectRunnable(b), new TestProjectRunnable(c)));
executor.resume();
executor.awaitShutdown();
Assertions.assertEquals(Arrays.asList(a, c, a, b), executed);
}
// copy&paste from ThreadPoolExecutor javadoc (use of Guava is a nice touch there)
private static class PausibleProjectExecutorService extends org.mvndaemon.mvnd.builder.ProjectExecutorService {
private final Monitor monitor = new Monitor();
private boolean isPaused = true;
private final Monitor.Guard paused = new Monitor.Guard(monitor) {
@Override
public boolean isSatisfied() {
return isPaused;
}
};
private final Monitor.Guard notPaused = new Monitor.Guard(monitor) {
@Override
public boolean isSatisfied() {
return !isPaused;
}
};
public PausibleProjectExecutorService(int degreeOfConcurrency, Comparator<MavenProject> projectComparator) {
super(degreeOfConcurrency, projectComparator);
}
@Override
protected void beforeExecute(Thread t, Runnable r) {
monitor.enterWhenUninterruptibly(notPaused);
try {
monitor.waitForUninterruptibly(notPaused);
} finally {
monitor.leave();
}
}
public void resume() {
monitor.enterIf(paused);
try {
isPaused = false;
} finally {
monitor.leave();
}
}
}
}

74
daemon/src/test/java/org/mvndaemon/mvnd/builder/ReactorBuildQueueTest.java
View File

@@ -1,74 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import org.apache.maven.project.MavenProject;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class ReactorBuildQueueTest extends AbstractSmartBuilderTest {
@Test
public void testBasic() {
MavenProject a = newProject("a"), b = newProject("b"), c = newProject("c");
TestProjectDependencyGraph graph = new TestProjectDependencyGraph(a, b, c);
graph.addDependency(b, a);
DependencyGraph<MavenProject> dp = DependencyGraph.fromMaven(graph);
ReactorBuildQueue schl = new ReactorBuildQueue(graph.getSortedProjects(), dp);
assertProjects(schl.getRootProjects(), a, c);
Assertions.assertFalse(schl.isEmpty());
assertProjects(schl.onProjectFinish(a), b);
Assertions.assertTrue(schl.isEmpty());
}
@Test
public void testNoDependencies() {
MavenProject a = newProject("a"), b = newProject("b"), c = newProject("c");
TestProjectDependencyGraph graph = new TestProjectDependencyGraph(a, b, c);
DependencyGraph<MavenProject> dp = DependencyGraph.fromMaven(graph);
ReactorBuildQueue schl = new ReactorBuildQueue(graph.getSortedProjects(), dp);
assertProjects(schl.getRootProjects(), a, b, c);
Assertions.assertTrue(schl.isEmpty());
}
@Test
public void testMultipleUpstreamDependencies() {
MavenProject a = newProject("a"), b = newProject("b"), c = newProject("c");
TestProjectDependencyGraph graph = new TestProjectDependencyGraph(a, b, c);
graph.addDependency(b, a);
graph.addDependency(b, c);
DependencyGraph<MavenProject> dp = DependencyGraph.fromMaven(graph);
ReactorBuildQueue schl = new ReactorBuildQueue(graph.getSortedProjects(), dp);
assertProjects(schl.getRootProjects(), a, c);
Assertions.assertFalse(schl.isEmpty());
assertProjects(schl.onProjectFinish(a), new MavenProject[0]);
Assertions.assertFalse(schl.isEmpty());
assertProjects(schl.onProjectFinish(c), b);
Assertions.assertTrue(schl.isEmpty());
}
}

78
daemon/src/test/java/org/mvndaemon/mvnd/builder/TestProjectDependencyGraph.java
View File

@@ -1,78 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.mvndaemon.mvnd.builder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
import org.apache.maven.execution.ProjectDependencyGraph;
import org.apache.maven.project.MavenProject;
import org.junit.jupiter.api.Assertions;
public class TestProjectDependencyGraph implements ProjectDependencyGraph {
private final List<MavenProject> projects = new ArrayList<MavenProject>();
private final ListMultimap<MavenProject, MavenProject> downstream = ArrayListMultimap.create();
private final ListMultimap<MavenProject, MavenProject> upstream = ArrayListMultimap.create();
public TestProjectDependencyGraph(MavenProject... projects) {
if (projects != null) {
this.projects.addAll(Arrays.asList(projects));
}
}
@Override
public List<MavenProject> getAllProjects() {
return projects;
}
@Override
public List<MavenProject> getSortedProjects() {
return projects;
}
@Override
public List<MavenProject> getDownstreamProjects(MavenProject project, boolean transitive) {
Assertions.assertFalse(transitive, "not implemented");
return downstream.get(project);
}
@Override
public List<MavenProject> getUpstreamProjects(MavenProject project, boolean transitive) {
Assertions.assertFalse(transitive, "not implemented");
return upstream.get(project);
}
public void addProject(MavenProject project) {
projects.add(project);
}
public void addDependency(MavenProject from, MavenProject to) {
// 'from' depends on 'to'
// 'from' is a downstream dependency of 'to'
// 'to' is upstream dependency of 'from'
this.upstream.put(from, to);
this.downstream.put(to, from);
}
}

3
dist-m39/src/main/provisio/maven-distro.xml
View File

@@ -34,6 +34,9 @@
</artifactSet> </artifactSet>
<artifactSet to="/lib/ext"> <artifactSet to="/lib/ext">
<artifact id="io.takari.maven:takari-smart-builder:${takari-smart-builder.version}">
<exclusion id="*:*"/>
</artifact>
<artifact id="org.apache.maven.daemon:mvnd-daemon:${project.version}"> <artifact id="org.apache.maven.daemon:mvnd-daemon:${project.version}">
<exclusion id="*:*"/> <exclusion id="*:*"/>
</artifact> </artifact>

3
dist-m40/src/main/provisio/maven-distro.xml
View File

@@ -34,6 +34,9 @@
</artifactSet> </artifactSet>
<artifactSet to="/lib/ext"> <artifactSet to="/lib/ext">
<artifact id="io.takari.maven:takari-smart-builder:${takari-smart-builder.version}">
<exclusion id="*:*"/>
</artifact>
<artifact id="org.apache.maven.daemon:mvnd-daemon:${project.version}"> <artifact id="org.apache.maven.daemon:mvnd-daemon:${project.version}">
<exclusion id="*:*"/> <exclusion id="*:*"/>
</artifact> </artifact>

2
integration-tests/src/test/resources/logback/logback.xml
View File

@@ -36,7 +36,7 @@
<appender-ref ref="DAEMON" /> <appender-ref ref="DAEMON" />
</logger> </logger>
<logger name="org.mvndaemon.mvnd.builder.SmartBuilder" level="DEBUG"/> <logger name="io.takari.maven.builder.smart.SmartBuilder" level="DEBUG"/>
<logger name="Sisu" level="INFO" /> <logger name="Sisu" level="INFO" />

7
pom.xml
View File

@@ -115,6 +115,7 @@
<maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version> <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
<xstream.version>1.4.20</xstream.version> <xstream.version>1.4.20</xstream.version>
<plexus-interactivity-api.version>1.0</plexus-interactivity-api.version> <plexus-interactivity-api.version>1.0</plexus-interactivity-api.version>
<takari-smart-builder.version>0.6.2</takari-smart-builder.version>
</properties> </properties>
<dependencyManagement> <dependencyManagement>
@@ -341,6 +342,12 @@
</exclusion> </exclusion>
</exclusions> </exclusions>
</dependency> </dependency>
<dependency>
<groupId>io.takari.maven</groupId>
<artifactId>takari-smart-builder</artifactId>
<version>${takari-smart-builder.version}</version>
</dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>