apache/maven-mvnd
1
0
Fork 0
mirror of https://github.com/apache/maven-mvnd.git synced 2025-09-28 08:47:29 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DAG width wrong for triple interdependent graph #287

Browse Source
This commit is contained in:
Peter Palaga
2020-12-25 00:36:22 +01:00
parent f046bb6119
commit a81aa7c397
2 changed files with 361 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
daemon/src/main/java/org/mvndaemon/mvnd/builder/DependencyGraph.java
View File

@@ -57,6 +57,7 @@ public class DependencyGraph<K> {
private final List<K> projects;
private final Map<K, List<K>> upstreams;
private final Map<K, Set<K>> transitiveUpstreams;
private final Map<K, List<K>> downstreams;
public static DependencyGraph<MavenProject> fromMaven(MavenSession session) {
@@ -116,7 +117,7 @@ public class DependencyGraph<K> {
binding.setProperty("session", session);
Object result = shell.evaluate(providerScript);
if (result instanceof Iterable) {
for (Object r : (Iterable) result) {
for (Object r : (Iterable<?>) result) {
list.add(r.toString());
}
} else if (result != null) {
@@ -210,6 +211,17 @@ public class DependencyGraph<K> {
this.projects = projects;
this.upstreams = upstreams;
this.downstreams = downstreams;
this.transitiveUpstreams = new HashMap<>();
projects.forEach(this::transitiveUpstreams); // topological ordering of projects matters
}
DependencyGraph(List<K> projects, Map<K, List<K>> upstreams, Map<K, List<K>> downstreams,
Map<K, Set<K>> transitiveUpstreams) {
this.projects = projects;
this.upstreams = upstreams;
this.downstreams = downstreams;
this.transitiveUpstreams = transitiveUpstreams;
}
public Stream<K> getDownstreamProjects(K project) {
@@ -234,29 +246,157 @@ public class DependencyGraph<K> {
public void store(Function<K, String> toString, Path path) {
try (Writer w = Files.newBufferedWriter(path)) {
getProjects().forEach(k -> {
try {
w.write(toString.apply(k));
w.write(" = ");
w.write(getUpstreamProjects(k).map(toString).collect(Collectors.joining(",")));
w.write(System.lineSeparator());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
store(toString, w);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public void store(Function<K, String> toString, Appendable w) {
getProjects().forEach(k -> {
try {
w.append(toString.apply(k));
w.append(" = ");
w.append(getUpstreamProjects(k).map(toString).collect(Collectors.joining(",")));
w.append(System.lineSeparator());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
store(k -> k.toString(), sb);
return sb.toString();
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((downstreams == null) ? 0 : downstreams.hashCode());
result = prime * result + ((projects == null) ? 0 : projects.hashCode());
result = prime * result + ((upstreams == null) ? 0 : upstreams.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
@SuppressWarnings("unchecked")
DependencyGraph<K> other = (DependencyGraph<K>) obj;
if (downstreams == null) {
if (other.downstreams != null)
return false;
} else if (!downstreams.equals(other.downstreams))
return false;
if (projects == null) {
if (other.projects != null)
return false;
} else if (!projects.equals(other.projects))
return false;
if (upstreams == null) {
if (other.upstreams != null)
return false;
} else if (!upstreams.equals(other.upstreams))
return false;
return true;
}
/**
* Creates a new {@link DependencyGraph} which is a <a href="https://en.wikipedia.org/wiki/Transitive_reduction">
* transitive reduction</a> of this {@link DependencyGraph}. The reduction operation keeps the set of graph nodes
* unchanged and it reduces the set of edges in the following way: An edge {@code C -> A} is removed if an edge
* {@code C -> B} exists such that {@code A != B} and the set of nodes reachable from {@code B} contains {@code A};
* otherwise the edge {@code C -> A} is kept in the reduced graph.
* <p>
* Examples:
*
* <pre>
* Original Reduced
*
* A A
* /| /
* B | B
* \| \
* C C
*
*
* A A
* /|\ /
* B | | B
* \| | \
* C | C
* \| \
* D D
*
* </pre>
*
*
* @return a transitive reduction of this {@link DependencyGraph}
*/
DependencyGraph<K> reduce() {
final Map<K, List<K>> newUpstreams = new HashMap<>();
final Map<K, List<K>> newDownstreams = new HashMap<>();
for (K node : projects) {
final List<K> oldNodeUpstreams = upstreams.get(node);
final List<K> newNodeUpstreams;
newDownstreams.computeIfAbsent(node, k -> new ArrayList<>());
if (oldNodeUpstreams.size() == 0) {
newNodeUpstreams = new ArrayList<>(oldNodeUpstreams);
} else if (oldNodeUpstreams.size() == 1) {
newNodeUpstreams = new ArrayList<>(oldNodeUpstreams);
newDownstreams.computeIfAbsent(newNodeUpstreams.get(0), k -> new ArrayList<>()).add(node);
} else {
newNodeUpstreams = new ArrayList<>(oldNodeUpstreams.size());
for (K leftNode : oldNodeUpstreams) {
if (oldNodeUpstreams.stream()
.filter(rightNode -> leftNode != rightNode)
.noneMatch(rightNode -> transitiveUpstreams.get(rightNode).contains(leftNode))) {
newNodeUpstreams.add(leftNode);
newDownstreams.computeIfAbsent(leftNode, k -> new ArrayList<>()).add(node);
}
}
}
newUpstreams.put(node, newNodeUpstreams);
}
return new DependencyGraph<K>(projects, newUpstreams, newDownstreams, transitiveUpstreams);
}
/**
* Compute the set of nodes reachable from the given {@code node} through the {@code is upstream of} relation. The
* {@code node} itself is not a part of the returned set.
*
* @param node the node for which the transitive upstream should be computed
* @return the set of transitive upstreams
*/
Set<K> transitiveUpstreams(K node) {
Set<K> result = transitiveUpstreams.get(node);
if (result == null) {
final List<K> firstOrderUpstreams = this.upstreams.get(node);
result = new HashSet<>(firstOrderUpstreams);
firstOrderUpstreams.stream()
.map(this::transitiveUpstreams)
.forEach(result::addAll);
transitiveUpstreams.put(node, result);
}
return result;
}
static class DagWidth<K> {
private final DependencyGraph<K> graph;
private final Map<K, Set<K>> allUpstreams = new HashMap<>();
public DagWidth(DependencyGraph<K> graph) {
this.graph = graph;
graph.getProjects().forEach(this::allUpstreams);
this.graph = graph.reduce();
}
public int getMaxWidth() {
@@ -269,10 +409,10 @@ public class DependencyGraph<K> {
public int getMaxWidth(int maxmax, long maxTimeMillis) {
int max = 0;
if (maxmax < allUpstreams.size()) {
if (maxmax < graph.transitiveUpstreams.size()) {
// try inverted upstream bound
Map<Set<K>, Set<K>> mapByUpstreams = new HashMap<>();
allUpstreams.forEach((k, ups) -> {
graph.transitiveUpstreams.forEach((k, ups) -> {
mapByUpstreams.computeIfAbsent(ups, n -> new HashSet<>()).add(k);
});
max = mapByUpstreams.values().stream()
@@ -327,36 +467,15 @@ public class DependencyGraph<K> {
.collect(Collectors.toList());
}
/**
* Get a stream of all subset of descendants of the given nodes
*/
private Stream<List<K>> childEnsembles(List<K> list) {
return Stream.concat(
Stream.of(list),
list.parallelStream()
.map(node -> ensembleWithChildrenOf(list, node))
.flatMap(this::childEnsembles));
}
List<K> ensembleWithChildrenOf(List<K> list, K node) {
return Stream.concat(
final List<K> result = Stream.concat(
list.stream().filter(k -> !Objects.equals(k, node)),
graph.getDownstreamProjects(node)
.filter(k -> allUpstreams(k).stream()
.filter(k -> graph.transitiveUpstreams.get(k)
.stream()
.noneMatch(k2 -> !Objects.equals(k2, node) && list.contains(k2))))
.distinct().collect(Collectors.toList());
}
private Set<K> allUpstreams(K node) {
Set<K> aups = allUpstreams.get(node);
if (aups == null) {
aups = Stream.concat(
graph.getUpstreamProjects(node),
graph.getUpstreamProjects(node).map(this::allUpstreams).flatMap(Set::stream))
.collect(Collectors.toSet());
allUpstreams.put(node, aups);
}
return aups;
return result;
}
}

265
daemon/src/test/java/org/mvndaemon/mvnd/builder/DagWidthTest.java
View File

@@ -33,6 +33,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
public class DagWidthTest {
@Test
void testSimpleGraph() {
DependencyGraph<String> graph = newSimpleGraph();
assertEquals(4, new DagWidth<>(graph).getMaxWidth(12));
}
/**
* <pre>
* A B
@@ -42,20 +48,21 @@ public class DagWidthTest {
* G
* </pre>
*/
@Test
void testSimpleGraph() {
//
Map<String, List<String>> upstreams = new HashMap<>();
upstreams.put("A", Collections.emptyList());
upstreams.put("B", Collections.emptyList());
upstreams.put("C", Collections.singletonList("A"));
upstreams.put("D", Collections.singletonList("A"));
upstreams.put("E", Arrays.asList("A", "B"));
upstreams.put("F", Collections.singletonList("B"));
upstreams.put("G", Arrays.asList("D", "E"));
DependencyGraph<String> graph = newGraph(upstreams);
private DependencyGraph<String> newSimpleGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.emptyList(),
"C", Collections.singletonList("A"),
"D", Collections.singletonList("A"),
"E", Arrays.asList("A", "B"),
"F", Collections.singletonList("B"),
"G", Arrays.asList("D", "E"));
}
assertEquals(4, new DagWidth<>(graph).getMaxWidth(12));
@Test
void tripleLinearGraph() {
DependencyGraph<String> graph = newTripleLinearGraph();
assertEquals(1, new DagWidth<>(graph).getMaxWidth());
}
/**
@@ -67,16 +74,69 @@ public class DagWidthTest {
* C
* </pre>
*/
private DependencyGraph<String> newTripleLinearGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Arrays.asList("A", "B"));
}
@Test
void tripleLinearGraph() {
Map<String, List<String>> upstreams = new HashMap<>();
upstreams.put("A", Collections.emptyList());
upstreams.put("B", Collections.singletonList("A"));
upstreams.put("C", Arrays.asList("A", "B"));
DependencyGraph<String> graph = newGraph(upstreams);
void quadrupleLinearGraph() {
DependencyGraph<String> graph = newQuadrupleLinearGraph();
assertEquals(1, new DagWidth<>(graph).getMaxWidth());
}
/**
* <pre>
* A
* /|\
* B | |
* \| |
* C |
* \|
* D
* </pre>
*/
private DependencyGraph<String> newQuadrupleLinearGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Arrays.asList("B", "A"),
"D", Arrays.asList("C", "A"));
}
@Test
void quadrupleLinearGraph2() {
DependencyGraph<String> graph = newQuadrupleLinearGraph2();
assertEquals(1, new DagWidth<>(graph).getMaxWidth());
}
/**
* <pre>
* A
* /|\
* B | |
* |\| |
* | C |
* \|/
* D
* </pre>
*/
private DependencyGraph<String> newQuadrupleLinearGraph2() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Arrays.asList("B", "A"),
"D", Arrays.asList("B", "C", "A"));
}
@Test
void multilevelSum() {
DependencyGraph<String> graph = newMultilevelSumGraph();
assertEquals(5, new DagWidth<>(graph).getMaxWidth());
}
/**
* <pre>
* A
@@ -86,19 +146,22 @@ public class DagWidthTest {
* E F G H
* </pre>
*/
private DependencyGraph<String> newMultilevelSumGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Collections.singletonList("A"),
"D", Collections.singletonList("A"),
"E", Collections.singletonList("B"),
"F", Collections.singletonList("B"),
"G", Collections.singletonList("B"),
"H", Arrays.asList("C", "D"));
}
@Test
void multilevelSum() {
Map<String, List<String>> upstreams = new HashMap<>();
upstreams.put("A", Collections.emptyList());
upstreams.put("B", Collections.singletonList("A"));
upstreams.put("C", Collections.singletonList("A"));
upstreams.put("D", Collections.singletonList("A"));
upstreams.put("E", Collections.singletonList("B"));
upstreams.put("F", Collections.singletonList("B"));
upstreams.put("G", Collections.singletonList("B"));
upstreams.put("H", Arrays.asList("C", "D"));
DependencyGraph<String> graph = newGraph(upstreams);
assertEquals(5, new DagWidth<>(graph).getMaxWidth());
void wideGraph() {
DependencyGraph<String> graph = newWideGraph();
assertEquals(3, new DagWidth<>(graph).getMaxWidth());
}
/**
@@ -110,16 +173,20 @@ public class DagWidthTest {
* E
* </pre>
*/
private DependencyGraph<String> newWideGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Collections.singletonList("A"),
"D", Collections.singletonList("A"),
"E", Collections.singletonList("D"));
}
@Test
void wide() {
Map<String, List<String>> upstreams = new HashMap<>();
upstreams.put("A", Collections.emptyList());
upstreams.put("B", Collections.singletonList("A"));
upstreams.put("C", Collections.singletonList("A"));
upstreams.put("D", Collections.singletonList("A"));
upstreams.put("E", Collections.singletonList("D"));
DependencyGraph<String> graph = newGraph(upstreams);
assertEquals(3, new DagWidth<>(graph).getMaxWidth());
void testSingle() {
DependencyGraph<String> graph = newSingleGraph();
assertEquals(1, new DagWidth<>(graph).getMaxWidth(12));
}
/**
@@ -127,32 +194,49 @@ public class DagWidthTest {
* A
* </pre>
*/
@Test
void testSingle() {
Map<String, List<String>> upstreams = new HashMap<>();
upstreams.put("A", Collections.emptyList());
DependencyGraph<String> graph = newGraph(upstreams);
assertEquals(1, new DagWidth<>(graph).getMaxWidth(12));
private DependencyGraph<String> newSingleGraph() {
return newGraph(
"A", Collections.emptyList());
}
@Test
void testLinear() {
//
// A -> B -> C -> D
//
Map<String, List<String>> upstreams = new HashMap<>();
upstreams.put("A", Collections.emptyList());
upstreams.put("B", Collections.singletonList("A"));
upstreams.put("C", Collections.singletonList("B"));
upstreams.put("D", Collections.singletonList("C"));
DependencyGraph<String> graph = newGraph(upstreams);
DependencyGraph<String> graph = newLinearGraph();
assertEquals(1, new DagWidth<>(graph).getMaxWidth(12));
}
/**
* <pre>
* A
* |
* B
* |
* C
* |
* D
* </pre>
*/
private DependencyGraph<String> newLinearGraph() {
return newGraph(
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Collections.singletonList("B"),
"D", Collections.singletonList("C"));
}
@Test
public void testHugeGraph() throws IOException {
public void testHugeGraph() {
DependencyGraph<String> graph = newHugeGraph();
DagWidth<String> w = new DagWidth<>(graph);
List<String> d = w.ensembleWithChildrenOf(
graph.getDownstreamProjects("org.apache.camel:camel").collect(Collectors.toList()),
"org.apache.camel:camel-parent");
assertEquals(12, w.getMaxWidth(12));
}
private DependencyGraph<String> newHugeGraph() {
Map<String, List<String>> upstreams = new HashMap<>();
try (BufferedReader r = new BufferedReader(
new InputStreamReader(getClass().getResourceAsStream("huge-graph.properties")))) {
@@ -167,15 +251,57 @@ public class DagWidthTest {
upstreams.put(k, list);
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
DependencyGraph<String> graph = newGraph(upstreams);
return newGraph(upstreams);
}
DagWidth<String> w = new DagWidth<>(graph);
List<String> d = w.ensembleWithChildrenOf(
graph.getDownstreamProjects("org.apache.camel:camel").collect(Collectors.toList()),
"org.apache.camel:camel-parent");
@Test
public void reduce() {
assertSameReduced(newSimpleGraph());
assertEquals(12, w.getMaxWidth(12));
assertReduced(
newTripleLinearGraph(),
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Arrays.asList("B"));
assertReduced(newQuadrupleLinearGraph(),
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Arrays.asList("B"),
"D", Arrays.asList("C"));
assertReduced(newQuadrupleLinearGraph2(),
"A", Collections.emptyList(),
"B", Collections.singletonList("A"),
"C", Arrays.asList("B"),
"D", Arrays.asList("C"));
assertSameReduced(newMultilevelSumGraph());
assertSameReduced(newWideGraph());
assertSameReduced(newSingleGraph());
assertSameReduced(newLinearGraph());
}
@Test
public void testToString() {
DependencyGraph<String> graph = newSingleGraph();
assertEquals("A = " + System.lineSeparator(), graph.toString());
}
@SuppressWarnings("unchecked")
static DependencyGraph<String> newGraph(Object... upstreams) {
final Map<String, List<String>> upstreamsMap = new HashMap<>();
for (int i = 0; i < upstreams.length; i++) {
upstreamsMap.put((String) upstreams[i++], (List<String>) upstreams[i]);
}
return newGraph(upstreamsMap);
}
static <K> DependencyGraph<K> newGraph(Map<K, List<K>> upstreams) {
@@ -189,4 +315,15 @@ public class DagWidthTest {
});
return new DependencyGraph<>(nodes, upstreams, downstreams);
}
static void assertReduced(DependencyGraph<String> graph, Object... expectedUpstreams) {
final DependencyGraph<String> reduced = graph.reduce();
final DependencyGraph<String> expectedGraph = newGraph(expectedUpstreams);
assertEquals(expectedGraph, reduced);
}
static void assertSameReduced(DependencyGraph<String> graph) {
final DependencyGraph<String> reduced = graph.reduce();
assertEquals(graph, reduced);
}
}