Maven-like rolling output when the build happens to be linear #269

common/src/main/java/org/mvndaemon/mvnd/common/logging/TerminalOutput.java

@@ -173,6 +173,11 @@ public class TerminalOutput implements ClientOutput {
             this.maxThreads = bs.getMaxThreads();
             final int maxThreadsDigits = (int) (Math.log10(maxThreads) + 1);
             this.threadsFormat = "%" + (maxThreadsDigits * 3 + 2) + "s";
+            if (maxThreads <= 1 || totalProjects <= 1) {
+                this.noBuffering = true;
+                display.update(Collections.emptyList(), 0);
+                applyNoBuffering();
+            }
             break;
         }
         case Message.CANCEL_BUILD: {
@@ -322,8 +327,7 @@ public class TerminalOutput implements ClientOutput {
             case CTRL_B:
                 noBuffering = !noBuffering;
                 if (noBuffering) {
-                    projects.values().stream().flatMap(p -> p.log.stream()).forEach(log);
-                    projects.clear();
+                    applyNoBuffering();
                 } else {
                     clearDisplay();
                 }
@@ -345,6 +349,11 @@ public class TerminalOutput implements ClientOutput {
         return true;
     }
 
+    private void applyNoBuffering() {
+        projects.values().stream().flatMap(p -> p.log.stream()).forEach(log);
+        projects.clear();
+    }
+
     @Override
     public void describeTerminal() {
         StringBuilder sb = new StringBuilder();

daemon/src/main/java/org/mvndaemon/mvnd/builder/DependencyGraph.java

@@ -15,28 +15,151 @@
  */
 package org.mvndaemon.mvnd.builder;
 
+import groovy.lang.Binding;
+import groovy.lang.GroovyShell;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Writer;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import org.apache.maven.execution.MavenSession;
 import org.apache.maven.execution.ProjectDependencyGraph;
 import org.apache.maven.project.MavenProject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * File origin:
  * https://github.com/takari/takari-smart-builder/blob/takari-smart-builder-0.6.1/src/main/java/io/takari/maven/builder/smart/DependencyGraph.java
  */
-interface DependencyGraph<K> {
+public class DependencyGraph<K> {
+
+    private static final Logger logger = LoggerFactory.getLogger(DependencyGraph.class);
+    static final Pattern mvndRuleSanitizerPattern = Pattern.compile("[,\\s]+");
+
+    private final List<K> projects;
+    private final Map<K, List<K>> upstreams;
+    private final Map<K, List<K>> downstreams;
+
+    public static DependencyGraph<MavenProject> fromMaven(MavenSession session) {
+
+        final ProjectDependencyGraph graph = session.getProjectDependencyGraph();
+        final List<MavenProject> projects = graph.getSortedProjects();
+        return fromMaven(graph, getRules(projects, session));
+    }
+
+    static String getRules(List<MavenProject> projects, MavenSession session) {
+        List<String> list = new ArrayList<>();
+
+        String providerScript = null;
+        final MavenProject topLevelProject = projects.get(0);
+        String providerUrl = topLevelProject.getProperties()
+                .getProperty(SmartBuilder.MVND_BUILDER_RULES_PROVIDER_URL);
+        if (providerUrl != null) {
+            logger.warn(SmartBuilder.MVND_BUILDER_RULES_PROVIDER_URL
+                    + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
+
+            URL url;
+            try {
+                url = new URL(providerUrl);
+            } catch (MalformedURLException e) {
+                try {
+                    url = new File(providerUrl).toURI().toURL();
+                } catch (MalformedURLException ex) {
+                    url = null;
+                }
+            }
+            if (url == null) {
+                throw new RuntimeException("Bad syntax for " + SmartBuilder.MVND_BUILDER_RULES_PROVIDER_URL, null);
+            }
+            try (BufferedReader r = new BufferedReader(new InputStreamReader(url.openStream()))) {
+                StringBuilder sb = new StringBuilder();
+                char[] buf = new char[8192];
+                int l;
+                while ((l = r.read(buf)) >= 0) {
+                    sb.append(buf, 0, l);
+                }
+                providerScript = sb.toString();
+            } catch (IOException e) {
+                throw new RuntimeException("Unable to read provider url " + SmartBuilder.MVND_BUILDER_RULES_PROVIDER_URL,
+                        e);
+            }
+        }
+        if (providerScript == null) {
+            providerScript = topLevelProject.getProperties()
+                    .getProperty(SmartBuilder.MVND_BUILDER_RULES_PROVIDER_SCRIPT);
+        }
+        if (providerScript != null) {
+            logger.warn(SmartBuilder.MVND_BUILDER_RULES_PROVIDER_SCRIPT
+                    + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
+
+            Binding binding = new Binding();
+            GroovyShell shell = new GroovyShell(binding);
+            binding.setProperty("session", session);
+            Object result = shell.evaluate(providerScript);
+            if (result instanceof Iterable) {
+                for (Object r : (Iterable) result) {
+                    list.add(r.toString());
+                }
+            } else if (result != null) {
+                list.add(result.toString());
+            } else {
+                throw new RuntimeException("The provider script did not return a valid string or string collection", null);
+            }
+            list.add(result.toString());
+        }
+
+        String topRule = topLevelProject.getProperties().getProperty(SmartBuilder.MVND_BUILDER_RULES);
+        if (topRule != null) {
+            logger.warn(SmartBuilder.MVND_BUILDER_RULES
+                    + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
+            list.add(topRule);
+        }
+
+        projects.forEach(p -> {
+            String rule = p.getProperties().getProperty(SmartBuilder.MVND_BUILDER_RULE);
+            if (rule != null) {
+                logger.warn(SmartBuilder.MVND_BUILDER_RULE
+                        + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
+                rule = rule.trim();
+                if (!rule.isEmpty()) {
+                    rule = mvndRuleSanitizerPattern.matcher(rule).replaceAll(",");
+                    list.add(rule + " before " + p.getGroupId() + ":" + p.getArtifactId());
+                }
+            }
+        });
+        String rules = null;
+        if (!list.isEmpty()) {
+            rules = String.join("\n", list);
+        }
+        return rules;
+    }
 
     static DependencyGraph<MavenProject> fromMaven(ProjectDependencyGraph graph, String rules) {
-        List<MavenProject> projects = graph.getSortedProjects();
+        final List<MavenProject> projects = graph.getSortedProjects();
         Map<MavenProject, List<MavenProject>> upstreams = projects.stream()
                 .collect(Collectors.toMap(p -> p, p -> graph.getUpstreamProjects(p, false)));
         Map<MavenProject, List<MavenProject>> downstreams = projects.stream()
-                .collect(Collectors.toMap(p -> p, p -> graph.getDownstreamProjects(p, false)));
+                .collect(
+                        Collectors.toMap(p -> p, p -> graph.getDownstreamProjects(p, false)));
 
         if (rules != null) {
             for (String rule : rules.split("\\s*;\\s*|\n")) {
@@ -80,35 +203,162 @@ interface DependencyGraph<K> {
                 deps.get(0).forEach(p -> downstreams.get(p).addAll(deps.get(1)));
             }
         }
-        return new DependencyGraph<MavenProject>() {
-            @Override
-            public Stream<MavenProject> getDownstreamProjects(MavenProject project) {
+        return new DependencyGraph<MavenProject>(Collections.unmodifiableList(projects), upstreams, downstreams);
+    }
+
+    public DependencyGraph(List<K> projects, Map<K, List<K>> upstreams, Map<K, List<K>> downstreams) {
+        this.projects = projects;
+        this.upstreams = upstreams;
+        this.downstreams = downstreams;
+    }
+
+    public Stream<K> getDownstreamProjects(K project) {
         return downstreams.get(project).stream();
     }
 
-            @Override
-            public Stream<MavenProject> getProjects() {
-                return projects.stream();
-            }
-
-            @Override
-            public Stream<MavenProject> getUpstreamProjects(MavenProject project) {
+    public Stream<K> getUpstreamProjects(K project) {
         return upstreams.get(project).stream();
     }
 
-            @Override
-            public boolean isRoot(MavenProject project) {
+    public boolean isRoot(K project) {
         return upstreams.get(project).isEmpty();
     }
-        };
+
+    public Stream<K> getProjects() {
+        return projects.stream();
     }
 
-    Stream<K> getProjects();
+    public int computeMaxWidth(int max, long maxTimeMillis) {
+        return new DagWidth<>(this).getMaxWidth(max, maxTimeMillis);
+    }
 
-    boolean isRoot(K project);
+    public void store(Function<K, String> toString, Path path) {
+        try (Writer w = Files.newBufferedWriter(path)) {
+            getProjects().forEach(k -> {
+                try {
+                    w.write(toString.apply(k));
+                    w.write(" = ");
+                    w.write(getUpstreamProjects(k).map(toString).collect(Collectors.joining(",")));
+                    w.write(System.lineSeparator());
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
 
-    Stream<K> getDownstreamProjects(K project);
+    static class DagWidth<K> {
 
-    Stream<K> getUpstreamProjects(K project);
+        private final DependencyGraph<K> graph;
+        private final Map<K, Set<K>> allUpstreams = new HashMap<>();
+
+        public DagWidth(DependencyGraph<K> graph) {
+            this.graph = graph;
+            graph.getProjects().forEach(this::allUpstreams);
+        }
+
+        public int getMaxWidth() {
+            return getMaxWidth(Integer.MAX_VALUE);
+        }
+
+        public int getMaxWidth(int maxmax) {
+            return getMaxWidth(maxmax, Long.MAX_VALUE);
+        }
+
+        public int getMaxWidth(int maxmax, long maxTimeMillis) {
+            int max = 0;
+            if (maxmax < allUpstreams.size()) {
+                // try inverted upstream bound
+                Map<Set<K>, Set<K>> mapByUpstreams = new HashMap<>();
+                allUpstreams.forEach((k, ups) -> {
+                    mapByUpstreams.computeIfAbsent(ups, n -> new HashSet<>()).add(k);
+                });
+                max = mapByUpstreams.values().stream()
+                        .mapToInt(Set::size)
+                        .max()
+                        .orElse(0);
+                if (max >= maxmax) {
+                    return maxmax;
+                }
+            }
+            long tmax = System.currentTimeMillis() + maxTimeMillis;
+            int tries = 0;
+            SubsetIterator iterator = new SubsetIterator(getRoots());
+            while (max < maxmax && iterator.hasNext()) {
+                if (++tries % 100 == 0 && System.currentTimeMillis() < tmax) {
+                    return maxmax;
+                }
+                List<K> l = iterator.next();
+                max = Math.max(max, l.size());
+            }
+            return Math.min(max, maxmax);
+        }
+
+        private class SubsetIterator implements Iterator<List<K>> {
+
+            final List<List<K>> nexts = new ArrayList<>();
+            final Set<List<K>> visited = new HashSet<>();
+
+            public SubsetIterator(List<K> roots) {
+                nexts.add(roots);
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !nexts.isEmpty();
+            }
+
+            @Override
+            public List<K> next() {
+                List<K> list = nexts.remove(0);
+                list.stream()
+                        .map(node -> ensembleWithChildrenOf(list, node))
+                        .filter(visited::add)
+                        .forEach(nexts::add);
+                return list;
+            }
+        }
+
+        private List<K> getRoots() {
+            return graph.getProjects()
+                    .filter(graph::isRoot)
+                    .collect(Collectors.toList());
+        }
+
+        /**
+         * Get a stream of all subset of descendants of the given nodes
+         */
+        private Stream<List<K>> childEnsembles(List<K> list) {
+            return Stream.concat(
+                    Stream.of(list),
+                    list.parallelStream()
+                            .map(node -> ensembleWithChildrenOf(list, node))
+                            .flatMap(this::childEnsembles));
+        }
+
+        List<K> ensembleWithChildrenOf(List<K> list, K node) {
+            return Stream.concat(
+                    list.stream().filter(k -> !Objects.equals(k, node)),
+                    graph.getDownstreamProjects(node)
+                            .filter(k -> allUpstreams(k).stream()
+                                    .noneMatch(k2 -> !Objects.equals(k2, node) && list.contains(k2))))
+                    .distinct().collect(Collectors.toList());
+        }
+
+        private Set<K> allUpstreams(K node) {
+            Set<K> aups = allUpstreams.get(node);
+            if (aups == null) {
+                aups = Stream.concat(
+                        graph.getUpstreamProjects(node),
+                        graph.getUpstreamProjects(node).map(this::allUpstreams).flatMap(Set::stream))
+                        .collect(Collectors.toSet());
+                allUpstreams.put(node, aups);
+            }
+            return aups;
+        }
+
+    }
 
 }

daemon/src/main/java/org/mvndaemon/mvnd/builder/SmartBuilder.java

@@ -15,14 +15,6 @@
  */
 package org.mvndaemon.mvnd.builder;
 
-import groovy.lang.Binding;
-import groovy.lang.GroovyShell;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.List;
@@ -105,96 +97,15 @@ public class SmartBuilder implements Builder {
     public synchronized void build(final MavenSession session, final ReactorContext reactorContext,
             ProjectBuildList projectBuilds, final List<TaskSegment> taskSegments,
             ReactorBuildStatus reactorBuildStatus) throws ExecutionException, InterruptedException {
-        List<String> list = new ArrayList<>();
 
-        String providerScript = null;
-        String providerUrl = session.getTopLevelProject().getProperties()
-                .getProperty(MVND_BUILDER_RULES_PROVIDER_URL);
-        if (providerUrl != null) {
-            logger.warn(MVND_BUILDER_RULES_PROVIDER_URL
-                    + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
-
-            URL url;
-            try {
-                url = new URL(providerUrl);
-            } catch (MalformedURLException e) {
-                try {
-                    url = new File(providerUrl).toURI().toURL();
-                } catch (MalformedURLException ex) {
-                    url = null;
-                }
-            }
-            if (url == null) {
-                throw new ExecutionException("Bad syntax for " + MVND_BUILDER_RULES_PROVIDER_URL, null);
-            }
-            try (BufferedReader r = new BufferedReader(new InputStreamReader(url.openStream()))) {
-                StringBuilder sb = new StringBuilder();
-                char[] buf = new char[8192];
-                int l;
-                while ((l = r.read(buf)) >= 0) {
-                    sb.append(buf, 0, l);
-                }
-                providerScript = sb.toString();
-            } catch (IOException e) {
-                throw new ExecutionException("Unable to read provider url " + MVND_BUILDER_RULES_PROVIDER_URL, e);
-            }
-        }
-        if (providerScript == null) {
-            providerScript = session.getTopLevelProject().getProperties()
-                    .getProperty(MVND_BUILDER_RULES_PROVIDER_SCRIPT);
-        }
-        if (providerScript != null) {
-            logger.warn(MVND_BUILDER_RULES_PROVIDER_SCRIPT
-                    + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
-
-            Binding binding = new Binding();
-            GroovyShell shell = new GroovyShell(binding);
-            binding.setProperty("session", session);
-            Object result = shell.evaluate(providerScript);
-            if (result instanceof Iterable) {
-                for (Object r : (Iterable) result) {
-                    list.add(r.toString());
-                }
-            } else if (result != null) {
-                list.add(result.toString());
-            } else {
-                throw new ExecutionException("The provider script did not return a valid string or string collection", null);
-            }
-            list.add(result.toString());
-        }
-
-        String topRule = session.getTopLevelProject().getProperties()
-                .getProperty(MVND_BUILDER_RULES);
-        if (topRule != null) {
-            logger.warn(MVND_BUILDER_RULES
-                    + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
-            list.add(topRule);
-        }
-
-        session.getAllProjects().forEach(p -> {
-            String rule = p.getProperties().getProperty(MVND_BUILDER_RULE);
-            if (rule != null) {
-                logger.warn(MVND_BUILDER_RULE
-                        + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
-                rule = rule.trim();
-                if (!rule.isEmpty()) {
-                    rule = mvndRuleSanitizerPattern.matcher(rule).replaceAll(",");
-                    list.add(rule + " before " + p.getGroupId() + ":" + p.getArtifactId());
-                }
-            }
-        });
-        String rules = null;
-        if (!list.isEmpty()) {
-            rules = String.join("\n", list);
-        }
-
-        DependencyGraph<MavenProject> graph = DependencyGraph.fromMaven(session.getProjectDependencyGraph(), rules);
+        DependencyGraph<MavenProject> graph = (DependencyGraph<MavenProject>) session.getRequest().getData()
+                .get(DependencyGraph.class.getName());
 
         // log overall build info
         final int degreeOfConcurrency = session.getRequest().getDegreeOfConcurrency();
         logger.info("Task segments : " + taskSegments.stream().map(Object::toString).collect(Collectors.joining(" ")));
         logger.info("Build maximum degree of concurrency is " + degreeOfConcurrency);
-        logger.info("Total number of projects is " + graph.getProjects().count());
+        logger.info("Total number of projects is " + session.getProjects().size());
 
         // the actual build execution
         List<Map.Entry<TaskSegment, ReactorBuildStats>> allstats = new ArrayList<>();

daemon/src/main/java/org/mvndaemon/mvnd/daemon/Server.java

@@ -43,6 +43,7 @@ import java.util.stream.Collectors;
 import org.apache.maven.cli.DaemonMavenCli;
 import org.apache.maven.execution.MavenSession;
 import org.apache.maven.project.MavenProject;
+import org.mvndaemon.mvnd.builder.DependencyGraph;
 import org.mvndaemon.mvnd.builder.SmartBuilder;
 import org.mvndaemon.mvnd.common.DaemonConnection;
 import org.mvndaemon.mvnd.common.DaemonException;
@@ -655,8 +656,12 @@ public class Server implements AutoCloseable, Runnable {
 
         @Override
         protected void onStartSession(MavenSession session) {
-            queue.add(new BuildStarted(getCurrentProject(session).getName(), session.getProjects().size(),
-                    session.getRequest().getDegreeOfConcurrency()));
+            final int degreeOfConcurrency = session.getRequest().getDegreeOfConcurrency();
+            final DependencyGraph<MavenProject> dependencyGraph = DependencyGraph.fromMaven(session);
+            session.getRequest().getData().put(DependencyGraph.class.getName(), dependencyGraph);
+
+            final int maxThreads = degreeOfConcurrency == 1 ? 1 : dependencyGraph.computeMaxWidth(degreeOfConcurrency, 1000);
+            queue.add(new BuildStarted(getCurrentProject(session).getName(), session.getProjects().size(), maxThreads));
         }
 
         private MavenProject getCurrentProject(MavenSession mavenSession) {

daemon/src/test/java/org/mvndaemon/mvnd/builder/DagWidthTest.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mvndaemon.mvnd.builder;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Test;
+import org.mvndaemon.mvnd.builder.DependencyGraph.DagWidth;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class DagWidthTest {
+
+    @Test
+    void testSimpleGraph() {
+        //
+        // A B
+        // / | \ / \
+        // C D E F
+        // \/
+        // G
+        Map<String, List<String>> upstreams = new HashMap<>();
+        upstreams.put("A", Collections.emptyList());
+        upstreams.put("B", Collections.emptyList());
+        upstreams.put("C", Collections.singletonList("A"));
+        upstreams.put("D", Collections.singletonList("A"));
+        upstreams.put("E", Arrays.asList("A", "B"));
+        upstreams.put("F", Collections.singletonList("B"));
+        upstreams.put("G", Arrays.asList("D", "E"));
+        DependencyGraph<String> graph = newGraph(upstreams);
+
+        assertEquals(4, new DagWidth<>(graph).getMaxWidth(12));
+    }
+
+    @Test
+    void testSingle() {
+        //
+        // A
+        //
+        Map<String, List<String>> upstreams = new HashMap<>();
+        upstreams.put("A", Collections.emptyList());
+        DependencyGraph<String> graph = newGraph(upstreams);
+
+        assertEquals(1, new DagWidth<>(graph).getMaxWidth(12));
+    }
+
+    @Test
+    void testLinear() {
+        //
+        // A -> B -> C -> D
+        //
+        Map<String, List<String>> upstreams = new HashMap<>();
+        upstreams.put("A", Collections.emptyList());
+        upstreams.put("B", Collections.singletonList("A"));
+        upstreams.put("C", Collections.singletonList("B"));
+        upstreams.put("D", Collections.singletonList("C"));
+        DependencyGraph<String> graph = newGraph(upstreams);
+
+        assertEquals(1, new DagWidth<>(graph).getMaxWidth(12));
+    }
+
+    @Test
+    public void testHugeGraph() throws IOException {
+        Map<String, List<String>> upstreams = new HashMap<>();
+        try (BufferedReader r = new BufferedReader(
+                new InputStreamReader(getClass().getResourceAsStream("huge-graph.properties")))) {
+            r.lines().forEach(l -> {
+                int idxEq = l.indexOf(" = ");
+                if (!l.startsWith("#") && idxEq > 0) {
+                    String k = l.substring(0, idxEq).trim();
+                    String[] ups = l.substring(idxEq + 3).trim().split(",");
+                    List<String> list = Stream.of(ups).map(String::trim)
+                            .filter(s -> !s.isEmpty())
+                            .collect(Collectors.toList());
+                    upstreams.put(k, list);
+                }
+            });
+        }
+        DependencyGraph<String> graph = newGraph(upstreams);
+
+        DagWidth<String> w = new DagWidth<>(graph);
+        List<String> d = w.ensembleWithChildrenOf(
+                graph.getDownstreamProjects("org.apache.camel:camel").collect(Collectors.toList()),
+                "org.apache.camel:camel-parent");
+
+        assertEquals(12, w.getMaxWidth(12));
+    }
+
+    static <K> DependencyGraph<K> newGraph(Map<K, List<K>> upstreams) {
+        List<K> nodes = Stream.concat(upstreams.keySet().stream(), upstreams.values().stream().flatMap(List::stream))
+                .distinct()
+                .sorted()
+                .collect(Collectors.toList());
+        Map<K, List<K>> downstreams = nodes.stream().collect(Collectors.toMap(k -> k, k -> new ArrayList<>()));
+        upstreams.forEach((k, ups) -> {
+            ups.forEach(up -> downstreams.get(up).add(k));
+        });
+        return new DependencyGraph<>(nodes, upstreams, downstreams);
+    }
+}