From 3c23c154d7e3314aa2053ac121f89b6c6afb9ddd Mon Sep 17 00:00:00 2001 From: Peter Palaga Date: Fri, 12 Jun 2020 13:11:56 +0200 Subject: [PATCH] Let CI deploy the artifacts --- .github/workflows/verify.yaml | 158 +++++++++++++- client/pom.xml | 6 + .../jboss/fuse/mvnd/client/DefaultClient.java | 85 +++++--- .../org/jboss/fuse/mvnd/client/Installer.java | 194 ++++++++++++++++++ .../org/jboss/fuse/mvnd/client/Layout.java | 22 +- .../fuse/mvnd/client/mvnd.properties.template | 5 + daemon/src/main/provisio/maven-distro.xml | 2 +- pom.xml | 7 + 8 files changed, 437 insertions(+), 42 deletions(-) create mode 100644 client/src/main/java/org/jboss/fuse/mvnd/client/Installer.java create mode 100644 client/src/main/resources/org/jboss/fuse/mvnd/client/mvnd.properties.template diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index fc9de652..fec6f167 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -15,62 +15,204 @@ # limitations under the License. # -name: Linux, Windows and MacOS CI - +name: GitHub Actions on: [push, pull_request] +env: + GRAALVM_VERSION: '20.1.0.java11' + jobs: linux: runs-on: ubuntu-18.04 steps: + - uses: actions/checkout@v1 + - name: setup-graalvm-ce uses: DeLaGuardo/setup-graalvm@3 with: - graalvm-version: '20.1.0.java11' + graalvm-version: ${{ env.GRAALVM_VERSION }} + - name: gu install native-image run: gu install native-image + - name: mvn clean verify run: ./mvnw clean verify -Pnative -B -ntp -e + - name: Upload mvnd + uses: actions/upload-artifact@v2 + with: + name: mvnd-linux-amd64 + path: client/target/mvnd + - name: Upload mvnd-dist.zip + uses: actions/upload-artifact@v2 + with: + name: mvnd-dist.zip + path: daemon/target/mvnd-dist-*.zip + + windows: runs-on: windows-2019 steps: + + - name: Set CHOCO_CACHE_PATH + run: | + echo "::set-env name=CHOCO_CACHE_PATH::C:\Users\$env:UserName\AppData\Local\Temp\chocolatey" + + - name: Cache chocolatey localCache + uses: actions/cache@v2 + with: + path: ${{ env.CHOCO_CACHE_PATH }} + key: ${{ runner.os }}-choco-cache-2 + + - name: choco install visualstudio2017-workload-vctools + run: choco install visualstudio2017-workload-vctools --no-progress + - name: setup-graalvm-ce uses: DeLaGuardo/setup-graalvm@3 with: - graalvm-version: '20.1.0.java11' + graalvm-version: ${{ env.GRAALVM_VERSION }} + - name: gu install native-image shell: cmd run: gu install native-image - - name: choco install visualstudio2017-workload-vctools - run: choco install visualstudio2017-workload-vctools + + - uses: actions/cache@v2 + with: + path: ${{ env.JAVA_HOME }}\bin\native-image.exe + key: ${{ runner.os }}-native-image-${{ env.GRAALVM_VERSION }} + + - name: + id: native_image_exe_exists + uses: andstor/file-existence-action@v1 + with: + files: ${{ env.JAVA_HOME }}\bin\native-image.exe + - name: Compile native-image.cmd to native-image.exe + if: ${{ steps.native_image_exe_exists.outputs.files_exists == 'false' }} shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat" "%JAVA_HOME%\bin\native-image.cmd" -jar "%JAVA_HOME%\lib\graalvm\svm-driver.jar" native-image + - name: move native-image.exe %JAVA_HOME%\bin\ + if: ${{ steps.native_image_exe_exists.outputs.files_exists == 'false' }} shell: cmd run: | move native-image.exe "%JAVA_HOME%\bin\" + - uses: actions/checkout@v1 + - name: mvn clean verify shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat" ./mvnw clean verify -Pnative -B -ntp -e - macos: + - name: Upload mvnd + uses: actions/upload-artifact@v2 + with: + name: mvnd-windows-amd64.exe + path: client/target/mvnd.exe + + + darwin: runs-on: macos-10.15 steps: + - uses: actions/checkout@v1 + - name: setup-graalvm-ce uses: DeLaGuardo/setup-graalvm@3 with: - graalvm-version: '20.1.0.java11' + graalvm-version: ${{ env.GRAALVM_VERSION }} + - name: gu install native-image run: gu install native-image + - name: mvn clean verify run: ./mvnw clean verify -Pnative -B -ntp -e + + - name: Upload mvnd + uses: actions/upload-artifact@v2 + with: + name: mvnd-darwin-amd64 + path: client/target/mvnd + + + deploy: + runs-on: ubuntu-18.04 + needs: [linux, windows, darwin] + if: startsWith(github.ref, 'refs/tags') # deploy only for tags + steps: + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: ls -R + run: ls -R + + - name: Set environment + run: | + if [[ ${GITHUB_REF} = refs/heads/* ]] + then + VERSION=${GITHUB_REF##*/}-${GITHUB_SHA::8} + else + VERSION=${GITHUB_REF##*/} + fi + echo "Using VERSION=$VERSION" + echo "::set-env name=VERSION::$VERSION" + mv ./mvnd-dist.zip/mvnd-dist-*.zip ./mvnd-dist.zip/mvnd-dist.zip + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ env.VERSION }} + release_name: Release ${{ env.VERSION }} + draft: false + prerelease: false + + - name: Deploy mvnd-linux-amd64 + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: mvnd-linux-amd64/mvnd + asset_name: mvnd-linux-amd64 + asset_content_type: application/x-executable + + - name: Deploy mvnd-dist.zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: mvnd-dist.zip/mvnd-dist.zip + asset_name: mvnd-dist.zip + asset_content_type: application/zip + + - name: Deploy mvnd-darwin-amd64 + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: mvnd-darwin-amd64/mvnd + asset_name: mvnd-darwin-amd64 + asset_content_type: application/x-executable + + - name: Deploy mvnd-windows-amd64.exe + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: mvnd-windows-amd64.exe/mvnd.exe + asset_name: mvnd-windows-amd64.exe + asset_content_type: application/vnd.microsoft.portable-executable + diff --git a/client/pom.xml b/client/pom.xml index e68c5cca..562d0b6a 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -15,9 +15,15 @@ + + org.apache.commons + commons-compress + + org.graalvm.nativeimage svm + provided diff --git a/client/src/main/java/org/jboss/fuse/mvnd/client/DefaultClient.java b/client/src/main/java/org/jboss/fuse/mvnd/client/DefaultClient.java index e4080412..7e66198e 100644 --- a/client/src/main/java/org/jboss/fuse/mvnd/client/DefaultClient.java +++ b/client/src/main/java/org/jboss/fuse/mvnd/client/DefaultClient.java @@ -17,6 +17,7 @@ package org.jboss.fuse.mvnd.client; import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -24,7 +25,6 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Properties; import java.util.UUID; @@ -55,16 +55,24 @@ public class DefaultClient implements Client { private final Properties buildProperties; public static void main(String[] argv) throws Exception { - final List args = new ArrayList<>(Arrays.asList(argv)); + final List args = new ArrayList<>(argv.length); Path logFile = null; - for (int i = 0; i < args.size() - 2; i++) { - String arg = args.get(i); + int i = 0; + while (i < argv.length) { + final String arg = argv[i++]; if ("-l".equals(arg) || "--log-file".equals(arg)) { - logFile = Paths.get(args.get(i + 1)); - args.remove(i); - args.remove(i); - break; + if (i < argv.length) { + logFile = Paths.get(argv[i++]); + } else { + throw new IllegalArgumentException("-l and --log-file need to befollowed by a path"); + } + } else if ("--install".equals(arg)) { + install(false); + } else if ("--update".equals(arg)) { + install(true); + } else { + args.add(arg); } } @@ -73,14 +81,36 @@ public class DefaultClient implements Client { } } + private static void install(boolean overwrite) { + final Properties buildProps = loadBuildProperties(); + final String version = buildProps.getProperty("version"); + final String rawZipUri = System.getProperty("mvnd.zip.uri", "https://github.com/ppalaga/mvnd/releases/download/"+ version +"/mvnd-" + version + ".zip"); + final URI zipUri = URI.create(rawZipUri); + final Path mvndHome; + final String rawMvndHome = Layout.findEnvMavenHome(); + if (rawMvndHome == null) { + mvndHome = Paths.get(System.getProperty("user.home")).resolve(".m2/mvnd/" + version); + } else { + mvndHome = Paths.get(rawMvndHome); + } + final String rawJavaHome = System.getProperty("java.home"); + final Path javaHome = rawJavaHome != null ? Paths.get(rawJavaHome) : null; + Installer.installServer(zipUri, Layout.MVND_PROPS_PATH, mvndHome, javaHome, overwrite); + } + public DefaultClient(ClientLayout layout) { this.layout = layout; - this.buildProperties = new Properties(); + this.buildProperties = loadBuildProperties(); + } + + static Properties loadBuildProperties() { + final Properties result = new Properties(); try (InputStream is = DefaultClient.class.getResourceAsStream("build.properties")) { - buildProperties.load(is); + result.load(is); } catch (IOException e) { throw new RuntimeException("Could not read build.properties"); } + return result; } @Override @@ -95,9 +125,13 @@ public class DefaultClient implements Client { boolean debug = args.contains("-X") || args.contains("--debug"); if (version || showVersion || debug) { final String nativeSuffix = Layout.isNative() ? " (native)" : ""; - final String v = Ansi.ansi().bold().a("Maven Daemon " + buildProperties.getProperty("version") + nativeSuffix).reset().toString(); + final String v = Ansi.ansi().bold().a("Maven Daemon " + buildProperties.getProperty("version") + nativeSuffix) + .reset().toString(); output.accept(v); - /* Do not return, rather pass -v to the server so that the client module does not need to depend on any Maven artifacts */ + /* + * Do not return, rather pass -v to the server so that the client module does not need to depend on any + * Maven artifacts + */ } final Path javaHome = layout.javaHome(); @@ -162,17 +196,17 @@ public class DefaultClient implements Client { } else if (m instanceof BuildEvent) { BuildEvent be = (BuildEvent) m; switch (be.getType()) { - case BuildStarted: - break; - case BuildStopped: - return new DefaultResult(argv, true); - case ProjectStarted: - case MojoStarted: - case MojoStopped: - output.projectStateChanged(be.projectId, be.display); - break; - case ProjectStopped: - output.projectFinished(be.projectId); + case BuildStarted: + break; + case BuildStopped: + return new DefaultResult(argv, true); + case ProjectStarted: + case MojoStarted: + case MojoStopped: + output.projectStateChanged(be.projectId, be.display); + break; + case ProjectStopped: + output.projectFinished(be.projectId); } } else if (m instanceof BuildMessage) { BuildMessage bm = (BuildMessage) m; @@ -236,13 +270,14 @@ public class DefaultClient implements Client { } catch (Exception e) { throw new DaemonException.StartException( String.format("Error starting daemon: uid = %s, workingDir = %s, daemonArgs: %s", - uid, workingDir, command), e); + uid, workingDir, command), + e); } } Path findClientJar(Path mavenHome) { final Path ext = mavenHome.resolve("lib/ext"); - final String clientJarName = "mvnd-client-"+ buildProperties.getProperty("version") + ".jar"; + final String clientJarName = "mvnd-client-" + buildProperties.getProperty("version") + ".jar"; try (Stream files = Files.list(ext)) { return files .filter(f -> f.getFileName().toString().equals(clientJarName)) diff --git a/client/src/main/java/org/jboss/fuse/mvnd/client/Installer.java b/client/src/main/java/org/jboss/fuse/mvnd/client/Installer.java new file mode 100644 index 00000000..2248f840 --- /dev/null +++ b/client/src/main/java/org/jboss/fuse/mvnd/client/Installer.java @@ -0,0 +1,194 @@ +package org.jboss.fuse.mvnd.client; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; + +public class Installer { + private static final int BUFFER_SIZE = 4096; + private static final int MAX_PERMISSIONS = 0777; + public static void installServer(URI zipUri, Path mvndPropsPath, Path mvndHome, Path javaHome, boolean overwrite) { + final boolean mvndHomeExists = Files.exists(mvndHome); + if (!overwrite && mvndHomeExists) { + throw new IllegalStateException( + "Cannot install if mvnd.home " + mvndHome + " exists. Consider --update instead of --install."); + } + if (!overwrite && Files.exists(mvndPropsPath)) { + throw new IllegalStateException( + "Cannot install if " + mvndPropsPath + " exists. Consider --update instead of --install."); + } + deleteIfExists(mvndHome); + deleteIfExists(mvndPropsPath); + + final Path localZip = download(zipUri); + unzip(localZip, mvndHome); + writeMvndProperties(mvndPropsPath, mvndHome, javaHome); + } + + private static void deleteIfExists(Path path) { + if (Files.isRegularFile(path)) { + try { + Files.delete(path); + } catch (IOException e) { + throw new RuntimeException("Could not delete " + path); + } + } else if (Files.isDirectory(path)) { + try (Stream files = Files.walk(path)) { + files.sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.delete(p); + } catch (Exception e) { + throw new RuntimeException("Could not delete " + p, e); + } + }); + } catch (IOException e) { + throw new RuntimeException("Could not delete " + path, e); + } + } + } + + static void writeMvndProperties(Path mvndPropsPath, Path mvndHome, Path javaHome) { + final String template = readTemplate(); + final String javaHomeLine = javaHome == null ? "" : "java.home = " + javaHome.toString(); + final String content = String.format(template, mvndHome.toString(), javaHomeLine); + try { + Files.write(mvndPropsPath, content.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new RuntimeException("Could not write to " + mvndPropsPath); + } + } + + static String readTemplate() { + try (InputStream in = Installer.class.getResourceAsStream("mvnd.properties.template"); + ByteArrayOutputStream out = new ByteArrayOutputStream(256)) { + copy(in, out); + return new String(out.toByteArray(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException("Could not read build.properties"); + } + } + + static void unzip(Path localZip, Path mvndHome) { + try { + Files.createDirectories(mvndHome); + } catch (IOException e) { + throw new RuntimeException("Could not create directories " + mvndHome, e); + } + try (ZipFile zip = new ZipFile(Files.newByteChannel(localZip))) { + final Map> permissionCache = new HashMap<>(); + final Enumeration entries = zip.getEntries(); + while (entries.hasMoreElements()) { + final ZipArchiveEntry entry = entries.nextElement(); + final Path dest = mvndHome.resolve(entry.getName()).normalize(); + if (!dest.startsWith(mvndHome)) { + /* Avoid writing to paths outside of mvndHome */ + throw new IllegalStateException("Tainted ZIP entry name " + entry.getName()); + } + if (entry.isDirectory()) { + Files.createDirectories(dest); + } else { + Files.createDirectories(dest.getParent()); + try ( + InputStream in = new BufferedInputStream(zip.getInputStream(entry), BUFFER_SIZE); + OutputStream out = new BufferedOutputStream(Files.newOutputStream(dest), BUFFER_SIZE)) { + copy(in, out); + } catch (IOException e) { + throw new RuntimeException( + "Could not unzip entry " + entry.getName() + " from " + localZip + " to " + dest); + } + } + final PosixFileAttributeView attributes = Files.getFileAttributeView(dest, PosixFileAttributeView.class); + if (attributes != null) { + int mode = (int) (entry.getUnixMode() & MAX_PERMISSIONS); + Files.setPosixFilePermissions(dest, permissionCache.computeIfAbsent(mode, Installer::toPermissionSet)); + } + Files.setLastModifiedTime(dest, FileTime.from(entry.getTime(), TimeUnit.MILLISECONDS)); + } + } catch (IOException e) { + throw new RuntimeException("Could not unzip " + localZip, e); + } + } + + static Path download(URI zipUri) { + try { + final Path localZip = Files.createTempFile("", "-mvnd-dist.zip"); + try ( + InputStream in = new BufferedInputStream(zipUri.toURL().openStream(), BUFFER_SIZE); + OutputStream out = new BufferedOutputStream(Files.newOutputStream(localZip), BUFFER_SIZE)) { + copy(in, out); + } catch (IOException e) { + throw new RuntimeException("Could not download " + zipUri + " to " + localZip); + } + return localZip; + } catch (IOException e) { + throw new RuntimeException("Could not create temp file", e); + } + } + + static void copy(InputStream in, OutputStream out) throws IOException { + final byte buf[] = new byte[BUFFER_SIZE]; + int len; + while ((len = in.read(buf)) >= 0) { + out.write(buf, 0, len); + } + } + + static Set toPermissionSet(Integer mode) { + final Set result = EnumSet.noneOf(PosixFilePermission.class); + /* others */ + if ((mode & 0001) != 0) { + result.add(PosixFilePermission.OTHERS_EXECUTE); + } + if ((mode & 0002) != 0) { + result.add(PosixFilePermission.OTHERS_WRITE); + } + if ((mode & 0004) != 0) { + result.add(PosixFilePermission.OTHERS_READ); + } + /* group */ + if ((mode & 0010) != 0) { + result.add(PosixFilePermission.GROUP_EXECUTE); + } + if ((mode & 0020) != 0) { + result.add(PosixFilePermission.GROUP_WRITE); + } + if ((mode & 0040) != 0) { + result.add(PosixFilePermission.GROUP_READ); + } + /* user */ + if ((mode & 0100) != 0) { + result.add(PosixFilePermission.OWNER_EXECUTE); + } + if ((mode & 0200) != 0) { + result.add(PosixFilePermission.OWNER_WRITE); + } + if ((mode & 0400) != 0) { + result.add(PosixFilePermission.OWNER_READ); + } + return Collections.unmodifiableSet(result); + } + +} diff --git a/client/src/main/java/org/jboss/fuse/mvnd/client/Layout.java b/client/src/main/java/org/jboss/fuse/mvnd/client/Layout.java index dfb0e8b4..a883681a 100644 --- a/client/src/main/java/org/jboss/fuse/mvnd/client/Layout.java +++ b/client/src/main/java/org/jboss/fuse/mvnd/client/Layout.java @@ -26,6 +26,8 @@ import java.util.stream.Stream; public class Layout { + public static final Path MVND_PROPS_PATH = Paths.get(System.getProperty("user.home")).resolve(".m2/mvnd.properties"); + private static Layout ENV_INSTANCE; private final Path mavenHome; @@ -79,22 +81,18 @@ public class Layout { static Properties loadMvndProperties() { final Properties result = new Properties(); - final Path mvndPropsPath = Paths.get(System.getProperty("user.home")).resolve(".m2/mvnd.properties"); - if (Files.exists(mvndPropsPath)) { - try (InputStream in = Files.newInputStream(mvndPropsPath)) { + if (Files.exists(MVND_PROPS_PATH)) { + try (InputStream in = Files.newInputStream(MVND_PROPS_PATH)) { result.load(in); } catch (IOException e) { - throw new RuntimeException("Could not read " + mvndPropsPath); + throw new RuntimeException("Could not read " + MVND_PROPS_PATH); } } return result; } static Path findMavenHome(Properties mvndProperties) { - String rawValue = System.getenv("MAVEN_HOME"); - if (rawValue == null) { - rawValue = System.getProperty("maven.home"); - } + String rawValue = findEnvMavenHome(); if (isNative()) { try { final Path nativeExecutablePath = Paths.get(Class.forName("org.graalvm.nativeimage.ProcessProperties").getMethod("getExecutableName").invoke(null).toString()).toAbsolutePath().normalize(); @@ -126,6 +124,14 @@ public class Layout { return Paths.get(rawValue).toAbsolutePath().normalize(); } + public static String findEnvMavenHome() { + String rawValue = System.getenv("MAVEN_HOME"); + if (rawValue == null) { + rawValue = System.getProperty("maven.home"); + } + return rawValue; + } + static Path findMultiModuleProjectDirectory(Path pwd) { final String multiModuleProjectDirectory = System.getProperty("maven.multiModuleProjectDirectory"); if (multiModuleProjectDirectory != null) { diff --git a/client/src/main/resources/org/jboss/fuse/mvnd/client/mvnd.properties.template b/client/src/main/resources/org/jboss/fuse/mvnd/client/mvnd.properties.template new file mode 100644 index 00000000..7005cc40 --- /dev/null +++ b/client/src/main/resources/org/jboss/fuse/mvnd/client/mvnd.properties.template @@ -0,0 +1,5 @@ +# An absolute path to your Maven Daemon installation +maven.home = %s + +# java.home is optional if you have JAVA_HOME environment variable set +%s diff --git a/daemon/src/main/provisio/maven-distro.xml b/daemon/src/main/provisio/maven-distro.xml index fe9e3351..00124336 100644 --- a/daemon/src/main/provisio/maven-distro.xml +++ b/daemon/src/main/provisio/maven-distro.xml @@ -16,7 +16,7 @@ - diff --git a/pom.xml b/pom.xml index d723554a..34385961 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,7 @@ 3.16.1 + 1.20 20.1.0 3.0.0 1.0 @@ -57,6 +58,12 @@ ${slf4j.version} + + org.apache.commons + commons-compress + ${commons-compress.version} + + org.graalvm.sdk graal-sdk