From dcf409aecbd7e11d270806fe9b5ab56ac860ee3b Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 3 Apr 2019 14:24:00 -0500 Subject: [PATCH] Improve CI caching (#416) * Adjust linux distro to ubuntu 14.04 * Cache lib directory for speedy builds * Fix path linking for default extensions * Update reset * Reset to head * Improve caching * Still run yarn in CI * Update yarn before install * Increase cache timeout * Install vscode from vstar * Undo data-dir changes to CLI, add back clean, remove unused CI func * Remove additional flags added * Remove unused dependency * Reset vscode install dir so patching always works --- .gitignore | 1 + .travis.yml | 9 ++- build/tasks.ts | 114 +++++++++--------------------- package.json | 4 ++ scripts/vstar.sh | 24 +++++++ scripts/webpack.general.config.js | 14 ++++ yarn.lock | 63 ++++++++++++++++- 7 files changed, 145 insertions(+), 84 deletions(-) create mode 100755 scripts/vstar.sh diff --git a/.gitignore b/.gitignore index 8795da8..d2ee77a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dist out .DS_Store release +.cache \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 37ccfea..d144c09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,12 @@ env: matrix: include: - os: linux - dist: ubuntu + dist: trusty - os: osx before_install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev libsecret-1-dev; fi +- npm install -g yarn@1.12.3 script: - scripts/build.sh before_deploy: @@ -35,4 +36,8 @@ deploy: on: repo: codercom/code-server branch: master -cache: yarn +cache: + yarn: true + timeout: 1000 + directories: + - .cache diff --git a/build/tasks.ts b/build/tasks.ts index a784fbf..ae28512 100644 --- a/build/tasks.ts +++ b/build/tasks.ts @@ -4,21 +4,22 @@ import * as fse from "fs-extra"; import * as os from "os"; import * as path from "path"; import * as zlib from "zlib"; +import * as https from "https"; +import * as tar from "tar"; const isWin = os.platform() === "win32"; const libPath = path.join(__dirname, "../lib"); const vscodePath = path.join(libPath, "vscode"); +const defaultExtensionsPath = path.join(libPath, "extensions"); const pkgsPath = path.join(__dirname, "../packages"); -const defaultExtensionsPath = path.join(libPath, "VSCode-linux-x64/resources/app/extensions"); const vscodeVersion = process.env.VSCODE_VERSION || "1.32.0"; +const vsSourceUrl = `https://codesrv-ci.cdr.sh/vstar-${vscodeVersion}.tar.gz`; const buildServerBinary = register("build:server:binary", async (runner) => { await ensureInstalled(); - await copyForDefaultExtensions(); await Promise.all([ buildBootstrapFork(), buildWeb(), - buildDefaultExtensions(), buildServerBundle(), buildAppBrowser(), ]); @@ -129,97 +130,50 @@ const buildWeb = register("build:web", async (runner) => { await runner.execute(isWin ? "npm.cmd" : "npm", ["run", "build"]); }); -const extDirPath = path.join("lib", "vscode-default-extensions"); -const copyForDefaultExtensions = register("build:copy-vscode", async (runner) => { - if (!fs.existsSync(defaultExtensionsPath)) { - await ensureClean(); - await ensureInstalled(); - await new Promise((resolve, reject): void => { - fse.remove(extDirPath, (err) => { - if (err) { - return reject(err); - } - - resolve(); - }); - }); - await new Promise((resolve, reject): void => { - fse.copy(vscodePath, extDirPath, (err) => { - if (err) { - return reject(err); - } - - resolve(); - }); - }); - } -}); - -const buildDefaultExtensions = register("build:default-extensions", async (runner) => { - if (!fs.existsSync(defaultExtensionsPath)) { - await copyForDefaultExtensions(); - runner.cwd = extDirPath; - const resp = await runner.execute(isWin ? "npx.cmd" : "npx", [isWin ? "gulp.cmd" : "gulp", "vscode-linux-x64", "--max-old-space-size=32384"]); - if (resp.exitCode !== 0) { - throw new Error(`Failed to build default extensions: ${resp.stderr}`); - } - } -}); - const ensureInstalled = register("vscode:install", async (runner) => { - await ensureCloned(); + runner.cwd = libPath; - runner.cwd = vscodePath; - const install = await runner.execute(isWin ? "yarn.cmd" : "yarn", []); - if (install.exitCode !== 0) { - throw new Error(`Failed to install vscode dependencies: ${install.stderr}`); - } -}); + if (fs.existsSync(vscodePath) && fs.existsSync(defaultExtensionsPath)) { + const pkgVersion = JSON.parse(fs.readFileSync(path.join(vscodePath, "package.json")).toString("utf8")).version; + if (pkgVersion === vscodeVersion) { + runner.cwd = vscodePath; -const ensureCloned = register("vscode:clone", async (runner) => { - if (fs.existsSync(vscodePath)) { - await ensureClean(); - } else { - fse.mkdirpSync(libPath); - runner.cwd = libPath; - const clone = await runner.execute("git", ["clone", "https://github.com/microsoft/vscode", "--branch", vscodeVersion, "--single-branch", "--depth=1"]); - if (clone.exitCode !== 0) { - throw new Error(`Failed to clone: ${clone.exitCode}`); + const reset = await runner.execute("git", ["reset", "--hard"]); + if (reset.exitCode !== 0) { + throw new Error(`Failed to clean git repository: ${reset.stderr}`); + } + + return; } } - runner.cwd = vscodePath; - const checkout = await runner.execute("git", ["checkout", vscodeVersion]); - if (checkout.exitCode !== 0) { - throw new Error(`Failed to checkout: ${checkout.stderr}`); - } -}); + fse.removeSync(libPath); + fse.mkdirpSync(libPath); -const ensureClean = register("vscode:clean", async (runner) => { - runner.cwd = vscodePath; + await new Promise((resolve, reject): void => { + https.get(vsSourceUrl, (res) => { + if (res.statusCode !== 200) { + return reject(res.statusMessage); + } - const status = await runner.execute("git", ["status", "--porcelain"]); - if (status.stdout.trim() !== "") { - const clean = await runner.execute("git", ["clean", "-f", "-d", "-X"]); - if (clean.exitCode !== 0) { - throw new Error(`Failed to clean git repository: ${clean.stderr}`); - } - const removeUnstaged = await runner.execute("git", ["checkout", "--", "."]); - if (removeUnstaged.exitCode !== 0) { - throw new Error(`Failed to remove unstaged files: ${removeUnstaged.stderr}`); - } - } - const fetch = await runner.execute("git", ["fetch", "--prune"]); - if (fetch.exitCode !== 0) { - throw new Error(`Failed to fetch latest changes: ${fetch.stderr}`); - } + res.pipe(tar.x({ + C: libPath, + }).on("finish", () => { + resolve(); + }).on("error", (err: Error) => { + reject(err); + })); + }).on("error", (err) => { + reject(err); + }); + }); }); const ensurePatched = register("vscode:patch", async (runner) => { if (!fs.existsSync(vscodePath)) { throw new Error("vscode must be cloned to patch"); } - await ensureClean(); + await ensureInstalled(); runner.cwd = vscodePath; const patchPath = path.join(__dirname, "../scripts/vscode.patch"); diff --git a/package.json b/package.json index 8fb17fc..72e5007 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,9 @@ "devDependencies": { "@types/fs-extra": "^5.0.4", "@types/node": "^10.12.18", + "@types/tar": "^4.0.0", "@types/trash": "^4.3.1", + "cache-loader": "^2.0.1", "cross-env": "^5.2.0", "crypto-browserify": "^3.12.0", "css-loader": "^2.1.0", @@ -34,6 +36,8 @@ "sass-loader": "^7.1.0", "string-replace-loader": "^2.1.1", "style-loader": "^0.23.1", + "tar": "^4.4.8", + "terser-webpack-plugin": "^1.2.3", "ts-loader": "^5.3.3", "ts-node": "^7.0.1", "tsconfig-paths": "^3.8.0", diff --git a/scripts/vstar.sh b/scripts/vstar.sh new file mode 100755 index 0000000..478bc81 --- /dev/null +++ b/scripts/vstar.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euxo pipefail + +# Builds a tarfile containing vscode sourcefiles neccessary for CI. +# Done outside the CI and uploaded to object storage to reduce CI time. + +branch=1.32.0 +dir=/tmp/vstar +outfile=/tmp/vstar-$branch.tar.gz +rm -rf $dir +mkdir -p $dir + +cd $dir +git clone https://github.com/microsoft/vscode --branch $branch --single-branch --depth=1 +cd vscode + +yarn + +npx gulp vscode-linux-x64 --max-old-space-size=32384 +rm -rf extensions build out* test +cd .. +mv *-x64/resources/app/extensions ./extensions +rm -rf *-x64 +tar -czvf $outfile . diff --git a/scripts/webpack.general.config.js b/scripts/webpack.general.config.js index 83215fa..4da6ca8 100644 --- a/scripts/webpack.general.config.js +++ b/scripts/webpack.general.config.js @@ -3,6 +3,7 @@ const os = require("os"); const environment = process.env.NODE_ENV || "development"; const HappyPack = require("happypack"); const webpack = require("webpack"); +const TerserPlugin = require("terser-webpack-plugin"); const root = path.join(__dirname, ".."); @@ -107,6 +108,11 @@ module.exports = (options = {}) => ({ id: "ts", threads: Math.max(os.cpus().length - 1, 1), loaders: [{ + path: "cache-loader", + query: { + cacheDirectory: path.join(__dirname, "..", ".cache"), + }, + }, { path: "ts-loader", query: { happyPackMode: true, @@ -121,6 +127,14 @@ module.exports = (options = {}) => ({ "process.env.VERSION": `"${process.env.VERSION || ""}"`, }), ], + optimization: { + minimizer: [ + new TerserPlugin({ + cache: path.join(__dirname, "..", ".cache", "terser"), + parallel: true, + }), + ], + }, stats: { all: false, // Fallback for options not defined. errors: true, diff --git a/yarn.lock b/yarn.lock index 5add43c..4c998bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -24,6 +24,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.18.tgz#1d3ca764718915584fcd9f6344621b7672665c67" integrity sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ== +"@types/tar@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@types/tar/-/tar-4.0.0.tgz#e3239d969eeb693a012200613860d0eb871c94f0" + integrity sha512-YybbEHNngcHlIWVCYsoj7Oo1JU9JqONuAlt1LlTH/lmL8BMhbzdFUgReY87a05rY1j8mfK47Del+TCkaLAXwLw== + dependencies: + "@types/node" "*" + "@types/trash@^4.3.1": version "4.3.1" resolved "https://registry.yarnpkg.com/@types/trash/-/trash-4.3.1.tgz#4880ff17c4eb467f1a26774ea6328428403b5c57" @@ -723,6 +730,17 @@ cache-base@^1.0.1: union-value "^1.0.0" unset-value "^1.0.0" +cache-loader@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/cache-loader/-/cache-loader-2.0.1.tgz#5758f41a62d7c23941e3c3c7016e6faeb03acb07" + integrity sha512-V99T3FOynmGx26Zom+JrVBytLBsmUCzVG2/4NnUKgvXN4bEV42R1ERl1IyiH/cvFIDA1Ytq2lPZ9tXDSahcQpQ== + dependencies: + loader-utils "^1.1.0" + mkdirp "^0.5.1" + neo-async "^2.6.0" + normalize-path "^3.0.0" + schema-utils "^1.0.0" + camel-case@3.0.x: version "3.0.0" resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-3.0.0.tgz#ca3c3688a4e9cf3a4cda777dc4dcbc713249cf73" @@ -916,6 +934,11 @@ commander@^2.12.1, commander@^2.18.0: resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a" integrity sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg== +commander@^2.19.0: + version "2.20.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422" + integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ== + commondir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" @@ -3305,7 +3328,7 @@ negotiator@0.6.1: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" integrity sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk= -neo-async@^2.5.0: +neo-async@^2.5.0, neo-async@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.0.tgz#b9d15e4d71c6762908654b5183ed38b753340835" integrity sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA== @@ -3452,6 +3475,11 @@ normalize-path@^2.1.1: dependencies: remove-trailing-separator "^1.0.1" +normalize-path@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" + integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== + npm-bundled@^1.0.1: version "1.0.5" resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.5.tgz#3c1732b7ba936b3a10325aef616467c0ccbcc979" @@ -4625,6 +4653,14 @@ source-map-support@^0.5.6, source-map-support@~0.5.6: buffer-from "^1.0.0" source-map "^0.6.0" +source-map-support@~0.5.10: + version "0.5.11" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.11.tgz#efac2ce0800355d026326a0ca23e162aeac9a4e2" + integrity sha512-//sajEx/fGL3iw6fltKMdPvy8kL3kJ2O3iuYlRoT3k9Kb4BjOoZ+BZzaNHeuaruSt+Kf3Zk9tnfAQg9/AJqUVQ== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + source-map-url@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3" @@ -4924,7 +4960,7 @@ tar@^2.0.0: fstream "^1.0.2" inherits "2" -tar@^4: +tar@^4, tar@^4.4.8: version "4.4.8" resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.8.tgz#b19eec3fde2a96e64666df9fdb40c5ca1bc3747d" integrity sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ== @@ -4951,6 +4987,29 @@ terser-webpack-plugin@^1.1.0: webpack-sources "^1.1.0" worker-farm "^1.5.2" +terser-webpack-plugin@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.2.3.tgz#3f98bc902fac3e5d0de730869f50668561262ec8" + integrity sha512-GOK7q85oAb/5kE12fMuLdn2btOS9OBZn4VsecpHDywoUC/jLhSAKOiYo0ezx7ss2EXPMzyEWFoE0s1WLE+4+oA== + dependencies: + cacache "^11.0.2" + find-cache-dir "^2.0.0" + schema-utils "^1.0.0" + serialize-javascript "^1.4.0" + source-map "^0.6.1" + terser "^3.16.1" + webpack-sources "^1.1.0" + worker-farm "^1.5.2" + +terser@^3.16.1: + version "3.17.0" + resolved "https://registry.yarnpkg.com/terser/-/terser-3.17.0.tgz#f88ffbeda0deb5637f9d24b0da66f4e15ab10cb2" + integrity sha512-/FQzzPJmCpjAH9Xvk2paiWrFq+5M6aVOf+2KRbwhByISDX/EujxsK+BAvrhb6H+2rtrLCHK9N01wO014vrIwVQ== + dependencies: + commander "^2.19.0" + source-map "~0.6.1" + source-map-support "~0.5.10" + terser@^3.8.1: version "3.14.1" resolved "https://registry.yarnpkg.com/terser/-/terser-3.14.1.tgz#cc4764014af570bc79c79742358bd46926018a32"