Bundle dependencies (sveltejs#67)

* zip dependencies instead of installing with turbo

* convince esbuild to work

* remove logging

* install deps

* try this

* exclude sourcemaps

* minimise bundle

* remove jszip, which ended up being surplus to requirements

* remove unused code

Author: Rich-Harris

.gitignore

@@ -8,3 +8,6 @@
 !.env.example
 .apps
 .vercel
+/content/tutorial/common/.svelte-kit
+/content/tutorial/common/node_modules
+/content/tutorial/common/package-lock.json

content/tutorial/common/package.json

@@ -9,8 +9,8 @@
 		"prepare": "svelte-kit sync"
 	},
 	"devDependencies": {
-		"@sveltejs/adapter-auto": "next",
 		"@sveltejs/kit": "next",
+		"esbuild-wasm": "^0.14.43",
 		"svelte": "^3.44.0"
 	},
 	"type": "module"

content/tutorial/common/svelte.config.js

@@ -1,10 +1,6 @@
-import adapter from '@sveltejs/adapter-auto';
-
 /** @type {import('@sveltejs/kit').Config} */
 const config = {
-	kit: {
-		adapter: adapter()
-	}
+	kit: {}
 };
 
 export default config;

package.json

@@ -3,7 +3,7 @@
 	"version": "0.0.1",
 	"scripts": {
 		"dev": "svelte-kit dev",
-		"build": "svelte-kit build && node scripts/create-middleware.js",
+		"build": "node scripts/create-common-bundle && svelte-kit build && node scripts/create-middleware",
 		"package": "svelte-kit package",
 		"preview": "svelte-kit preview",
 		"prepare": "svelte-kit sync",
@@ -17,16 +17,19 @@
 		"@sveltejs/kit": "next",
 		"@sveltejs/site-kit": "^2.1.0",
 		"diff": "^5.1.0",
+		"esbuild": "^0.14.43",
 		"prettier": "^2.6.2",
 		"prettier-plugin-svelte": "^2.7.0",
 		"svelte": "^3.48.0",
 		"svelte-check": "^2.7.2",
+		"tiny-glob": "^0.2.9",
 		"typescript": "~4.6.4"
 	},
 	"type": "module",
 	"dependencies": {
 		"@fontsource/roboto-mono": "^4.5.7",
 		"@webcontainer/api": "^0.0.1",
+		"adm-zip": "^0.5.9",
 		"base64-js": "^1.5.1",
 		"marked": "^4.0.16",
 		"monaco-editor": "^0.33.0",

pnpm-lock.yaml

@@ -0,0 +1,13 @@
+import fs from 'fs';
+import AdmZip from 'adm-zip';
+
+const zip = new AdmZip('common.zip');
+zip.extractAllTo('.');
+
+fs.mkdirSync('node_modules/.bin');
+
+fs.symlinkSync('../@sveltejs/kit/svelte-kit.js', 'node_modules/.bin/svelte-kit');
+fs.chmodSync('node_modules/.bin/svelte-kit', 0o777);
+
+fs.symlinkSync('../esbuild/bin/esbuild', 'node_modules/.bin/esbuild');
+fs.chmodSync('node_modules/.bin/esbuild', 0o777);

scripts/create-common-bundle/boot.js

@@ -0,0 +1,69 @@
+import fs from 'fs';
+import esbuild from 'esbuild';
+import AdmZip from 'adm-zip';
+import glob from 'tiny-glob/sync.js';
+import { fileURLToPath } from 'url';
+import { execSync } from 'child_process';
+
+const cwd = 'content/tutorial/common';
+
+if (!fs.existsSync(`${cwd}/node_modules`)) {
+	execSync('npm install', { cwd });
+}
+
+const zip = new AdmZip();
+
+// we selectively exclude certain files to minimise the bundle size.
+// this is a bit ropey, but it works
+const ignored_basenames = ['.DS_Store', 'LICENSE'];
+const ignored_extensions = ['.d.ts', '.map', '.md'];
+const ignored_directories = ['.svelte-kit', 'node_modules/.bin', 'node_modules/rollup/dist/shared'];
+
+const ignored_files = new Set([
+	'node_modules/vite/dist/node/terser.js',
+	'node_modules/rollup/dist/es/rollup.browser.js',
+	'node_modules/rollup/dist/rollup.browser.js',
+	'node_modules/svelte/compiler.js'
+]);
+
+for (const file of glob('**', { cwd, filesOnly: true, dot: true })) {
+	if (ignored_extensions.find((ext) => file.endsWith(ext))) continue;
+	if (ignored_basenames.find((basename) => file.endsWith('/' + basename))) continue;
+	if (ignored_directories.find((dir) => file.startsWith(dir + '/'))) continue;
+
+	if (ignored_files.has(file)) {
+		ignored_files.delete(file);
+		continue;
+	}
+
+	// esbuild is a special case
+	if (file.startsWith('node_modules/esbuild-wasm/')) {
+		zip.addFile(
+			file.replace('node_modules/esbuild-wasm', 'node_modules/esbuild'),
+			fs.readFileSync(`${cwd}/${file}`)
+		);
+		continue;
+	} else if (file.startsWith('node_modules/esbuild')) {
+		continue;
+	}
+
+	zip.addFile(file, fs.readFileSync(`${cwd}/${file}`));
+}
+
+if (ignored_files.size > 0) {
+	throw new Error(`expected to find ${Array.from(ignored_files).join(', ')}`);
+}
+
+const out = zip.toBuffer();
+
+fs.writeFileSync(`src/lib/client/adapters/common/common.zip`, out);
+
+// bundle adm-zip so we can use it in the webcontainer
+esbuild.buildSync({
+	entryPoints: [fileURLToPath(new URL('./boot.js', import.meta.url))],
+	bundle: true,
+	platform: 'node',
+	minify: true,
+	outfile: 'src/lib/client/adapters/common/boot.cjs',
+	format: 'cjs'
+});

scripts/create-common-bundle/index.js

@@ -0,0 +1,2 @@
+/boot.cjs
+/common.zip

src/lib/client/adapters/common/.gitignore

@@ -0,0 +1,16 @@
+import zipped from './common.zip?url';
+import boot from './boot.cjs?url';
+
+async function load() {
+	const result = await Promise.all([
+		fetch(zipped).then((r) => r.arrayBuffer()),
+		fetch(boot).then((r) => r.text())
+	]);
+
+	return {
+		zipped: result[0],
+		boot: result[1]
+	};
+}
+
+export const ready = load();

src/lib/client/adapters/common/index.js

@@ -1,5 +1,7 @@
 import { load } from '@webcontainer/api';
+
 import base64 from 'base64-js';
+import { ready } from '../common/index.js';
 
 const WebContainer = await load();
 
@@ -8,29 +10,42 @@ const WebContainer = await load();
  * @returns {Promise<import('$lib/types').Adapter>}
  */
 export async function create(stubs) {
-	const vm = await WebContainer.boot();
-
 	const tree = convert_stubs_to_tree(stubs);
+
+	const common = await ready;
+	tree['common.zip'] = file(new Uint8Array(common.zipped));
+	tree['boot.cjs'] = file(common.boot);
+
+	const vm = await WebContainer.boot();
 	await vm.loadFiles(tree);
 
+	const boot = await vm.run(
+		{
+			command: 'node',
+			args: ['boot.cjs']
+		},
+		{
+			stderr: (data) => console.error(`[boot] ${data}`)
+		}
+	);
+
+	const code = await boot.onExit;
+
+	if (code !== 0) {
+		throw new Error('Failed to initialize WebContainer');
+	}
+
 	const base = await new Promise(async (fulfil, reject) => {
 		vm.on('server-ready', (port, base) => {
 			fulfil(base);
 		});
 
-		const install = await vm.run({
-			command: 'turbo',
-			args: ['install']
-		});
-
-		const code = await install.onExit;
-
-		if (code !== 0) {
-			reject(new Error('Installation failed'));
-			return;
-		}
-
-		await vm.run({ command: 'turbo', args: ['run', 'dev'] });
+		await vm.run(
+			{ command: 'turbo', args: ['run', 'dev'] },
+			{
+				stderr: (data) => console.error(`[dev] ${data}`)
+			}
+		);
 	});
 
 	let current = stubs;
@@ -97,11 +112,7 @@ export async function create(stubs) {
 					tree = /** @type {import('@webcontainer/api').DirectoryEntry} */ (tree[part]).directory;
 				}
 
-				tree[basename] = {
-					file: {
-						contents: stub.text ? stub.contents : base64.toByteArray(stub.contents)
-					}
-				};
+				tree[basename] = file(stub.text ? stub.contents : base64.toByteArray(stub.contents));
 			}
 
 			await vm.loadFiles(root);
@@ -132,14 +143,17 @@ function convert_stubs_to_tree(stubs, depth = 1) {
 					directory: convert_stubs_to_tree(children, depth + 1)
 				};
 			} else {
-				tree[stub.basename] = {
-					file: {
-						contents: stub.text ? stub.contents : base64.toByteArray(stub.contents)
-					}
-				};
+				tree[stub.basename] = file(stub.text ? stub.contents : base64.toByteArray(stub.contents));
 			}
 		}
 	}
 
 	return tree;
 }
+
+/** @param {string | Uint8Array} contents */
+function file(contents) {
+	return {
+		file: { contents }
+	};
+}

src/lib/client/adapters/webcontainer/index.js

@@ -14,7 +14,7 @@ const text_files = new Set([
 	'.md'
 ]);
 
-const excluded = new Set(['.DS_Store', '.gitkeep']);
+const excluded = new Set(['.DS_Store', '.gitkeep', '.svelte-kit']);
 
 /** @param {string} file */
 function json(file) {
@@ -104,7 +104,7 @@ export function get_section(slug) {
 				if (section.slug !== slug) continue;
 
 				const a = {
-					...walk('content/tutorial/common'),
+					...walk('content/tutorial/common', { exclude: ['node_modules'] }),
 					...walk(`content/tutorial/${part.slug}/common`),
 					...walk(`${section.dir}/app-a`)
 				};
@@ -163,8 +163,11 @@ function extract_frontmatter(markdown, dir) {
 /**
  * Get a list of all files in a directory
  * @param {string} cwd - the directory to walk
+ * @param {{
+ *   exclude?: string[]
+ * }} options
  */
-export function walk(cwd) {
+export function walk(cwd, options = {}) {
 	/** @type {Record<string, import('$lib/types').FileStub | import('$lib/types').DirectoryStub>} */
 	const result = {};
 
@@ -179,6 +182,7 @@ export function walk(cwd) {
 
 		for (const basename of files) {
 			if (excluded.has(basename)) continue;
+			if (options.exclude?.includes(basename)) continue;
 
 			const name = dir + basename;
 			const resolved = path.join(cwd, name);