feat: build esm modules using ipjs (#865)

This builds up on @achingbrain 's work on #863 with build improvements and full support

This adds:
- `ipjs` for ESM modules
  - auto-detected
- `types` property transformation, allowing real time ts check in dev and path update for dist folder (TLDR no dist in the path)
- `release` for ESM modules will navigate to the dist to publish its content
- Dockerfile to bundlesize action per actions/runner#772 (comment) as we need node14+ for ESM

One of the problematic modules in skypack using aegir is `uint8arrays`. It is a CJS module that depends on a ESM first module (multiformats), which makes skypack to get bad dependency paths.

I tested this out shipping `uint8arrays` achingbrain/uint8arrays#22 PR and everything working smoothly 🎉 

Original release: https://codepen.io/vascosantos/pen/KKmXoPV?editors=0011
Scoped release using `aegir`: https://codepen.io/vascosantos/pen/bGWoONq?editors=0011

(see browser built in console for errors)

Co-authored-by: achingbrain <alex@achingbrain.net>

.gitignore

@@ -1,6 +1,7 @@
 package-lock.json
 yarn.lock
-node_modules
+/node_modules
+/actions/bundle-size/node_modules
 /coverage
 /dist
 /docs

actions/bundle-size/.dockerignore

@@ -0,0 +1,2 @@
+**
+!/dist

actions/bundle-size/Dockerfile

@@ -0,0 +1,6 @@
+# Just enough docker until github gets a new node16 runner
+# see: https://github.com/actions/runner/issues/772
+FROM node:16-alpine
+WORKDIR /usr/src/app
+COPY dist/index.js .
+CMD [ "node", "index.js" ]

actions/bundle-size/action.yml

@@ -8,5 +8,9 @@ inputs:
     description: A directory to run the bundle check in
     required: false
 runs:
-  using: 'node12'
-  main: 'dist/index.js'
+  # TODO: we need node14.14 minimum.
+  # https://github.com/actions/runner/issues/772
+  # using: 'node12'
+  # main: 'dist/index.js'
+  using: 'docker'
+  image: 'Dockerfile'

md/esm.md

@@ -0,0 +1,27 @@
+# ESM support
+
+## Setup
+
+`aegir` leverages [ipjs](https://github.com/mikeal/ipjs) to output a build with `cjs` and `esm` for maximum compatibility. The general guidelines for writing a module in `esm` are detailed on the `ipjs` README. `aegir` will automatically identify a `esm` repo by the `module` property in `package.json`.
+
+## Electron testing
+
+Electron does [not support ESM](https://github.com/electron/electron/issues/21457) at the time of writing. When writing a module using ESM, we need to compile the tests to `cjs` and rely on them. For generating the build including the tests:
+
+```bash
+aegir build --esm-tests
+```
+
+## Lerna Monorepo
+
+When using a lerna monorepo, local dependencies are symlinked by lerna on install. This means that an `esm` module will not use the resulting `dist` folder as symlink. This can become a problem if we are testing the `cjs` build of a module.
+
+To work around the above problem, we can use `publishConfig.directory = "dist"` in `package.json` to notice lerna about the symlink path. After running the `aegir build` command, it is necessary to run `lerna link` to update the symlinks.
+
+## Release
+
+When releasing an `esm` module, the published content will be the generated `dist` folder content, as indicated by [ipjs](https://github.com/mikeal/ipjs).
+
+## Examples
+
+TODO: List examples when merged (`ipfs-unixfs`, `uint8arrays`)

package.json

@@ -87,13 +87,14 @@
     "esbuild-register": "^2.3.0",
     "eslint": "^7.23.0",
     "eslint-config-ipfs": "^2.0.0",
-    "execa": "^5.0.0",
+    "execa": "^5.1.1",
     "extract-zip": "^2.0.1",
     "fs-extra": "^10.0.0",
     "gh-pages": "^3.1.0",
     "git-authors-cli": "^1.0.33",
     "globby": "^11.0.3",
     "ipfs-utils": "^8.1.0",
+    "ipjs": "^5.0.5",
     "it-glob": "~0.0.10",
     "kleur": "^4.1.4",
     "lilconfig": "^2.0.2",
@@ -119,7 +120,7 @@
     "typedoc": "^0.21.2",
     "typescript": "^4.3.5",
     "update-notifier": "^5.0.0",
-    "yargs": "^17.0.1"
+    "yargs": "^17.1.1"
   },
   "devDependencies": {
     "@types/bytes": "^3.1.0",

src/build/index.js

@@ -56,11 +56,44 @@ const build = async (argv) => {
   return outfile
 }
 
+/**
+ * Build command
+ *
+ * @param {GlobalOptions & BuildOptions} argv
+ */
+const buildEsm = async (argv) => {
+  const dist = path.join(process.cwd(), 'dist')
+  // @ts-ignore no types
+  const ipjs = await import('ipjs')
+
+  await ipjs.default({
+    dist,
+    onConsole: (/** @type {any[]} */...args) => console.info.apply(console, args),
+    cwd: process.cwd(),
+    main: argv.esmMain,
+    tests: argv.esmTests
+  })
+}
+
 const tasks = new Listr([
   {
     title: 'Clean ./dist',
     task: async () => del(path.join(process.cwd(), 'dist'))
   },
+  {
+    title: 'Build ESM',
+    enabled: ctx => {
+      return pkg.type === 'module'
+    },
+    /**
+     *
+     * @param {GlobalOptions & BuildOptions} ctx
+     * @param {Task} task
+     */
+    task: async (ctx, task) => {
+      await buildEsm(ctx)
+    }
+  },
   {
     title: 'Bundle',
     enabled: ctx => ctx.bundle,

src/cmds/build.js

@@ -37,6 +37,18 @@ module.exports = {
           type: 'boolean',
           describe: 'Build the Typescripts type declarations.',
           default: userConfig.build.types
+        },
+        esmMain: {
+          alias: 'esm-main',
+          type: 'boolean',
+          describe: 'Include a main field in a built esm project',
+          default: userConfig.build.esmMain
+        },
+        esmTests: {
+          alias: 'esm-tests',
+          type: 'boolean',
+          describe: 'Include tests in a built esm project',
+          default: userConfig.build.esmTests
         }
       })
   },

src/config/user.js

@@ -40,7 +40,9 @@ const defaults = {
     bundlesize: false,
     bundlesizeMax: '100kB',
     types: true,
-    config: {}
+    config: {},
+    esmMain: true,
+    esmTests: false
   },
   // linter cmd options
   lint: {

src/release/publish.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const execa = require('execa')
-const { otp } = require('../utils')
+const { otp, pkg, repoDirectory } = require('../utils')
 /**
  * @typedef {import('./../types').ReleaseOptions} ReleaseOptions
  * @typedef {import('listr').ListrTaskWrapper} ListrTask
@@ -25,16 +25,22 @@ function publish (ctx, task) {
     task.title += ` (npm ${publishArgs.join(' ')})`
   }
 
-  return execa('npm', publishArgs)
-    .catch(async (error) => {
-      if (error.toString().includes('provide a one-time password')) {
-        const code = await otp()
-        task.title += '. Trying again with OTP.'
-        return await execa('npm', publishArgs.concat('--otp', code))
+  // Publish from dist if ESM
+  const execaOptions = pkg.type === 'module'
+    ? {
+        cwd: `${repoDirectory}/dist`
       }
+    : {}
 
-      throw error
-    })
+  return execa('npm', publishArgs, execaOptions).catch(async (error) => {
+    if (error.toString().includes('provide a one-time password')) {
+      const code = await otp()
+      task.title += '. Trying again with OTP.'
+      return await execa('npm', publishArgs.concat('--otp', code), execaOptions)
+    }
+
+    throw error
+  })
 }
 
 module.exports = publish

src/types.d.ts

@@ -99,6 +99,13 @@ interface GlobalOptions {
   fileConfig: Options
 }
 
+interface ESMHooks {
+  onParse: () => {}
+  onParsed: () => {}
+  onDeflateStart: () => {}
+  onDeflateEnd: () => {}
+}
+
 interface BuildOptions {
   /**
    * Build the JS standalone bundle.
@@ -120,6 +127,14 @@ interface BuildOptions {
    * esbuild build options
    */
   config: esbuild.BuildOptions
+  /**
+   * Include tests in the ipjs output directory
+   */
+  esmTests: boolean
+  /**
+   * Include a main field in the ipjs output package.json
+   */
+  esmMain: boolean
 }
 
 interface TSOptions {

test/build.js

@@ -0,0 +1,37 @@
+/* eslint-env mocha */
+'use strict'
+
+const { expect } = require('../utils/chai')
+const execa = require('execa')
+const { copy, existsSync } = require('fs-extra')
+const { join } = require('path')
+const bin = require.resolve('../')
+const tempy = require('tempy')
+
+describe('build', () => {
+  describe('esm', () => {
+    let projectDir = ''
+
+    before(async () => {
+      projectDir = tempy.directory()
+
+      await copy(join(__dirname, 'fixtures', 'esm', 'an-esm-project'), projectDir)
+    })
+
+    it('should build an esm project', async function () {
+      this.timeout(20 * 1000) // slow ci is slow
+
+      await execa(bin, ['build'], {
+        cwd: projectDir
+      })
+
+      expect(existsSync(join(projectDir, 'dist', 'esm'))).to.be.true()
+      expect(existsSync(join(projectDir, 'dist', 'cjs'))).to.be.true()
+
+      const module = require(join(projectDir, 'dist'))
+
+      expect(module).to.have.property('useHerp').that.is.a('function')
+      expect(module).to.have.property('useDerp').that.is.a('function')
+    })
+  })
+})

test/fixtures/esm/an-esm-project/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "an-esm-project",
+  "version": "1.0.0",
+  "description": "",
+  "main": "src/index.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "eslintConfig": {
+    "extends": "ipfs",
+    "parserOptions": {
+      "sourceType": "module"
+    }
+  }
+}

test/fixtures/esm/an-esm-project/src/index.js

@@ -0,0 +1,10 @@
+import herp from 'a-cjs-dep'
+import derp from 'an-esm-dep'
+
+export const useHerp = () => {
+  herp()
+}
+
+export const useDerp = () => {
+  derp()
+}

test/node.js

@@ -1,5 +1,6 @@
 'use strict'
 
+require('./build')
 require('./lint')
 require('./fixtures')
 require('./dependants')