Manage native build toolchain (#1506)

Summary:
Pull Request resolved: https://github.com/facebook/yoga/pull/1506

In order to build the website, you must build a Yoga binary. This usually requires installing native toolchains.

We have warning messages for these right now, but an even better solution is to just manage the dependencies ourselves. This does that, bringing in specific CMake and Ninja binaries from NPM, and caching a local copy of Emscripten during the build.

A downside is that the CMake packages are chunky, so we add 130MB to node_modules (for a repo total around 350MB). This also delays acquiring Emscripten (which is even chunkier) in CI builds until it is needed, and I added some caching for it as well.

The upside of JS users being able to run and test (inc the website) without installing and managing their own versions of toolchains is a real time-saver though, and is probably worth it.

allow-large-files

Reviewed By: yungsters

Differential Revision: D52013026

fbshipit-source-id: 3d307f751463a21c5e5d5b98b8e9e63db9d3d52e

.github/actions/cache-emsdk/action.yml

@@ -0,0 +1,10 @@
+name: Cache the installed copy of emsdk and its build artifacts
+
+runs:
+  using: "composite"
+  steps:
+    - name: Cache emsdk
+      uses: actions/cache@v3
+      with:
+        path: javascript/.emsdk
+        key: emsdk-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('yarn.lock', 'javascript/package.json', 'javascript/just.config.cjs'}}

.github/actions/install-emsdk/action.yml

@@ -1,23 +0,0 @@
-name: Install emsdk (including emcc)
-inputs:
-  version:
-    description: EMCC Version to install
-    required: false
-    default: 3.1.28
-
-runs:
-  using: "composite"
-  steps:
-    - name: Clone emsdk repo
-      working-directory: ${{ runner.temp }}
-      shell: bash
-      run: git clone https://github.com/emscripten-core/emsdk.git
-
-    - name: emdsk install
-      working-directory: ${{ runner.temp }}/emsdk
-      shell: bash
-      run: |
-        ./emsdk install ${{ inputs.version }}
-        ./emsdk activate ${{ inputs.version }}
-        echo $RUNNER_TEMP/emsdk >> $GITHUB_PATH
-        echo $RUNNER_TEMP/emsdk/upstream/emscripten >> $GITHUB_PATH

.github/actions/setup-js/action.yml

@@ -3,12 +3,6 @@ name: Setup JavaScript envirionment
 runs:
   using: "composite"
   steps:
-    - name: Install emsdk
-      uses: ./.github/actions/install-emsdk
-
-    - name: Install Ninja
-      uses: ./.github/actions/install-ninja
-
     - name: Setup Node environment
       uses: actions/setup-node@v3
       with:

.github/workflows/validate-js.yml

@@ -24,6 +24,9 @@ jobs:
       - name: Setup
         uses: ./.github/actions/setup-js
 
+      - name: Restore emsdk
+        uses: ./.github/actions/cache-emsdk
+
       - name: yarn benchmark
         run: yarn benchmark
         working-directory: javascript
@@ -40,8 +43,11 @@ jobs:
       - name: Setup
         uses: ./.github/actions/setup-js
 
+      - name: Restore emsdk
+        uses: ./.github/actions/cache-emsdk
+
       - name: yarn build
-        run: yarn build
+        run: yarn build --verbose
         working-directory: javascript
 
   test:
@@ -56,6 +62,9 @@ jobs:
       - name: Setup
         uses: ./.github/actions/setup-js
 
+      - name: Restore emsdk
+        uses: ./.github/actions/cache-emsdk
+
       - name: yarn test
         run: yarn test
         working-directory: javascript
@@ -94,6 +103,9 @@ jobs:
       - name: Setup
         uses: ./.github/actions/setup-js
 
+      - name: Restore emsdk
+        uses: ./.github/actions/cache-emsdk
+
       - name: yarn pack
         run: yarn pack --filename yoga-layout.tar.gz
         working-directory: javascript

.github/workflows/validate-website.yml

@@ -34,6 +34,9 @@ jobs:
       - name: Setup
         uses: ./.github/actions/setup-js
 
+      - name: Restore emsdk
+        uses: ./.github/actions/cache-emsdk
+
       - name: Build Website
         run: yarn build
         working-directory: website-next

javascript/.gitignore

@@ -1,3 +1,4 @@
+/.emsdk
 /binaries
 /build
 /dist

javascript/CMakeLists.txt

@@ -44,7 +44,11 @@ add_link_options(
     "SHELL:-s MODULARIZE=1"
     "SHELL:-s EXPORT_ES6=1"
     "SHELL:-s WASM=1"
-    "SHELL:-s TEXTDECODER=0")
+    "SHELL:-s TEXTDECODER=0"
+    # SINGLE_FILE=1 combined with ENVIRONMENT='web' creates code that works on
+    # both bundlders and Node.
+    "SHELL:-s SINGLE_FILE=1"
+    "SHELL:-s ENVIRONMENT='web'")
 
 link_libraries(embind)
 
@@ -52,9 +56,4 @@ add_library(yogaObjLib OBJECT ${SOURCES})
 
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/binaries)
 
-add_executable(web $<TARGET_OBJECTS:yogaObjLib>)
-target_link_options(web PRIVATE
-    # SINGLE_FILE=1 combined with ENVIRONMENT='web' creates code that works on
-    # both bundlders and Node.
-    "SHELL:-s SINGLE_FILE=1"
-    "SHELL:-s ENVIRONMENT='web'")
+add_executable(yoga-wasm-base64-esm $<TARGET_OBJECTS:yogaObjLib>)

javascript/just.config.cjs

@@ -20,9 +20,9 @@ const {
   tscTask,
 } = require('just-scripts');
 
-const {readFile, writeFile} = require('fs/promises');
+const {existsSync} = require('fs');
+const {readFile, writeFile, rm} = require('fs/promises');
 
-const chalk = require('chalk');
 const glob = require('glob');
 const path = require('path');
 const which = require('which');
@@ -31,39 +31,26 @@ const node = process.execPath;
 
 option('fix');
 
-task('clean', cleanTask({paths: ['build', 'dist']}));
+task('clean', cleanTask({paths: ['.emsdk', 'binaries', 'build']}));
 
-function defineFlavor(flavor, env) {
-  task(`cmake-build:${flavor}`, cmakeBuildTask({targets: [flavor]}));
-  task(
-    `jest:${flavor}`,
+task(
+  'build',
+  series(installEmsdkTask(), emcmakeGenerateTask(), cmakeBuildTask()),
+);
+
+task(
+  'test',
+  series(
+    'build',
     jestTask({
       config: path.join(__dirname, 'jest.config.js'),
       nodeArgs: ['--experimental-vm-modules'],
-      env,
     }),
-  );
-  task(
-    `test:${flavor}`,
-    series(emcmakeGenerateTask(), `cmake-build:${flavor}`, `jest:${flavor}`),
-  );
-}
-
-defineFlavor('web');
-
-task('build', series(emcmakeGenerateTask(), cmakeBuildTask()));
-
-task('test', series(emcmakeGenerateTask(), 'cmake-build:web', 'jest:web'));
-
-task(
-  'benchmark',
-  series(
-    emcmakeGenerateTask(),
-    cmakeBuildTask({targets: ['web']}),
-    runBenchTask(),
   ),
 );
 
+task('benchmark', series('build', runBenchTask()));
+
 task('clang-format', clangFormatTask({fix: argv().fix}));
 
 task('prepack-package-json', async () => {
@@ -133,36 +120,74 @@ function runBenchTask() {
   };
 }
 
-function findExecutable(name, failureMessage) {
-  const exec = which.sync(name, {nothrow: true});
-  if (exec) {
-    return exec;
-  }
+const emsdkVersion = '3.1.28';
+const emsdkPath = path.join(__dirname, '.emsdk');
+const emsdkBin = path.join(
+  emsdkPath,
+  process.platform === 'win32' ? 'emsdk.bat' : 'emsdk',
+);
+const emcmakeBin = path.join(
+  emsdkPath,
+  'upstream',
+  'emscripten',
+  process.platform === 'win32' ? 'emcmake.bat' : 'emcmake',
+);
 
-  logger.error(chalk.bold.red(failureMessage));
-  process.exit(1);
+function installEmsdkTask() {
+  return async () => {
+    if (await isEmsdkReadyAndActivated()) {
+      logger.verbose(
+        `emsdk ${emsdkVersion} is already installed and activated`,
+      );
+      return false;
+    }
+
+    logger.info(`installing emsdk ${emsdkVersion} to ${emsdkPath}`);
+    await rm(emsdkPath, {recursive: true, force: true});
+
+    await spawn(
+      'git',
+      ['clone', 'https://github.com/emscripten-core/emsdk.git', emsdkPath],
+      {stdio: 'inherit'},
+    );
+
+    await spawn(emsdkBin, ['install', emsdkVersion], {stdio: 'inherit'});
+
+    await spawn(emsdkBin, ['activate', emsdkVersion], {
+      stdio: logger.enableVerbose ? 'inherit' : 'ignore',
+    });
+  };
 }
 
-function tryFindExecutable(name, failureMessage) {
-  const exec = which.sync(name, {nothrow: true});
-  if (exec) {
-    return exec;
+async function isEmsdkReadyAndActivated() {
+  if (!existsSync(emcmakeBin)) {
+    return false;
   }
 
-  logger.warn(chalk.bold.yellow(failureMessage));
-  return exec;
+  try {
+    const emsdkReleases = JSON.parse(
+      await readFile(path.join(emsdkPath, 'emscripten-releases-tags.json')),
+    ).releases;
+
+    const versionHash = emsdkReleases[emsdkVersion];
+    if (!versionHash) {
+      return false;
+    }
+
+    const activatedVersion = await readFile(
+      path.join(emsdkPath, 'upstream', '.emsdk_version'),
+    );
+
+    return activatedVersion.toString().includes(versionHash);
+  } catch {
+    // Something is wrong. Pave and redo.
+    return false;
+  }
 }
 
 function emcmakeGenerateTask() {
   return () => {
-    const ninja = tryFindExecutable(
-      'ninja',
-      'Warning: Install Ninja (e.g. "brew install ninja") for faster builds',
-    );
-    const emcmake = findExecutable(
-      'emcmake',
-      'Error: Please install the emscripten SDK: https://emscripten.org/docs/getting_started/',
-    );
+    logger.verbose(`emcmake path: ${emcmakeBin}`);
 
     const args = [
       'cmake',
@@ -170,20 +195,21 @@ function emcmakeGenerateTask() {
       '.',
       '-B',
       'build',
-      ...(ninja ? ['-G', 'Ninja'] : []),
+      ...(process.platform === 'win32' ? [] : ['-G', 'Ninja']),
     ];
     logger.info(['emcmake', ...args].join(' '));
 
-    return spawn(emcmake, args, {stdio: 'inherit'});
+    return spawn(emcmakeBin, args, {
+      stdio: logger.enableVerbose ? 'inherit' : 'ignore',
+    });
   };
 }
 
 function cmakeBuildTask(opts) {
   return () => {
-    const cmake = findExecutable(
-      'cmake',
-      'Error: Please install CMake (e.g. "brew install cmake")',
-    );
+    const cmake = which.sync('cmake');
+    logger.verbose(`cmake path: ${cmake}`);
+
     const args = [
       '--build',
       'build',

javascript/package.json

@@ -36,11 +36,13 @@
     "@types/jest": "^29.5.1",
     "@types/node": "^16.18.25",
     "@types/which": "^3.0.0",
+    "@yogalayout/cmake-bin": "3.28.0-1",
     "babel-register-esm": "^1.2.5",
     "clang-format": "^1.8.0",
     "glob": "^8.0.3",
     "jest": "^29.3.1",
     "just-scripts": "^2.1.0",
+    "ninja-binaries": "^1.11.1",
     "which": "^3.0.0"
   }
 }

javascript/src/index.ts

@@ -8,7 +8,7 @@
  */
 
 // @ts-ignore untyped from Emscripten
-import loadYoga from '../binaries/web.js';
+import loadYoga from '../binaries/yoga-wasm-base64-esm.js';
 import wrapAssembly from './wrapAssembly.ts';
 
 export type {

yarn.lock

@@ -2901,6 +2901,30 @@
   resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d"
   integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==
 
+"@yogalayout/cmake-bin-darwin-universal@3.28.0":
+  version "3.28.0"
+  resolved "https://registry.yarnpkg.com/@yogalayout/cmake-bin-darwin-universal/-/cmake-bin-darwin-universal-3.28.0.tgz#727e10d62ae4fca960e3502465b68f948841dd6c"
+  integrity sha512-grgtVOaJZT5fCvCaLVbYmVKqsBqtphQffG6WTjDCdyJ++giM4AvrhiRgFKnyr4rx3pj42DrWTvpIWJJl2nBc6w==
+
+"@yogalayout/cmake-bin-linux-x64@3.28.0":
+  version "3.28.0"
+  resolved "https://registry.yarnpkg.com/@yogalayout/cmake-bin-linux-x64/-/cmake-bin-linux-x64-3.28.0.tgz#9bbd414a24a1264a6b8b824c6eb582d623d4ae8e"
+  integrity sha512-RQlsWl1si7+yi87NI2tV13Ca7RYxhjw1AmoULLwhPNkNSLu9Laja2IFvjgvz6iJ3zD9fWSMUl5TwA/ugiNj7OQ==
+
+"@yogalayout/cmake-bin-windows-x64@3.28.0":
+  version "3.28.0"
+  resolved "https://registry.yarnpkg.com/@yogalayout/cmake-bin-windows-x64/-/cmake-bin-windows-x64-3.28.0.tgz#c60787a7148d54fe0d579dcb4fd80d1f7aef56ed"
+  integrity sha512-wMWwUbw/NkXynVRcrFElbFJD/p9HQHmScuv9amZAKMM/JhLjrxDQaivQ+7ZmEOlRQ1uWZjiue9QCkaJERt9Bog==
+
+"@yogalayout/cmake-bin@3.28.0-1":
+  version "3.28.0-1"
+  resolved "https://registry.yarnpkg.com/@yogalayout/cmake-bin/-/cmake-bin-3.28.0-1.tgz#af366eb6205ec07c1ede2bcec668a2be083869b9"
+  integrity sha512-2bnXhH4qt+bxjm+fgkjTHYhvB2quaS874XkYmUL1Obo6S0kWtDXEp7s0ZgtKvOfW3+y4Q5eomNMIobOeGLi9rw==
+  optionalDependencies:
+    "@yogalayout/cmake-bin-darwin-universal" "3.28.0"
+    "@yogalayout/cmake-bin-linux-x64" "3.28.0"
+    "@yogalayout/cmake-bin-windows-x64" "3.28.0"
+
 accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8:
   version "1.3.8"
   resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e"
@@ -7823,6 +7847,11 @@ next-tick@^1.1.0:
   resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb"
   integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==
 
+ninja-binaries@^1.11.1:
+  version "1.11.1"
+  resolved "https://registry.yarnpkg.com/ninja-binaries/-/ninja-binaries-1.11.1.tgz#9089191512ab44f990b1ee03792b47f79aedeaaa"
+  integrity sha512-/fjozfApkgrHe2IGbSMhwiDYdz0AUpvsm1szWWu9cy4NJmwPDuhd4mvfFMlCp35CPexHim3jSWn1BjHddxFkkA==
+
 no-case@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/no-case/-/no-case-3.0.4.tgz#d361fd5c9800f558551a8369fc0dcd4662b6124d"