From 0d03d8a06d9715f41504996c5336fdd27da8fba4 Mon Sep 17 00:00:00 2001 From: Nick Gerleman Date: Tue, 12 Dec 2023 09:06:58 -0800 Subject: [PATCH] Manage native build toolchain (#1506) Summary: Pull Request resolved: https://github.com/facebook/yoga/pull/1506 In order to build the website, you must build a Yoga binary. This usually requires installing native toolchains. We have warning messages for these right now, but an even better solution is to just manage the dependencies ourselves. This does that, bringing in specific CMake and Ninja binaries from NPM, and caching a local copy of Emscripten during the build. A downside is that the CMake packages are chunky, so we add 130MB to node_modules (for a repo total around 350MB). This also delays acquiring Emscripten (which is even chunkier) in CI builds until it is needed, and I added some caching for it as well. The upside of JS users being able to run and test (inc the website) without installing and managing their own versions of toolchains is a real time-saver though, and is probably worth it. allow-large-files Reviewed By: yungsters Differential Revision: D52013026 fbshipit-source-id: 3d307f751463a21c5e5d5b98b8e9e63db9d3d52e --- .github/actions/cache-emsdk/action.yml | 10 ++ .github/actions/install-emsdk/action.yml | 23 ---- .github/actions/setup-js/action.yml | 6 - .github/workflows/validate-js.yml | 14 ++- .github/workflows/validate-website.yml | 3 + javascript/.gitignore | 1 + javascript/CMakeLists.txt | 13 +-- javascript/just.config.cjs | 134 ++++++++++++++--------- javascript/package.json | 2 + javascript/src/index.ts | 2 +- yarn.lock | 29 +++++ 11 files changed, 145 insertions(+), 92 deletions(-) create mode 100644 .github/actions/cache-emsdk/action.yml delete mode 100644 .github/actions/install-emsdk/action.yml diff --git a/.github/actions/cache-emsdk/action.yml b/.github/actions/cache-emsdk/action.yml new file mode 100644 index 00000000..da3c55fa --- /dev/null +++ b/.github/actions/cache-emsdk/action.yml @@ -0,0 +1,10 @@ +name: Cache the installed copy of emsdk and its build artifacts + +runs: + using: "composite" + steps: + - name: Cache emsdk + uses: actions/cache@v3 + with: + path: javascript/.emsdk + key: emsdk-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('yarn.lock', 'javascript/package.json', 'javascript/just.config.cjs'}} diff --git a/.github/actions/install-emsdk/action.yml b/.github/actions/install-emsdk/action.yml deleted file mode 100644 index 8f0be736..00000000 --- a/.github/actions/install-emsdk/action.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Install emsdk (including emcc) -inputs: - version: - description: EMCC Version to install - required: false - default: 3.1.28 - -runs: - using: "composite" - steps: - - name: Clone emsdk repo - working-directory: ${{ runner.temp }} - shell: bash - run: git clone https://github.com/emscripten-core/emsdk.git - - - name: emdsk install - working-directory: ${{ runner.temp }}/emsdk - shell: bash - run: | - ./emsdk install ${{ inputs.version }} - ./emsdk activate ${{ inputs.version }} - echo $RUNNER_TEMP/emsdk >> $GITHUB_PATH - echo $RUNNER_TEMP/emsdk/upstream/emscripten >> $GITHUB_PATH diff --git a/.github/actions/setup-js/action.yml b/.github/actions/setup-js/action.yml index c3140d72..a248ff66 100644 --- a/.github/actions/setup-js/action.yml +++ b/.github/actions/setup-js/action.yml @@ -3,12 +3,6 @@ name: Setup JavaScript envirionment runs: using: "composite" steps: - - name: Install emsdk - uses: ./.github/actions/install-emsdk - - - name: Install Ninja - uses: ./.github/actions/install-ninja - - name: Setup Node environment uses: actions/setup-node@v3 with: diff --git a/.github/workflows/validate-js.yml b/.github/workflows/validate-js.yml index 489064c7..2b215f6f 100644 --- a/.github/workflows/validate-js.yml +++ b/.github/workflows/validate-js.yml @@ -24,6 +24,9 @@ jobs: - name: Setup uses: ./.github/actions/setup-js + - name: Restore emsdk + uses: ./.github/actions/cache-emsdk + - name: yarn benchmark run: yarn benchmark working-directory: javascript @@ -40,8 +43,11 @@ jobs: - name: Setup uses: ./.github/actions/setup-js + - name: Restore emsdk + uses: ./.github/actions/cache-emsdk + - name: yarn build - run: yarn build + run: yarn build --verbose working-directory: javascript test: @@ -56,6 +62,9 @@ jobs: - name: Setup uses: ./.github/actions/setup-js + - name: Restore emsdk + uses: ./.github/actions/cache-emsdk + - name: yarn test run: yarn test working-directory: javascript @@ -94,6 +103,9 @@ jobs: - name: Setup uses: ./.github/actions/setup-js + - name: Restore emsdk + uses: ./.github/actions/cache-emsdk + - name: yarn pack run: yarn pack --filename yoga-layout.tar.gz working-directory: javascript diff --git a/.github/workflows/validate-website.yml b/.github/workflows/validate-website.yml index cdfbd4a5..8a542b36 100644 --- a/.github/workflows/validate-website.yml +++ b/.github/workflows/validate-website.yml @@ -34,6 +34,9 @@ jobs: - name: Setup uses: ./.github/actions/setup-js + - name: Restore emsdk + uses: ./.github/actions/cache-emsdk + - name: Build Website run: yarn build working-directory: website-next diff --git a/javascript/.gitignore b/javascript/.gitignore index 54e8f18f..504215ab 100644 --- a/javascript/.gitignore +++ b/javascript/.gitignore @@ -1,3 +1,4 @@ +/.emsdk /binaries /build /dist diff --git a/javascript/CMakeLists.txt b/javascript/CMakeLists.txt index 4eb4e67a..b1a1278f 100644 --- a/javascript/CMakeLists.txt +++ b/javascript/CMakeLists.txt @@ -44,7 +44,11 @@ add_link_options( "SHELL:-s MODULARIZE=1" "SHELL:-s EXPORT_ES6=1" "SHELL:-s WASM=1" - "SHELL:-s TEXTDECODER=0") + "SHELL:-s TEXTDECODER=0" + # SINGLE_FILE=1 combined with ENVIRONMENT='web' creates code that works on + # both bundlders and Node. + "SHELL:-s SINGLE_FILE=1" + "SHELL:-s ENVIRONMENT='web'") link_libraries(embind) @@ -52,9 +56,4 @@ add_library(yogaObjLib OBJECT ${SOURCES}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/binaries) -add_executable(web $) -target_link_options(web PRIVATE - # SINGLE_FILE=1 combined with ENVIRONMENT='web' creates code that works on - # both bundlders and Node. - "SHELL:-s SINGLE_FILE=1" - "SHELL:-s ENVIRONMENT='web'") +add_executable(yoga-wasm-base64-esm $) diff --git a/javascript/just.config.cjs b/javascript/just.config.cjs index 663555e9..4ca485e3 100644 --- a/javascript/just.config.cjs +++ b/javascript/just.config.cjs @@ -20,9 +20,9 @@ const { tscTask, } = require('just-scripts'); -const {readFile, writeFile} = require('fs/promises'); +const {existsSync} = require('fs'); +const {readFile, writeFile, rm} = require('fs/promises'); -const chalk = require('chalk'); const glob = require('glob'); const path = require('path'); const which = require('which'); @@ -31,39 +31,26 @@ const node = process.execPath; option('fix'); -task('clean', cleanTask({paths: ['build', 'dist']})); +task('clean', cleanTask({paths: ['.emsdk', 'binaries', 'build']})); -function defineFlavor(flavor, env) { - task(`cmake-build:${flavor}`, cmakeBuildTask({targets: [flavor]})); - task( - `jest:${flavor}`, +task( + 'build', + series(installEmsdkTask(), emcmakeGenerateTask(), cmakeBuildTask()), +); + +task( + 'test', + series( + 'build', jestTask({ config: path.join(__dirname, 'jest.config.js'), nodeArgs: ['--experimental-vm-modules'], - env, }), - ); - task( - `test:${flavor}`, - series(emcmakeGenerateTask(), `cmake-build:${flavor}`, `jest:${flavor}`), - ); -} - -defineFlavor('web'); - -task('build', series(emcmakeGenerateTask(), cmakeBuildTask())); - -task('test', series(emcmakeGenerateTask(), 'cmake-build:web', 'jest:web')); - -task( - 'benchmark', - series( - emcmakeGenerateTask(), - cmakeBuildTask({targets: ['web']}), - runBenchTask(), ), ); +task('benchmark', series('build', runBenchTask())); + task('clang-format', clangFormatTask({fix: argv().fix})); task('prepack-package-json', async () => { @@ -133,36 +120,74 @@ function runBenchTask() { }; } -function findExecutable(name, failureMessage) { - const exec = which.sync(name, {nothrow: true}); - if (exec) { - return exec; - } +const emsdkVersion = '3.1.28'; +const emsdkPath = path.join(__dirname, '.emsdk'); +const emsdkBin = path.join( + emsdkPath, + process.platform === 'win32' ? 'emsdk.bat' : 'emsdk', +); +const emcmakeBin = path.join( + emsdkPath, + 'upstream', + 'emscripten', + process.platform === 'win32' ? 'emcmake.bat' : 'emcmake', +); - logger.error(chalk.bold.red(failureMessage)); - process.exit(1); +function installEmsdkTask() { + return async () => { + if (await isEmsdkReadyAndActivated()) { + logger.verbose( + `emsdk ${emsdkVersion} is already installed and activated`, + ); + return false; + } + + logger.info(`installing emsdk ${emsdkVersion} to ${emsdkPath}`); + await rm(emsdkPath, {recursive: true, force: true}); + + await spawn( + 'git', + ['clone', 'https://github.com/emscripten-core/emsdk.git', emsdkPath], + {stdio: 'inherit'}, + ); + + await spawn(emsdkBin, ['install', emsdkVersion], {stdio: 'inherit'}); + + await spawn(emsdkBin, ['activate', emsdkVersion], { + stdio: logger.enableVerbose ? 'inherit' : 'ignore', + }); + }; } -function tryFindExecutable(name, failureMessage) { - const exec = which.sync(name, {nothrow: true}); - if (exec) { - return exec; +async function isEmsdkReadyAndActivated() { + if (!existsSync(emcmakeBin)) { + return false; } - logger.warn(chalk.bold.yellow(failureMessage)); - return exec; + try { + const emsdkReleases = JSON.parse( + await readFile(path.join(emsdkPath, 'emscripten-releases-tags.json')), + ).releases; + + const versionHash = emsdkReleases[emsdkVersion]; + if (!versionHash) { + return false; + } + + const activatedVersion = await readFile( + path.join(emsdkPath, 'upstream', '.emsdk_version'), + ); + + return activatedVersion.toString().includes(versionHash); + } catch { + // Something is wrong. Pave and redo. + return false; + } } function emcmakeGenerateTask() { return () => { - const ninja = tryFindExecutable( - 'ninja', - 'Warning: Install Ninja (e.g. "brew install ninja") for faster builds', - ); - const emcmake = findExecutable( - 'emcmake', - 'Error: Please install the emscripten SDK: https://emscripten.org/docs/getting_started/', - ); + logger.verbose(`emcmake path: ${emcmakeBin}`); const args = [ 'cmake', @@ -170,20 +195,21 @@ function emcmakeGenerateTask() { '.', '-B', 'build', - ...(ninja ? ['-G', 'Ninja'] : []), + ...(process.platform === 'win32' ? [] : ['-G', 'Ninja']), ]; logger.info(['emcmake', ...args].join(' ')); - return spawn(emcmake, args, {stdio: 'inherit'}); + return spawn(emcmakeBin, args, { + stdio: logger.enableVerbose ? 'inherit' : 'ignore', + }); }; } function cmakeBuildTask(opts) { return () => { - const cmake = findExecutable( - 'cmake', - 'Error: Please install CMake (e.g. "brew install cmake")', - ); + const cmake = which.sync('cmake'); + logger.verbose(`cmake path: ${cmake}`); + const args = [ '--build', 'build', diff --git a/javascript/package.json b/javascript/package.json index 600eb60d..c06b6a20 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -36,11 +36,13 @@ "@types/jest": "^29.5.1", "@types/node": "^16.18.25", "@types/which": "^3.0.0", + "@yogalayout/cmake-bin": "3.28.0-1", "babel-register-esm": "^1.2.5", "clang-format": "^1.8.0", "glob": "^8.0.3", "jest": "^29.3.1", "just-scripts": "^2.1.0", + "ninja-binaries": "^1.11.1", "which": "^3.0.0" } } diff --git a/javascript/src/index.ts b/javascript/src/index.ts index af5b665a..3028b225 100644 --- a/javascript/src/index.ts +++ b/javascript/src/index.ts @@ -8,7 +8,7 @@ */ // @ts-ignore untyped from Emscripten -import loadYoga from '../binaries/web.js'; +import loadYoga from '../binaries/yoga-wasm-base64-esm.js'; import wrapAssembly from './wrapAssembly.ts'; export type { diff --git a/yarn.lock b/yarn.lock index 0b635273..562dea86 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2901,6 +2901,30 @@ resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== +"@yogalayout/cmake-bin-darwin-universal@3.28.0": + version "3.28.0" + resolved "https://registry.yarnpkg.com/@yogalayout/cmake-bin-darwin-universal/-/cmake-bin-darwin-universal-3.28.0.tgz#727e10d62ae4fca960e3502465b68f948841dd6c" + integrity sha512-grgtVOaJZT5fCvCaLVbYmVKqsBqtphQffG6WTjDCdyJ++giM4AvrhiRgFKnyr4rx3pj42DrWTvpIWJJl2nBc6w== + +"@yogalayout/cmake-bin-linux-x64@3.28.0": + version "3.28.0" + resolved "https://registry.yarnpkg.com/@yogalayout/cmake-bin-linux-x64/-/cmake-bin-linux-x64-3.28.0.tgz#9bbd414a24a1264a6b8b824c6eb582d623d4ae8e" + integrity sha512-RQlsWl1si7+yi87NI2tV13Ca7RYxhjw1AmoULLwhPNkNSLu9Laja2IFvjgvz6iJ3zD9fWSMUl5TwA/ugiNj7OQ== + +"@yogalayout/cmake-bin-windows-x64@3.28.0": + version "3.28.0" + resolved "https://registry.yarnpkg.com/@yogalayout/cmake-bin-windows-x64/-/cmake-bin-windows-x64-3.28.0.tgz#c60787a7148d54fe0d579dcb4fd80d1f7aef56ed" + integrity sha512-wMWwUbw/NkXynVRcrFElbFJD/p9HQHmScuv9amZAKMM/JhLjrxDQaivQ+7ZmEOlRQ1uWZjiue9QCkaJERt9Bog== + +"@yogalayout/cmake-bin@3.28.0-1": + version "3.28.0-1" + resolved "https://registry.yarnpkg.com/@yogalayout/cmake-bin/-/cmake-bin-3.28.0-1.tgz#af366eb6205ec07c1ede2bcec668a2be083869b9" + integrity sha512-2bnXhH4qt+bxjm+fgkjTHYhvB2quaS874XkYmUL1Obo6S0kWtDXEp7s0ZgtKvOfW3+y4Q5eomNMIobOeGLi9rw== + optionalDependencies: + "@yogalayout/cmake-bin-darwin-universal" "3.28.0" + "@yogalayout/cmake-bin-linux-x64" "3.28.0" + "@yogalayout/cmake-bin-windows-x64" "3.28.0" + accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8: version "1.3.8" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" @@ -7823,6 +7847,11 @@ next-tick@^1.1.0: resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb" integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ== +ninja-binaries@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/ninja-binaries/-/ninja-binaries-1.11.1.tgz#9089191512ab44f990b1ee03792b47f79aedeaaa" + integrity sha512-/fjozfApkgrHe2IGbSMhwiDYdz0AUpvsm1szWWu9cy4NJmwPDuhd4mvfFMlCp35CPexHim3jSWn1BjHddxFkkA== + no-case@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/no-case/-/no-case-3.0.4.tgz#d361fd5c9800f558551a8369fc0dcd4662b6124d"