build(libzip): use Docker (#5056)

* build(libzip): use docker

* ci: validate libzip build

* build: make build directory on the host

* chore: update artifacts

* chore: versions

Author: merceyz

.github/workflows/integration-workflow.yml

@@ -60,6 +60,18 @@ jobs:
       env:
         TARGET_BRANCH: ${{github.event.pull_request.base.ref}}
 
+    - name: 'Check that libzip artifacts are consistent with a fresh build (fix w/ "git merge master && bash packages/yarnpkg-libzip/artifacts/build.sh")'
+      run: |
+        if [[ $(git diff --name-only "$(git merge-base origin/"$TARGET_BRANCH" HEAD)" HEAD -- 'packages/yarnpkg-libzip/artifacts' 'packages/yarnpkg-libzip/sources/*.js' | wc -l) -gt 0 ]]; then
+          bash packages/yarnpkg-libzip/artifacts/build.sh
+          [[ $(git diff --name-only 'packages/yarnpkg-libzip/artifacts' 'packages/yarnpkg-libzip/sources/*.js' | wc -l) -eq 0 ]]
+        fi
+      shell: bash
+      if: |
+        (success() || failure()) && github.event.pull_request != ''
+      env:
+        TARGET_BRANCH: ${{github.event.pull_request.base.ref}}
+
     - name: 'Check that the PnP hooks are consistent with a fresh build (fix w/ "git merge master && yarn update:pnp:hook")'
       run: |
         if [[ $(git diff --name-only "$(git merge-base origin/"$TARGET_BRANCH" HEAD)" HEAD -- 'packages/yarnpkg-{fslib,libzip,pnp}/sources/**/*' | wc -l) -gt 0 ]]; then

.yarn/versions/95d9f16a.yml

@@ -0,0 +1,3 @@
+declined:
+  - "@yarnpkg/core"
+  - "@yarnpkg/libzip"

packages/yarnpkg-core/sources/worker-zip/index.js

@@ -2,12 +2,8 @@
 
 set -e
 
-EMSDK_ENV=~/emsdk/emsdk_env.sh
 EMSDK_VERSION=2.0.22
 
-~/emsdk/emsdk install $EMSDK_VERSION
-~/emsdk/emsdk activate $EMSDK_VERSION
-
 THIS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 cd "$THIS_DIR"
 
@@ -17,35 +13,46 @@ LIBZIP_VERSION=1.5.2 # Ignored at the moment; we use a fork, cf next params
 LIBZIP_REPO=arcanis/libzip
 LIBZIP_COMMIT=664462465d2730d51f04437c90ed7ebcbe19a36f
 
-[[ -f ./zlib-"$ZLIB_VERSION"/libz.a ]] || (
+ZLIB_ROOT="$THIS_DIR/zlib-ng-$ZLIB_VERSION"
+LIBZIP_ROOT="$THIS_DIR/libzip-$LIBZIP_VERSION"
+
+[[ -f "$ZLIB_ROOT/build/libz.a" ]] || (
   if ! [[ -e zlib-"$ZLIB_VERSION".zip ]]; then
     wget -O ./zlib-"$ZLIB_VERSION".zip "https://github.com/zlib-ng/zlib-ng/archive/refs/tags/$ZLIB_VERSION.zip"
   fi
 
-  if ! [[ -e zlib-"$ZLIB_VERSION" ]]; then
+  if ! [[ -e "$ZLIB_ROOT" ]]; then
     unzip ./zlib-"$ZLIB_VERSION".zip
-    mv ./zlib-ng-"$ZLIB_VERSION" ./zlib-"$ZLIB_VERSION"
   fi
 
-  cd "$THIS_DIR"/zlib-"$ZLIB_VERSION"
-
-  source "$EMSDK_ENV"
-
-  mkdir -p build
-  cd build
-
-  CHOST="wasm32" CFLAGS="-static" LDFLAGS="-static" emconfigure ../configure --warn --zlib-compat --static
-
-  emmake make -j2
-
+  mkdir -p "$ZLIB_ROOT/build"
+
+  docker run --rm \
+    -v "$ZLIB_ROOT:/zlib" \
+    -u $(id -u):$(id -g) \
+    --env CHOST="wasm32" \
+    --env CFLAGS="-static" \
+    --env LDFLAGS="-static" \
+    -w "/zlib/build" \
+    emscripten/emsdk:$EMSDK_VERSION \
+    emconfigure ../configure --warn --zlib-compat --static
+
+  docker run --rm \
+    -v "$ZLIB_ROOT:/zlib" \
+    -u $(id -u):$(id -g) \
+    -w "/zlib/build" \
+    emscripten/emsdk:$EMSDK_VERSION \
+    emmake make -j2
+
+  cd "$ZLIB_ROOT/build"
   mkdir -p local/lib local/include
   cp libz.a local/lib/
   cp zconf.h ../zlib.h local/include/
 
   echo Built zlib
 )
 
-[[ -f ./libzip-"$LIBZIP_VERSION"/build/lib/libzip.a ]] || (
+[[ -f "$LIBZIP_ROOT/build/lib/libzip.a" ]] || (
   if [[ -n "$LIBZIP_REPO" ]]; then
     if ! [[ -e libzip-"$LIBZIP_VERSION" ]]; then
       git clone https://github.com/"$LIBZIP_REPO" libzip-"$LIBZIP_VERSION"
@@ -64,27 +71,35 @@ LIBZIP_COMMIT=664462465d2730d51f04437c90ed7ebcbe19a36f
     fi
   fi
 
-  cd "$THIS_DIR"/libzip-"$LIBZIP_VERSION"
-
-  source "$EMSDK_ENV"
-
-  mkdir -p build
-  cd build
+  mkdir -p "$LIBZIP_ROOT/build"
 
-  emcmake cmake -Wno-dev \
+  docker run --rm \
+    -v "$LIBZIP_ROOT:/libzip" \
+    -v "$ZLIB_ROOT:/zlib" \
+    -u $(id -u):$(id -g) \
+    -w "/libzip/build" \
+    emscripten/emsdk:$EMSDK_VERSION \
+    emcmake cmake -Wno-dev \
     -DBUILD_SHARED_LIBS=OFF \
     -DENABLE_LOCALTIME=OFF \
     -DENABLE_COMMONCRYPTO=OFF \
     -DENABLE_GNUTLS=OFF \
     -DENABLE_MBEDTLS=OFF \
     -DENABLE_OPENSSL=OFF \
     -DENABLE_WINDOWS_CRYPTO=OFF \
-    -DZLIB_LIBRARY="$THIS_DIR"/zlib-"$ZLIB_VERSION"/build/local/lib/libz.a \
-    -DZLIB_INCLUDE_DIR="$THIS_DIR"/zlib-"$ZLIB_VERSION"/build/local/include \
+    -DZLIB_LIBRARY=/zlib/build/local/lib/libz.a \
+    -DZLIB_INCLUDE_DIR=/zlib/build/local/include \
     ..
 
-  emmake make zip
+  docker run --rm \
+    -v "$LIBZIP_ROOT:/libzip" \
+    -v "$ZLIB_ROOT:/zlib" \
+    -u $(id -u):$(id -g) \
+    -w "/libzip/build" \
+    emscripten/emsdk:$EMSDK_VERSION \
+    emmake make zip
 
+  cd "$LIBZIP_ROOT/build"
   mkdir -p local/lib local/include
   cp lib/libzip.a local/lib/
   cp config.h zipconf.h ../lib/zip.h ../lib/compat.h ../lib/zipint.h local/include/
@@ -96,11 +111,16 @@ build() {
   local name=$1
   shift
 
-  source "$EMSDK_ENV"
-
   # Options are documented at https://github.com/emscripten-core/emscripten/blob/86131037aa4f1c7bf6021081dd28fae12bdedba1/src/settings.js
-  emcc \
-    -o ./build.js \
+  docker run --rm \
+    -v "$THIS_DIR:/src" \
+    -v "$LIBZIP_ROOT:/libzip" \
+    -v "$ZLIB_ROOT:/zlib" \
+    -u $(id -u):$(id -g) \
+    -w "/src" \
+    emscripten/emsdk:$EMSDK_VERSION \
+    emcc \
+    -o ./$name.js \
     -s WASM=1 \
     -s EXPORTED_FUNCTIONS="$(cat ./exported.json)" \
     -s EXPORTED_RUNTIME_METHODS='["cwrap", "getValue"]' \
@@ -111,26 +131,28 @@ build() {
     -s NODERAWFS=1 \
     -s SINGLE_FILE=1 \
     -s MODULARIZE=1 \
+    -s STRICT=1 \
     -s EXPORT_NAME="createModule" \
     -s NODEJS_CATCH_EXIT=0 \
     -s NODEJS_CATCH_REJECTION=0 \
     "$@" \
-    -I./libzip-"$LIBZIP_VERSION"/build/local/include \
-    -I./zlib-"$ZLIB_VERSION"/build/local/include \
+    -I/libzip/build/local/include \
+    -I/zlib/build/local/include \
     -O3 \
     ./lib/*.c \
-    ./libzip-"$LIBZIP_VERSION"/build/local/lib/libzip.a \
-    ./zlib-"$ZLIB_VERSION"/build/local/lib/libz.a
+    /libzip/build/local/lib/libzip.a \
+    /zlib/build/local/lib/libz.a
 
   cat >../sources/"$name".js \
     <(echo "var frozenFs = Object.assign({}, require('fs'));") \
-    <(sed 's/require("fs")/frozenFs/g' ./build.js |
+    <(sed 's/require("fs")/frozenFs/g' ./$name.js |
       sed 's/process\["binding"\]("constants")/({"fs":fs.constants})/g')
 
-  yarn prettier --write ../sources/"$name".js
-
   echo "Built wasm ($name)"
 }
 
-build libzipSync -s BINARYEN_ASYNC_COMPILATION=0
-build libzipAsync -s BINARYEN_ASYNC_COMPILATION=1
+build libzipSync -s WASM_ASYNC_COMPILATION=0 &
+build libzipAsync -s WASM_ASYNC_COMPILATION=1 &
+wait
+
+yarn prettier --write ../sources/*.js