diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index ce71ab2fecb..d49fd880474 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -21,6 +21,7 @@ env: # Update this prior to requiring a higher minor version in go.mod GO_VERSION: "1.21" # 1.xx == latest patch of 1.xx TINYGO_VERSION: "0.30.0" ZIG_VERSION: "0.11.0" + BINARYEN_VERSION: "116" concurrency: # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-concurrency-to-cancel-any-in-progress-job-or-run @@ -36,6 +37,7 @@ jobs: env: ZIG_INSTALL: ~/zig-install ZIG_SOURCE: ~/zig-source + BINARYEN_INSTALL: ~/binaryen-install steps: - name: Checkout wazero @@ -48,7 +50,7 @@ jobs: enableCrossOsArchive: true key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }} path: - ./zigbin/ + ./internal/integration_test/stdlibs/testdata/zig - name: Install Zig build if: steps.binary-cache.outputs.cache-hit != 'true' @@ -62,16 +64,20 @@ jobs: mkdir -p ${{ env.ZIG_SOURCE }} curl -sSL https://ziglang.org/download/${{ env.ZIG_VERSION }}/zig-${{ env.ZIG_VERSION }}.tar.xz | tar -xJ --strip-components=1 -C ${{ env.ZIG_SOURCE }} + - name: Install Binaryen build + if: steps.binary-cache.outputs.cache-hit != 'true' + run: | + mkdir -p ${{ env.BINARYEN_INSTALL }} + curl -sSL https://github.com/WebAssembly/binaryen/releases/download/version_${{ env.BINARYEN_VERSION }}/binaryen-version_${{ env.BINARYEN_VERSION }}-x86_64-linux.tar.gz | tar -xz --strip-components=1 -C ${{ env.BINARYEN_INSTALL }} + - name: Build Stdlib test binary if: steps.binary-cache.outputs.cache-hit != 'true' # --test-no-exec allows building of the test Wasm binary without executing command. # We use find because the test.wasm will be something like ./zig-cache/o/dd6df1361b2134adc5eee9d027495436/test.wasm run: | - mkdir ${{ github.workspace }}/zigbin - cd ${{ env.ZIG_SOURCE }} - ${{ env.ZIG_INSTALL }}/zig test --test-no-exec -target wasm32-wasi --zig-lib-dir ./lib ./lib/std/std.zig - _ZIG_TEST_BINARY_PATH=$(find . -name test.wasm) - cp ${_ZIG_TEST_BINARY_PATH} ${{ github.workspace }}/zigbin/test.wasm + PATH=${{ env.ZIG_INSTALL }}:${{ env.BINARYEN_INSTALL }}/bin:$PATH + cd internal/integration_test/stdlibs + make build.zig zigroot=${{ env.ZIG_SOURCE }} zig: needs: build_zig_test_binary @@ -101,7 +107,7 @@ jobs: fail-on-cache-miss: true key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }} path: - ./zigbin/ + ./internal/integration_test/stdlibs/testdata/zig - uses: actions/setup-go@v4 with: @@ -136,12 +142,12 @@ jobs: - name: Run built test binaries (container) if: ${{ matrix.compiler == 'optimizing' }} run: | - docker run --platform linux/${{ matrix.arch }} -v $(pwd)/zigbin:/test -v $(pwd)/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ + docker run --platform linux/${{ matrix.arch }} -v $(pwd)/internal/integration_test/stdlibs/testdata/zig:/test -v $(pwd)/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ /wazero run -optimizing-compiler -mount=:/ ./test/test.wasm - name: Run built test binaries if: ${{ matrix.compiler != 'optimizing' }} - run: ./wazerocli run -mount=:/ ./zigbin/test.wasm + run: ./wazerocli run -mount=:/ ./internal/integration_test/stdlibs/testdata/zig/test.wasm build_tinygo_test_binary: name: Build TinyGo test binary @@ -157,7 +163,7 @@ jobs: enableCrossOsArchive: true key: tinygo-test-binaries-${{ env.TINYGO_VERSION }} path: - ./tinygobin/ + ./internal/integration_test/stdlibs/testdata/tinygo - name: Install TinyGo if: steps.binary-cache.outputs.cache-hit != 'true' @@ -181,48 +187,8 @@ jobs: # - compress/zlib is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/compress/zlib/writer_test.go#L16-L30 # - debug/macho is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/debug/macho/file_test.go#L25 run: | - mkdir ./tinygobin - for value in container/heap \ - container/list \ - container/ring \ - crypto/des \ - crypto/md5 \ - crypto/rc4 \ - crypto/sha1 \ - crypto/sha256 \ - crypto/sha512 \ - embed/internal/embedtest \ - encoding \ - encoding/ascii85 \ - encoding/base32 \ - encoding/csv \ - encoding/hex \ - go/scanner \ - hash \ - hash/adler32 \ - hash/crc64 \ - hash/fnv \ - html \ - internal/itoa \ - internal/profile \ - math \ - math/cmplx \ - net \ - net/http/internal/ascii \ - net/mail \ - os \ - path \ - reflect \ - sync \ - testing \ - testing/iotest \ - text/scanner \ - unicode \ - unicode/utf16 \ - unicode/utf8 - do - tinygo test -target wasi -c -o ./tinygobin/${value/\//_}.test $value - done + cd internal/integration_test/stdlibs + make build.tinygo tinygo: needs: build_tinygo_test_binary @@ -252,7 +218,7 @@ jobs: fail-on-cache-miss: true key: tinygo-test-binaries-${{ env.TINYGO_VERSION }} path: - ./tinygobin/ + ./internal/integration_test/stdlibs/testdata/tinygo - uses: actions/setup-go@v4 with: @@ -286,7 +252,7 @@ jobs: if: ${{ matrix.compiler == 'optimizing' }} # This runs all tests compiled above in sequence. Note: This mounts /tmp to allow t.TempDir() in tests. run: | - cd ./tinygobin + cd ./internal/integration_test/stdlibs/testdata/tinygo for bin in *.test; do echo $bin docker run --platform linux/${{ matrix.arch }} -v $(pwd):/test -v ~/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ @@ -296,7 +262,7 @@ jobs: - name: Run test binaries if: ${{ matrix.compiler != 'optimizing' }} run: | - cd ./tinygobin + cd ./internal/integration_test/stdlibs/testdata/tinygo for bin in *.test; do echo $bin ~/wazerocli run -mount=:/ -mount=:/tmp $bin -- -test.v @@ -413,96 +379,29 @@ jobs: with: go-version: ${{ matrix.go-version }} + - name: Checkout wazero + uses: actions/checkout@v3 + - name: Cache Go test binaries id: cache-go-test-binaries uses: actions/cache@v3 with: - path: ~/tests + path: + ./internal/integration_test/stdlibs/testdata/go # Use precise Go version from setup-go as patch version differences can effect tests. key: go-wasip1-binaries-${{ matrix.os }}-${{ steps.setup-go.outputs.go-version }} - - if: ${{ steps.cache-go-test-binaries.outputs.cache-hit != 'true' }} - name: Build Test Binaries - run: | - mkdir ~/tests - cd $(go env GOROOT) - # Choose important packages to limit execution time. - for value in src/archive/tar \ - src/bufio \ - src/bytes \ - src/context \ - src/encoding/ascii85 \ - src/encoding/asn1 \ - src/encoding/base32 \ - src/encoding/base64 \ - src/encoding/binary \ - src/encoding/csv \ - src/encoding/gob \ - src/encoding/hex \ - src/encoding/json \ - src/encoding/pem \ - src/encoding/xml \ - src/errors \ - src/expvar \ - src/flag \ - src/fmt \ - src/hash \ - src/hash/adler32 \ - src/hash/crc32 \ - src/hash/crc64 \ - src/hash/fnv \ - src/hash/maphash \ - src/io \ - src/io/fs \ - src/io/ioutil \ - src/log \ - src/log/syslog \ - src/maps \ - src/math \ - src/math/big \ - src/math/bits \ - src/math/cmplx \ - src/math/rand \ - src/mime \ - src/mime/multipart \ - src/mime/quotedprintable \ - src/os \ - src/os/exec \ - src/os/signal \ - src/os/user \ - src/path \ - src/reflect \ - src/regexp \ - src/regexp/syntax \ - src/runtime \ - src/runtime/internal/atomic \ - src/runtime/internal/math \ - src/runtime/internal/sys \ - src/slices \ - src/sort \ - src/strconv \ - src/strings \ - src/sync \ - src/sync/atomic \ - src/syscall \ - src/testing \ - src/testing/fstest \ - src/testing/iotest \ - src/testing/quick \ - src/time - do - echo "GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value" - GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value - done - - - name: Checkout wazero - uses: actions/checkout@v3 - - name: Build wazero run: go build -o ~/wazerocli ./cmd/wazero env: GOARCH: ${{ matrix.arch }} + - if: ${{ steps.cache-go-test-binaries.outputs.cache-hit != 'true' }} + name: Build Test Binaries + run: | + cd ./internal/integration_test/stdlibs + make build.gowasip1 + - name: Set up QEMU if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }} uses: docker/setup-qemu-action@v2 @@ -519,13 +418,14 @@ jobs: - if: ${{ matrix.compiler == 'optimizing' }} name: Run test binaries (container) +# uses: mxschmitt/action-tmate@v3 run: | - echo "Running $(find ~/tests -name *.test | wc -l) test binaries" + echo "Running $(find ./internal/integration_test/stdlibs/testdata/go -name '*.test' | wc -l) test binaries" # Skip tests that are hideously slow (mostly compilation time) for running inside QEMU. skip_targets="src_encoding_json|src_runtime|src_os_exec|src_math_big|src_encoding_gob|src_time|src_archive_tar|src_math_rand|src_expvar|src_testing|src_os" # Go tests often look for files relative to the source. Change to the corresponding directory. - for bin in $(find ~/tests -name "*.test" | grep -vE "$skip_targets"); do + for bin in $(find $PWD/internal/integration_test/stdlibs/testdata/go -name '*.test' | grep -vE "$skip_targets"); do dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} pushd $(go env GOROOT)/$dir # Mount / as /ROOT in docker and then remount /ROOT as / in wazero; $bin is now in /ROOT/$bin. @@ -537,10 +437,10 @@ jobs: - if: ${{ matrix.compiler != 'optimizing' && runner.os != 'Windows' }} name: Run test binaries run: | - echo "Running $(find ~/tests -name *.test | wc -l) test binaries" + echo "Running $(find ./internal/integration_test/stdlibs/testdata/go -name *.test | wc -l) test binaries" # Go tests often look for files relative to the source. Change to the corresponding directory. - for bin in ~/tests/*.test; do + for bin in $PWD/internal/integration_test/stdlibs/testdata/go/*.test; do dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} pushd $(go env GOROOT)/$dir ~/wazerocli run -mount=/:/ -env PWD=$PWD $bin -- -test.short -test.v @@ -553,7 +453,7 @@ jobs: continue-on-error: true run: | GOOS=$(go env GOOS) - echo "Running $(find ~/tests -name *.test | wc -l) test binaries" + echo "Running $(find $PWD/internal/integration_test/stdlibs/testdata/go -name *.test | wc -l) test binaries" MOUNT=c:\\:/ SCRIPT="$HOME/tests.cmd" @@ -566,7 +466,7 @@ jobs: EXTRAPARAMS="-mount=%TEMP%:/tmp" # Go tests often look for files relative to the source. Change to the corresponding directory. - for bin in ~/tests/*.test; do + for bin in $PWD/internal/integration_test/stdlibs/testdata/go/*.test; do dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} pushd $REAL_GOROOT/$dir diff --git a/internal/integration_test/stdlibs/Makefile b/internal/integration_test/stdlibs/Makefile index 01b5f9650a3..5199cccfc0e 100644 --- a/internal/integration_test/stdlibs/Makefile +++ b/internal/integration_test/stdlibs/Makefile @@ -122,23 +122,27 @@ run: .PHONY: build.zig build.zig: + # --test-no-exec allows building of the test Wasm binary without executing command. + # We use find because the test.wasm will be something like ./zig-cache/o/dd6df1361b2134adc5eee9d027495436/test.wasm @mkdir -p $(zig_bin) @cd $(zigroot) && zig test --test-no-exec -target wasm32-wasi --zig-lib-dir $(zigroot)/lib $(zigroot)/lib/std/std.zig - @cp $(shell find $(zigroot) -name test.wasm) $(zig_bin)/test.wasm + @cp $$(find $(zigroot) -name test.wasm) $(zig_bin)/test.wasm + # The generated test binary is large and produces skewed results in favor of the optimized compiler. + # We also generate a stripped, optimized binary with wasm-opt. @wasm-opt $(zig_bin)/test.wasm -O --strip-dwarf -o $(zig_bin)/test-opt.wasm .PHONY: build.tinygo build.tinygo: @mkdir -p $(tinygo_bin) - @for value in $(tinygo_tests); do\ + for value in $(tinygo_tests); do\ echo Building $${value}... ;\ - tinygo test -target wasi -c -o $(tinygo_bin)/$${value//\//_}.test $${value} 2>&1 >/dev/null ;\ + tinygo test -target wasi -c -o $(tinygo_bin)/$$(echo $$value | sed -e 's/\//_/g').test $${value} 2>&1 >/dev/null ;\ done .PHONY: build.gowasip1 build.gowasip1: @mkdir -p $(gowasip1_bin) - @cd $(goroot); \ + @cd '$(goroot)'; \ for value in $(gowasip1_tests); do\ - GOOS=wasip1 GOARCH=wasm go test -v -c -o $(gowasip1_bin)/$${value//\//_}.test ./$${value}; \ + GOOS=wasip1 GOARCH=wasm go test -v -c -o $(gowasip1_bin)/$$(echo $$value | sed -e 's/\//_/g').test ./$${value}; \ done