diff --git a/.github/actions/install-cmake-linux/action.yml b/.github/actions/install-cmake-linux/action.yml new file mode 100644 index 000000000..50d43583f --- /dev/null +++ b/.github/actions/install-cmake-linux/action.yml @@ -0,0 +1,11 @@ +name: Install protobuf linux +runs: + using: composite + steps: + - name: Install protobuf linux + shell: bash + run: | + apt-get -yq update && apt-get install -y build-essential unzip libssl-dev + curl -fsSL http://cmake.org/files/v3.19/cmake-3.19.0.tar.gz | tar xz && cd cmake-3.19.0 + ./configure --prefix=/opt/cmake && make -j2 && make install && cd .. && rm -r cmake-3.19.0 + echo "/opt/cmake/bin" >> $GITHUB_PATH \ No newline at end of file diff --git a/.github/actions/update-deps-linux/action.yml b/.github/actions/update-deps-linux/action.yml new file mode 100644 index 000000000..0ba7c79da --- /dev/null +++ b/.github/actions/update-deps-linux/action.yml @@ -0,0 +1,63 @@ +name: Install protobuf linux +runs: + using: composite + steps: + - name: Install protobuf linux + shell: bash + run: | + sudo apt-get install build-essential make zlib1g-dev wget + wget https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.36.tar.bz2 + tar jxf libgpg-error-1.36.tar.bz2 + cd libgpg-error-1.36 + ./configure + make + sudo make install + cd .. + wget https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.4.tar.bz2 + tar jxf libgcrypt-1.8.4.tar.bz2 + cd libgcrypt-1.8.4 + ./configure + make + sudo make install + cd .. + wget https://www.gnupg.org/ftp/gcrypt/libksba/libksba-1.3.5.tar.bz2 + tar jxf libksba-1.3.5.tar.bz2 + cd libksba-1.3.5 + ./configure + make + sudo make install + cd .. + wget https://www.gnupg.org/ftp/gcrypt/libassuan/libassuan-2.5.3.tar.bz2 + tar jxf libassuan-2.5.3.tar.bz2 + cd libassuan-2.5.3 + ./configure + make + sudo make install + cd .. + wget https://www.gnupg.org/ftp/gcrypt/ntbtls/ntbtls-0.1.2.tar.bz2 + tar jxf ntbtls-0.1.2.tar.bz2 + cd ntbtls-0.1.2 + ./configure + make + sudo make install + cd .. + wget https://www.gnupg.org/ftp/gcrypt/npth/npth-1.6.tar.bz2 + tar jxf npth-1.6.tar.bz2 + cd npth-1.6 + ./configure + make + sudo make install + cd .. + echo 'include /usr/local/lib/' | sudo tee -a /etc/ld.so.conf + sudo ldconfig -v + wget https://www.gnupg.org/ftp/gcrypt/gnupg/gnupg-2.2.17.tar.bz2 + tar jxf gnupg-2.2.17.tar.bz2 + cd gnupg-2.2.17/ + ./configure + make + sudo make install + cd .. + sudo apt-get install pinentry-curses + #echo 'pinentry-program /usr/bin/pinentry-curses' | tee -a ~/.gnupg/gpg-agent.conf + #gpg-connect-agent reloadagent /bye + sudo apt-get install ca-certificates diff --git a/.github/workflows/build-android-x86_64.yml b/.github/workflows/build-android-x86_64.yml index b4e529203..100785699 100644 --- a/.github/workflows/build-android-x86_64.yml +++ b/.github/workflows/build-android-x86_64.yml @@ -3,8 +3,20 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} android-x86_64: - runs-on: ubuntu-18.04 + needs: pre-ci + runs-on: ubuntu-16.04 steps: - uses: AutoModality/action-clean@v1 - name: Cancel Previous Runs @@ -16,7 +28,9 @@ jobs: with: ndk-version: r18b - uses: actions/checkout@v2 + - uses: ./.github/actions/update-deps-linux - uses: ./.github/actions/install-protobuf-linux + - uses: ./.github/actions/install-cmake-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: @@ -29,11 +43,13 @@ jobs: - name: Build on linux-x86_64 env: ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }} - LIBND4J_HOME: "${GITHUB_WORKSPACE}/libnd4j" + LIBND4J_HOME: ${GITHUB_WORKSPACE}/libnd4j GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + OPENBLAS_PATH: ${HOME}/.javacpp/cache/openblas-0.3.13-1.5.5-android-x86_64.jar/lib/ + DEPLOY_TO: ossrh run: | echo "Verifying programs on path. Path is $PATH" echo "Path post update is $PATH. Maven is at `which mvn` cmake is at `which cmake` protoc is at `which protoc`" @@ -41,6 +57,8 @@ jobs: cmake --version protoc --version clang --version + export LIBND4J_HOME=${GITHUB_WORKSPACE}/libnd4j + export OPENBLAS_PATH=${HOME}/.javacpp/cache/openblas-0.3.13-1.5.5-android-x86_64.jar/lib/ mvn -X -Dorg.bytedeco.javacpp.logger.debug=true -Possrh -pl ":nd4j-native,:libnd4j" --also-make \ -Djavacpp.platform=android-x86_64 \ -Dlibnd4j.platform=android-x86_64 -Dlibnd4j.chip=cpu \ diff --git a/.github/workflows/build-deploy-android-arm32.yml b/.github/workflows/build-deploy-android-arm32.yml index 3f4a0fc04..5390efc11 100644 --- a/.github/workflows/build-deploy-android-arm32.yml +++ b/.github/workflows/build-deploy-android-arm32.yml @@ -3,9 +3,21 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. android-arm32: - runs-on: ubuntu-18.04 + needs: pre-ci + runs-on: ubuntu-16.04 steps: - uses: AutoModality/action-clean@v1 - name: Cancel Previous Runs @@ -13,7 +25,9 @@ jobs: with: access_token: ${{ github.token }} - uses: actions/checkout@v2 + - uses: ./.github/actions/update-deps-linux - uses: ./.github/actions/install-protobuf-linux + - uses: ./.github/actions/install-cmake-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: diff --git a/.github/workflows/build-deploy-android-arm64.yml b/.github/workflows/build-deploy-android-arm64.yml index d2eba04ea..58d5dcdee 100644 --- a/.github/workflows/build-deploy-android-arm64.yml +++ b/.github/workflows/build-deploy-android-arm64.yml @@ -3,9 +3,21 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. android-arm64: - runs-on: ubuntu-18.04 + needs: pre-ci + runs-on: ubuntu-16.04 steps: - uses: AutoModality/action-clean@v1 - name: Cancel Previous Runs @@ -13,7 +25,9 @@ jobs: with: access_token: ${{ github.token }} - uses: actions/checkout@v2 + - uses: ./.github/actions/update-deps-linux - uses: ./.github/actions/install-protobuf-linux + - uses: ./.github/actions/install-cmake-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: @@ -36,6 +50,7 @@ jobs: MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + DEPLOY_TO: ossrh run: | mvn --version cmake --version diff --git a/.github/workflows/build-deploy-linux-arm32.yml b/.github/workflows/build-deploy-linux-arm32.yml index 88884aa5b..fc229f3fe 100644 --- a/.github/workflows/build-deploy-linux-arm32.yml +++ b/.github/workflows/build-deploy-linux-arm32.yml @@ -3,9 +3,21 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. linux-arm32: - runs-on: ubuntu-18.04 + needs: pre-ci + runs-on: ubuntu-16.04 steps: - uses: AutoModality/action-clean@v1 - name: Cancel Previous Runs @@ -13,7 +25,9 @@ jobs: with: access_token: ${{ github.token }} - uses: actions/checkout@v2 + - uses: ./.github/actions/update-deps-linux - uses: ./.github/actions/install-protobuf-linux + - uses: ./.github/actions/install-cmake-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: diff --git a/.github/workflows/build-deploy-linux-arm64.yml b/.github/workflows/build-deploy-linux-arm64.yml index d806203ce..65bb426b8 100644 --- a/.github/workflows/build-deploy-linux-arm64.yml +++ b/.github/workflows/build-deploy-linux-arm64.yml @@ -3,9 +3,21 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. linux-arm64: - runs-on: ubuntu-18.04 + needs: pre-ci + runs-on: ubuntu-16.04 steps: - uses: AutoModality/action-clean@v1 - name: Cancel Previous Runs @@ -13,7 +25,9 @@ jobs: with: access_token: ${{ github.token }} - uses: actions/checkout@v2 + - uses: ./.github/actions/update-deps-linux - uses: ./.github/actions/install-protobuf-linux + - uses: ./.github/actions/install-cmake-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: diff --git a/.github/workflows/build-deploy-linux-cuda-11.0.yml b/.github/workflows/build-deploy-linux-cuda-11.0.yml index 1a559c01f..98ca27951 100644 --- a/.github/workflows/build-deploy-linux-cuda-11.0.yml +++ b/.github/workflows/build-deploy-linux-cuda-11.0.yml @@ -3,9 +3,20 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: - + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-16.04 + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} linux-x86_64-cuda_11-0: - runs-on: ubuntu-18.04 + needs: pre-ci + runs-on: ubuntu-16.04 steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.8.0 @@ -23,7 +34,9 @@ jobs: env: cuda: 11.0.167 GCC: 9 + - uses: ./.github/actions/update-deps-linux - uses: ./.github/actions/install-protobuf-linux + - uses: ./.github/actions/install-cmake-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: diff --git a/.github/workflows/build-deploy-linux-cuda-11.2.yml b/.github/workflows/build-deploy-linux-cuda-11.2.yml index 3387740eb..03e4752c0 100644 --- a/.github/workflows/build-deploy-linux-cuda-11.2.yml +++ b/.github/workflows/build-deploy-linux-cuda-11.2.yml @@ -3,8 +3,20 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-16.04 + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} linux-x86_64-cuda-11-2: - runs-on: ubuntu-18.04 + needs: pre-ci + runs-on: ubuntu-16.04 steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.8.0 @@ -22,7 +34,9 @@ jobs: env: cuda: 11.2.1_461 GCC: 9 + - uses: ./.github/actions/update-deps-linux - uses: ./.github/actions/install-protobuf-linux + - uses: ./.github/actions/install-cmake-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: diff --git a/.github/workflows/build-deploy-linux-x86_64.yml b/.github/workflows/build-deploy-linux-x86_64.yml index 68fe86012..d0a399b68 100644 --- a/.github/workflows/build-deploy-linux-x86_64.yml +++ b/.github/workflows/build-deploy-linux-x86_64.yml @@ -4,16 +4,30 @@ on: workflow_dispatch: push: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. linux-x86_64: - runs-on: ubuntu-18.04 + needs: pre-ci + runs-on: ubuntu-16.04 steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.8.0 with: access_token: ${{ github.token }} - uses: actions/checkout@v2 + - uses: ./.github/actions/update-deps-linux - uses: ./.github/actions/install-protobuf-linux + - uses: ./.github/actions/install-cmake-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: @@ -40,4 +54,3 @@ jobs: sudo apt-get clean mvn -X -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cpu -Pcpu --batch-mode deploy -DskipTests - diff --git a/.github/workflows/build-deploy-mac.yml b/.github/workflows/build-deploy-mac.yml index 93bd3898e..866f25713 100644 --- a/.github/workflows/build-deploy-mac.yml +++ b/.github/workflows/build-deploy-mac.yml @@ -3,7 +3,19 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} mac-x86_64: + needs: pre-ci runs-on: macos-10.15 steps: - name: Cancel Previous Runs diff --git a/.github/workflows/build-deploy-windows-cuda-11.0.yml b/.github/workflows/build-deploy-windows-cuda-11.0.yml index b2f61ed9a..4b847aa4f 100644 --- a/.github/workflows/build-deploy-windows-cuda-11.0.yml +++ b/.github/workflows/build-deploy-windows-cuda-11.0.yml @@ -3,7 +3,19 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} windows-x86_64-cuda-11-0: + needs: pre-ci runs-on: windows-2019 steps: - name: Cancel Previous Runs diff --git a/.github/workflows/build-deploy-windows-cuda-11.2.yml b/.github/workflows/build-deploy-windows-cuda-11.2.yml index 8cd435732..8e651d2d1 100644 --- a/.github/workflows/build-deploy-windows-cuda-11.2.yml +++ b/.github/workflows/build-deploy-windows-cuda-11.2.yml @@ -3,7 +3,19 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} windows-x86_64-cuda-11-2: + needs: pre-ci runs-on: windows-2019 steps: - name: Cancel Previous Runs diff --git a/.github/workflows/build-deploy-windows.yml b/.github/workflows/build-deploy-windows.yml index 51ad31e7e..5ef668698 100644 --- a/.github/workflows/build-deploy-windows.yml +++ b/.github/workflows/build-deploy-windows.yml @@ -3,8 +3,19 @@ on: - cron: "0 */12 * * *" workflow_dispatch: jobs: - + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} windows-x86_64: + needs: pre-ci runs-on: windows-2019 steps: - name: Cancel Previous Runs diff --git a/.github/workflows/cpu-integration-tests.yaml b/.github/workflows/cpu-integration-tests.yaml index 8d57fa2e7..edb240dc4 100644 --- a/.github/workflows/cpu-integration-tests.yaml +++ b/.github/workflows/cpu-integration-tests.yaml @@ -1,7 +1,19 @@ on: push: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} linux-x86_64: + needs: pre-ci runs-on: ubuntu-18.04 steps: - name: Cancel Previous Runs diff --git a/.github/workflows/cpu-sanity-check-tests.yaml b/.github/workflows/cpu-sanity-check-tests.yaml index 0fc3120a9..46dccea75 100644 --- a/.github/workflows/cpu-sanity-check-tests.yaml +++ b/.github/workflows/cpu-sanity-check-tests.yaml @@ -1,7 +1,19 @@ on: push: jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} linux-x86_64: + needs: pre-ci runs-on: ubuntu-18.04 steps: - name: Cancel Previous Runs diff --git a/.github/workflows/run-gpu-tests-sanity-checks.yml b/.github/workflows/run-gpu-tests-sanity-checks.yml new file mode 100644 index 000000000..0e9e437f5 --- /dev/null +++ b/.github/workflows/run-gpu-tests-sanity-checks.yml @@ -0,0 +1,40 @@ +on: + push: + +jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + linux-x86_64-cuda-11-2: + needs: pre-ci + runs-on: [self-hosted] + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: ./.github/actions/download-dl4j-test-resources-linux + - uses: actions/checkout@v2 + - name: Run cuda tests + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + run: | + export PATH="/opt/protobuf/bin:/usr/local/cuda-11.2/bin:$PATH" + nvcc --version + mvn --version + cmake --version + protoc --version + bash ./change-cuda-versions.sh 11.2 + export OMP_NUM_THREADS=1 + mvn -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-cuda-11.2,:samediff-import,:libnd4j" -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Ptest-nd4j-cuda --also-make -Dlibnd4j.chip=cuda clean test + diff --git a/.github/workflows/test_multiple_arch.yaml b/.github/workflows/test_multiple_arch.yaml new file mode 100644 index 000000000..9165d4eff --- /dev/null +++ b/.github/workflows/test_multiple_arch.yaml @@ -0,0 +1,92 @@ +on: + push: +jobs: + # Wait for up to a minute for previous run to complete, abort if not done by then + pre-ci: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: 'Block Concurrent Executions' + uses: softprops/turnstyle@v1 + with: + poll-interval-seconds: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + build_and_test_cross_platform: + needs: pre-ci + # The host should always be linux + runs-on: ubuntu-16.04 + name: Build on ${{ matrix.distro }} ${{ matrix.arch }} + + # Run steps on a matrix of 2 arch/distro combinations + strategy: + fail-fast: false + matrix: + include: + - arch: aarch64 + distro: ubuntu16.04 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/download-dl4j-test-resources-linux + - uses: ./.github/actions/install-protobuf-linux + - uses: uraimo/run-on-arch-action@v2.0.8 + name: Build and test with ${{ matrix.arch }} ${{ matrix.distro }} + id: build + with: + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} + # Pass some environment variables to the container + env: | # YAML, but pipe character is necessary + DEBIAN_FRONTEND: noninteractive + M2_HOME: /opt/maven + + # The shell to run commands with in the container + shell: /bin/sh + run: | + echo "Running install on architecture ${{ matrix.arch }}" + apt-get -yq update && apt-get install -y build-essential unzip libssl-dev curl openjdk-8-jdk-headless + echo "After install on architecture ${{ matrix.arch }}" + mkdir -p "/opt/maven" + curl -fsSL http://apache.osuosl.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz | tar -xzC "/opt/maven" --strip-components=1 + apt-get -yq update && apt-get install -y build-essential unzip libssl-dev + curl -fsSL http://cmake.org/files/v3.19/cmake-3.19.0.tar.gz | tar xz && cd cmake-3.19.0 + ./configure --prefix=/opt/cmake && make -j2 && make install && cd .. && rm -r cmake-3.19.0 + export PATH="/opt/maven/bin:$PATH" + echo "RUNNING ARCH ${{ matrix.arch }}" + case "${{ matrix.arch }}" in + armv7) + export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-armhf/jre + ;; + aarch64) + export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-arm64/jre" + ;; + ppc64le) + export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-ppc64el/jre + ;; + esac + mvn --version + export PATH="/opt/cmake/bin:/opt/maven/bin:/opt/protobuf/bin:$PATH" + cmake --version + wget https://github.com/KonduitAI/dl4j-test-resources/archive/master.zip && unzip master.zip + cd dl4j-test-resources-master + mvn clean install -DskipTests + echo "Extracted test resources" + echo "Running build in ${pwd}" + curl -fsSL https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-cpp-3.5.1.tar.gz \ + | tar xz && \ + cd protobuf-3.5.1 && \ + ./configure --prefix=/opt/protobuf && \ + make -j2 && \ + make install && \ + cd .. && \ + rm -rf protobuf-3.5.1 + echo "/opt/protobuf/bin" >> $GITHUB_PATH + mvn -X -Possrh -pl :nd4j-tests -Djavacpp.platform=linux-${{matrix.arch}} -Dlibnd4j.chip=cpu -Pcpu --batch-mode clean test -Dtest=org.nd4j.smoketests.SmokeTest + + + + diff --git a/libnd4j/buildnativeoperations.sh b/libnd4j/buildnativeoperations.sh index 0719a910a..8e721fd47 100755 --- a/libnd4j/buildnativeoperations.sh +++ b/libnd4j/buildnativeoperations.sh @@ -430,9 +430,9 @@ fi if [ -z "$COMPUTE" ]; then if [ "$ARCH" == "x86-64" ]; then - COMPUTE="5.0 5.2 5.3 6.0 6.2 8.0" + COMPUTE="5.0 5.2 5.3 6.0 8.0" else - COMPUTE="5.0 5.2 5.3 6.0 6.2" + COMPUTE="5.0 5.2 5.3 6.0" fi fi diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/ir/TensorNamespace.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/ir/TensorNamespace.java index a0899ddbe..434bda3a8 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/ir/TensorNamespace.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/ir/TensorNamespace.java @@ -5411,14 +5411,12 @@ public final class TensorNamespace { * Serializations can either use one of the fields above, or use this * raw bytes field. The only exception is the string case, where one is * required to store the content in the repeated bytes string_data field. - * * When this raw_data field is used to store tensor value, elements MUST * be stored in as fixed-width, little-endian order. * Floating-point data types MUST be stored in IEEE 754 format. * Complex64 elements must be written as two consecutive FLOAT values, real component first. * Complex128 elements must be written as two consecutive DOUBLE values, real component first. * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false). - * * Note: the advantage of specific field rather than the raw_data field is * that in some cases (e.g. int data), protobuf does a better packing via * variable length storage, and may lead to smaller binary footprint. @@ -5657,7 +5655,6 @@ public final class TensorNamespace { /** * 
    * Tensors
-   *
    * A serialized tensor value.
    *
* @@ -7013,14 +7010,12 @@ public final class TensorNamespace { * Serializations can either use one of the fields above, or use this * raw bytes field. The only exception is the string case, where one is * required to store the content in the repeated bytes string_data field. - * * When this raw_data field is used to store tensor value, elements MUST * be stored in as fixed-width, little-endian order. * Floating-point data types MUST be stored in IEEE 754 format. * Complex64 elements must be written as two consecutive FLOAT values, real component first. * Complex128 elements must be written as two consecutive DOUBLE values, real component first. * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false). - * * Note: the advantage of specific field rather than the raw_data field is * that in some cases (e.g. int data), protobuf does a better packing via * variable length storage, and may lead to smaller binary footprint. @@ -7771,7 +7766,6 @@ public final class TensorNamespace { /** * 
      * Tensors
-     *
      * A serialized tensor value.
      *
* @@ -9086,14 +9080,12 @@ public final class TensorNamespace { * Serializations can either use one of the fields above, or use this * raw bytes field. The only exception is the string case, where one is * required to store the content in the repeated bytes string_data field. - * * When this raw_data field is used to store tensor value, elements MUST * be stored in as fixed-width, little-endian order. * Floating-point data types MUST be stored in IEEE 754 format. * Complex64 elements must be written as two consecutive FLOAT values, real component first. * Complex128 elements must be written as two consecutive DOUBLE values, real component first. * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false). - * * Note: the advantage of specific field rather than the raw_data field is * that in some cases (e.g. int data), protobuf does a better packing via * variable length storage, and may lead to smaller binary footprint. @@ -9110,14 +9102,12 @@ public final class TensorNamespace { * Serializations can either use one of the fields above, or use this * raw bytes field. The only exception is the string case, where one is * required to store the content in the repeated bytes string_data field. - * * When this raw_data field is used to store tensor value, elements MUST * be stored in as fixed-width, little-endian order. * Floating-point data types MUST be stored in IEEE 754 format. * Complex64 elements must be written as two consecutive FLOAT values, real component first. * Complex128 elements must be written as two consecutive DOUBLE values, real component first. * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false). - * * Note: the advantage of specific field rather than the raw_data field is * that in some cases (e.g. int data), protobuf does a better packing via * variable length storage, and may lead to smaller binary footprint. @@ -9140,14 +9130,12 @@ public final class TensorNamespace { * Serializations can either use one of the fields above, or use this * raw bytes field. The only exception is the string case, where one is * required to store the content in the repeated bytes string_data field. - * * When this raw_data field is used to store tensor value, elements MUST * be stored in as fixed-width, little-endian order. * Floating-point data types MUST be stored in IEEE 754 format. * Complex64 elements must be written as two consecutive FLOAT values, real component first. * Complex128 elements must be written as two consecutive DOUBLE values, real component first. * Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false). - * * Note: the advantage of specific field rather than the raw_data field is * that in some cases (e.g. int data), protobuf does a better packing via * variable length storage, and may lead to smaller binary footprint. diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/pom.xml b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/pom.xml index 836387031..0b4220f8b 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/pom.xml +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/pom.xml @@ -287,10 +287,7 @@ maven-jar-plugin - - org.apache.maven.plugins - maven-surefire-plugin - + org.apache.maven.plugins maven-enforcer-plugin diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/smoketests/SmokeTest.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/smoketests/SmokeTest.java new file mode 100644 index 000000000..91419ebef --- /dev/null +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/smoketests/SmokeTest.java @@ -0,0 +1,59 @@ +/* + * + * * ****************************************************************************** + * * * + * * * + * * * This program and the accompanying materials are made available under the + * * * terms of the Apache License, Version 2.0 which is available at + * * * https://www.apache.org/licenses/LICENSE-2.0. + * * * + * * * See the NOTICE file distributed with this work for additional + * * * information regarding copyright ownership. + * * * Unless required by applicable law or agreed to in writing, software + * * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * * License for the specific language governing permissions and limitations + * * * under the License. + * * * + * * * SPDX-License-Identifier: Apache-2.0 + * * ***************************************************************************** + * + * + */ + +package org.nd4j.smoketests; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.profiler.ProfilerConfig; + +@Slf4j +public class SmokeTest { + + + @Test + public void testBasic() { + Nd4j.getEnvironment().setDebug(true); + Nd4j.getExecutioner().setProfilingConfig(ProfilerConfig.builder() + .checkForNAN(true) + .checkForINF(true) + .checkLocality(true) + .checkElapsedTime(true) + .checkWorkspaces(true) + .build()); + INDArray arr = Nd4j.randn(2,2); + INDArray arr2 = Nd4j.randn(2,2); + for(DataType dataType : DataType.values()) { + log.info("Testing matrix multiply on data type {}",dataType); + INDArray casted = arr.castTo(dataType); + INDArray casted2 = arr2.castTo(dataType); + INDArray result = casted.mmul(casted2); + log.info("Result for data type {} was {}",dataType,result); + + } + } + +}