.gitlab-ci.yml

workflow:
  rules:
    # always run on master
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    # always run for merge requests
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    # always run if triggered by "run pipleline" in gitlab
    - if: $CI_PIPELINE_SOURCE == "web"
    # always run if the ci variable CI_FORCE is 1.
    # for example by pushing with: git push -o ci.variable=CI_FORCE=1
    - if: $CI_FORCE == "1"
      # Run distro tests if CI_DISTRO_TESTS=1 is set, or when it is a scheduled run
    - if: $CI_PIPELINE_SOURCE == "schedule"
      variables:
        CI_DISTRO_TESTS: "1"
    # do not run for anything else


variables:
  OMP_NUM_THREADS: "1"
# mpirun in docker has a problem with its default transfer mechanism,
# so we disable it:
  OMPI_MCA_btl_vader_single_copy_mechanism: "none"
# make DEBUG_DWARF the default
  DEBUG: "1"
  DEBUG_DWARF: "1"
# utests give a backtrace + abort on the first error
  BART_UTEST_ABORT: "1"
# For gitlab-runner using docker: create diretories as the USER in the docker image, not as root
  FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: "true"

image: registry.gitlab.tugraz.at/ibi/reproducibility/gitlab-ci-containers/ibi_cuda_bart


default:
  # Make Builds interruptible by default
  interruptible: true

stages:
  - distro_build
  - build
  - test1
  - test2

Build_NoDEBUG:
  stage: build
  script:
    - DEBUG=0 DEBUG_DWARF=0 WERROR=1 make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
    expire_in: 45 minutes

Build:
  stage: build
  script:
    - WERROR=1 make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes

Build_riscv:
  stage: build
  tags:
    - riscv
  image: registry.gitlab.tugraz.at/ibi/reproducibility/gitlab-ci-containers/ibi_riscv_noroot
  script:
    - make -j 4 all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 180 minutes


Build_Clang:
  stage: build
  script:
    - CC=clang-19 make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes

Build_Clang_GPU:
  stage: build
  script:
    - CC=clang-19 CUDA_CC=clang-14 CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes

Build_Static:
  stage: build
  script:
#    - apt-get update -qq && apt-get install -y libgfortran-12-dev
    - SLINK=1 make
  artifacts:
    paths:
      - bart
    expire_in: 45 minutes


Build_Shared:
  stage: build
  script:
    - make libbart.so
  artifacts:
    paths:
      - libbart.so
    expire_in: 45 minutes

Build_Shared_GPU:
  stage: build
  script:
    - CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" make libbart.so
  artifacts:
    paths:
      - libbart.so
    expire_in: 45 minutes


Build_WinDLL:
  stage: build
  script:
    #- apt-get update -qq && apt-get install -y mingw-w64
    - BARTDLL=1 make bart.dll
  artifacts:
    paths:
      - bart.dll
    expire_in: 45 minutes


Build_UBSan:
  stage: build
  script:
    - UBSAN=1 ASAN=1 make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes


Build_GPU:
  stage: build
  script:
    - CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" WERROR=1 make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes

Build_MPI_GPU:
  stage: build
  script:
    - MPI=1 CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes

Build_MPI:
  stage: build
  script:
    - MPI=1 make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes

Build_LTO:
  stage: build
  script:
    - CFLAGS="-flto -O2 -g" DEBUG=0 DEBUG_DWARF=0 make
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes


Build_Analyzer:
  stage: build
  image: registry.gitlab.tugraz.at/ibi/reproducibility/gitlab-ci-containers/ibi_gcc-git_bart
  script:
    - gcc-git --version
    - CC=gcc-git CFLAGS="-O2 -fanalyzer -Werror -fsanitize=vla-bound -fsanitize-trap=undefined -Wno-maybe-uninitialized -Wno-analyzer-use-of-uninitialized-value -Wno-analyzer-imprecise-fp-arithmetic" DEBUG=0 DEBUG_DWARF=0 make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
    expire_in: 45 minutes


Build_Tensorflow:
  stage: build
  script:
# Already part of the Docker image
#    - wget --no-verbose https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-2.4.0.tar.gz
#    - mkdir tensorflow_dir && tar -C tensorflow_dir -xvzf libtensorflow-gpu-linux-x86_64-2.4.0.tar.gz
    - TENSORFLOW=1 TENSORFLOW_BASE=/tensorflow_dir/ CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes


Build_Pytorch:
  stage: build
  script:
    #    - apt-get install -y libtorch-dev
    - PYTORCH=1 make all
  artifacts:
    paths:
      - bart
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
    expire_in: 45 minutes



Build_wasm:
  stage: build
  image: registry.gitlab.tugraz.at/ibi/reproducibility/gitlab-ci-containers/ibi_wasm_bart
  variables:
    DEPS_FOLDER: /wasm_libs
  script:
    - echo -e
        "FFTW_BASE=$DEPS_FOLDER\n
        BLAS_BASE=$DEPS_FOLDER\n
        CC=emcc\n
        LDFLAGS=-s EXPORTED_FUNCTIONS=_main,__Block_object_dispose -s ALLOW_MEMORY_GROWTH=1 -s MAXIMUM_MEMORY=4GB -s INITIAL_MEMORY=256MB -s TOTAL_STACK=64MB $DEPS_FOLDER/lib/libfftw3f.a $DEPS_FOLDER/lib/libopenblas.a $DEPS_FOLDER/usr/local/lib/libBlocksRuntime.a -lnodefs.js -lnoderawfs.js --pre-js src/pre.js\n
        OPENBLAS=1\n
        FORTRAN=0\n
        FFTWTHREADS=0\n
        USE_THREAD=1\n
        PNG=0\n
        OMP=0\n
        TENSORFLOW=0\n
        DEBUG=0\n
        MKL=0\n
        ACML=0\n
        CUDA=0\n
        CUDNN=0\n
        ISMRMRD=0\n" > Makefiles/Makefile.wasm
    - echo -e
        "Module.preRun = () => {\n
          ENV.BART_BINARY_STREAM=process.env.BART_BINARY_STREAM;\n
        };" > src/pre.js
    - bash -c "source /emsdk/emsdk_env.sh && make -j 4 DEBUG=0 DEBUG_DWARF=0 all"
  artifacts:
    paths:
      - bart
      - bart.wasm
      - "./lib/*.a"
      - ./commands/
      - "./src/*.o"
      - ./src/misc/version.inc
      - ./Makefiles/Makefile.wasm
      - ./src/pre.js
    expire_in: 45 minutes



UTest:
  stage: test1
  script:
    - AUTOCLEAN=0 WERROR=1 make utest
  needs: [Build]
  dependencies:
     - Build

UTest_riscv:
  stage: test1
  tags:
    - riscv
  image: registry.gitlab.tugraz.at/ibi/reproducibility/gitlab-ci-containers/ibi_riscv_noroot
  script:
    - AUTOCLEAN=0 make -j 4 utest
  needs: [Build_riscv]
  dependencies:
     - Build_riscv


UTest_Clang:
  stage: test1
  script:
    - AUTOCLEAN=0 CC=clang-19 make utest
  needs: [Build_Clang]
  dependencies:
     - Build_Clang

UTest_Clang_GPU:
  stage: test1
  script:
    - if ! nvidia-smi ; then printf "No usable GPU found, skipping GPU tests!\n"; exit 0; fi
    - AUTOCLEAN=0 CC=clang-19 CUDA_CC=clang-14 CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" make utest_gpu
  needs: [Build_Clang_GPU]
  dependencies:
    - Build_Clang_GPU



UTest_Valgrind:
  stage: test1
  script:
    - AUTOCLEAN=0 UTESTLEAK=1 make utest
  needs: [Build]
  dependencies:
     - Build


UTest_UBSan:
  stage: test1
  script:
    - AUTOCLEAN=0 UBSAN=1 ASAN=1 make utest
  needs: [Build_UBSan]
  dependencies:
     - Build_UBSan


UTest_GPU:
  stage: test1
  script:
    - if ! nvidia-smi ; then printf "No usable GPU found, skipping GPU tests!\n"; exit 0; fi
    - AUTOCLEAN=0 CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" WERROR=1 make utest_gpu
  needs: [Build_GPU]
  dependencies:
    - Build_GPU

UTest_MPI:
  stage: test1
  script:
    - AUTOCLEAN=0 MPI=1 make utest
  needs: [Build_MPI]
  dependencies:
     - Build_MPI


UTest_Tensorflow:
  stage: test1
  script:
# Already part of the Docker image
#    - wget --no-verbose https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-2.4.0.tar.gz
#    - mkdir tensorflow_dir && tar -C tensorflow_dir -xvzf libtensorflow-gpu-linux-x86_64-2.4.0.tar.gz
    - AUTOCLEAN=0 TENSORFLOW=1 TENSORFLOW_BASE=/tensorflow_dir/ CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" make utest
  needs: [Build_Tensorflow]
  dependencies:
    - Build_Tensorflow


UTest_LTO:
  stage: test1
  script:
    - AUTOCLEAN=0 CFLAGS="-flto -O2 -g" DEBUG=0 DEBUG_DWARF=0 make utest
  needs: [Build_LTO]
  dependencies:
    - Build_LTO


UTest_wasm:
  stage: test1
  image: registry.gitlab.tugraz.at/ibi/reproducibility/gitlab-ci-containers/ibi_wasm_bart
  script:
    - bash -c "source /emsdk/emsdk_env.sh && AUTOCLEAN=0 DEBUG=0 DEBUG_DWARF=0 make utest"
  needs: [Build_wasm]
  dependencies:
     - Build_wasm


UTest_Wine:
  stage: test1
  script:
    #- apt-get update -qq && apt-get install -y wine mingw-w64
    - AUTOCLEAN=0 WERROR=1 BARTDLL=1 make utest_wine
  needs: [Build_WinDLL]
  dependencies:
     - Build_WinDLL


IntTest:
  stage: test2
  script:
    - AUTOCLEAN=0 AGUE_REF=/reference_data/ WERROR=1 make test testslow testague
  needs: [Build]
  dependencies:
    - Build


IntTest_riscv:
  stage: test2
  tags:
    - riscv
  image: registry.gitlab.tugraz.at/ibi/reproducibility/gitlab-ci-containers/ibi_riscv_noroot
  script:
    - AUTOCLEAN=0 make -Otarget -j 4 test
  needs: [Build_riscv]
  dependencies:
    - Build_riscv
  timeout: 6 hours



IntTest_Python:
  stage: test2
  script:
    - AUTOCLEAN=0 make pythontest
  needs: [Build]
  dependencies:
    - Build
  allow_failure: true


IntTest_GPU:
  stage: test2
  script:
    - if ! nvidia-smi ; then printf "No usable GPU found, skipping GPU tests!\n"; exit 0; fi
    - AUTOCLEAN=0 CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" WERROR=1 make gputest
  needs: [Build_GPU]
  dependencies:
    - Build_GPU


IntTest_Clang:
  stage: test2
  script:
    - AUTOCLEAN=0 CC=clang-19 make test
  needs: [Build_Clang]
  dependencies:
    - Build_Clang


IntTest_Clang_GPU:
  stage: test2
  script:
    - if ! nvidia-smi ; then printf "No usable GPU found, skipping GPU tests!\n"; exit 0; fi
    - AUTOCLEAN=0 CC=clang-19 CUDA_CC=clang-14 CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" make gputest
  needs: [Build_Clang_GPU]
  dependencies:
    - Build_Clang_GPU


IntTest_UBSan:
  stage: test2
  script:
    - AUTOCLEAN=0 UBSAN=1 ASAN=1 DEBUG=1 UBSAN_OPTIONS=print_stacktrace=1 make test
  needs: [Build_UBSan]
  dependencies:
    - Build_UBSan


IntTest_MPI:
  stage: test2
  script:
    - AUTOCLEAN=0 MPI=1 make mpitest
  needs: [Build_MPI]
  dependencies:
    - Build_MPI

IntTest_MPI_GPU:
  stage: test2
  script:
    - if ! nvidia-smi ; then printf "No usable GPU found, skipping GPU tests!\n"; exit 0; fi
    - AUTOCLEAN=0 CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" MPI=1 make gputest
  needs: [Build_MPI_GPU]
  dependencies:
    - Build_MPI_GPU


IntTest_Tensorflow:
  stage: test2
  script:
# Already part of the Docker image
#    - wget --no-verbose https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-2.4.0.tar.gz
#    - mkdir tensorflow_dir && tar -C tensorflow_dir -xvzf libtensorflow-gpu-linux-x86_64-2.4.0.tar.gz
    - python3 -m pip install tensorflow --break-system-packages
    - AUTOCLEAN=0 TENSORFLOW=1 TENSORFLOW_BASE=/tensorflow_dir/ CUDA=1 CUDA_LIB=lib64 GPUARCH_FLAGS="-arch sm_35 -Wno-deprecated-gpu-targets" make test
  needs: [Build_Tensorflow]
  dependencies:
    - Build_Tensorflow
  allow_failure: true


#IntTest_LTO:
#  stage: test2
#  script:
#    - CFLAGS="-flto -O2 -g" DEBUG=0 DEBUG_DWARF=0 make test
#  needs: [Build_LTO]
#  dependencies:
#    - Build_LTO

IntTest_wasm:
  stage: test2
  image: registry.gitlab.tugraz.at/ibi/reproducibility/gitlab-ci-containers/ibi_wasm_bart
  script:
    - bash -c "source /emsdk/emsdk_env.sh && AUTOCLEAN=0 DEBUG=0 DEBUG_DWARF=0 make test"
  needs: [Build_wasm]
  dependencies:
     - Build_wasm


Debian_Bookworm:
  variables:
      DEBUG: "0"
      DEBUG_DWARF: "0"
  rules:
      - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: debian:bookworm
  script:
      - apt-get update
      - apt-get install -y gcc make libfftw3-dev liblapacke-dev libpng-dev libopenblas-dev gfortran
      - make all -j8
      - make utest -j8

Debian_Trixie:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
    - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: debian:trixie
  script:
    - apt-get update
    - apt-get install -y gcc make libfftw3-dev liblapacke-dev libpng-dev libopenblas-dev gfortran
    - make all -j8
    - make utest -j8


Debian_Trixie_CUDA13:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
    - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: registry.gitlab.tugraz.at/ibi/reproducibility/gitlab-ci-containers/ibi_cuda13
  script:
    - apt-get update
    - apt-get install -y gcc make libfftw3-dev liblapacke-dev libpng-dev libopenblas-dev gfortran
    - CUDA=1 CUDA_BASE=/usr/local/cuda-13/ CUDA_LIB=lib64 make all -j8
    - CUDA=1 CUDA_BASE=/usr/local/cuda-13/ CUDA_LIB=lib64 make utest -j8
    - if ! nvidia-smi ; then printf "No usable GPU found, skipping GPU tests!\n"; exit 0; fi
    - CUDA=1 CUDA_BASE=/usr/local/cuda-13/ CUDA_LIB=lib64 make utest_gpu -j8


Fedora_39:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
    - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: fedora:39
  script:
    - dnf install -y gcc make fftw-devel lapack-devel openblas-devel atlas-devel libpng-devel
    - make all -j8
    - make utest -j8

Fedora_40:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
    - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: fedora:40
  script:
    - dnf install -y gcc make fftw-devel lapack-devel openblas-devel atlas-devel libpng-devel util-linux-core
    - make all -j8
    - make utest -j8

RockyLinux_8:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
      - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: docker.io/rockylinux:8
  script:
    - dnf --enablerepo=powertools install -y fftw-devel atlas-devel libpng-devel lapack-devel gcc-toolset-12
    - source scl_source enable gcc-toolset-12 # redhat magic for enabling gcc-12
    - make all -j8
    - make utest -j8

RockyLinux_9:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
    - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: docker.io/rockylinux:9
  script:
    - dnf --enablerepo=crb install -y fftw-devel atlas-devel libpng-devel lapack-devel gcc-toolset-12
    - source scl_source enable gcc-toolset-12 # redhat magic for enabling gcc-12
    - make all -j8
    - make utest -j8

RockyLinux_10:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
    - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: docker.io/rockylinux/rockylinux:10
  script:
    - dnf --enablerepo=crb install -y fftw-devel openblas-devel libpng-devel lapack-devel gcc
    - echo -e -n "OPENBLAS=1\nCPPFLAGS=-Wall -Wextra -I/usr/include/openblas\n" > Makefiles/Makefile.local
    - make all -j8
    - make utest -j8

Ubuntu_22.04:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
    - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: docker.io/ubuntu:22.04
  script:
    - apt-get update
    - apt-get install -y gcc-12 make libfftw3-dev liblapacke-dev libpng-dev libopenblas-dev gfortran
    - echo -e -n "CC=gcc-12\n" > Makefiles/Makefile.local
    - make all -j8
    - make utest -j8

Ubuntu_24.04:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
    - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: docker.io/ubuntu:24.04
  script:
    - apt-get update
    - apt-get install -y gcc make libfftw3-dev liblapacke-dev libpng-dev libopenblas-dev gfortran
    - make all -j8
    - make utest -j8

Arch_Latest:
  variables:
    DEBUG: "0"
    DEBUG_DWARF: "0"
  rules:
    - if: $CI_DISTRO_TESTS == "1"
  stage: distro_build
  image: docker.io/archlinux:base-devel
  script:
    - pacman -Sy --noconfirm blas-openblas fftw libpng
    - echo -e -n "CPPFLAGS+=-I/usr/include/openblas\nAR_LOCK_NEEDED=0\n" > Makefiles/Makefile.local
    - make all -j8
    - make utest -j8
  allow_failure: true