pipeline {
    agent none

    environment {
        CCACHE_DIR = '/tmp/ccache'
        CCACHE_MAXSIZE = '10G'
        CCACHE_CPP2 = 'true'
    }
    stages {
        stage('Build') {
            parallel {
                stage('HIP-3.1-HCC') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.hipcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--pull --build-arg BASE=rocm/dev-ubuntu-18.04:3.1'
                            label 'rocm-docker && vega'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video'
                        }
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''rm -rf build && mkdir -p build && cd build && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Debug \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=hipcc \
                                -DCMAKE_CXX_FLAGS="-Werror -Wno-unused-command-line-argument -Wno-braced-scalar-init" \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_HIP=ON \
                                -DKokkos_ENABLE_LIBDL=OFF \
                                -DKokkos_ENABLE_PROFILING=OFF \
                              .. && \
                              make -j8 && ctest --output-on-failure'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                        }
                    }
                }
                stage('CUDA-9.2-Clang') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.clang'
                            dir 'scripts/docker'
                            additionalBuildArgs '--pull'
                            label 'nvidia-docker && volta'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache'
                        }
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''rm -rf build && mkdir -p build && cd build && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Release \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=clang++ \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_CUDA=ON \
                                -DKokkos_ENABLE_CUDA_LAMBDA=ON \
                                -DKokkos_ARCH_VOLTA70=ON \
                              .. && \
                              make -j8 && ctest --output-on-failure'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                        }
                    }
                }
                stage('CUDA-9.2-NVCC') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.nvcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--pull --build-arg BASE=nvidia/cuda:9.2-devel'
                            label 'nvidia-docker && volta'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache'
                        }
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''rm -rf build && mkdir -p build && cd build && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Release \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=$WORKSPACE/bin/nvcc_wrapper \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_CUDA=ON \
                                -DKokkos_ENABLE_CUDA_LAMBDA=ON \
                                -DKokkos_ARCH_VOLTA70=ON \
                              .. && \
                              make -j8 && ctest --output-on-failure'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                        }
                    }
                }
                stage('CUDA-10.1-NVCC-RDC') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.nvcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--pull --build-arg BASE=nvidia/cuda:10.1-devel'
                            label 'nvidia-docker && volta'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache'
                        }
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''rm -rf build && mkdir -p build && cd build && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Release \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=$WORKSPACE/bin/nvcc_wrapper \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_OPENMP=ON \
                                -DKokkos_ENABLE_CUDA=ON \
                                -DKokkos_ENABLE_CUDA_LAMBDA=ON \
                                -DKokkos_ENABLE_CUDA_UVM=ON \
                                -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON \
                                -DKokkos_ARCH_VOLTA70=ON \
                              .. && \
                              make -j8 && ctest --output-on-failure'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                        }
                    }
                }
                stage('CUDA-10.1-NVCC-DEBUG') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.nvcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--pull --build-arg BASE=nvidia/cuda:10.1-devel --build-arg CMAKE_VERSION=3.15.5'
                            label 'nvidia-docker && volta'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache'
                        }
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''rm -rf build && mkdir -p build && cd build && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Debug \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=$WORKSPACE/bin/nvcc_wrapper \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEBUG=ON \
                                -DKokkos_ENABLE_DEBUG_BOUNDS_CHECK=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_CUDA=ON \
                                -DKokkos_ENABLE_CUDA_LAMBDA=ON \
                                -DKokkos_ARCH_VOLTA70=ON \
                              .. && \
                              make -j8 && ctest --output-on-failure'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                        }
                    }
                }
                stage('GCC-4.8.4') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.gcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--pull'
                            label 'docker'
                        }
                    }
                    environment {
                        OMP_NUM_THREADS = 8
                        OMP_PLACES = 'threads'
                        OMP_PROC_BIND = 'spread'
                    }
                    steps {
                        sh '''rm -rf build && mkdir -p build && cd build && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Release \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_OPENMP=ON \
                              .. && \
                              make -j8 && ctest --output-on-failure'''
                    }
                }
                stage('Clang-Format') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.clang'
                            dir 'scripts/docker'
                            additionalBuildArgs '--pull'
                            label 'nvidia-docker || docker'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache'
                        }
                    }
                    steps {
                        sh './scripts/docker/check_format_cpp.sh'
                    }
                }
            }
        }
    }
}
