# 10.6 is "buster"
# Use a slim image as base
ARG BASE_IMAGE=python
ARG BASE_VERSION=3.11-slim-bookworm
FROM $BASE_IMAGE:$BASE_VERSION AS base
LABEL maintainer="Karel van de Plassche <karelvandeplassche@gmail.com>"

ENV DEBIAN_FRONTEND=noninteractive SHELL=/bin/bash
#USER <user>:<group> root:root by defalt

COPY bashrc_root /root/.bashrc
COPY inputrc_root /root/.inputrc
WORKDIR /root

# Install apt-utils
RUN apt-get update -y && \
    apt-get install -y --no-install-recommends \
        apt-utils apt-transport-https \
    && apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

# Add useful linux and debugging tools
RUN apt-get update --fix-missing && \
    apt-get install -y vim-tiny sudo wget curl pv patch bzip2 make && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*


##########################
# Install Intel compiler #
##########################
FROM intel/hpckit:2024.2.1-0-devel-ubuntu22.04 AS intel-build

ENV INTEL_INSTALL_DIR=/opt/intel/oneapi
RUN rm -rf \
    $INTEL_INSTALL_DIR/ipp \
    $INTEL_INSTALL_DIR/advisor \
    $INTEL_INSTALL_DIR/dal \
    $INTEL_INSTALL_DIR/vtune \
    $INTEL_INSTALL_DIR/dpcpp-ct \
    $INTEL_INSTALL_DIR/ccl \
    $INTEL_INSTALL_DIR/ippcp \
    $INTEL_INSTALL_DIR/tbb \
    $INTEL_INSTALL_DIR/tcm \
    $INTEL_INSTALL_DIR/compiler/latest/bin/llc \
    $INTEL_INSTALL_DIR/compiler/latest/bin/lld \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llc \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/lld \
    $INTEL_INSTALL_DIR/compiler/latest/lib/clc \
    $INTEL_INSTALL_DIR/compiler/latest/lib/clang \
    $INTEL_INSTALL_DIR/compiler/latest/lib/libimf_msan.a \
    $INTEL_INSTALL_DIR/compiler/latest/lib/libomptarget.rtl.x86_64.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/libomptarget.rtl.unified_runtime.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/libomptarget.rtl.opencl.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/libomptarget.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/libomptarget.rtl.level0.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/clbltfnshared.rtl \
    $INTEL_INSTALL_DIR/compiler/latest/lib/icx-lto.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/libintelocl.so.2024.18.7.0 \
    $INTEL_INSTALL_DIR/compiler/latest/lib/libcommon_clang.so.2024.18.7.0 \
    $INTEL_INSTALL_DIR/compiler/latest/bin/clang-* \
    $INTEL_INSTALL_DIR/compiler/latest/bin/clangd \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/clang* \
    $INTEL_INSTALL_DIR/compiler/latest/bin/llvm-* \
    $INTEL_INSTALL_DIR/compiler/latest/bin/clang-* \
    $INTEL_INSTALL_DIR/compiler/latest/bin/clangd \
    $INTEL_INSTALL_DIR/compiler/latest/bin/llvm-* \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/clang* \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm* \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-ar \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-cov \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-dwp \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-foreach \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-link \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-ml \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-nm \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-objcopy \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-profdata \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-profgen \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-ranlib \
    $INTEL_INSTALL_DIR/compiler/latest/bin/compiler/llvm-spirv \
    $INTEL_INSTALL_DIR/compiler/latest/lib/__ocl_svml_e9.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/__ocl_svml_h8.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/__ocl_svml_l9.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/__ocl_svml_z0.so \
    $INTEL_INSTALL_DIR/compiler/latest/lib/__ocl_svml_z1.so \
    $INTEL_INSTALL_DIR/compiler/2024.2/lib/libsvml_msan.a \
    $INTEL_INSTALL_DIR/compiler/2024.2/lib/libsvml.so \
    $INTEL_INSTALL_DIR/debugger/2024.2/opt/debugger/src \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_scalapack_ilp64.a \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_scalapack_lp64.a \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_gf_lp64.a \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_gnu_thread.a \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_intel_lp64.a \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_pgi_thread.a \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_tbb_thread.a \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_intel_thread.a \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_sycl.a \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_sycl_blas.so.4 \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_sycl_rng.so.4 \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_sycl_sparse.so.4 \
    $INTEL_INSTALL_DIR/mkl/latest/lib/libmkl_sycl_vm.so.4

# We need
# $INTEL_INSTALL_DIR/compiler/2024.2/lib/libonnxruntime.1.12.22.721.so
# $INTEL_INSTALL_DIR/compiler/2024.2/lib/libsvml.a \

# Patch bashrc and assorted sourced scripts
# Set intel variables
RUN printf '\n#intel variables\n\
source /opt/intel/oneapi/setvars.sh\n \
export VT_ROOT=/opt/intel/oneapi/compiler/2024.2/bin/\n' \
    >> /root/.bashrc

# You should now have mpiifort and ifort
WORKDIR /root

########################
# Install PGI compiler #
########################

# Copied from nvcr.io/nvidia/nvhpc:25.1-devel-cuda12.6-ubuntu22.04
FROM ubuntu:22.04 AS pgi-build
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
COPY entrypoint.sh /usr/local/bin
ARG TARGETARCH=amd64
ARG NVHPC_ARCH=x86_64
ARG LAUNCHPAD_BUILD_ARCH
ARG RELEASE
CMD ["/bin/bash"]
LABEL org.opencontainers.image.version=22.04
LABEL org.opencontainers.image.ref.name=ubuntu
LABEL com.nvidia.hpcsdk.release=2025 com.nvidia.hpcsdk.version=25.1
# Use NVIDIA version of OpenCL
RUN mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd # buildkit 20.5kB buildkit.dockerfile.v0
ENV MELLANOX_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=compute,utility NVIDIA_REQUIRE_CUDA=cuda>=12.0 brand=unknown,driver>=470,driver<471 brand=grid,driver>=470,driver<471 brand=tesla,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=vapps,driver>=470,driver<471 brand=vpc,driver>=470,driver<471 brand=vcs,driver>=470,driver<471 brand=vws,driver>=470,driver<471 brand=cloudgaming,driver>=470,driver<471 brand=unknown,driver>=535,driver<536 brand=grid,driver>=535,driver<536 brand=tesla,driver>=535,driver<536 brand=nvidia,driver>=535,driver<536 brand=quadro,driver>=535,driver<536 brand=quadrortx,driver>=535,driver<536 brand=nvidiartx,driver>=535,driver<536 brand=vapps,driver>=535,driver<536 brand=vpc,driver>=535,driver<536 brand=vcs,driver>=535,driver<536 brand=vws,driver>=535,driver<536 brand=cloudgaming,driver>=535,driver<536 brand=unknown,driver>=550,driver<551 brand=grid,driver>=550,driver<551 brand=tesla,driver>=550,driver<551 brand=nvidia,driver>=550,driver<551 brand=quadro,driver>=550,driver<551 brand=quadrortx,driver>=550,driver<551 brand=nvidiartx,driver>=550,driver<551 brand=vapps,driver>=550,driver<551 brand=vpc,driver>=550,driver<551 brand=vcs,driver>=550,driver<551 brand=vws,driver>=550,driver<551 brand=cloudgaming,driver>=550,driver<551 NVIDIA_VISIBLE_DEVICES=all
# Remove older NVIDIA data, we are rebuilding
#ARG NVHPC_ARCH=smth
#RUN rm -rf /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/REDIST/cuda/11.0/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/10.1/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/10.1/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/10.2/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/10.2/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.0/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.0/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.0/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.0/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.1/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.1/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.1/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.1/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.2/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.2/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.2/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.2/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.3/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.3/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.3/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.3/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.4/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.4/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.4/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.4/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.5/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.5/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.5/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.5/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.6/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.6/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.6/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.6/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.7/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.7/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.7/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.7/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.8/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.8/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.8/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/11.8/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.0/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.0/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.0/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.0/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.1/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.1/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.1/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.1/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.2/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.2/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.2/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.2/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.3/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.3/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.3/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.3/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.4/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.4/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.4/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.4/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.5/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.5/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.5/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.5/libnvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.6/bin/computeprof /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.6/bin/nvvp /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.6/doc/man/man1/nvvp.1 /opt/nvidia/hpc_sdk/Linux_${NVHPC_ARCH}/25.1/cuda/12.6/libnvvp # buildkit 36.9kB buildkit.dockerfile.v0
# https://developer.nvidia.com/hpc-sdk
ENV HPCSDK_RELEASE=2025 HPCSDK_VERSION=25.1

# Install prerequisites
RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
    wget gpg make gcc build-essential vim-tiny ca-certificates # Karel additions
RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
    ca-certificates gnupg wget && rm -rf /var/lib/apt/lists/*
RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends -t \
    jammy dapl2-utils ibutils ibverbs-providers ibverbs-utils infiniband-diags libdapl-dev libdapl2 libibmad-dev \
    libibmad5 libibverbs-dev libibverbs1 librdmacm-dev librdmacm1 rdmacm-utils \
    && rm -rf /var/lib/apt/lists/*
RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
    autoconf autoconf-archive automake binutils bzip2 ca-certificates cmake diffutils file gdb git gzip libaec-dev libnuma-dev libsz2 libtool lmod make numactl patch tar vim wget xz-utils zlib1g-dev && rm -rf /var/lib/apt/lists/*

RUN echo "source /usr/share/lmod/lmod/init/bash" >> /etc/bash.bashrc && echo "/opt/nvidia/hpc_sdk/modulefiles" >> /etc/lmod/modulespath && echo "module load nvhpc" >> /etc/bash.bashrc
# Building the brunk of NVIDIA tools
# Takes 750s to build on Karel Super using buildx
# nvhpc-25-1 takes up 15.5GB uncompressed disk space without rm
# Takes a lot of time to copy to other layers
ENV MODULEPATH=/opt/nvidia/hpc_sdk/modulefiles
ENV CPATH=/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/nvshmem/include:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/nccl/include:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/extras/qd/include/qd:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/math_libs/include:/usr/local/gdrcopy/include: LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/nvshmem/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/nccl/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/math_libs/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/cuda/lib64: MANPATH=/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/man: PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/nvshmem/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/nccl/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/profilers/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/cuda/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/mpi/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 OMPI_ALLOW_RUN_AS_ROOT=1
RUN mkdir -p /usr/share/keyrings && rm -f /usr/share/keyrings/DEB-GPG-KEY-NVIDIA-HPC-SDK.gpg \
    && wget -qO - https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK \
    | gpg --dearmor -o /usr/share/keyrings/DEB-GPG-KEY-NVIDIA-HPC-SDK.gpg \
    && echo "deb [signed-by=/usr/share/keyrings/DEB-GPG-KEY-NVIDIA-HPC-SDK.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/${TARGETARCH} /" >> /etc/apt/sources.list.d/hpccm.list \
    && apt-get update -y \
    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends nvhpc-25-1 && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/openmpi4 && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/nvshmem && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/nccl && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libcudafor_110.so && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libcudafor_110.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libcudafor_113.so && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libcudafor_113.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libacchost.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libacchosts.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libcudafor_120.so && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libcudafor_120.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libcudafor_118.so && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libcudafor_118.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/examples && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/profilers && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/math_libs && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/cuda && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/nvvm34 && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/nvvm70 && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/liblapack_ilp64.so.0 && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/liblapack_lp64.so.0 && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libaccdevices.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libaccdevice.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/liblapack_ilp64.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/liblapack_lp64.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libblas_ilp64.so.0 && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libblas_lp64.so.0 && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libblas_ilp64.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libblas_lp64.a && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/openmpi4 && \
    #rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/nccl && \
    #rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/nvshmem && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/hpcx/hpcx-2.20 && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/hpcx/hpcx-2.21/sharp && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/hpcx/hpcx-2.21/ucc && \
    rm -r /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/hpcx/hpcx-2.21/ucx && \
    rm /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/nvshmem && \
    rm /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/nccl && \
    rm /opt/nvidia/hpc_sdk/Linux_x86_64/2025 && \
    rm /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/hpcx/bin/mpiexec && \
    /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/hpcx/bin/mpicc && \
    /opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/hpcx/bin/mpicxx  && \
    rm -rf /var/lib/apt/lists/*
# To run QLKNN-fortran with MPI we need
# See the repository at https://gitlab.com/qualikiz-group/QLKNN-fortran
#/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/compilers/lib/libacchost.so && \
#/opt/nvidia/hpc_sdk/Linux_x86_64/25.1/comm_libs/12.6/hpcx && \

# Some more tools
ENV CPATH=/usr/local/gdrcopy/include: LIBRARY_PATH=/usr/local/gdrcopy/lib:
RUN mkdir -p /var/tmp && \
    wget -q -nc --no-check-certificate -P /var/tmp https://github.com/NVIDIA/gdrcopy/archive/v2.4.4.tar.gz && \
    mkdir -p /var/tmp && tar -x -f /var/tmp/v2.4.4.tar.gz -C /var/tmp -z && cd /var/tmp/gdrcopy-2.4.4 && \
    mkdir -p /usr/local/gdrcopy/include /usr/local/gdrcopy/lib && make prefix=/usr/local/gdrcopy lib lib_install && \
    echo "/usr/local/gdrcopy/lib" >> /etc/ld.so.conf.d/hpccm.conf && \
    ldconfig && rm -rf /var/tmp/gdrcopy-2.4.4 /var/tmp/v2.4.4.tar.gz


# Update and upgrade
RUN apt-get update -y && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
#ADD file:ebe009f86035c175ba244badd298a2582914415cf62783d510eab3a311a5d4e1 in / 87.5MB 

#ENV NVCOMPILERS="/opt/nvidia/hpc_sdk" \
#    NVIDIA_HPC_SDK_VERSION="2024"
#ENV NV_INSTALL_LOCATION=$NVCOMPILERS/Linux_x86_64/$NVIDIA_HPC_SDK_VERSION
#ENV NV_MATHLIB=$NV_INSTALL_LOCATION/math_libs/12.6/targets/x86_64-linux/lib/ \
#    NV_COMPILERS=$NV_INSTALL_LOCATION/compilers \
#    NV_COMM_LIBS=$NV_INSTALL_LOCATION/comm_libs \
#    NV_PROFILERS=$NV_INSTALL_LOCATION/profilers
#
## Remove unneeded libraries
## No GPUs -> no CUDA
## No system GPU profiling -> no Nsight_Compute
## No CUDA and OptiX profiling -> no Nsight_Systems
#RUN rm -rf \
#    $NV_INSTALL_LOCATION/cuda \
#    $NV_PROFILERS/Nsight_Compute \
#    $NV_PROFILERS/Nsight_Systems \
#    $NV_PROFILERS \
#    $NV_COMM_LIBS/nccl \
#    $NV_COMM_LIBS/nvshmem \
#    $NV_COMM_LIBS/openmpi4 \
#    $NV_COMM_LIBS/12.6/nccl \
#    $NV_COMM_LIBS/12.6/nvshmem \
#    $NV_COMM_LIBS/12.6/openmpi4 \
#    $NV_COMM_LIBS/12.6/hpcx/latest/hcoll \
#    $NV_COMM_LIBS/12.6/hpcx/latest/ucx \
#    $NV_COMM_LIBS/12.6/hpcx/latest/ucc \
#    $NV_COMM_LIBS/12.6/hpcx/latest/sharp \
#    $NV_COMPILERS/bin/tools/nvcpfe \
#    $NV_COMPILERS/bin/tools/nvnvvmd \
#    $NV_COMPILERS/bin/tools/ipa \
#    $NV_COMPILERS/lib/libcudafor_110.a \
#    $NV_COMPILERS/lib/libcudafor_110.so \
#    $NV_COMPILERS/lib/libcudafor_113.a \
#    $NV_COMPILERS/lib/libcudafor_113.so \
#    $NV_COMPILERS/lib/libcudafor_118.a \
#    $NV_COMPILERS/lib/libcudafor_118.so \
#    $NV_COMPILERS/lib/libcudafor_120.a \
#    $NV_COMPILERS/lib/libcudafor_120.so \
#    $NV_COMPILERS/lib/libacchost.a \
#    $NV_COMPILERS/lib/libacchosts.a \
#    $NV_COMPILERS/lib/liblapack_lp64.so.0 \
#    $NV_COMPILERS/lib/liblapack_ilp64.so.0 \
#    $NV_COMPILERS/lib/libaccdevices.a \
#    $NV_COMPILERS/lib/libaccdevice.a \
#    $NV_COMPILERS/lib/liblapack_ilp64.a \
#    $NV_COMPILERS/lib/liblapack_lp64.a \
#    $NV_COMPILERS/lib/libblas_lp64.so.0 \
#    $NV_COMPILERS/lib/libblas_ilp64.so.0 \
#    $NV_COMPILERS/lib/libblas_ilp64.a \
#    $NV_COMPILERS/lib/libblas_lp64.a \
#    $NV_MATHLIB/libcutensorMg.so.2.0.2 \
#    $NV_MATHLIB/libcublasmp.so.0.2.1 \
#    $NV_MATHLIB/libcutensorMg_static.a \
#    $NV_MATHLIB/libcusolver_lapack_static.a \
#    $NV_MATHLIB/libcusolverMp.so.0.5.1.0 \
#    $NV_MATHLIB/libcusolverMg.so.11.7.1.2 \
#    $NV_MATHLIB/libcurand.so.10.3.7.77 \
#    $NV_MATHLIB/libcurand_static.a \
#    $NV_MATHLIB/libcublas.so.12.6.3.3 \
#    $NV_MATHLIB/libcusolver.so.11.7.1.2 \
#    $NV_MATHLIB/libcublas_static.a \
#    $NV_MATHLIB/libcusolver_static.a \
#    $NV_MATHLIB/libcufftMp.so.11.2.6 \
#    $NV_MATHLIB/libcufftMp.so.11 \
#    $NV_MATHLIB/libcufftMp.so \
#    $NV_MATHLIB/libcutensor.so.2.0.2 \
#    $NV_MATHLIB/libcufft.so.11.3.0.4 \
#    $NV_MATHLIB/libcufft_static_nocallback.a \
#    $NV_MATHLIB/libcufft_static.a \
#    $NV_MATHLIB/libcusparse.so.12.5.4.2 \
#    $NV_MATHLIB/libcutensor_static.a \
#    $NV_MATHLIB/libcusparse_static.a \
#    $NV_MATHLIB/libcublasLt.so.12.6.3.3 \
#    $NV_MATHLIB/libcublasLt_static.a
#
## makelocalrc will fail, as we do not have g++, but we do not care
#RUN $NV_INSTALL_LOCATION/compilers/bin/makelocalrc -x $NV_INSTALL_LOCATION/compilers/
#
## Set NVIDIA variables in /root/.bashrc
#RUN printf '\n#PGI variables\n\
#export LFC=-lgfortran;\n\
#export LDSO=/lib64/ld-linux-x86-64.so.2;\n\
#export GCCDIR=/usr/lib/gcc/x86_64-linux-gnu/13/;\n\
#export G77DIR=/usr/lib/gcc/x86_64-linux-gnu/13/;\n\
#export OEM_INFO="64-bit target on x86-64 Linux $INFOTPVAL";\n\
#export INFOTPVAL=blerg;\n\
#export GNUATOMIC=-latomic;\n\
#export GCCINC="/usr/lib/gcc/x86_64-linux-gnu/13/include /usr/local/include /usr/include/x86_64-linux-gnu /usr/include";\n\
#export GPPDIR="/usr/include/c++/13 /usr/include/x86_64-linux-gnu/c++/13 /usr/include/c++/13/backward /usr/lib/gcc/x86_64-linux-gnu/13/include /usr/local/include /usr/include/x86_64-linux-gnu /usr/include";\n\
#export NUMALIBNAME=-lnuma;\n\
#export LOCALRC=YES;\n\
#export EXTENSION=__extension__=;\n\
#export LC="-lgcc -lc $if(-Bstatic,-lgcc_eh, -lgcc_s)";\n\
#export DEFSTDOBJDIR=/usr/lib/x86_64-linux-gnu;\n\
#export DEFCUDAVERSION=12.6;\n\
#export DEFSTDPARCOMPUTECAP=;\n\
## GLIBC version 2.39\n\
## GCC version 13.2.0\n\
#export GCCVERSION=130200;\n\
#export PGI=$COMPBASE;\n\
#export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/24.11/compilers/bin/:$PATH;\n\
#\n'>> /root/.bashrc

# You should now have pgfortran and mpi


##########################
# Set up developer image #
##########################
FROM pgi-build AS devenv
# Intel Build 20250222 3.76GB
COPY --from=intel-build /opt/intel/ /opt/intel
COPY --from=pgi-build /opt/nvidia/ /opt/nvidia/

########################
# Install gcc compiler #
########################

# Install gfortran
RUN apt-get update -y && \
    apt-get install -y --no-install-recommends \
        gfortran \
    && apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

# Install openmpi for gfortran
RUN apt-get update -y && \
    apt-get install -y --no-install-recommends \
        libopenmpi-dev \
    && apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

###########################
# Set up user environment #
###########################

# Copy useful statements from base image
# Easiest to just repeat over here
# base image copy start
#
ENV SHELL=/bin/bash
#USER <user>:<group> root:root by defalt

COPY bashrc_root /root/.bashrc
COPY inputrc_root /root/.inputrc
WORKDIR /root

# Install apt-utils
RUN apt-get update -y && \
    apt-get install -y --no-install-recommends \
        apt-utils apt-transport-https \
    && apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

# Add useful linux and debugging tools
RUN apt-get update --fix-missing && \
    apt-get install -y vim-tiny sudo wget curl pv patch bzip2 make && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*
#
# base image copy end

# sh is the "command" shell, drop into "bash" for convinience. Use "exec" form
# CMD ["bash"] is the default from Debian Docker, see
# https://github.com/debuerreotype/docker-debian-artifacts/blob/b31d4538a203c4128acf04399e96b33bc6fdc0a0/bullseye/Dockerfile
CMD ["bash"]


################################################################
# Patch bashrc and assorted sourced scripts, from other images #
################################################################

# intel-build image copy start
#
# Patch bashrc and assorted sourced scripts
# Set intel variables
RUN printf '\n#intel variables\n\
source /opt/intel/oneapi/setvars.sh\n \
export VT_ROOT=/opt/intel/oneapi/compiler/2024.2/bin/\n' \
    >> /root/.bashrc

# Patch problematic line for sourcing .bashrc in gitlab CI
#RUN sed -i -r 's#MANPATH="\$\{I_MPI_ROOT\}/man":`manpath 2>/dev/null`; export MANPATH#sleep 0#' /opt/intel/compilers_and_libraries_2018/linux/mpi/intel64/bin/mpivars.sh
#
# intel-build image copy end

# pgi-build image copy start
#
# Set NVIDIA variables in /root/.bashrc
RUN printf '\n#PGI variables\n\
export LFC=-lgfortran;\n\
export LDSO=/lib64/ld-linux-x86-64.so.2;\n\
export GCCDIR=/usr/lib/gcc/x86_64-linux-gnu/13/;\n\
export G77DIR=/usr/lib/gcc/x86_64-linux-gnu/13/;\n\
export OEM_INFO="64-bit target on x86-64 Linux $INFOTPVAL";\n\
export INFOTPVAL=blerg;\n\
export GNUATOMIC=-latomic;\n\
export GCCINC="/usr/lib/gcc/x86_64-linux-gnu/13/include /usr/local/include /usr/include/x86_64-linux-gnu /usr/include";\n\
export GPPDIR="/usr/include/c++/13 /usr/include/x86_64-linux-gnu/c++/13 /usr/include/c++/13/backward /usr/lib/gcc/x86_64-linux-gnu/13/include /usr/local/include /usr/include/x86_64-linux-gnu /usr/include";\n\
export NUMALIBNAME=-lnuma;\n\
export LOCALRC=YES;\n\
export EXTENSION=__extension__=;\n\
export LC="-lgcc -lc $if(-Bstatic,-lgcc_eh, -lgcc_s)";\n\
export DEFSTDOBJDIR=/usr/lib/x86_64-linux-gnu;\n\
export DEFCUDAVERSION=12.6;\n\
export DEFSTDPARCOMPUTECAP=;\n\
# GLIBC version 2.39\n\
# GCC version 13.2.0\n\
export GCCVERSION=130200;\n\
export PGI=$COMPBASE;\n\
export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/24.11/compilers/bin/:$PATH;\n\
unset FC\n\
\n'>> /root/.bashrc

ENV OMPI_PREFIX=/opt/ompi-pgi

RUN ln -sf $OMPI_PREFIX/bin/mpifort /usr/bin/mpifort.pgi && \
    #ln -sf $OMPI_PREFIX/share/openmpi/mpifort-wrapper-data.txt $OMPI_PREFIX/share/openmpi/mpifort.pgi-wrapper-data.txt && \
    ln -sf $OMPI_PREFIX/bin/mpirun /usr/bin/mpirun.pgi && \
    ln -sf $OMPI_PREFIX/bin/ompi_info /usr/bin/ompi_info.pgi
#
# pgi-build image copy end

# Need this depending on base image
## Set up Python
#RUN apt-get update -y && \
#    apt-get install -y --no-install-recommends \
#        python3 python3-pip python3-venv \
#    && apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
#
#RUN ln -s /usr/bin/python3 /usr/bin/python && \
#    ln -s /usr/bin/pip3 /usr/bin/pip

# Install QuaLiKiz-pythontools dependencies
# About half a Gig
RUN apt-get update -y && \
    apt-get install -y --no-install-recommends \
        python3-venv \
        python3-docopt python3-ipython python3-numpy python3-pandas python3-xarray python3-scipy python3-tabulate python3-matplotlib python3-netcdf4 python3-psutil python3-pytest python3-pytest-cov \
    && apt-get clean && apt-get purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

## Set up user environment
#ENV HOSTNAME=docker-debian
#ENV USER=mpirun
#RUN adduser --disabled-password --gecos "" ${USER}
#RUN echo "export PATH=/opt/pgi/linux86-64/2019/mpi/openmpi-3.1.3/bin:/opt/pgi/linux86-64/2019/bin:\$PATH" >> /root/.bashrc
#RUN echo "export PATH=/opt/pgi/linux86-64/2019/mpi/openmpi-3.1.3/bin:/opt/pgi/linux86-64/2019/bin:\$PATH" >> /home/mpirun/.bashrc
# Exporting layers should take ~600s
# Unpacking should take ~200s
