diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0216c3078e51d41e9beebfebddfe6dc6881070c3..e50059322f9880940220ec9070cc9784fcb54b54 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,3 @@ -# yaml-language-server: $schema=gitlab-ci -# yaml-language-server: $format.enable=false - stages: - linters - checkout @@ -22,6 +19,7 @@ workflow: variables: TERM: ansi LANG: en_US.UTF-8 + REGISTRY_URL: "registry.hpc.epfl.ch" .parallel_definition: parallel: @@ -29,34 +27,34 @@ variables: # - environment: [helvetios] # slurm_options: ["-c 36"] # stack: ["pinot-noir"] - # app_image: ["registry.c4science.ch/scitas-stack/rhel9-jed"] + # app_image: ["registry.hpc.epfl.ch/scitas/scitas-stack:rhel94-kuma"] # path_suffix: [''] - environment: [izar] slurm_options: ['-c 20 --gpus 1 -p gpu -q scitas'] apptainer_options: ['--nv'] stack: ["pinot-noir"] - app_image: ["registry.c4science.ch/scitas-stack/rhel9-jed"] + app_image_label: ["rhel94-kuma"] path_suffix: ['izar/'] - environment: [jed] slurm_options: ["-c 72 -p jed -q jed"] stack: ["pinot-noir"] - app_image: ["registry.c4science.ch/scitas-stack/rhel9-jed"] + app_image_label: ["rhel94-kuma"] path_suffix: [''] - environment: [kuma-h100] slurm_options: ["-c 16 -q scitas"] stack: ["pinot-noir"] apptainer_options: ['--nv'] - app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"] + app_image_label: ["rhel94-kuma"] path_suffix: [''] - environment: [kuma-l40s] slurm_options: ["-c 16 -q scitas"] stack: ["pinot-noir"] apptainer_options: ['--nv'] - app_image: ["registry.c4science.ch/scitas-stack/rhel9-kuma"] + app_image_label: ["rhel94-kuma"] path_suffix: [''] tags: - stack @@ -70,7 +68,14 @@ variables: - if: $CI_PIPELINE_SOURCE == "merge_request_event" variables: SQUASHFS_ID: ${CI_MERGE_REQUEST_IID} + changes: + compare_to: $CI_DEFAULT_BRANCH + paths: + - stacks/${stack}/**/* + - ci/**/* + variables: + app_image: "${REGISTRY_URL}/scitas-stack/software-stack:${app_image_label}" CI_DATA_LT: "/work/scitas-ge/scitas-stack/ci" CI_DATA_ST: "/scratch/${path_suffix}scitas-stack/ci" COMMAND_OPTIONS_SBATCH: ${slurm_options} @@ -81,6 +86,36 @@ variables: image_name: ${stack}-${environment}-${SQUASHFS_ID}-${CI_PIPELINE_ID} image: ${app_image} + +# ------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ +build_images: + stage: .pre + image: + name: gcr.io/kaniko-project/executor:v1.23.2-debug + entrypoint: [""] + parallel: + matrix: + - IMAGE_NAME: ["rhel94-kuma"] + script: + - echo $ACTIVATION_KEY > /kaniko/ACTIVATION_KEY + - echo "cp ${REGISTRY_AUTH_CREDENTIALS} /kaniko/.docker/config.json" + - cp ${REGISTRY_AUTH_CREDENTIALS} /kaniko/.docker/config.json + - /kaniko/executor + --single-snapshot + --context "${CI_PROJECT_DIR}/dockerfiles/${IMAGE_NAME}" + --dockerfile "Dockerfile" + --destination "${REGISTRY_URL}/scitas-stack/software-stack:${IMAGE_NAME}" + # --destination "${REGISTRY_URL}/scitas/software-stack:${IMAGE_NAME}" + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # changes: + # compare_to: ${CI_DEFAULT_BRANCH} + # paths: + # - dockerfiles/${IMAGE_NAME}/* + # when: manual + allow_failure: true + # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ .parallel_job: @@ -96,6 +131,7 @@ variables: -H ${FAKEHOME}:/home/$(id -un) --bind ${CI_DATA_LT}/buildcache:${MOUNT_POINT}/buildcache --bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror + --bind /ssoft/spack/spack-mirror-restricted:${MOUNT_POINT}/spack-mirror-restricted --bind /ssoft/spack/external:${MOUNT_POINT}/external --bind ${CI_DATA_LT}/squashfs-cache/:/squashfs-cache --bind ${CI_DATA_ST}/overlayfs:/overlayfs diff --git a/ci/apptainer_by_hand.sh b/ci/apptainer_by_hand.sh index 4d724f677c9d1e37aed4525e24b8b973ce2b2c25..39fca49e58b72e54bcf84962aafdf68de3692102 100755 --- a/ci/apptainer_by_hand.sh +++ b/ci/apptainer_by_hand.sh @@ -51,6 +51,8 @@ srun ${slurm_options[$environment]} --time 1-0:0:0 --pty apptainer run \ --cleanenv \ -H $(mktemp -d -p ${FAKEHOME}/):/home/$(id -un) \ --bind ${CI_DATA_LT}/buildcache:${MOUNT_POINT}/buildcache \ + --bind /ssoft/spack/spack-mirror-restricted:${MOUNT_POINT}/spack-mirror-restricted \ + --bind /ssoft/spack/external:${MOUNT_POINT}/external \ --bind ${CI_DATA_LT}/spack-mirror:${MOUNT_POINT}/spack-mirror \ --bind ${CI_DATA_LT}/squashfs-cache/:/squashfs-cache \ --bind ${CI_DATA_ST}/overlayfs:/overlayfs \ diff --git a/ci/setup_spack.sh b/ci/setup_spack.sh index 4c50de38ba0d0d8aa79347faa3382a08716dbd29..3ccfec36674d56a2b92f0bb22c93d0adb6c9f1e1 100755 --- a/ci/setup_spack.sh +++ b/ci/setup_spack.sh @@ -35,11 +35,14 @@ file_mapping["_stack"]="_stack" file_mapping["_acc_${acc_type}"]="_accel" file_mapping["_${environment}"]="_env" + + replace_placholders() { dst=$1 declare -A placeholders placeholders["{target}"]="${target}" placeholders["{gcc_version}"]=$(jq -Mrc '.stack.compilers.gcc.version' ${STACK_CONFIG}) + placeholders["{system_gcc_version}"]=$(jq -Mrc '.stack.system_compiler | to_entries | .[].value.version' ${STACK_CONFIG}) if [ "${accelerator}" != "none" ] then placeholders["{${acc_type}_arch}"]="${acc_arch}" diff --git a/dockerfiles/rhel94-kuma/Dockerfile b/dockerfiles/rhel94-kuma/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..3c846d0a1c2c897eeb68573dbac6a9286f36ee30 --- /dev/null +++ b/dockerfiles/rhel94-kuma/Dockerfile @@ -0,0 +1,31 @@ +FROM redhat/ubi9:9.4 + + +ARG SLURM_VERSION=24.11.3-1.scitas.el9 +ARG NVIDIA_MODULE_VERSION=560-open +ARG NVIDIA_DRIVER_VERSION=3:560.35.03-1.el9 +ARG FUSE_OVERLAYFS_VERSION=1.9-1.el9 +ARG SUBSCRIPTION_URL=https://foreman1.hpc.epfl.ch/rhsm +ARG REPO_BASE_URL=https://foreman1.hpc.epfl.ch/pulp/content +ARG FOREMAN_ORG=SCITAS + +SHELL ["/bin/bash", "-ec", "-o", "pipefail"] + +WORKDIR /tmp + +COPY ./foreman1-ca.pem /etc/rhsm/ca/katello-server-ca.pem + +COPY ./scripts /scripts + +RUN --mount=type=secret,id=ACTIVATION_KEY \ + export ACTIVATION_KEY="$(cat /run/secrets/ACTIVATION_KEY /kaniko/ACTIVATION_KEY 2>/dev/null)" \ + && /scripts/00_register.sh \ + && /scripts/10_install_spack_deps.sh \ + && /scripts/11_install_slurm.sh \ + && /scripts/12_install_pmix.sh \ + && /scripts/13_install_mlnx.sh \ + && /scripts/14_install_nvidia.sh \ + && /scripts/15_install_gpfs.sh \ + && /scripts/16_install_local_extras.sh \ + && /scripts/99_unregister.sh \ + && rm -rf /scripts diff --git a/dockerfiles/rhel94-kuma/docker-rhel94-kuma.pkr.hcl b/dockerfiles/rhel94-kuma/docker-rhel94-kuma.pkr.hcl new file mode 100644 index 0000000000000000000000000000000000000000..86bbc8e07cd94228cfb2697a5b45415c24277d80 --- /dev/null +++ b/dockerfiles/rhel94-kuma/docker-rhel94-kuma.pkr.hcl @@ -0,0 +1,70 @@ +packer { + required_plugins { + docker = { + version = ">= 1.1.1" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubi94" { + image = "redhat/ubi9:9.4" + platform = "linux/x86_64" + commit = true + changes = [ + "ENTRYPOINT /bin/bash" + ] +} + +build { + name = "rhel94-kuma" + sources = [ + "source.docker.ubi94" + ] + # Copy Foreman's CA certificate + provisioner "file" { + source = "./foreman1-ca.pem" + destination = "/etc/rhsm/ca/katello-server-ca.pem" + } + + # Register with Foreman with the specified activation key + # + # The activation key determines which repository we can see and access. + # This step has its own provisioner block because we don't need or want to + # share the activation key with the other scripts. + provisioner "shell" { + environment_vars = [ + "ACTIVATION_KEY=${var.activation_key}", + "REPO_BASE_URL=${var.repo_base_url}", + "FOREMAN_ORG=${var.subscription_org}", + "SUBSCRIPTION_URL=${var.subscription_url}" + ] + script = "./scripts/00_register.sh" + } + + provisioner "shell" { + environment_vars = [ + "SLURM_VERSION=${var.slurm_version}", + "NVIDIA_MODULE_VERSION=${var.nvidia_module_version}", + "NVIDIA_DRIVER_VERSION=${var.nvidia_driver_version}", + "FUSE_OVERLAYFS_VERSION=${var.fuse_overlayfs_version}" + ] + scripts = [ + "./scripts/10_install_spack_deps.sh", + "./scripts/11_install_slurm.sh", + "./scripts/12_install_pmix.sh", + "./scripts/13_install_mlnx.sh", + "./scripts/14_install_nvidia.sh", + "./scripts/15_install_gpfs.sh", + "./scripts/16_install_local_extras.sh", + "./scripts/99_unregister.sh" + ] + } + + post-processors { + post-processor "docker-tag" { + repository = "${var.docker_registry_url}/kuma-rhel" + tags = ["9.4"] + } + } +} diff --git a/dockerfiles/rhel94-kuma/foreman1-ca.pem b/dockerfiles/rhel94-kuma/foreman1-ca.pem new file mode 100644 index 0000000000000000000000000000000000000000..b329674fb92b2aaeb4be56d1d3315af184e2af32 --- /dev/null +++ b/dockerfiles/rhel94-kuma/foreman1-ca.pem @@ -0,0 +1,40 @@ +-----BEGIN CERTIFICATE----- +MIIG9jCCBN6gAwIBAgIUa86m2uz7zaCeQiEAVKc+6HyB4j8wDQYJKoZIhvcNAQEL +BQAwfzELMAkGA1UEBhMCVVMxFzAVBgNVBAgMDk5vcnRoIENhcm9saW5hMRAwDgYD +VQQHDAdSYWxlaWdoMRAwDgYDVQQKDAdLYXRlbGxvMRQwEgYDVQQLDAtTb21lT3Jn +VW5pdDEdMBsGA1UEAwwUZm9yZW1hbjEuaHBjLmVwZmwuY2gwHhcNMjMwOTIyMTMz +ODEyWhcNMzgwMTE4MTMzODEyWjB/MQswCQYDVQQGEwJVUzEXMBUGA1UECAwOTm9y +dGggQ2Fyb2xpbmExEDAOBgNVBAcMB1JhbGVpZ2gxEDAOBgNVBAoMB0thdGVsbG8x +FDASBgNVBAsMC1NvbWVPcmdVbml0MR0wGwYDVQQDDBRmb3JlbWFuMS5ocGMuZXBm +bC5jaDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJu9+4IWRetw9weO +VWQstdJPavpN5JfjaoLogdUtCooszEwUNh9hLV5eX7Qm97TF4D7apYC/Zh/hWdij +1HMWeew7vKX0J2uLz4aow719HIO/LIWtTP+U212QS5bCv4PnQOqtoT8ISOwyVnHF +x2U4ICI+gJ6hQaRdIFOpfk06H786UCua+28VuRnMB0moTuwS8VjQwr+sJBaIBjR+ +vYcUGfkfn0zEekOypTJYl1LqDq6wcHit7Cq2zQ1teA5JJHBgL1WxcBYCDGd9QvUq ++dIpskZJ0F+ovJWRbsRbqnNlCb9nDZhV+FK3MLHmmDcV5miI5dM1AUdjEMxsx/z/ +mrzjqmxyhKURQy+pRZ/QJye+uhaDgdEBKUYl44PbQjE357raaKKSs3GrdNSvCgO7 +SRQmsKf8gFjvC3xYlljB0pih2IvS3gnUUM6YzeC8K8GaVCM9eokceHdcStVPr2R9 +Fkjd48RNEKQpIirKT96pCIAgmTga91MOd4aaTXV2T2WyO7NBlnYOakPM1LidfGod +vynoMHRCaXSukbTLh9Jo2951JJK5Ssdp2LrIQOmwcXld0+9y7gfn2Y44m/8buX43 +BVstLqYNW4cHLy8LJArE/mlBq0I44sd8iAfVXZhsKG4FQ5v5F4EbtO55cyZZz36z +5YPN29m6tN14AQsAa4SGDuofIYCTAgMBAAGjggFoMIIBZDAMBgNVHRMEBTADAQH/ +MAsGA1UdDwQEAwIBpjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwEQYJ +YIZIAYb4QgEBBAQDAgJEMDUGCWCGSAGG+EIBDQQoFiZLYXRlbGxvIFNTTCBUb29s +IEdlbmVyYXRlZCBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUILTc8HrvuUgsMgPbhump +3tRHOwAwgb4GA1UdIwSBtjCBs4AUILTc8HrvuUgsMgPbhump3tRHOwChgYSkgYEw +fzELMAkGA1UEBhMCVVMxFzAVBgNVBAgMDk5vcnRoIENhcm9saW5hMRAwDgYDVQQH +DAdSYWxlaWdoMRAwDgYDVQQKDAdLYXRlbGxvMRQwEgYDVQQLDAtTb21lT3JnVW5p +dDEdMBsGA1UEAwwUZm9yZW1hbjEuaHBjLmVwZmwuY2iCFGvOptrs+82gnkIhAFSn +Puh8geI/MA0GCSqGSIb3DQEBCwUAA4ICAQCQs4hEkX0mAjPd0cTjJsF7THck2X8u +IFmB1Rcxg5dhT4jw0Id7fyzKhoRlGKk9CZ2ksENv6AZpWr2LyMeiz7q56ULyUewr +Si1IRGghwwKYIYV/u+kRqs8kYjhmexi5iavdMiiDNknIudBxV+ogKe8jK/+7mnjS +sSsnXG0UiUTg9jrAvlM6ggYE41yNGJCRSEGbW0hP/bqEbjtpowmCaH6/DICn38Tv +/qPVLSTlFMo0gxrBhImB6Ocj8EqWuwuivkGTT9AmENN85S7ru+2BoZ0XdGPs8qp0 +9aGk9Ym3UR7Xq+UZwxuRyi+WelPwwB2wQ+qW9HoN9Llfi1pMRgw+OCTVkGZB/zrK +BqgXXktP9FrJBk1U15mzz6V6J+dH2e0PLxG58BrpJmexKlURogsk6wz2rTAZWPwA +ou5pbWcY02KBC5Ho1DWUHKm1bWIeNjQmTaPkYpLnxGFZwqWZDqV62pJp5oQVRS25 +fa3rHRerQMTEZGmrXb/u+bep/WKKA1kfyYwR9LUO6gnUjk1T4H/JaCAX4/3SXpjJ +eQsZ3X30g1Egg73e0khXGl9IJKPccXUUP5KT+6WlSb5MLiECQRAVGuVzDYilYr6i +aeFH/s7Sw3LihZQk+c5sOngWvhxiEOsuyHT1R/g7ijIiJAcKFp0j8AFvxnFblvN2 +fhTIDaRqDRg1pg== +-----END CERTIFICATE----- diff --git a/dockerfiles/rhel94-kuma/scripts/00_register.sh b/dockerfiles/rhel94-kuma/scripts/00_register.sh new file mode 100755 index 0000000000000000000000000000000000000000..9e152d4fc58a3f7e032fd6b175fc756d7d87de41 --- /dev/null +++ b/dockerfiles/rhel94-kuma/scripts/00_register.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +subscription-manager config --rhsm.repo_ca_cert /etc/rhsm/ca/katello-server-ca.pem + +subscription-manager register --serverurl "${SUBSCRIPTION_URL:?}" \ + --org="${FOREMAN_ORG:?}" --activationkey="${ACTIVATION_KEY:?}" \ + --baseurl="${REPO_BASE_URL:?}" + +dnf config-manager --set-disabled "ubi*" +dnf makecache diff --git a/dockerfiles/rhel94-kuma/scripts/10_install_spack_deps.sh b/dockerfiles/rhel94-kuma/scripts/10_install_spack_deps.sh new file mode 100755 index 0000000000000000000000000000000000000000..4e8a6b6abd08b5dab625c52dc67bb64fa3c25338 --- /dev/null +++ b/dockerfiles/rhel94-kuma/scripts/10_install_spack_deps.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +dnf --setopt=install_weak_deps=False install -y \ + gcc-c++ gcc-gfortran \ + patchelf findutils patch \ + xz bzip2 file gnupg2 hostname iproute unzip zlib-devel \ + python3 python3-pip python3-setuptools python3-boto3 \ + git subversion mercurial \ + ninja-build \ + libglvnd-devel \ + glibc-gconv-extra \ + libatomic \ + squashfs-tools \ + squashfuse "fuse-overlayfs-${FUSE_OVERLAYFS_VERSION:?}" diff --git a/dockerfiles/rhel94-kuma/scripts/11_install_slurm.sh b/dockerfiles/rhel94-kuma/scripts/11_install_slurm.sh new file mode 100755 index 0000000000000000000000000000000000000000..1e9a78ec407fb429319118b8055a017311c50545 --- /dev/null +++ b/dockerfiles/rhel94-kuma/scripts/11_install_slurm.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +dnf --setopt=install_weak_deps=False install -y \ + "slurm-devel-${SLURM_VERSION:?}" \ + "slurm-libpmi-${SLURM_VERSION:?}" diff --git a/dockerfiles/rhel94-kuma/scripts/12_install_pmix.sh b/dockerfiles/rhel94-kuma/scripts/12_install_pmix.sh new file mode 100755 index 0000000000000000000000000000000000000000..7c5202c4a8813ba3d17e27f1ebf83185adc5a49c --- /dev/null +++ b/dockerfiles/rhel94-kuma/scripts/12_install_pmix.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +dnf --setopt=install_weak_deps=False install -y pmix diff --git a/dockerfiles/rhel94-kuma/scripts/13_install_mlnx.sh b/dockerfiles/rhel94-kuma/scripts/13_install_mlnx.sh new file mode 100755 index 0000000000000000000000000000000000000000..632c1fcfac012a4b9e9e75a4b65d1b005f6a6252 --- /dev/null +++ b/dockerfiles/rhel94-kuma/scripts/13_install_mlnx.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +dnf --setopt=install_weak_deps=False install -y libibverbs libibverbs-utils \ + rdma-core rdma-core-devel hcoll xpmem kmod-knem diff --git a/dockerfiles/rhel94-kuma/scripts/14_install_nvidia.sh b/dockerfiles/rhel94-kuma/scripts/14_install_nvidia.sh new file mode 100755 index 0000000000000000000000000000000000000000..986a3713ef3fb52d4ed9b8fd338d92d71e047f2a --- /dev/null +++ b/dockerfiles/rhel94-kuma/scripts/14_install_nvidia.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +dnf module enable -y "nvidia-driver:${NVIDIA_MODULE_VERSION:?}" + +dnf --setopt=install_weak_deps=False install -y "nvidia-driver-cuda-${NVIDIA_DRIVER_VERSION:?}" diff --git a/dockerfiles/rhel94-kuma/scripts/15_install_gpfs.sh b/dockerfiles/rhel94-kuma/scripts/15_install_gpfs.sh new file mode 100755 index 0000000000000000000000000000000000000000..2d3be2e28eb90b58b2693d18723285beac212ad5 --- /dev/null +++ b/dockerfiles/rhel94-kuma/scripts/15_install_gpfs.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +dnf --setopt=install_weak_deps=False install -y gpfs.base diff --git a/dockerfiles/rhel94-kuma/scripts/16_install_local_extras.sh b/dockerfiles/rhel94-kuma/scripts/16_install_local_extras.sh new file mode 100755 index 0000000000000000000000000000000000000000..2bacc048a1198bda2763dc0cf378987a64dc1fc5 --- /dev/null +++ b/dockerfiles/rhel94-kuma/scripts/16_install_local_extras.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +# NOTE Those will _not_ be present on the clusters. +curl -sL -o /usr/local/bin/jq https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64 \ + && chmod +x /usr/local/bin/jq + +curl -sL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.44.2/yq_linux_amd64 \ + && chmod +x /usr/local/bin/yq diff --git a/dockerfiles/rhel94-kuma/scripts/99_unregister.sh b/dockerfiles/rhel94-kuma/scripts/99_unregister.sh new file mode 100755 index 0000000000000000000000000000000000000000..b0a6dc2f4f53b8e99495ecb42a23d3e46354db67 --- /dev/null +++ b/dockerfiles/rhel94-kuma/scripts/99_unregister.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +dnf clean all && rm -rf /var/cache +subscription-manager unregister diff --git a/dockerfiles/rhel94-kuma/variables.pkr.hcl b/dockerfiles/rhel94-kuma/variables.pkr.hcl new file mode 100644 index 0000000000000000000000000000000000000000..d9d10d18318ea02ac4cdddaeba9c2307c0eab32e --- /dev/null +++ b/dockerfiles/rhel94-kuma/variables.pkr.hcl @@ -0,0 +1,44 @@ +variable "slurm_version" { + type = string + default = "24.11.3-1.scitas.el9" +} + +variable "nvidia_module_version" { + type = string + default = "560-open" +} + +variable "nvidia_driver_version" { + type = string + default = "3:560.35.03-1.el9" +} + +variable "fuse_overlayfs_version" { + type = string + default = "1.9-1.el9" +} + +variable "subscription_url" { + type = string + default = "https://foreman1.hpc.epfl.ch/rhsm" +} + +variable "repo_base_url" { + type = string + default = "https://foreman1.hpc.epfl.ch/pulp/content" +} + +variable "subscription_org" { + type = string + default = "SCITAS" +} + +variable "activation_key" { + type = string + sensitive = true +} + +variable "docker_registry_url" { + type = string + default = "registry.hpc.epfl.ch/scitas" +} diff --git a/stacks/pinot-noir/concretizer_common.yaml b/stacks/pinot-noir/concretizer_common.yaml index cb79471be5f2fc33edb56059c5247ffee5d8e237..f4872132c545ced00350563e064b8f97b5b82032 100644 --- a/stacks/pinot-noir/concretizer_common.yaml +++ b/stacks/pinot-noir/concretizer_common.yaml @@ -2,7 +2,7 @@ concretizer: reuse: false # roots: true # include: - # - "%gcc@11.2.1" + # - "%gcc@{system_gcc_version}" # from: # - type: local unify: false diff --git a/stacks/pinot-noir/config.json b/stacks/pinot-noir/config.json index 6d6decefde422800c526850d543ab35005c58fc4..865737cc8455b7cc0b30c44b1081f5c69e0fdb52 100644 --- a/stacks/pinot-noir/config.json +++ b/stacks/pinot-noir/config.json @@ -17,20 +17,20 @@ "type": "relative" }, "restricted": { - "url": "spack-mirror-restriced", + "url": "spack-mirror-restricted", "type": "relative" } } }, "stack": { - "version": "v1", + "version": "v2", "mount_point": "/ssoft/spack", "system_arch": "target=x86_64_v3", "system_compiler": { "gcc@11": { "compiler": "gcc", - "spec": "gcc@11.2.1", - "version": "11.2.1" + "spec": "gcc@11.4.1", + "version": "11.4.1" } }, "environments": { @@ -66,18 +66,18 @@ "compilers": { "oneapi": { "compiler": "oneapi", - "spec": "intel-oneapi-compilers@2024.1.0 %gcc@11.2.1", + "spec": "intel-oneapi-compilers@2024.1.0 %gcc@11.4.1", "version": "2024.1.0", "constraint": ["jed", "helvetios"] }, "gcc": { "compiler": "gcc", - "spec": "gcc@13.2.0 %gcc@11.2.1", + "spec": "gcc@13.2.0 %gcc@11.4.1", "version": "13.2.0" }, "nvhpc": { "compiler": "nvhpc", - "spec": "nvhpc+mpi+lapack+blas default_cuda=12.5 @24.7 %gcc@11.2.1", + "spec": "nvhpc+mpi+lapack+blas default_cuda=12.5 @24.7 %gcc@11.4.1", "version": "24.7", "constraint": ["kuma-h100", "kuma-l40s", "izar"] } diff --git a/stacks/pinot-noir/mark_explicit_kuma b/stacks/pinot-noir/mark_explicit_kuma index af0b507d523bf926e7ec4194d9a71d1673855661..0dfa8b0cc9efa65cc5c2e26b5b7cec97a2dbeb87 100644 --- a/stacks/pinot-noir/mark_explicit_kuma +++ b/stacks/pinot-noir/mark_explicit_kuma @@ -1,5 +1,4 @@ cuda %gcc@13.2.0 -cuda %nvhpc nccl %gcc@13.2.0 py-matplotlib %gcc@13.2.0 py-numpy@1.26.4 %gcc@13.2.0 ^openblas threads=openmp diff --git a/stacks/pinot-noir/mark_explicit_kuma-nvhpc b/stacks/pinot-noir/mark_explicit_kuma-nvhpc new file mode 100644 index 0000000000000000000000000000000000000000..06ffc4698d2da32b2ce45097aef0aab2cd4ca697 --- /dev/null +++ b/stacks/pinot-noir/mark_explicit_kuma-nvhpc @@ -0,0 +1 @@ +cuda %nvhpc diff --git a/stacks/pinot-noir/modules_common.yaml b/stacks/pinot-noir/modules_common.yaml index 135e4d565bea584df6ef0f3dd3438807e2ad29e6..d57ee4ffbfeb240c137cace492e3e5777ace4fc2 100644 --- a/stacks/pinot-noir/modules_common.yaml +++ b/stacks/pinot-noir/modules_common.yaml @@ -10,7 +10,7 @@ modules: hash_length: 0 core_compilers: - - gcc@11.2.1 + - gcc@{system_gcc_version} all: environment: @@ -114,11 +114,13 @@ modules: - UCX_ROOT comsol: + template: modules/group_restricted.lua environment: prepend_path: MATLABPATH: "{prefix}/mli" fdtd@2022-R2.1-2963: + template: modules/group_restricted.lua environment: prepend_path: PATH: "{prefix}/mpich2/nemesis/bin" diff --git a/stacks/pinot-noir/packages_common.yaml b/stacks/pinot-noir/packages_common.yaml index df8edbc056b14c0b629e50e5b76c793c601f3b2b..85ec35aa15ab1a0e428f9e83e037aa632f87c3bd 100644 --- a/stacks/pinot-noir/packages_common.yaml +++ b/stacks/pinot-noir/packages_common.yaml @@ -17,7 +17,7 @@ packages: glibc: - prefer: ['%gcc@11.2.1'] + prefer: ['%gcc@{system_gcc_version}'] jpeg: require: ['libjpeg'] diff --git a/stacks/pinot-noir/spack_aocc.yaml b/stacks/pinot-noir/spack_aocc.yaml index b6e7276c6afff30eae898e3e7019f55dd6525050..51147cf7821328d0d97934b808d254213ab06c38 100644 --- a/stacks/pinot-noir/spack_aocc.yaml +++ b/stacks/pinot-noir/spack_aocc.yaml @@ -34,7 +34,7 @@ spack: aocc: require: ['@4.2.0', 'license-agreed=true' - '%gcc@11.2.1', 'target=x86_64_v3'] + '%gcc@{system_gcc_version}', 'target=x86_64_v3'] icu4c: require: ['%gcc'] diff --git a/stacks/pinot-noir/spack_nvhpc.yaml b/stacks/pinot-noir/spack_nvhpc.yaml index 8f5e962e2e962ac4afec6bd1ca18c282c8994936..9804e3a76bee229b43c57e54ede6878f0752f128 100644 --- a/stacks/pinot-noir/spack_nvhpc.yaml +++ b/stacks/pinot-noir/spack_nvhpc.yaml @@ -34,7 +34,7 @@ spack: nvhpc: require: ['@24.7', '+mpi', '+blas', '+lapack', 'default_cuda=12.5', - '%gcc@11.2.1', 'target=x86_64_v3'] + '%gcc@{system_gcc_version}', 'target=x86_64_v3'] icu4c: require: ['%gcc'] @@ -50,7 +50,7 @@ spack: - "[virtuals=blas,lapack] nvhpc +blas +lapack" - python_nvhpc: - - python@3.11.7 %gcc@11.2.1 + - python@3.11.7 %gcc@{system_gcc_version} specs: - matrix: diff --git a/stacks/pinot-noir/spack_oneapi.yaml b/stacks/pinot-noir/spack_oneapi.yaml index 67f92a44c1545928e22ff099f7e20561f34a1842..bd04a11020bc4bfa00c3453043f3cd8f7693bfe5 100644 --- a/stacks/pinot-noir/spack_oneapi.yaml +++ b/stacks/pinot-noir/spack_oneapi.yaml @@ -64,7 +64,7 @@ spack: - "intel-oneapi-mkl" - python_oneapi: - - python@3.11.7 %gcc@11.2.1 + - python@3.11.7 %gcc@{system_gcc_version} specs: - matrix: