diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca5fce7d..0fcb90ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,9 @@ # We define the following GitLab pipeline variables: variables: +# Required information about GitHub repository + GITHUB_PROJECT_NAME: "CHAI" + GITHUB_PROJECT_ORG: "LLNL" # Use the umdev LLNL service user to run CI. This prevents from running # pipelines as an actual user. LLNL_SERVICE_USER: umdev @@ -34,12 +37,18 @@ variables: # We build the projects in the CI clone directory. # TODO: add a clean-up mechanism BUILD_ROOT: ${CI_PROJECT_DIR} +# Set the build-and-test command. + BUILD_AND_TEST_CMD: "./scripts/gitlab/build_and_test.sh" +# Override the list of branch that will skip the "draft PR test". +# Add protected branches here. Defaults to "develop main master". +# ALWAYS_RUN_LIST: "" # We organize the build-and-test stage in sub-pipelines. Each sub-pipeline # corresponds to a test batch on a given machine. # High level stages stages: + - machine-checks - build-and-test # Template for jobs triggering a build-and-test sub-pipelines: @@ -49,13 +58,17 @@ stages: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' - ref: v2022.09.0 + ref: v2023.03.1 file: '${CI_MACHINE}-build-and-test.yml' - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml' strategy: depend forward: pipeline_variables: true -# pipelines subscribed by the project include: + # [Optional] checks preliminary to running the actual CI test + #- project: 'radiuss/radiuss-shared-ci' + # ref: v2023.03.1 + # file: 'preliminary-ignore-draft-pr.yml' + # pipelines subscribed by the project - local: .gitlab/subscribed-pipelines.yml diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index e5eda30a..7126322d 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -15,30 +15,40 @@ variables: # Ruby # Arguments for top level allocation - RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --partition=pdebug --time=20 --nodes=1" + RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --reservation=ci --qos=ci_ruby --time=20 --nodes=1" # Arguments for job level allocation - RUBY_BUILD_AND_TEST_JOB_ALLOC: "--overlap --time=20 --nodes=1" + RUBY_BUILD_AND_TEST_JOB_ALLOC: "--overlap --reservation=ci --time=20 --nodes=1" # Project specific variants for ruby - PROJECT_RUBY_VARIANTS: "" + PROJECT_RUBY_VARIANTS: "~shared +raja tests=basic " # Project specific deps for ruby PROJECT_RUBY_DEPS: "" # Corona # Arguments for top level allocation - CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--time-limit=30m --nodes=1" + CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=45m --nodes=1" # Arguments for job level allocation - CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=30m --nodes=1" + CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=30m --nodes=1 --begin-time=+5s" # Project specific variants for corona - PROJECT_CORONA_VARIANTS: "" + PROJECT_CORONA_VARIANTS: "~shared +raja tests=basic " # Project specific deps for corona PROJECT_CORONA_DEPS: "" +# Tioga +# Arguments for top level allocation + TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=45m --nodes=1" +# Arguments for job level allocation + TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=30m --nodes=1 --begin-time=+5s" +# Project specific variants for tioga + PROJECT_TIOGA_VARIANTS: "~shared +raja tests=basic " +# Project specific deps for tioga + PROJECT_TIOGA_DEPS: "" + # Lassen and Butte use a different job scheduler (spectrum lsf) that does not # allow pre-allocation the same way slurm does. # Arguments for job level allocation LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 120" # Project specific variants for lassen - PROJECT_LASSEN_VARIANTS: "" + PROJECT_LASSEN_VARIANTS: "~shared +raja cuda_arch=70 tests=basic" # Project specific deps for lassen PROJECT_LASSEN_DEPS: "" diff --git a/.gitlab/lassen-build-and-test-extra.yml b/.gitlab/lassen-build-and-test-extra.yml index 606e6c0b..fede40d2 100644 --- a/.gitlab/lassen-build-and-test-extra.yml +++ b/.gitlab/lassen-build-and-test-extra.yml @@ -12,18 +12,11 @@ # We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that # the comparison with the original job is easier. -# Overriding shared spec: Longer allocation + Allow failures : known to fail at raja install -pgi_20_4_gcc_8_3_1: - extends: .build_and_test_on_lassen +# Override job with old cuda to allow it to fail. +clang_12_0_1_gcc_8_3_1_cuda_10_1_243: variables: - SPEC: "${PROJECT_LASSEN_VARIANTS} %pgi@20.4 cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ${PROJECT_LASSEN_DEPS}" - allow_failure: true - -# Overriding shared spec: Allow failures : known to fail at raja install -ibm_clang_9_0_0_gcc_8_3_1_cuda_10_1_168: + SPEC: "${PROJECT_LASSEN_VARIANTS} +cuda %clang@12.0.1.gcc.8.3.1 ^cuda@10.1.243+allow-unsupported-compilers ${PROJECT_LASSEN_DEPS}" extends: .build_and_test_on_lassen - variables: - SPEC: " ${PROJECT_LASSEN_VARIANTS} +cuda %clang@ibm.9.0.0 cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^cuda@10.1.168 ${PROJECT_LASSEN_DEPS}" allow_failure: true @@ -34,26 +27,20 @@ ibm_clang_9_0_0_gcc_8_3_1_cuda_10_1_168: # ${PROJECT__DEPS} in the extra jobs. There is no reason not to fully # describe the spec here. -# In RAJA, this spec has an updated cuda (11.0.2), with cuda_arch=70 set. -ibm_clang_9_0_0_cuda_10_1_168: - variables: - SPEC: "+cuda %clang@ibm.9.0.0 ^cuda@10.1.168" +clang_12_0_1_libcpp: extends: .build_and_test_on_lassen - -clang_9_0_0_libcpp (build and test on lassen): variables: - SPEC: "%clang@9.0.0 cxxflags=\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\" cflags=\"-DGTEST_HAS_CXXABI_H_=0\"" - extends: .build_and_test_on_lassen + SPEC: "~shared +raja tests=basic %clang@12.0.1 cflags==\"-DGTEST_HAS_CXXABI_H_=0\" cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\"" -clang_9_0_0_memleak (build and test on lassen): +clang_12_0_1_gcc_8_3_1_memleak: + extends: .build_and_test_on_lassen variables: - SPEC: "%clang@9.0.0 cxxflags=-fsanitize=address" + SPEC: "~shared +raja tests=basic %clang@12.0.1.gcc.8.3.1 cxxflags==\"-fsanitize=address\"" ASAN_OPTIONS: "detect_leaks=1" - extends: .build_and_test_on_lassen -clang_9_0_0_datarace (build and test on lassen): +clang_12_0_1_datarace: + extends: .build_and_test_on_lassen variables: - SPEC: "%clang@9.0.0 cxxflags=-fsanitize=thread" + SPEC: "~shared +raja tests=basic %clang@12.0.1 cxxflags==\"-fsanitize=thread\"" allow_failure: true - extends: .build_and_test_on_lassen diff --git a/.gitlab/ruby-build-and-test-extra.yml b/.gitlab/ruby-build-and-test-extra.yml index 366916ef..452928d9 100644 --- a/.gitlab/ruby-build-and-test-extra.yml +++ b/.gitlab/ruby-build-and-test-extra.yml @@ -12,13 +12,7 @@ # We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that # the comparison with the original job is easier. -# Overriding shared spec: Allow failures -pgi_20_1_gcc_local_8_3_1: - variables: - SPEC: "${PROJECT_RUBY_VARIANTS} %pgi@20.1 cxxflags\"=-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" cflags\"=-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" fflags=\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" ${PROJECT_RUBY_DEPS}" - extends: .build_and_test_on_ruby - allow_failure: true - +# No overridden jobs so far. ############ # Extra jobs @@ -27,13 +21,8 @@ pgi_20_1_gcc_local_8_3_1: # ${PROJECT__DEPS} in the extra jobs. There is no reason not to fully # describe the spec here. -gcc_4_9_3: +clang_14_0_6_develop_tpls: variables: - SPEC: "%gcc@4.9.3" + SPEC: " %clang@14.0.6 ^umpire@develop ^raja@develop ^camp@main" extends: .build_and_test_on_ruby -#clang_10_develop_tpls: -# variables: -# SPEC: " %clang@10.0.1 ^umpire@develop ^raja@develop ^camp@main" -# extends: .build_and_test_on_ruby - diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index 24a02b0d..9d292469 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -5,18 +5,74 @@ # SPDX-License-Identifier: (MIT) ############################################################################### -# Trigger a build-and-test pipeline for ruby, corona and lassen +# The template job to test whether a machine is up. +# Expects CI_MACHINE defined to machine name. +.machine-check: + stage: machine-checks + tags: [shell, oslic] + variables: + GIT_STRATEGY: none + script: + - | + if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]] + then + echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m" + curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \ + --header 'Content-Type: application/json' \ + --header "authorization: Bearer ${GITHUB_TOKEN}" \ + --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }" + exit 1 + fi + +### +# Trigger a build-and-test pipeline for a machine. +# Comment the jobs for machines you don’t need. +### + +# RUBY +ruby-up-check: + variables: + CI_MACHINE: "ruby" + extends: [.machine-check] + ruby-build-and-test: variables: CI_MACHINE: "ruby" + needs: [ruby-up-check] extends: [.build-and-test] +# CORONA +corona-up-check: + variables: + CI_MACHINE: "corona" + extends: [.machine-check] + corona-build-and-test: variables: CI_MACHINE: "corona" + needs: [corona-up-check] extends: [.build-and-test] +# TIOGA +tioga-up-check: + variables: + CI_MACHINE: "tioga" + extends: [.machine-check] + +tioga-build-and-test: + variables: + CI_MACHINE: "tioga" + needs: [tioga-up-check] + extends: [.build-and-test] + +# LASSEN +lassen-up-check: + variables: + CI_MACHINE: "lassen" + extends: [.machine-check] + lassen-build-and-test: variables: CI_MACHINE: "lassen" + needs: [lassen-up-check] extends: [.build-and-test] diff --git a/.gitlab/tioga-build-and-test-extra.yml b/.gitlab/tioga-build-and-test-extra.yml new file mode 100644 index 00000000..c96489c5 --- /dev/null +++ b/.gitlab/tioga-build-and-test-extra.yml @@ -0,0 +1,9 @@ +############################################################################## +# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################## + +variables: + INCLUDED_FILE_CANNOT_BE_EMPTY: "True" diff --git a/.uberenv_config.json b/.uberenv_config.json index 93854a5f..34531069 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -1,13 +1,14 @@ { "package_name" : "chai", "package_version" : "develop", -"package_final_phase" : "hostconfig", +"package_final_phase" : "initconfig", "package_source_dir" : "../..", "spack_url": "https://github.com/spack/spack.git", -"spack_branch": "v0.18.1", +"spack_branch": "e4s-23.02", "spack_activate" : {}, "spack_configs_path": "scripts/radiuss-spack-configs", -"spack_packages_path": "scripts/spack_packages", +"spack_packages_path": "scripts/radiuss-spack-configs/packages", "spack_concretizer": "clingo", -"spack_setup_clingo": false +"spack_setup_clingo": false, +"debug": true } diff --git a/blt b/blt index 84fe0d15..09f7a4be 160000 --- a/blt +++ b/blt @@ -1 +1 @@ -Subproject commit 84fe0d15d841a79d923e97a9105b86b59147bfed +Subproject commit 09f7a4be48886e1175885e10ed6af1225ae12a90 diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs index 1ce0f442..f6626f79 160000 --- a/scripts/radiuss-spack-configs +++ b/scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 1ce0f4421cfe6be4200ae9aa8abd113e09ee4c2d +Subproject commit f6626f790b07fbd399f238aa9792a11ca78dfa97 diff --git a/scripts/spack_packages/blt/package.py b/scripts/spack_packages/blt/package.py deleted file mode 100644 index 507b02aa..00000000 --- a/scripts/spack_packages/blt/package.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - - -class Blt(Package): - """BLT is a streamlined CMake-based foundation for Building, Linking and - Testing large-scale high performance computing (HPC) applications.""" - - homepage = "https://github.com/LLNL/blt" - url = "https://github.com/LLNL/blt/archive/v0.4.0.tar.gz" - git = "https://github.com/LLNL/blt.git" - tags = ['radiuss'] - - maintainers = ['white238', 'davidbeckingsale'] - - version('develop', branch='develop') - version('main', branch='main') - # Note: 0.4.0+ contains a breaking change to BLT created targets - # if you export targets this could cause problems in downstream - # projects if not handled properly. More info here: - # https://llnl-blt.readthedocs.io/en/develop/tutorial/exporting_targets.html - version('0.5.0', sha256='5f680ef922d0e0a7ff1b1a5fc8aa107cd4f543ad888cbc9b12639bea72a6ab1f') - version('0.4.1', sha256='16cc3e067ddcf48b99358107e5035a17549f52dcc701a35cd18a9d9f536826c1') - version('0.4.0', sha256='f3bc45d28b9b2eb6df43b75d4f6f89a1557d73d012da7b75bac1be0574767193') - version('0.3.6', sha256='6276317c29e7ff8524fbea47d9288ddb40ac06e9f9da5e878bf9011e2c99bf71') - version('0.3.5', sha256='68a1c224bb9203461ae6f5ab0ff3c50b4a58dcce6c2d2799489a1811f425fb84') - version('0.3.0', sha256='bb917a67cb7335d6721c997ba9c5dca70506006d7bba5e0e50033dd0836481a5') - version('0.2.5', sha256='3a000f60194e47b3e5623cc528cbcaf88f7fea4d9620b3c7446ff6658dc582a5') - version('0.2.0', sha256='c0cadf1269c2feb189e398a356e3c49170bc832df95e5564e32bdbb1eb0fa1b3') - - depends_on('cmake', type='run') - - def install(self, spec, prefix): - install_tree('.', prefix) \ No newline at end of file diff --git a/scripts/spack_packages/camp/package.py b/scripts/spack_packages/camp/package.py deleted file mode 100644 index 3bff14ef..00000000 --- a/scripts/spack_packages/camp/package.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import glob - -from spack.package import * - - -class Camp(CMakePackage, CudaPackage, ROCmPackage): - """ - Compiler agnostic metaprogramming library providing concepts, - type operations and tuples for C++ and cuda - """ - - homepage = "https://github.com/LLNL/camp" - git = "https://github.com/LLNL/camp.git" - url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz" - - maintainers = ['trws'] - - version('main', branch='main', submodules='True') - version('2022.03.0', sha256='e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721') - version('0.3.0', sha256='129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb') - version('0.2.3', sha256='58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7') - version('0.2.2', sha256='194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819') - version('0.1.0', sha256='fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc') - - # TODO: figure out gtest dependency and then set this default True. - variant('tests', default=False, description='Build tests') - variant('openmp', default=False, description='Build with OpenMP support') - - depends_on('cub', when='+cuda') - - depends_on('blt') - - def cmake_args(self): - spec = self.spec - - options = [] - - options.append("-DBLT_SOURCE_DIR={0}".format(spec['blt'].prefix)) - - if '+cuda' in spec: - options.extend([ - '-DENABLE_CUDA=ON', - '-DCUDA_TOOLKIT_ROOT_DIR=%s' % (spec['cuda'].prefix)]) - - if not spec.satisfies('cuda_arch=none'): - cuda_arch = spec.variants['cuda_arch'].value - options.append('-DCMAKE_CUDA_ARCHITECTURES={0}'.format(cuda_arch[0])) - options.append('-DCUDA_ARCH=sm_{0}'.format(cuda_arch[0])) - flag = '-arch sm_{0}'.format(cuda_arch[0]) - options.append('-DCMAKE_CUDA_FLAGS:STRING={0}'.format(flag)) - else: - options.append('-DENABLE_CUDA=OFF') - - if '+rocm' in spec: - options.extend([ - '-DENABLE_HIP=ON', - '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix) - ]) - archs = self.spec.variants['amdgpu_target'].value - if archs != 'none': - arch_str = ",".join(archs) - options.append( - '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch_str) - ) - # there is only one dir like this, but the version component is unknown - options.append( - "-DHIP_CLANG_INCLUDE_PATH=" + glob.glob( - "{}/lib/clang/*/include".format(spec['llvm-amdgpu'].prefix) - )[0] - ) - else: - options.append('-DENABLE_HIP=OFF') - - options.append(self.define_from_variant('ENABLE_TESTS', 'tests')) - options.append(self.define_from_variant('ENABLE_OPENMP', 'openmp')) - - - return options diff --git a/scripts/spack_packages/chai/package.py b/scripts/spack_packages/chai/package.py deleted file mode 100644 index cd6d7b8a..00000000 --- a/scripts/spack_packages/chai/package.py +++ /dev/null @@ -1,288 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - - -from spack import * - -import socket -import os - -from os import environ as env -from os.path import join as pjoin - -import re - -def cmake_cache_entry(name, value, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE PATH "%s")\n\n' % (name,value,comment) - - -def cmake_cache_string(name, string, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE STRING "%s")\n\n' % (name,string,comment) - - -def cmake_cache_option(name, boolean_value, comment=""): - """Generate a string for a cmake configuration option""" - - value = "ON" if boolean_value else "OFF" - return 'set(%s %s CACHE BOOL "%s")\n\n' % (name,value,comment) - - -def get_spec_path(spec, package_name, path_replacements = {}, use_bin = False) : - """Extracts the prefix path for the given spack package - path_replacements is a dictionary with string replacements for the path. - """ - - if not use_bin: - path = spec[package_name].prefix - else: - path = spec[package_name].prefix.bin - - path = os.path.realpath(path) - - for key in path_replacements: - path = path.replace(key, path_replacements[key]) - - return path - - -class Chai(CMakePackage, CudaPackage, ROCmPackage): - """ - Copy-hiding array interface for data migration between memory spaces - """ - - homepage = "https://github.com/LLNL/CHAI" - git = "https://github.com/LLNL/CHAI.git" - - version('develop', branch='develop', submodules='True') - version('master', branch='main', submodules='True') - version('2.1.1', tag='v2.1.1', submodules='True') - version('2.1.0', tag='v2.1.0', submodules='True') - version('2.0.0', tag='v2.0.0', submodules='True') - version('1.2.0', tag='v1.2.0', submodules='True') - version('1.1.0', tag='v1.1.0', submodules='True') - version('1.0', tag='v1.0', submodules='True') - - variant('shared', default=False, description='Build Shared Libs') - variant('raja', default=True, description='Build plugin for RAJA') - variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), - multi=False, description='Tests to run') - - depends_on('umpire') - depends_on('raja', when="+raja") - - depends_on('umpire@main', when='@main') - depends_on('raja@main', when="@main+raja") - - depends_on('cmake@3.14:', type='build') - depends_on('umpire+cuda', when="+cuda") - depends_on('raja+cuda', when="+raja+cuda") - - for val in ROCmPackage.amdgpu_targets: - depends_on('raja amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) - depends_on('umpire amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) - - for sm_ in CudaPackage.cuda_arch_values: - depends_on('raja cuda_arch={0}'.format(sm_), - when='cuda_arch={0}'.format(sm_)) - depends_on('umpire cuda_arch={0}'.format(sm_), - when='cuda_arch={0}'.format(sm_)) - - phases = ['hostconfig', 'cmake', 'build', 'install'] - - def _get_sys_type(self, spec): - sys_type = str(spec.architecture) - # if on llnl systems, we can use the SYS_TYPE - if "SYS_TYPE" in env: - sys_type = env["SYS_TYPE"] - return sys_type - - def _get_host_config_path(self, spec): - var='' - if '+cuda' in spec: - var= '-'.join([var,'cuda']) - - host_config_path = "hc-%s-%s-%s%s-%s.cmake" % (socket.gethostname().rstrip('1234567890'), - self._get_sys_type(spec), - spec.compiler, - var, - spec.dag_hash()) - dest_dir = self.stage.source_path - host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) - return host_config_path - - def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): - """ - This method creates a 'host-config' file that specifies - all of the options used to configure and build CHAI. - - For more details about 'host-config' files see: - http://software.llnl.gov/conduit/building.html - - Note: - The `py_site_pkgs_dir` arg exists to allow a package that - subclasses this package provide a specific site packages - dir when calling this function. `py_site_pkgs_dir` should - be an absolute path or `None`. - - This is necessary because the spack `site_packages_dir` - var will not exist in the base class. For more details - on this issue see: https://github.com/spack/spack/issues/6261 - """ - - ####################### - # Compiler Info - ####################### - c_compiler = env["SPACK_CC"] - cpp_compiler = env["SPACK_CXX"] - - # Even though we don't have fortran code in our project we sometimes - # use the Fortran compiler to determine which libstdc++ to use - f_compiler = "" - if "SPACK_FC" in env.keys(): - # even if this is set, it may not exist - # do one more sanity check - if os.path.isfile(env["SPACK_FC"]): - f_compiler = env["SPACK_FC"] - - ####################################################################### - # By directly fetching the names of the actual compilers we appear - # to doing something evil here, but this is necessary to create a - # 'host config' file that works outside of the spack install env. - ####################################################################### - - sys_type = self._get_sys_type(spec) - - ############################################## - # Find and record what CMake is used - ############################################## - - cmake_exe = spec['cmake'].command.path - cmake_exe = os.path.realpath(cmake_exe) - - host_config_path = self._get_host_config_path(spec) - cfg = open(host_config_path, "w") - cfg.write("###################\n".format("#" * 60)) - cfg.write("# Generated host-config - Edit at own risk!\n") - cfg.write("###################\n".format("#" * 60)) - cfg.write("# Copyright (c) 2020, Lawrence Livermore National Security, LLC and\n") - cfg.write("# other CHAI Project Developers. See the top-level LICENSE file for\n") - cfg.write("# details.\n") - cfg.write("#\n") - cfg.write("# SPDX-License-Identifier: (BSD-3-Clause) \n") - cfg.write("###################\n\n".format("#" * 60)) - - cfg.write("#------------------\n".format("-" * 60)) - cfg.write("# SYS_TYPE: {0}\n".format(sys_type)) - cfg.write("# Compiler Spec: {0}\n".format(spec.compiler)) - cfg.write("# CMake executable path: %s\n" % cmake_exe) - cfg.write("#------------------\n\n".format("-" * 60)) - - ####################### - # Compiler Settings - ####################### - - cfg.write("#------------------\n".format("-" * 60)) - cfg.write("# Compilers\n") - cfg.write("#------------------\n\n".format("-" * 60)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) - - # use global spack compiler flags - cflags = ' '.join(spec.compiler_flags['cflags']) - if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) - - cxxflags = ' '.join(spec.compiler_flags['cxxflags']) - if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) - - if ("gfortran" in f_compiler) and ("clang" in cpp_compiler): - libdir = pjoin(os.path.dirname( - os.path.dirname(f_compiler)), "lib") - flags = "" - for _libpath in [libdir, libdir + "64"]: - if os.path.exists(_libpath): - flags += " -Wl,-rpath,{0}".format(_libpath) - description = ("Adds a missing libstdc++ rpath") - if flags: - cfg.write(cmake_cache_entry("BLT_EXE_LINKER_FLAGS", flags, - description)) - - gcc_toolchain_regex = re.compile(".*gcc-toolchain.*") - gcc_name_regex = re.compile(".*gcc-name.*") - - using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) - using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) - compilers_using_toolchain = ["pgi", "xl", "icpc"] - if any(compiler in cpp_compiler for compiler in compilers_using_toolchain): - if using_toolchain or using_gcc_name: - cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", - "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) - - if "+cuda" in spec: - cfg.write("#------------------{0}\n".format("-" * 60)) - cfg.write("# Cuda\n") - cfg.write("#------------------{0}\n\n".format("-" * 60)) - - cfg.write(cmake_cache_option("ENABLE_CUDA", True)) - - cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry("CUDA_TOOLKIT_ROOT_DIR", - cudatoolkitdir)) - cudacompiler = "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" - cfg.write(cmake_cache_entry("CMAKE_CUDA_COMPILER", - cudacompiler)) - - if not spec.satisfies('cuda_arch=none'): - cuda_arch = spec.variants['cuda_arch'].value - cuda_arch = "sm_{0}".format(cuda_arch[0]) - flag = '-arch {0}'.format(cuda_arch) - cfg.write(cmake_cache_string("CUDA_ARCH",cuda_arch)) - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS", flag)) - - else: - cfg.write(cmake_cache_option("ENABLE_CUDA", False)) - - if "+raja" in spec: - cfg.write("#------------------{0}\n".format("-" * 60)) - cfg.write("# RAJA\n") - cfg.write("#------------------{0}\n\n".format("-" * 60)) - - cfg.write(cmake_cache_option("CHAI_ENABLE_RAJA_PLUGIN", True)) - raja_dir = spec['raja'].prefix - cfg.write(cmake_cache_entry("RAJA_DIR", raja_dir)) - else: - cfg.write(cmake_cache_option("CHAI_ENABLE_RAJA_PLUGIN", False)) - - # shared vs static libs - cfg.write(cmake_cache_option("BUILD_SHARED_LIBS","+shared" in spec)) - - cfg.write(cmake_cache_entry("umpire_DIR",spec['umpire'].prefix)) - camp_conf_path = spec['camp'].prefix + "/lib/cmake/camp" - cfg.write(cmake_cache_entry("camp_DIR",camp_conf_path)) - - cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec)) - cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec)) - - ####################### - # Close and save - ####################### - cfg.write("\n") - cfg.close() - - print("OUT: host-config file {0}".format(host_config_path)) - - def cmake_args(self): - spec = self.spec - host_config_path = self._get_host_config_path(spec) - - options = [] - options.extend(['-C', host_config_path]) - - return options diff --git a/scripts/spack_packages/raja/package.py b/scripts/spack_packages/raja/package.py deleted file mode 100644 index 88812cd6..00000000 --- a/scripts/spack_packages/raja/package.py +++ /dev/null @@ -1,383 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - - -from spack import * - -import glob -import os -import socket - -from os import environ as env -from os.path import join as pjoin - -import re - -def cmake_cache_entry(name, value, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE PATH "%s")\n\n' % (name,value,comment) - - -def cmake_cache_string(name, string, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE STRING "%s")\n\n' % (name,string,comment) - - -def cmake_cache_option(name, boolean_value, comment=""): - """Generate a string for a cmake configuration option""" - - value = "ON" if boolean_value else "OFF" - return 'set(%s %s CACHE BOOL "%s")\n\n' % (name,value,comment) - - -def get_spec_path(spec, package_name, path_replacements = {}, use_bin = False) : - """Extracts the prefix path for the given spack package - path_replacements is a dictionary with string replacements for the path. - """ - - if not use_bin: - path = spec[package_name].prefix - else: - path = spec[package_name].prefix.bin - - path = os.path.realpath(path) - - for key in path_replacements: - path = path.replace(key, path_replacements[key]) - - return path - - -class Raja(CMakePackage, CudaPackage, ROCmPackage): - """RAJA Performance Portability Abstractions for C++ HPC Applications.""" - - homepage = "https://github.com/LLNL/RAJA" - git = "https://github.com/LLNL/RAJA.git" - tags = ['radiuss', 'e4s'] - - maintainers = ['davidbeckingsale'] - - version('develop', branch='develop', submodules='True') - version('main', branch='main', submodules='True') - version('2022.03.0', tag='v2022.03.0', submodules='True') - version('0.14.1', tag='v0.14.1', submodules="True") - version('0.14.0', tag='v0.14.0', submodules="True") - version('0.13.0', tag='v0.13.0', submodules="True") - version('0.12.1', tag='v0.12.1', submodules="True") - version('0.12.0', tag='v0.12.0', submodules="True") - version('0.11.0', tag='v0.11.0', submodules="True") - version('0.10.1', tag='v0.10.1', submodules="True") - version('0.10.0', tag='v0.10.0', submodules="True") - version('0.9.0', tag='v0.9.0', submodules="True") - version('0.8.0', tag='v0.8.0', submodules="True") - version('0.7.0', tag='v0.7.0', submodules="True") - version('0.6.0', tag='v0.6.0', submodules="True") - version('0.5.3', tag='v0.5.3', submodules="True") - version('0.5.2', tag='v0.5.2', submodules="True") - version('0.5.1', tag='v0.5.1', submodules="True") - version('0.5.0', tag='v0.5.0', submodules="True") - version('0.4.1', tag='v0.4.1', submodules="True") - version('0.4.0', tag='v0.4.0', submodules="True") - - variant('openmp', default=True, description='Build OpenMP backend') - variant('shared', default=False, description='Build Shared Libs') - variant('libcpp', default=False, description='Uses libc++ instead of libstdc++') - variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), - multi=False, description='Tests to run') - variant('desul', default=False, description='Build Desul Atomics backend') - - depends_on('cmake@3.9:', type='build') - - depends_on('blt@0.4.1', type='build', when='@main') - depends_on('blt@0.4.1:', type='build') - - depends_on('camp') - depends_on('camp@main') # TODO: remove this ASAP - depends_on('camp+rocm', when='+rocm') - depends_on('camp+openmp', when='+openmp') - for val in ROCmPackage.amdgpu_targets: - depends_on('camp amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) - - depends_on('camp+cuda', when='+cuda') - for sm_ in CudaPackage.cuda_arch_values: - depends_on('camp cuda_arch={0}'.format(sm_), - when='cuda_arch={0}'.format(sm_)) - - conflicts('+openmp', when='+rocm') - depends_on('rocprim', when='+rocm') - - phases = ['hostconfig', 'cmake', 'build', 'install'] - - def _get_sys_type(self, spec): - sys_type = str(spec.architecture) - # if on llnl systems, we can use the SYS_TYPE - if "SYS_TYPE" in env: - sys_type = env["SYS_TYPE"] - return sys_type - - def _get_host_config_path(self, spec): - var='' - if '+cuda' in spec: - var= '-'.join([var,'cuda']) - if '+libcpp' in spec: - var='-'.join([var,'libcpp']) - - host_config_path = "hc-%s-%s-%s%s-%s.cmake" % (socket.gethostname().rstrip('1234567890'), - self._get_sys_type(spec), - spec.compiler, - var, - spec.dag_hash()) - dest_dir = self.stage.source_path - host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) - return host_config_path - - def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): - """ - This method creates a 'host-config' file that specifies - all of the options used to configure and build Umpire. - - For more details about 'host-config' files see: - http://software.llnl.gov/conduit/building.html - - Note: - The `py_site_pkgs_dir` arg exists to allow a package that - subclasses this package provide a specific site packages - dir when calling this function. `py_site_pkgs_dir` should - be an absolute path or `None`. - - This is necessary because the spack `site_packages_dir` - var will not exist in the base class. For more details - on this issue see: https://github.com/spack/spack/issues/6261 - """ - - ####################### - # Compiler Info - ####################### - c_compiler = env["SPACK_CC"] - cpp_compiler = env["SPACK_CXX"] - - # Even though we don't have fortran code in our project we sometimes - # use the Fortran compiler to determine which libstdc++ to use - f_compiler = "" - if "SPACK_FC" in env.keys(): - # even if this is set, it may not exist - # do one more sanity check - if os.path.isfile(env["SPACK_FC"]): - f_compiler = env["SPACK_FC"] - - ####################################################################### - # By directly fetching the names of the actual compilers we appear - # to doing something evil here, but this is necessary to create a - # 'host config' file that works outside of the spack install env. - ####################################################################### - - sys_type = self._get_sys_type(spec) - - ############################################## - # Find and record what CMake is used - ############################################## - - cmake_exe = spec['cmake'].command.path - cmake_exe = os.path.realpath(cmake_exe) - - host_config_path = self._get_host_config_path(spec) - cfg = open(host_config_path, "w") - cfg.write("###################\n".format("#" * 60)) - cfg.write("# Generated host-config - Edit at own risk!\n") - cfg.write("###################\n".format("#" * 60)) - cfg.write("# Copyright 2016-22, Lawrence Livermore National Security, LLC\n") - cfg.write("# and RAJA project contributors. See the RAJA/LICENSE file\n") - cfg.write("# for details.\n") - cfg.write("#\n") - cfg.write("# SPDX-License-Identifier: (BSD-3-Clause) \n") - cfg.write("###################\n\n".format("#" * 60)) - - cfg.write("#------------------\n".format("-" * 60)) - cfg.write("# SYS_TYPE: {0}\n".format(sys_type)) - cfg.write("# Compiler Spec: {0}\n".format(spec.compiler)) - cfg.write("# CMake executable path: %s\n" % cmake_exe) - cfg.write("#------------------\n\n".format("-" * 60)) - - cfg.write(cmake_cache_string("CMAKE_BUILD_TYPE", spec.variants['build_type'].value)) - - ####################### - # Compiler Settings - ####################### - - cfg.write("#------------------\n".format("-" * 60)) - cfg.write("# Compilers\n") - cfg.write("#------------------\n\n".format("-" * 60)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) - - # use global spack compiler flags - cflags = ' '.join(spec.compiler_flags['cflags']) - if "+libcpp" in spec: - cflags += ' '.join([cflags,"-DGTEST_HAS_CXXABI_H_=0"]) - if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) - - cxxflags = ' '.join(spec.compiler_flags['cxxflags']) - if "+libcpp" in spec: - cxxflags += ' '.join([cxxflags,"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0"]) - if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) - - # TODO (bernede1@llnl.gov): Is this useful for RAJA? - if ("gfortran" in f_compiler) and ("clang" in cpp_compiler): - libdir = pjoin(os.path.dirname( - os.path.dirname(f_compiler)), "lib") - flags = "" - for _libpath in [libdir, libdir + "64"]: - if os.path.exists(_libpath): - flags += " -Wl,-rpath,{0}".format(_libpath) - description = ("Adds a missing libstdc++ rpath") - #if flags: - # cfg.write(cmake_cache_string("BLT_EXE_LINKER_FLAGS", flags, - # description)) - - gcc_toolchain_regex = re.compile("--gcc-toolchain=(.*)") - gcc_name_regex = re.compile(".*gcc-name.*") - - using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) - if(using_toolchain): - gcc_toolchain_path = gcc_toolchain_regex.match(using_toolchain[0]) - using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) - compilers_using_toolchain = ["pgi", "xl", "icpc"] - if any(compiler in cpp_compiler for compiler in compilers_using_toolchain): - if using_toolchain or using_gcc_name: - cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", - "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) - - compilers_using_cxx14 = ["intel-17", "intel-18", "xl"] - if any(compiler in cpp_compiler for compiler in compilers_using_cxx14): - cfg.write(cmake_cache_entry("BLT_CXX_STD", "c++14")) - - if "+cuda" in spec: - cfg.write("#------------------{0}\n".format("-" * 60)) - cfg.write("# Cuda\n") - cfg.write("#------------------{0}\n\n".format("-" * 60)) - - cfg.write(cmake_cache_option("ENABLE_CUDA", True)) - - cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry("CUDA_TOOLKIT_ROOT_DIR", - cudatoolkitdir)) - cudacompiler = "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" - cfg.write(cmake_cache_entry("CMAKE_CUDA_COMPILER", - cudacompiler)) - - if ("xl" in cpp_compiler): - cfg.write(cmake_cache_entry("CMAKE_CUDA_FLAGS", "-Xcompiler -O3 -Xcompiler -qxlcompatmacros -Xcompiler -qalias=noansi " + - "-Xcompiler -qsmp=omp -Xcompiler -qhot -Xcompiler -qnoeh -Xcompiler -qsuppress=1500-029 " + - "-Xcompiler -qsuppress=1500-036 -Xcompiler -qsuppress=1500-030")) - cuda_release_flags = "-O3" - cuda_reldebinf_flags = "-O3 -g" - cuda_debug_flags = "-O0 -g" - - cfg.write(cmake_cache_string("BLT_CXX_STD", "c++14")) - elif ("gcc" in cpp_compiler): - cuda_release_flags = "-O3 -Xcompiler -Ofast -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" - cuda_reldebinf_flags = "-O3 -g -Xcompiler -Ofast -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" - cuda_debug_flags = "-O0 -g -Xcompiler -O0 -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" - else: - cuda_release_flags = "-O3 -Xcompiler -Ofast -Xcompiler -finline-functions" - cuda_reldebinf_flags = "-O3 -g -Xcompiler -Ofast -Xcompiler -finline-functions" - cuda_debug_flags = "-O0 -g -Xcompiler -O0 -Xcompiler -finline-functions" - - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELEASE", cuda_release_flags)) - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELWITHDEBINFO", cuda_reldebinf_flags)) - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_DEBUG", cuda_debug_flags)) - - if not spec.satisfies('cuda_arch=none'): - cuda_arch = spec.variants['cuda_arch'].value - cfg.write(cmake_cache_string("CUDA_ARCH", 'sm_{0}'.format(cuda_arch[0]))) - - else: - cfg.write(cmake_cache_option("ENABLE_CUDA", False)) - - if "+rocm" in spec: - cfg.write("#------------------{0}\n".format("-" * 60)) - cfg.write("# HIP\n") - cfg.write("#------------------{0}\n\n".format("-" * 60)) - - cfg.write(cmake_cache_option("ENABLE_HIP", True)) - - hip_root = spec['hip'].prefix - rocm_root = hip_root + "/.." - hip_arch = spec.variants['amdgpu_target'].value - cfg.write(cmake_cache_entry("HIP_ROOT_DIR", - hip_root)) - # there is only one dir like this, but the version component is unknown - cfg.write( - cmake_cache_path( - "HIP_CLANG_INCLUDE_PATH", - glob.glob( - "{}/lib/clang/*/include".format(spec['llvm-amdgpu'].prefix) - )[0] - ) - ) - cfg.write(cmake_cache_entry("ROCM_ROOT_DIR", - rocm_root)) - cfg.write(cmake_cache_entry("HIP_PATH", - rocm_root + '/llvm/bin')) - cfg.write(cmake_cache_entry("CMAKE_HIP_ARCHITECTURES", hip_arch[0])) - - if ('%gcc' in spec) or (using_toolchain): - if ('%gcc' in spec): - gcc_bin = os.path.dirname(self.compiler.cxx) - gcc_prefix = join_path(gcc_bin, '..') - else: - gcc_prefix = gcc_toolchain_path.group(1) - cfg.write(cmake_cache_entry("HIP_CLANG_FLAGS", - "--gcc-toolchain={0}".format(gcc_prefix))) - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", - " -Wl,-rpath {}/lib64".format(gcc_prefix))) - - else: - cfg.write(cmake_cache_option("ENABLE_HIP", False)) - - cfg.write("#------------------{0}\n".format("-" * 60)) - cfg.write("# Other\n") - cfg.write("#------------------{0}\n\n".format("-" * 60)) - - cfg.write(cmake_cache_string("RAJA_RANGE_ALIGN", "4")) - cfg.write(cmake_cache_string("RAJA_RANGE_MIN_LENGTH", "32")) - cfg.write(cmake_cache_string("RAJA_DATA_ALIGN", "64")) - - cfg.write(cmake_cache_option("RAJA_HOST_CONFIG_LOADED", True)) - - # shared vs static libs - cfg.write(cmake_cache_option("BUILD_SHARED_LIBS","+shared" in spec)) - cfg.write(cmake_cache_option("ENABLE_OPENMP","+openmp" in spec)) - cfg.write(cmake_cache_option("RAJA_ENABLE_DESUL_ATOMICS","+desul" in spec)) - - if "+desul" in spec: - cfg.write(cmake_cache_string("BLT_CXX_STD","c++14")) - if "+cuda" in spec: - cfg.write(cmake_cache_string("CMAKE_CUDA_STANDARD", "14")) - - cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec)) - cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec or self.run_tests)) - cfg.write(cmake_cache_string("camp_DIR", spec['camp'].prefix)) - - ####################### - # Close and save - ####################### - cfg.write("\n") - cfg.close() - - print("OUT: host-config file {0}".format(host_config_path)) - - def cmake_args(self): - spec = self.spec - host_config_path = self._get_host_config_path(spec) - - options = [] - options.extend(['-C', host_config_path]) - - return options diff --git a/scripts/spack_packages/umpire/camp_target_umpire_3.0.0.patch b/scripts/spack_packages/umpire/camp_target_umpire_3.0.0.patch deleted file mode 100644 index 2deb565c..00000000 --- a/scripts/spack_packages/umpire/camp_target_umpire_3.0.0.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/umpire-config.cmake.in b/umpire-config.cmake.in -index a98ad5fe..4e54e173 100644 ---- a/umpire-config.cmake.in -+++ b/umpire-config.cmake.in -@@ -7,6 +7,13 @@ - get_filename_component(UMPIRE_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) - set(UMPIRE_INCLUDE_DIRS "@CMAKE_INSTALL_PREFIX@/include") - -+if (NOT TARGET camp) -+ if (NOT DEFINED camp_DIR) -+ set(camp_DIR @CMAKE_INSTALL_PREFIX@/lib/cmake/camp) -+ endif () -+ find_package(camp REQUIRED) -+endif () -+ - set(Umpire_VERSION_MAJOR @Umpire_VERSION_MAJOR@) - set(Umpire_VERSION_MINOR @Umpire_VERSION_MINOR@) - set(Umpire_VERSION_PATCH @Umpire_VERSION_PATCH@) diff --git a/scripts/spack_packages/umpire/package.py b/scripts/spack_packages/umpire/package.py deleted file mode 100644 index c3deab95..00000000 --- a/scripts/spack_packages/umpire/package.py +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -import glob - -from spack import * - -import socket -import os - -from os import environ as env -from os.path import join as pjoin - -import re - - -class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage): - """An application-focused API for memory management on NUMA & GPU - architectures""" - - homepage = 'https://github.com/LLNL/Umpire' - git = 'https://github.com/LLNL/Umpire.git' - - version('develop', branch='develop', submodules='True') - version('main', branch='main', submodules='True') - version('2022.03.0', tag='v2022.03.0', submodules=True) - version('6.0.0', tag='v6.0.0', submodules=True) - version('5.0.1', tag='v5.0.1', submodules=True) - version('5.0.0', tag='v5.0.0', submodules=True) - version('4.1.2', tag='v4.1.2', submodules=True) - version('4.1.1', tag='v4.1.1', submodules=True) - version('4.1.0', tag='v4.1.0', submodules=True) - version('3.0.0', tag='v3.0.0', submodules='True') - version('2.1.0', tag='v2.1.0', submodules='True') - version('2.0.0', tag='v2.0.0', submodules='True') - version('1.1.0', tag='v1.1.0', submodules='True') - version('1.0.1', tag='v1.0.1', submodules='True') - version('1.0.0', tag='v1.0.0', submodules='True') - version('0.3.5', tag='v0.3.5', submodules='True') - version('0.3.4', tag='v0.3.4', submodules='True') - version('0.3.3', tag='v0.3.3', submodules='True') - version('0.3.2', tag='v0.3.2', submodules='True') - version('0.3.1', tag='v0.3.1', submodules='True') - version('0.3.0', tag='v0.3.0', submodules='True') - version('0.2.4', tag='v0.2.4', submodules='True') - version('0.2.3', tag='v0.2.3', submodules='True') - version('0.2.2', tag='v0.2.2', submodules='True') - version('0.2.1', tag='v0.2.1', submodules='True') - version('0.2.0', tag='v0.2.0', submodules='True') - version('0.1.4', tag='v0.1.4', submodules='True') - version('0.1.3', tag='v0.1.3', submodules='True') - - patch('camp_target_umpire_3.0.0.patch', when='@3.0.0') - - variant('fortran', default=False, description='Build C/Fortran API') - variant('c', default=True, description='Build C API') - variant('mpi', default=False, description='Enable MPI support') - variant('ipc_shmem', default=False, description='Enable POSIX shared memory') - variant('sqlite_experimental', default=False, description='Enable sqlite integration with umpire events (Experimental)') - variant('numa', default=False, description='Enable NUMA support') - variant('shared', default=False, description='Enable Shared libs') - variant('openmp', default=False, description='Build with OpenMP support') - variant('openmp_target', default=False, description='Build with OpenMP 4.5 support') - variant('deviceconst', default=False, - description='Enables support for constant device memory') - variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), - multi=False, description='Tests to run') - - variant('libcpp', default=False, description='Uses libc++ instead of libstdc++') - variant('tools', default=True, description='Enable tools') - variant('backtrace', default=False, description='Enable backtrace tools') - variant('dev_benchmarks', default=False, description='Enable Developer Benchmarks') - variant('device_alloc', default=False, description='Enable the DeviceAllocator') - variant('werror', default=True, description='Enable warnings as errors') - variant('asan', default=False, description='Enable ASAN') - variant('sanitizer_tests', default=False, description='Enable address sanitizer tests') - - depends_on('cmake@3.14:', type='build') - depends_on('sqlite', when='+sqlite_experimental') - depends_on('mpi', when='+mpi') - - depends_on('blt@0.5.0', type='build', when='@main') - depends_on('blt@0.5.0:', type='build') - - # variants +rocm and amdgpu_targets are not automatically passed to - # dependencies, so do it manually. - depends_on('camp+rocm', when='+rocm') - for val in ROCmPackage.amdgpu_targets: - depends_on('camp amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) - - depends_on('camp+cuda', when='+cuda') - for sm_ in CudaPackage.cuda_arch_values: - depends_on('camp cuda_arch={0}'.format(sm_), - when='cuda_arch={0}'.format(sm_)) - - depends_on('camp@main') - - conflicts('+numa', when='@:0.3.2') - conflicts('~c', when='+fortran', msg='Fortran API requires C API') - conflicts('~openmp', when='+openmp_target', msg='OpenMP target requires OpenMP') - conflicts('+cuda', when='+rocm') - conflicts('+rocm', when='+openmp_target', msg='Cant support both rocm and openmp device backends at once') - conflicts('+deviceconst', when='~rocm~cuda') - conflicts('+device_alloc', when='~rocm~cuda') - conflicts('~mpi', when='+ipc_shmem', msg='Shared Memory Allocator requires MPI') - conflicts('+ipc_shmem', when='@:5.0.1') - conflicts('+sqlite_experimental', when='@:6.0.0') - conflicts('+sanitizer_tests', when='~asan') - - def _get_sys_type(self, spec): - sys_type = str(spec.architecture) - # if on llnl systems, we can use the SYS_TYPE - if "SYS_TYPE" in env: - sys_type = env["SYS_TYPE"] - return sys_type - - @property - def cache_name(self): - hostname = socket.gethostname() - if "SYS_TYPE" in env: - hostname = hostname.rstrip('1234567890') - return "{0}-{1}-{2}@{3}.cmake".format( - hostname, - self._get_sys_type(self.spec), - self.spec.compiler.name, - self.spec.compiler.version - ) - - def spec_uses_toolchain(self, spec): - gcc_toolchain_regex = re.compile(".*gcc-toolchain.*") - using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) - - return using_toolchain - - def spec_uses_gccname(self, spec): - gcc_name_regex = re.compile(".*gcc-name.*") - using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) - - return using_gcc_name - - - - def initconfig_compiler_entries(self): - spec = self.spec - entries = super(Umpire, self).initconfig_compiler_entries() - - entries.append(cmake_cache_option("ENABLE_FORTRAN", - ('+fortran' in spec) and (self.compiler.fc is not None))) - entries.append(cmake_cache_option("UMPIRE_ENABLE_C", '+c' in spec)) - - fortran_compilers = ["gfortran", "xlf"] - if any(compiler in self.compiler.fc for compiler in fortran_compilers) and ("clang" in self.compiler.cxx): - entries.append(cmake_cache_string("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", - "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) - - libdir = pjoin(os.path.dirname( - os.path.dirname(self.compiler.fc)), "lib") - flags = "" - for _libpath in [libdir, libdir + "64"]: - if os.path.exists(_libpath): - flags += " -Wl,-rpath,{0}".format(_libpath) - description = ("Adds a missing libstdc++ rpath") - if flags: - entries.append(cmake_cache_string("BLT_EXE_LINKER_FLAGS", flags, description)) - - - compilers_using_toolchain = ["pgi", "xl", "icpc"] - if any(compiler in self.compiler.cxx for compiler in compilers_using_toolchain): - if self.spec_uses_toolchain(self.spec) or self.spec_uses_gccname(self.spec): - entries.append(cmake_cache_string("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", - "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) - - entries = [x for x in entries if not 'COMPILER_ID' in x] - return entries - - - def initconfig_hardware_entries(self): - spec = self.spec - entries = super(Umpire, self).initconfig_hardware_entries() - - entries.append(cmake_cache_option("ENABLE_CUDA", "+cuda" in spec)) - if "+cuda" in spec: - cuda_flags = [] - if not spec.satisfies('cuda_arch=none'): - cuda_arch = spec.variants['cuda_arch'].value - cuda_flags.append('-arch sm_{0}'.format(cuda_arch[0])) - - if self.spec_uses_toolchain(self.spec): - cuda_flags.append("-Xcompiler {}".format(self.spec_uses_toolchain(self.spec)[0])) - - if (spec.satisfies('%gcc@8.1: target=ppc64le')): - cuda_flags.append('-Xcompiler -mno-float128') - - entries.append(cmake_cache_string("CMAKE_CUDA_FLAGS", ' '.join(cuda_flags))) - - entries.append(cmake_cache_option("ENABLE_HIP", "+rocm" in spec)) - if "+rocm" in spec: - hip_root = spec['hip'].prefix - rocm_root = hip_root + "/.." - hip_arch = spec.variants['amdgpu_target'].value - entries.append(cmake_cache_path("HIP_ROOT_DIR", - hip_root)) - entries.append(cmake_cache_path("ROCM_ROOT_DIR", - rocm_root)) - entries.append(cmake_cache_string("CMAKE_HIP_ARCHITECTURES", - hip_arch[0])) - entries.append(cmake_cache_option("UMPIRE_ENABLE_TOOLS", False)) - # there is only one dir like this, but the version component is unknown - - entries.append( - cmake_cache_path("HIP_CLANG_INCLUDE_PATH", glob.glob( - "{}/lib/clang/*/include".format(spec['llvm-amdgpu'].prefix) - )[0]) - ) - hip_link_flags = "" - if '%gcc' in spec: - gcc_bin = os.path.dirname(self.compiler.cxx) - gcc_prefix = join_path(gcc_bin, '..') - entries.append(cmake_cache_string("HIP_CLANG_FLAGS", "--gcc-toolchain={0}".format(gcc_prefix))) - entries.append(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", hip_link_flags + " -Wl,-rpath {}/lib64".format(gcc_prefix))) - else: - entries.append(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-Wl,-rpath={0}/llvm/lib/".format(rocm_root))) - - entries.append(cmake_cache_option("UMPIRE_ENABLE_DEVICE_CONST", "+deviceconst" in spec)) - - entries.append(cmake_cache_option("UMPIRE_ENABLE_OPENMP_TARGET", "+openmp_target" in spec)) - if "+openmp_target" in spec: - if ('%xl' in spec): - entries.append(cmake_cache_string("OpenMP_CXX_FLAGS", "-qsmp;-qoffload")) - - return entries - - - def initconfig_mpi_entries(self): - spec = self.spec - - entries = super(Umpire, self).initconfig_mpi_entries() - entries.append(cmake_cache_option("ENABLE_MPI", '+mpi' in spec)) - - return entries - - - def initconfig_package_entries(self): - spec = self.spec - entries = [] - - entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec['blt'].prefix)) - entries.append(cmake_cache_path("camp_DIR" ,spec['camp'].prefix)) - entries.append(cmake_cache_string("CMAKE_BUILD_TYPE", spec.variants['build_type'].value)) - entries.append(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec or '+dev_benchmarks' in spec)) - entries.append(cmake_cache_option("UMPIRE_ENABLE_DEVELOPER_BENCHMARKS", '+dev_benchmarks' in spec)) - entries.append(cmake_cache_option("UMPIRE_ENABLE_DEVICE_ALLOCATOR", '+device_alloc' in spec)) - entries.append(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec)) - entries.append(cmake_cache_option("UMPIRE_ENABLE_TOOLS", '+tools' in spec)) - entries.append(cmake_cache_option("UMPIRE_ENABLE_BACKTRACE", '+backtrace' in spec)) - entries.append(cmake_cache_option("ENABLE_WARNINGS_AS_ERRORS", '+werror' in spec)) - entries.append(cmake_cache_option("UMPIRE_ENABLE_ASAN", '+asan' in spec)) - entries.append(cmake_cache_option("BUILD_SHARED_LIBS", '+shared' in spec)) - entries.append(cmake_cache_option("UMPIRE_ENABLE_SANITIZER_TESTS", '+sanitizer_tests' in spec)) - entries.append(cmake_cache_option("UMPIRE_ENABLE_NUMA", '+numa' in spec)) - entries.append(cmake_cache_option("ENABLE_OPENMP", '+openmp' in spec)) - entries.append(cmake_cache_option("UMPIRE_ENABLE_IPC_SHARED_MEMORY", '+ipc_shmem' in spec)) - entries.append(cmake_cache_option("UMPIRE_ENABLE_SQLITE_EXPERIMENTAL", '+sqlite_experimental' in spec)) - if "+sqlite_experimental" in spec: - entries.append(cmake_cache_path("SQLite3_ROOT" ,spec['sqlite'].prefix)) - - return entries - - - def cmake_args(self): - spec = self.spec - options = [] - return options diff --git a/scripts/uberenv b/scripts/uberenv index 4941c237..0d00dc8e 160000 --- a/scripts/uberenv +++ b/scripts/uberenv @@ -1 +1 @@ -Subproject commit 4941c237eec514d6d68872243efb9f4af8843f4d +Subproject commit 0d00dc8e19a889ba07ae433590b87533c4b5b3da diff --git a/src/chai/ManagedArray.hpp b/src/chai/ManagedArray.hpp index e8c10ca3..ac6b709c 100644 --- a/src/chai/ManagedArray.hpp +++ b/src/chai/ManagedArray.hpp @@ -283,11 +283,11 @@ class ManagedArray : public CHAICopyable CHAI_HOST_DEVICE ManagedArray& operator=(std::nullptr_t); - CHAI_HOST_DEVICE bool operator==(ManagedArray& rhs) const; - CHAI_HOST_DEVICE bool operator!=(ManagedArray& from) const; + CHAI_HOST_DEVICE bool operator==(const ManagedArray& rhs) const; + CHAI_HOST_DEVICE bool operator!=(const ManagedArray& from) const; - CHAI_HOST_DEVICE bool operator==(T* from) const; - CHAI_HOST_DEVICE bool operator!=(T* from) const; + CHAI_HOST_DEVICE bool operator==(const T* from) const; + CHAI_HOST_DEVICE bool operator!=(const T* from) const; CHAI_HOST_DEVICE bool operator==(std::nullptr_t from) const; CHAI_HOST_DEVICE bool operator!=(std::nullptr_t from) const; diff --git a/src/chai/ManagedArray.inl b/src/chai/ManagedArray.inl index 99ed2f44..2f5f3445 100644 --- a/src/chai/ManagedArray.inl +++ b/src/chai/ManagedArray.inl @@ -406,10 +406,14 @@ void ManagedArray::move(ExecutionSpace space, bool registerTouch) const CHAI_LOG(Debug, "Moved to " << m_active_pointer); #if defined(CHAI_ENABLE_UM) if (m_pointer_record->m_last_space == UM) { + // just because we were allocated in UM doesn't mean our CHAICopyable array values were + moveInnerImpl(); } else #endif #if defined(CHAI_ENABLE_PINNED) if (m_pointer_record->m_last_space == PINNED) { + // just because we were allocated in PINNED doesn't mean our CHAICopyable array values were + moveInnerImpl(); } else #endif if (registerTouch) { @@ -613,7 +617,7 @@ template CHAI_INLINE CHAI_HOST_DEVICE bool -ManagedArray::operator== (ManagedArray& rhs) const +ManagedArray::operator== (const ManagedArray& rhs) const { return (m_active_pointer == rhs.m_active_pointer); } @@ -622,7 +626,7 @@ template CHAI_INLINE CHAI_HOST_DEVICE bool -ManagedArray::operator!= (ManagedArray& rhs) const +ManagedArray::operator!= (const ManagedArray& rhs) const { return (m_active_pointer != rhs.m_active_pointer); } @@ -632,7 +636,7 @@ template CHAI_INLINE CHAI_HOST_DEVICE bool -ManagedArray::operator== (T * from) const { +ManagedArray::operator== (const T * from) const { return m_active_pointer == from; } @@ -640,7 +644,7 @@ template CHAI_INLINE CHAI_HOST_DEVICE bool -ManagedArray::operator!= (T * from) const { +ManagedArray::operator!= (const T * from) const { return m_active_pointer != from; } diff --git a/src/chai/ManagedArray_thin.inl b/src/chai/ManagedArray_thin.inl index e74c6256..8de7bee1 100644 --- a/src/chai/ManagedArray_thin.inl +++ b/src/chai/ManagedArray_thin.inl @@ -358,27 +358,27 @@ CHAI_INLINE CHAI_HOST_DEVICE ManagedArray& ManagedArray::operator=(std::nu template CHAI_INLINE CHAI_HOST_DEVICE bool ManagedArray::operator==( - ManagedArray& rhs) const + const ManagedArray& rhs) const { return (m_active_pointer == rhs.m_active_pointer); } template CHAI_INLINE CHAI_HOST_DEVICE bool ManagedArray::operator!=( - ManagedArray& rhs) const + const ManagedArray& rhs) const { return (m_active_pointer != rhs.m_active_pointer); } template -CHAI_INLINE CHAI_HOST_DEVICE bool ManagedArray::operator==(T* from) const +CHAI_INLINE CHAI_HOST_DEVICE bool ManagedArray::operator==(const T* from) const { return m_active_pointer == from; } template -CHAI_INLINE CHAI_HOST_DEVICE bool ManagedArray::operator!=(T* from) const +CHAI_INLINE CHAI_HOST_DEVICE bool ManagedArray::operator!=(const T* from) const { return m_active_pointer != from; } diff --git a/src/tpl/raja b/src/tpl/raja index 5f3282c9..4351fe6a 160000 --- a/src/tpl/raja +++ b/src/tpl/raja @@ -1 +1 @@ -Subproject commit 5f3282c90a069915f76799bc337c9104f39c6d14 +Subproject commit 4351fe6a50bd579511a625b017c9e054885e7fd2 diff --git a/src/tpl/umpire b/src/tpl/umpire index 93b1441a..2db6224a 160000 --- a/src/tpl/umpire +++ b/src/tpl/umpire @@ -1 +1 @@ -Subproject commit 93b1441aaa258c1dcd211a552b75cff6461a2a8a +Subproject commit 2db6224ae0c3f3e0bbd6722e95c1167b7f79be7b diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index b29a5046..e551cf21 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -39,14 +39,14 @@ if (CHAI_ENABLE_MANAGED_PTR) NAME managed_ptr_test COMMAND managed_ptr_tests) endif () - + if (CHAI_ENABLE_RAJA_PLUGIN) set(raja_test_depends ${chai_integration_test_depends} RAJA) blt_add_executable( - NAME raja-chai-tests + NAME raja-chai-tests SOURCES raja-chai-tests.cpp DEPENDS_ON ${raja_test_depends}) @@ -59,8 +59,9 @@ if (CHAI_ENABLE_RAJA_PLUGIN) PUBLIC ${PROJECT_BINARY_DIR}/include) if (CHAI_ENABLE_RAJA_NESTED_TEST) +#raja kernel tests blt_add_executable( - NAME raja-chai-nested-tests + NAME raja-chai-nested-tests SOURCES raja-chai-nested.cpp DEPENDS_ON ${raja_test_depends}) @@ -71,5 +72,21 @@ if (CHAI_ENABLE_RAJA_PLUGIN) target_include_directories( raja-chai-nested-tests PUBLIC ${PROJECT_BINARY_DIR}/include) + +#raja launch tests + blt_add_executable( + NAME raja-chai-launch-tests + SOURCES raja-chai-launch.cpp + DEPENDS_ON ${raja_test_depends}) + + blt_add_test( + NAME raja-chai-launch-tests + COMMAND raja-chai-launch-tests) + + target_include_directories( + raja-chai-launch-tests + PUBLIC ${PROJECT_BINARY_DIR}/include) + endif () + endif () diff --git a/tests/integration/raja-chai-launch.cpp b/tests/integration/raja-chai-launch.cpp new file mode 100644 index 00000000..13dfcf77 --- /dev/null +++ b/tests/integration/raja-chai-launch.cpp @@ -0,0 +1,409 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright (c) 2016-20, Lawrence Livermore National Security, LLC and CHAI +// project contributors. See the COPYRIGHT file for details. +// +// SPDX-License-Identifier: BSD-3-Clause +////////////////////////////////////////////////////////////////////////////// +/// +/// Source file containing tests for CHAI in RAJA nested loops. +/// +/// +#include +#include +#include + +#include +#include +#include + +#include "RAJA/RAJA.hpp" + +using namespace RAJA; +using namespace std; + +#include "chai/ArrayManager.hpp" +#include "chai/ManagedArrayView.hpp" +#include "chai/ManagedArray.hpp" + +#include "gtest/gtest.h" + +// TODO: add hip policy for these tests. +#if defined(RAJA_ENABLE_CUDA) +#define PARALLEL_RAJA_DEVICE __device__ +#elif defined(RAJA_ENABLE_OPENMP) +#define PARALLEL_RAJA_DEVICE +#else +#define PARALLEL_RAJA_DEVICE +#endif + +#define CUDA_TEST(X, Y) \ + static void cuda_test_##X##_##Y(); \ + TEST(X, Y) { cuda_test_##X##_##Y(); } \ + static void cuda_test_##X##_##Y() + +/* + * Simple tests using nested::forall and View + */ +CUDA_TEST(Chai, LaunchSimple) +{ + + using LAUNCH_POLICY = RAJA::LaunchPolicy; + using LOOP_POLICY_0 = RAJA::LoopPolicy; + using LOOP_POLICY_1 = RAJA::LoopPolicy; + +#if defined(RAJA_ENABLE_CUDA) + + const bool async = false; + using PARALLEL_LAUNCH_POLICY = RAJA::LaunchPolicy>; + using PARALLEL_LOOP_POLICY_0 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_1 = RAJA::LoopPolicy; + +#elif defined(RAJA_ENABLE_OPENMP) + + using PARALLEL_LAUNCH_POLICY = RAJA::LaunchPolicy; + using PARALLEL_LOOP_POLICY_0 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_1 = RAJA::LoopPolicy; + +#else + using PARALLEL_LAUNCH_POLICY = LAUNCH_POLICY; + using PARALLEL_LOOP_POLICY_0 = LOOP_POLICY_0; + using PARALLEL_LOOP_POLICY_1 = LOOP_POLICY_1; +#endif + + const int X = 16; + const int Y = 16; + + chai::ManagedArray v1(X * Y); + chai::ManagedArray v2(X * Y); + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::RangeSegment(0, X), [&] (int i) { + RAJA::loop(ctx, RAJA::RangeSegment(0, Y), [&] (int j) { + int index = j * X + i; + v1[index] = index; + }); + }); + }); + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::RangeSegment(0, X), [&] (int i) { + RAJA::loop(ctx, RAJA::RangeSegment(0, Y), [&] (int j) { + int index = j * X + i; + v2[index] = v1[index] * 2.0f; + }); + }); + }); + + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::RangeSegment(0, X), [&] (int i) { + RAJA::loop(ctx, RAJA::RangeSegment(0, Y), [&] (int j) { + int index = j * X + i; + ASSERT_FLOAT_EQ(v1[index], index * 1.0f); + ASSERT_FLOAT_EQ(v2[index], index * 2.0f); + }); + }); + }); + + v1.free(); + v2.free(); +} + +CUDA_TEST(Chai, LaunchView) +{ + + using LAUNCH_POLICY = RAJA::LaunchPolicy; + using LOOP_POLICY_0 = RAJA::LoopPolicy; + using LOOP_POLICY_1 = RAJA::LoopPolicy; + +#if defined(RAJA_ENABLE_CUDA) + + const bool async = false; + using PARALLEL_LAUNCH_POLICY = RAJA::LaunchPolicy>; + using PARALLEL_LOOP_POLICY_0 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_1 = RAJA::LoopPolicy; + +#elif defined(RAJA_ENABLE_OPENMP) + + using PARALLEL_LAUNCH_POLICY = RAJA::LaunchPolicy; + using PARALLEL_LOOP_POLICY_0 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_1 = RAJA::LoopPolicy; + +#else + + using PARALLEL_LAUNCH_POLICY = LAUNCH_POLICY; + using PARALLEL_LOOP_POLICY_0 = LOOP_POLICY_0; + using PARALLEL_LOOP_POLICY_1 = LOOP_POLICY_1; +#endif + + const int X = 16; + const int Y = 16; + + chai::ManagedArray v1_array(X * Y); + chai::ManagedArray v2_array(X * Y); + + using view = chai::ManagedArrayView>; + + view v1(v1_array, X, Y); + view v2(v2_array, X, Y); + + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::RangeSegment(0, X), [&] (int i) { + RAJA::loop(ctx, RAJA::RangeSegment(0, Y), [&] (int j) { + v1(i, j) = (i + (j * X)) * 1.0f; + }); + }); + }); + + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::RangeSegment(0, X), [&] (int i) { + RAJA::loop(ctx, RAJA::RangeSegment(0, Y), [&] (int j) { + v2(i, j) = v1(i, j) * 2.0f; + }); + }); + }); + + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::RangeSegment(0, X), [&] (int i) { + RAJA::loop(ctx, RAJA::RangeSegment(0, Y), [&] (int j) { + ASSERT_FLOAT_EQ(v2(i, j), v1(i, j) * 2.0f); + }); + }); + }); + + v1_array.free(); + v2_array.free(); +} + +CUDA_TEST(Chai, LaunchMultiView) +{ + + using LAUNCH_POLICY = RAJA::LaunchPolicy; + using LOOP_POLICY_0 = RAJA::LoopPolicy; + using LOOP_POLICY_1 = RAJA::LoopPolicy; + +#if defined(RAJA_ENABLE_CUDA) + + const bool async = false; + using PARALLEL_LAUNCH_POLICY = RAJA::LaunchPolicy>; + using PARALLEL_LOOP_POLICY_0 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_1 = RAJA::LoopPolicy; + +#elif defined(RAJA_ENABLE_OPENMP) + + using PARALLEL_LAUNCH_POLICY = RAJA::LaunchPolicy; + using PARALLEL_LOOP_POLICY_0 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_1 = RAJA::LoopPolicy; +#else + + using PARALLEL_LAUNCH_POLICY = LAUNCH_POLICY; + using PARALLEL_LOOP_POLICY_0 = LOOP_POLICY_0; + using PARALLEL_LOOP_POLICY_1 = LOOP_POLICY_1; +#endif + + const int X = 16; + const int Y = 16; + + chai::ManagedArray v1_array(X * Y); + chai::ManagedArray v2_array(X * Y); + + chai::ManagedArray all_arrays[2]; + all_arrays[0] = v1_array; + all_arrays[1] = v2_array; + + // default MultiView + using view = chai::ManagedArrayMultiView>; + view mview(all_arrays, RAJA::Layout<2>(X, Y)); + + // MultiView with index in 1st position + using view1p = chai::ManagedArrayMultiView, 1>; + view1p mview1p(all_arrays, RAJA::Layout<2>(X, Y)); + + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::RangeSegment(0, X), [&] (int i) { + RAJA::loop(ctx, RAJA::RangeSegment(0, Y), [&] (int j) { + mview(0, i, j) = (i + (j * X)) * 1.0f; + }); + }); + }); + + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::RangeSegment(0, X), [&] (int i) { + RAJA::loop(ctx, RAJA::RangeSegment(0, Y), [&] (int j) { + // use both MultiViews + mview(1, i, j) = mview1p(i, 0, j) * 2.0f; + }); + }); + }); + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::RangeSegment(0, X), [&] (int i) { + RAJA::loop(ctx, RAJA::RangeSegment(0, Y), [&] (int j) { + ASSERT_FLOAT_EQ(mview(1, i, j), mview(0, i, j) * 2.0f); + }); + }); + }); + + + v1_array.free(); + v2_array.free(); +} + +/////////////////////////////////////////////////////////////////////////// +// +// Example LTimes kernel test routines +// +// Demonstrates a 4-nested loop, the use of complex nested policies and +// the use of strongly-typed indices +// +// This routine computes phi(m, g, z) = SUM_d { ell(m, d)*psi(d,g,z) } +// +/////////////////////////////////////////////////////////////////////////// + +RAJA_INDEX_VALUE_T(IM, int, "IM"); +RAJA_INDEX_VALUE_T(ID, int, "ID"); +RAJA_INDEX_VALUE_T(IG, int, "IG"); +RAJA_INDEX_VALUE_T(IZ, int, "IZ"); + +void runLTimesTests(Index_type num_moments, + Index_type num_directions, + Index_type num_groups, + Index_type num_zones) +{ + // allocate data + // phi is initialized to all zeros, the others are randomized + chai::ManagedArray L_data(num_moments * num_directions); + chai::ManagedArray psi_data(num_directions * num_groups * num_zones); + chai::ManagedArray phi_data(num_moments * num_groups * num_zones); + + RAJA::forall( + RAJA::RangeSegment(0, (num_moments * num_directions)), + [=](int i) { + L_data[i] = i+2; + }); + + RAJA::forall( + RAJA::RangeSegment(0, (num_directions * num_groups * num_zones)), + [=](int i) { psi_data[i] = 2*i+1; }); + + RAJA::forall( + RAJA::RangeSegment(0, (num_moments * num_groups * num_zones)), + [=](int i) { phi_data[i] = 0.0; }); + + using LView = chai::TypedManagedArrayView, IM, ID>; + + // psi(d, g, z) : 2 -> z is stride-1 dimension + using PsiView = chai::TypedManagedArrayView, ID, IG, IZ>; + + // phi(m, g, z) : 2 -> z is stride-1 dimension + using PhiView = chai::TypedManagedArrayView, IM, IG, IZ>; + + std::array L_perm {{0, 1}}; + LView L(L_data, + RAJA::make_permuted_layout({{num_moments, num_directions}}, L_perm)); + + std::array psi_perm {{0, 1, 2}}; + PsiView psi(psi_data, + RAJA::make_permuted_layout({{num_directions, num_groups, num_zones}}, psi_perm)); + + std::array phi_perm {{0, 1, 2}}; + PhiView phi(phi_data, + RAJA::make_permuted_layout({{num_moments, num_groups, num_zones}}, phi_perm)); + +#if defined(RAJA_ENABLE_CUDA) + const bool async = false; + using PARALLEL_LAUNCH_POLICY = RAJA::LaunchPolicy>; + using PARALLEL_LOOP_POLICY_0 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_1 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_2 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_3 = RAJA::LoopPolicy; + +#elif defined(RAJA_ENABLE_OPENMP) + using PARALLEL_LAUNCH_POLICY = RAJA::LaunchPolicy; + using PARALLEL_LOOP_POLICY_0 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_1 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_2 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_3 = RAJA::LoopPolicy; +#else + using PARALLEL_LAUNCH_POLICY = RAJA::LaunchPolicy; + using PARALLEL_LOOP_POLICY_0 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_1 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_2 = RAJA::LoopPolicy; + using PARALLEL_LOOP_POLICY_3 = RAJA::LoopPolicy; +#endif + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(10)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + RAJA::loop(ctx, RAJA::TypedRangeSegment(0, num_moments), [&] (IM m) { + RAJA::loop(ctx, RAJA::TypedRangeSegment(0, num_groups), [&] (IG g) { + RAJA::loop(ctx, RAJA::TypedRangeSegment(0, num_zones ), [&] (IZ z) { + RAJA::loop(ctx, RAJA::TypedRangeSegment(0, num_directions ), [&] (ID d) { + phi(m, g, z) += L(m, d) * psi(d, g, z); + }); + }); + }); + }); + + }); + + RAJA::forall( + RAJA::TypedRangeSegment(0, num_moments), [=] (IM m) { + for (IG g(0); g < num_groups; ++g) { + for (IZ z(0); z < num_zones; ++z) { + double total = 0.0; + for (ID d(0); d < num_directions; ++d) { + double val = L(m, d) * psi(d, g, z); + total += val; + } + ASSERT_FLOAT_EQ(total, phi(m, g, z)); + } + } + }); + + L_data.free(); + psi_data.free(); + phi_data.free(); +} + +TEST(Chai, LaunchLTimes) +{ + // runLTimesTests(2, 0, 7, 3); + runLTimesTests(2, 3, 7, 3); + runLTimesTests(2, 3, 32, 4); + runLTimesTests(25, 96, 8, 32); + runLTimesTests(100, 15, 7, 13); +}