Merge pull request #421 from ohearnk/hip-f-func-porting-3

HIP and MPI+HIP Updates Part 3

Author: ohearnk

quick-cmake/QUICKCudaConfig.cmake

@@ -406,13 +406,13 @@ if(HIP)
     #  check ROCm version (as reported by hipcc),
     #  as the QUICK HIP codes trigger a known scalar register fill/spill bug
     #  in several ROCm versions
-    if (${HIP_VERSION} VERSION_GREATER_EQUAL 5.4.3)
+    if ((${HIP_VERSION} VERSION_GREATER_EQUAL 5.4.3) AND (${HIP_VERSION} VERSION_LESS 6.2.1))
         message(STATUS "")
         message("************************************************************")
 	message("Error: Incompatible ROCm/HIP version: ${HIP_VERSION}")
         message("  The QUICK HIP codes trigger a known compiler scalar register ")
-        message("  fill/spill bug in ROCm >= v5.4.3.")
-        message("  Please build QUICK with a known working ROCm version.")
+	message("  fill/spill bug in ROCm (>= v5.4.3, < v6.2.1).")
+        message("  Please build QUICK with a tested working ROCm version.")
         message("************************************************************")
         message(STATUS "")
         message(FATAL_ERROR)

src/gpu/cuda/gpu.cu

@@ -345,6 +345,15 @@ extern "C" void gpu_init_device_(int* ierr)
     status = cudaGetDeviceProperties(&deviceProp, device);
     PRINTERROR(status, "cudaGetDeviceProperties gpu_init failed!");
 
+#if defined(HIP) || defined(HIP_MPIV)
+    cudaDeviceSetCacheConfig(cudaFuncCachePreferL1);
+    /* NOTE: setting the stack size limit to 8K is required for correctness
+     * in HIP/MPI+HIP codes to workaround GPU kernel issues for recent ROCm versions (>= v6.2.1);
+     * ideally, this could be dropped in the future if ROCm properly addresses
+     * this issues internally */
+    cudaDeviceSetLimit(cudaLimitStackSize, 8192);
+#endif
+
 #if defined(DEBUG)
     size_t val;
 

src/gpu/cuda/mgpu.h

@@ -83,6 +83,15 @@ extern "C" void mgpu_init_device_(int *mpirank, int *mpisize, int *device, int*
     status = cudaGetDeviceProperties(&deviceProp, gpu->gpu_dev_id);
     PRINTERROR(status, "cudaGetDeviceProperties gpu_init failed!");
 
+#if defined(HIP) || defined(HIP_MPIV)
+    cudaDeviceSetCacheConfig(cudaFuncCachePreferL1);
+    /* NOTE: setting the stack size limit to 8K is required for correctness
+     * in HIP/MPI+HIP codes to workaround GPU kernel issues for recent ROCm versions (>= v6.2.1);
+     * ideally, this could be dropped in the future if ROCm properly addresses
+     * this issues internally */
+    cudaDeviceSetLimit(cudaLimitStackSize, 8192);
+#endif
+
     size_t val;
     cudaDeviceGetLimit(&val, cudaLimitStackSize);
 #ifdef DEBUG

src/gpu/hip/gpu.cu

@@ -345,6 +345,15 @@ extern "C" void gpu_init_device_(int* ierr)
     status = hipGetDeviceProperties(&deviceProp, device);
     PRINTERROR(status, "hipGetDeviceProperties gpu_init failed!");
 
+#if defined(HIP) || defined(HIP_MPIV)
+    hipDeviceSetCacheConfig(hipFuncCachePreferL1);
+    /* NOTE: setting the stack size limit to 8K is required for correctness
+     * in HIP/MPI+HIP codes to workaround GPU kernel issues for recent ROCm versions (>= v6.2.1);
+     * ideally, this could be dropped in the future if ROCm properly addresses
+     * this issues internally */
+    hipDeviceSetLimit(hipLimitStackSize, 8192);
+#endif
+
 #if defined(DEBUG)
     size_t val;
 

src/gpu/hip/mgpu.h

@@ -83,6 +83,15 @@ extern "C" void mgpu_init_device_(int *mpirank, int *mpisize, int *device, int*
     status = hipGetDeviceProperties(&deviceProp, gpu->gpu_dev_id);
     PRINTERROR(status, "hipGetDeviceProperties gpu_init failed!");
 
+#if defined(HIP) || defined(HIP_MPIV)
+    hipDeviceSetCacheConfig(hipFuncCachePreferL1);
+    /* NOTE: setting the stack size limit to 8K is required for correctness
+     * in HIP/MPI+HIP codes to workaround GPU kernel issues for recent ROCm versions (>= v6.2.1);
+     * ideally, this could be dropped in the future if ROCm properly addresses
+     * this issues internally */
+    hipDeviceSetLimit(hipLimitStackSize, 8192);
+#endif
+
     size_t val;
     hipDeviceGetLimit(&val, hipLimitStackSize);
 #ifdef DEBUG