#===============================================================================
# Copyright 2020-2022 Intel Corporation.
#
# This software and the related documents are Intel copyrighted  materials,  and
# your use of  them is  governed by the  express license  under which  they were
# provided to you (License).  Unless the License provides otherwise, you may not
# use, modify, copy, publish, distribute,  disclose or transmit this software or
# the related documents without Intel's prior written permission.
#
# This software and the related documents  are provided as  is,  with no express
# or implied  warranties,  other  than those  that are  expressly stated  in the
# License.
#===============================================================================

# This file shows how to use specific cluster targets MKL::MKL_CDFT, MKL::MKL_SCALAPACK, and MKL::MKL_BLACS.
# See f_mpi/CMakeLists.txt for how to enable cluster libraries in the MKL::MKL target.

cmake_minimum_required(VERSION 3.13)
enable_testing()

# Add cmake scripts and modules to CMake search path
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../cmake")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")

# Generate domainList and ${domain}_funcList
include(generate_examples_list)

# Allow MPI scripts as compiler for examples
set(ENABLE_MPI ON)

# Define language and compiler
set(TEST_LANG C)
set(TEST_EXT c)
set(DATA_EXT dat)
include(setup_examples)

project(MKL_Examples LANGUAGES ${TEST_LANG})

find_package(MKL CONFIG REQUIRED)

# W/A for known problem in Intel(R) MPI Library 2021.1
if(MKL_MPI STREQUAL "intelmpi" AND DEFINED ENV{I_MPI_ROOT})
  if(UNIX AND $ENV{I_MPI_ROOT} MATCHES "2021.1")
    set(MPI_C_ADDITIONAL_INCLUDE_DIRS $ENV{I_MPI_ROOT}/include)
  endif()
endif()

# Force mshpc to be the first in mpi find package
if(MKL_MPI STREQUAL "msmpi" OR MKL_MPI STREQUAL "mshpc")
  set(MPI_GUESS_LIBRARY_NAME MSMPI)
endif()

find_package(MPI REQUIRED)

# Override default compile/link lines for mpi script
if(USE_MPI_SCRIPT AND WIN32)
  set(CMAKE_C_COMPILE_OBJECT "<CMAKE_C_COMPILER> <DEFINES> <INCLUDES> <FLAGS> /Fo<OBJECT> -c <SOURCE>")
  set(CMAKE_C_LINK_EXECUTABLE "<CMAKE_C_COMPILER> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
endif()

# Try to identify MPI for correct BLACS if MKL_MPI is not defined
if(MPI_${TEST_LANG}_FOUND AND NOT MKL_MPI)
  if(UNIX AND MPI_${TEST_LANG}_COMPILER MATCHES "openmpi")
    set(CURRENT_MPI "openmpi")
  elseif(WIN32 AND MPI_${TEST_LANG}_COMPILER MATCHES "msmpi" OR MPI_${TEST_LANG}_COMPILER MATCHES "Microsoft MPI")
    set(CURRENT_MPI "msmpi")
  elseif(MPI_${TEST_LANG}_COMPILER MATCHES "mpich")
    set(CURRENT_MPI "mpich")
  elseif(MPI_${TEST_LANG}_COMPILER MATCHES ".*intel.*mpi.*")
    set(CURRENT_MPI "intelmpi")
    if(WIN32 AND MPI_LOCALONLY)
      set(MPI_LOCAL_OPT -localonly)
    endif()
  endif()
  set(MKL_MPI "${CURRENT_MPI}")
endif()

# Define target for each function from each domain
if(domainList)
foreach(domain IN LISTS domainList)
  set(TEST_INCLUDE "${MPI_${TEST_LANG}_INCLUDE_DIRS}")
  set(TEST_COPT "")
  set(TEST_LOPT "")

  list(APPEND TEST_INCLUDE "${PROJECT_SOURCE_DIR}/${domain}/sources")

  # Suppress CRT warning on Windows
  if(WIN32)
    list(APPEND TEST_COPT "-D_CRT_SECURE_NO_WARNINGS")
  endif()

  if(domain STREQUAL "cdft")
    add_library(cdft_support OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/cdft/source/cdft_example_support.c)
    target_include_directories(cdft_support PUBLIC ${TEST_INCLUDE} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    target_compile_options(cdft_support PUBLIC ${TEST_COPT} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    list(APPEND TEST_LOPT cdft_support)
  endif()
  if(domain STREQUAL "fftw3x_cdft")
    set(CMAKE_C_STANDARD 99)
    set(CMAKE_C_STANDARD_REQUIRED TRUE)
    set(libname ${domain}_${MKL_INTERFACE})
    set(objname ${domain}_${MKL_INTERFACE}_obj)
    file(GLOB WRAPPERS_SRC ${MKL_ROOT}/share/mkl/interfaces/${domain}/wrappers/*)
    add_library(${libname})
    add_library(${objname}_single OBJECT ${WRAPPERS_SRC})
    add_library(${objname}_double OBJECT ${WRAPPERS_SRC})
    target_compile_options(${objname}_single PRIVATE -DMKL_SINGLE
      $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    target_compile_options(${objname}_double PRIVATE -DMKL_DOUBLE
      $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    target_include_directories(${objname}_single PRIVATE ${TEST_INCLUDE} ${MKL_INCLUDE}/fftw
        $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    target_include_directories(${objname}_double PRIVATE ${TEST_INCLUDE} ${MKL_INCLUDE}/fftw
        $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    target_link_libraries(${libname} ${objname}_single ${objname}_double)
    list(APPEND TEST_LOPT ${libname})
    list(APPEND TEST_INCLUDE ${MKL_INCLUDE}/fftw)
  endif()

  # Build target for each example
  message(STATUS "Functions list ${domain}: ${${domain}_funcList}")
  foreach(func IN LISTS ${domain}_funcList)
    string(STRIP ${func} func_name)
    set(func "${domain}-${func_name}")

    file(GLOB_RECURSE ${domain}_${func}_SRC ${PROJECT_SOURCE_DIR}/${domain}/*/${func_name}.${TEST_EXT})
    if(NOT ${domain}_${func}_SRC)
      message(FATAL_ERROR "${domain} source file ${func_name}.${TEST_EXT} was not found")
    endif()

    add_executable(${func} ${${domain}_${func}_SRC})
    target_include_directories(${func} PUBLIC ${TEST_INCLUDE})
    target_compile_options(${func} PUBLIC ${TEST_COPT})

    if(domain STREQUAL "cdft" OR domain STREQUAL "fftw3x_cdft")
      # MKL::MKL_CDFT = MKL_CDFT + MKL::MKL + MKL_BLACS
      target_link_libraries(${func} PUBLIC ${TEST_LOPT} MKL::MKL_CDFT MPI::MPI_C)
    else()
      if(domain STREQUAL "scalapack" OR domain STREQUAL "pblas")
        # MKL::MKL_SCALAPACK = MKL_SCALAPACK + MKL::MKL + MKL_BLACS
        target_link_libraries(${func} PUBLIC ${TEST_LOPT} MKL::MKL_SCALAPACK MPI::MPI_C)
      else()
        # BLACS required for all cluster functionality
        # MKL::MKL_BLACS = MKL::MKL + MKL_BLACS
        target_link_libraries(${func} PUBLIC ${TEST_LOPT} MKL::MKL_BLACS MPI::MPI_C)
      endif()
    endif()

    # Register example as ctest
    if(domain STREQUAL "pblas" OR domain STREQUAL "scalapack")
      configure_file(${PROJECT_SOURCE_DIR}/${domain}/data/${func_name}.in ${CMAKE_CURRENT_BINARY_DIR} COPYONLY)
    endif()
    file(GLOB_RECURSE ${domain}_${func_name}_DATA ${PROJECT_SOURCE_DIR}/${domain}/*/${func_name}.${DATA_EXT})
    if(EXISTS ${${domain}_${func_name}_DATA})
      add_test(NAME ${func} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPI_LOCAL_OPT} ${PROJECT_BINARY_DIR}/${func} ${${domain}_${func_name}_DATA} )
    else()
      add_test(NAME ${func} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPI_LOCAL_OPT} ${PROJECT_BINARY_DIR}/${func})
    endif()

    # Add Environment variables
    if(MKL_ENV)
      set_tests_properties(${func} PROPERTIES ENVIRONMENT "${MKL_ENV}")
    endif()
  endforeach() #${domain}_funcList
endforeach() #domainList
endif() #not empty domainList
