#
# Copyright 2023-2025 NVIDIA Corporation.  All rights reserved.
#
# NOTICE TO LICENSEE:
#
# This source code and/or documentation ("Licensed Deliverables") are
# subject to NVIDIA intellectual property rights under U.S. and
# international Copyright laws.
#
# These Licensed Deliverables contained herein is PROPRIETARY and
# CONFIDENTIAL to NVIDIA and is being provided under the terms and
# conditions of a form of NVIDIA software license agreement by and
# between NVIDIA and Licensee ("License Agreement") or electronically
# accepted by Licensee.  Notwithstanding any terms or conditions to
# the contrary in the License Agreement, reproduction or disclosure
# of the Licensed Deliverables to any third party without the express
# written consent of NVIDIA is prohibited.
#
# NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
# LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
# SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
# PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
# NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
# DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
# NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
# NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
# LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
# SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
# OF THESE LICENSED DELIVERABLES.
#
# U.S. Government End Users.  These Licensed Deliverables are a
# "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
# 1995), consisting of "commercial computer software" and "commercial
# computer software documentation" as such terms are used in 48
# C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
# only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
# 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
# U.S. Government End Users acquire the Licensed Deliverables with
# only those rights set forth herein.
#
# Any use of the Licensed Deliverables in individual and commercial
# software must include, in the user documentation and internal
# comments to the code, the above Disclaimer and U.S. Government End
# Users Notice.
#

#
# cuDSS: Example CMake Project
#

cmake_minimum_required(VERSION 3.12)

project(cudss_examples LANGUAGES C CUDA CXX)

if ("${CMAKE_BUILD_TYPE}" STREQUAL "")
    set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type: Release, Debug, etc")
endif()

set(CMAKE_C_STANDARD 99)

option(BUILD_STATIC "Building cuDSS examples with static linking" ON)
option(BUILD_STANDARD  "Building standard cuDSS examples (without MGMN mode)" ON)
option(BUILD_MGMN   "Building cuDSS examples for the MGMN mode" OFF)
option(BUILD_MT     "Building cuDSS examples for the MT mode" ON)
set(MGMN_NPROC 1 CACHE STRING  "Number of MPI processes the MGMN mode examples")
set(MPIRUN_EXECUTABLE "mpirun" CACHE STRING "MPI executable (mpirun) for launching example binaries")
set(MPIRUN_NUMPROC_FLAG "-np" CACHE STRING "MPI flag for setting the number of processes")
set(MPIRUN_EXTRA_FLAGS "--allow-run-as-root" CACHE STRING "Extra flags to be passed to the MPI executable")
option(BUILD_MGMN_WITH_OPENMPI "Enable OpenMPI backend for the MGMN mode examples" ON)
option(BUILD_MGMN_WITH_NCCL    "Enable NCCL backend for the MGMN mode examples" ON)
option(BUILD_MT_WITH_OPENMP "Enable OpenMP backend for the MT mode examples" ON)
set(MTLAYER_NAME CACHE STRING "Threading layer library name for cuDSS (if not set, read from env via CUDSS_THREADING_LIB)")

if(BUILD_MGMN AND BUILD_MGMN_WITH_OPENMPI AND
    (NOT DEFINED OPENMPI_INCLUDE_DIRECTORIES OR
     NOT DEFINED OPENMPI_LINK_DIRECTORIES))
    message(FATAL_ERROR "For building MGMN examples with OpenMPI, path to \
OpenMPI headers and libraries must be set in \
OPENMPI_INCLUDE_DIRECTORIES and OPENMPI_LINK_DIRECTORIES" )
endif()

if(BUILD_MGMN AND BUILD_MGMN_WITH_NCCL AND
    (NOT DEFINED NCCL_INCLUDE_DIRECTORIES OR
     NOT DEFINED NCCL_LINK_DIRECTORIES))
    message(FATAL_ERROR "For building MGMN examples with NCCL, path to \
NCCL headers and libraries must be set in \
NCCL_INCLUDE_DIRECTORIES and NCCL_LINK_DIRECTORIES" )
endif()

# Find cuDSS
find_package(cudss REQUIRED) # COMPONENTS cudss)

if (BUILD_MT AND BUILD_MT_WITH_OPENMP)
    find_package(OpenMP REQUIRED)
    if (NOT OpenMP_CXX_FOUND)
        message(FATAL_ERROR "For building MT examples with OpenMP, OpenMP \
package must be detectable by cmake")
    endif()
endif()

enable_testing()

# Building standard examples (not for MGMN mode)
if (BUILD_STANDARD)

    set(CUDSS_EXAMPLE_SOURCES
        cudss_simple.cpp
        cudss_batch.cpp
        cudss_simple_complex.cpp
        cudss_get_set.cpp
        cudss_hybrid_memory_mode.cpp
        cudss_dense_matrix_helpers.cpp
        cudss_sparse_matrix_helpers.cpp
        cudss_memory_handlers.cpp
        cudss_batch_sparse_matrix_helpers.cpp
        cudss_hybrid_execute_mode.cpp
        cudss_reordering_phase.cpp
        cudss_uniform_batch.cpp
    )

    foreach(src ${CUDSS_EXAMPLE_SOURCES})
        set_source_files_properties(${src} PROPERTIES LANGUAGE CUDA)

        get_filename_component(name "${src}" NAME_WE)

        add_executable(${name} ${src})

        if (WIN32)
            target_include_directories(${name} PUBLIC ${cudss_INCLUDE_DIR})
        endif()
        target_link_directories(${name} PUBLIC ${cudss_LIBRARY_DIR} ${CUBLAS_LIB_PATH})

        target_compile_options(${name} PRIVATE
            "$<$<CONFIG:Debug>:-lineinfo -g>"
            "$<$<CONFIG:RelWithDebInfo>:-lineinfo -g>"
        )

        target_link_libraries(${name} PUBLIC
            cudss
            cublas
        )

        add_test(${name} ${name})

        install(TARGETS ${name}
                RUNTIME DESTINATION examples
                COMPONENT Examples)

        if (BUILD_STATIC)
            add_executable(${name}_static ${src})

            target_link_directories(${name}_static PUBLIC ${cudss_LIBRARY_DIR} ${CUBLAS_LIB_PATH})
            if (WIN32)
                target_include_directories(${name}_static PUBLIC ${cudss_INCLUDE_DIR})
                target_link_libraries(${name}_static PUBLIC
                    cudss
                    cublas
                )
            else()
                target_link_libraries(${name}_static PUBLIC
                    cudss_static
                    cublas
                )
            endif()


            add_test(${name}_static ${name}_static)

            install(TARGETS ${name}_static
                    RUNTIME DESTINATION example
                    COMPONENT Examples
            )

        endif()
    endforeach()
endif()

# Building examples for the MGMN mode

if(BUILD_MGMN)
    message(STATUS "CMake will use ${MPIRUN_EXECUTABLE} with ${MPIRUN_NUMPROC_FLAG} \
and extra flags ${MPIRUN_EXTRA_FLAGS} defined from MPIRUN_EXECUTABLE, MPIRUN_NUMPROC_FLAG \
and MPIRUN_EXTRA_FLAGS CMake variables to launch MGMN tests")
endif()

# Helper to make crun launch examples with mpirun
function(test_mpi_launcher target test Nproc)

    if(NOT (DEFINED MPIRUN_EXECUTABLE AND DEFINED MPIRUN_NUMPROC_FLAG))
        message(FATAL_ERROR "MPIRUN_EXECUTABLE and MPIRUN_NUMPROC_FLAG must be defined to use test_mpi_launcher")
    endif()

    if(NOT Nproc)
        message(FATAL_ERROR "Nproc must be defined to use test_mpi_launcher() function in cmake")
    endif()

    # This conversion is a workaround for the case when cmake passes multiple arguments to mpirin with extra ""
    string(REPLACE " " ";" MPIRUN_EXTRA_FLAGS_LIST ${MPIRUN_EXTRA_FLAGS})

    if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.29)
        set_property(TARGET ${target} PROPERTY TEST_LAUNCHER ${MPIRUN_EXECUTABLE} ${MPIRUN_EXTRA_FLAGS_LIST} ${MPIRUN_NUMPROC_FLAG} ${Nproc})
    else()
        set_property(TARGET ${target} PROPERTY CROSSCOMPILING_EMULATOR ${MPIRUN_EXECUTABLE} ${MPIRUN_EXTRA_FLAGS_LIST} ${MPIRUN_NUMPROC_FLAG} ${Nproc})
    endif()

    set_property(TEST ${test} PROPERTY PROCESSORS ${Nproc})

endfunction()

if (BUILD_MGMN)
    set(CUDSS_MGMN_COMM_BACKENDS "")
    if (BUILD_MGMN_WITH_OPENMPI)
        set(CUDSS_MGMN_COMM_BACKENDS ${CUDSS_MGMN_COMM_BACKENDS} "openmpi")
    endif()
    if (BUILD_MGMN_WITH_NCCL)
        set(CUDSS_MGMN_COMM_BACKENDS ${CUDSS_MGMN_COMM_BACKENDS} "nccl")
    endif()
    message(STATUS "CUDSS_MGMN_COMM_BACKENDS = ${CUDSS_MGMN_COMM_BACKENDS}")

    if (BUILD_MGMN_WITH_OPENMPI OR BUILD_MGMN_WITH_NCCL)
        set(CUDSS_MGMN_EXAMPLE_SOURCES
            cudss_mgmn_mode.cpp
            cudss_mgmn_distributed_matrix.cpp
        )
    else()
        set(CUDSS_MGMN_EXAMPLE_SOURCES "")
    endif()

    if (BUILD_MGMN_WITH_OPENMPI OR BUILD_MGMN_WITH_NCCL)
        set(CUDSS_USE_MPI 1)
    else()
        set(CUDSS_USE_MPI 0)
    endif()

    foreach(src ${CUDSS_MGMN_EXAMPLE_SOURCES})
        set_source_files_properties(${src} PROPERTIES LANGUAGE CUDA)

        get_filename_component(name "${src}" NAME_WE)

        foreach(backend ${CUDSS_MGMN_COMM_BACKENDS})

            add_executable(${name}_${backend} ${src})

            target_include_directories(${name}_${backend} PUBLIC
                ${cudss_INCLUDE_DIR}
            )

            target_link_directories(${name}_${backend} PUBLIC
                ${cudss_LIBRARY_DIR}
                ${CUBLAS_LIB_PATH}
            )

            target_compile_options(${name}_${backend} PRIVATE
                "$<$<CONFIG:Debug>:-lineinfo -g>"
                "$<$<CONFIG:RelWithDebInfo>:-lineinfo -g>"
            )

            target_link_libraries(${name}_${backend} PUBLIC
                cudss
                cublas
            )

            if (CUDSS_USE_MPI)
                target_compile_options(${name}_${backend} PRIVATE -DUSE_MPI)
            endif()

            if ("${backend}" STREQUAL "openmpi")
                target_compile_options(${name}_${backend} PRIVATE -DUSE_OPENMPI)
            elseif ("${backend}" STREQUAL "nccl")
                target_compile_options(${name}_${backend} PRIVATE -DUSE_NCCL)
            endif()

            # Currently, we only support OpenMPI here for MPI but this might change
            if ("${backend}" STREQUAL "openmpi" OR CUDSS_USE_MPI)
                target_include_directories(${name}_${backend} PUBLIC
                                           ${OPENMPI_INCLUDE_DIRECTORIES}
                )
            endif()
            if ("${backend}" STREQUAL "nccl")
                target_include_directories(${name}_${backend} PUBLIC
                                           ${NCCL_INCLUDE_DIRECTORIES}
                )
            endif()

            # Note: for NCCL we still need the MPI header (for mpi.h) and library (for MPI_Init)
            if ("${backend}" STREQUAL "openmpi" OR CUDSS_USE_MPI)
                target_link_directories(${name}_${backend} PUBLIC
                                        ${OPENMPI_LINK_DIRECTORIES}
                )
                target_link_libraries(${name}_${backend} PUBLIC mpi)
            endif()
            if ("${backend}" STREQUAL "nccl")
                target_link_directories(${name}_${backend} PUBLIC
                                        ${NCCL_LINK_DIRECTORIES}
                )
                target_link_libraries(${name}_${backend} PUBLIC nccl)
            endif()

            add_test(NAME ${name}_${backend} COMMAND ${name}_${backend} ${backend} ${cudss_LIBRARY_DIR}/libcudss_commlayer_${backend}.so)

            test_mpi_launcher(${name}_${backend} ${name}_${backend} ${MGMN_NPROC})

            install(TARGETS ${name}_${backend}
                    RUNTIME
                        DESTINATION examples
                        COMPONENT ExampleTests
                    ARCHIVE
                        DESTINATION examples
                        COMPONENT ExampleTests
            )

            if (BUILD_STATIC)
                add_executable(${name}_${backend}_static ${src})

                target_link_directories(${name}_${backend}_static PUBLIC ${CUBLAS_LIB_PATH} )

                target_link_libraries(${name}_${backend}_static PUBLIC
                    cudss_static
                    cublas
                )

                if (CUDSS_USE_MPI)
                    target_compile_options(${name}_${backend}_static PRIVATE -DUSE_MPI)
                endif()

                # Note: OpenMPI here has priority over NCCL ( we have to
                if ("${backend}" STREQUAL "openmpi")
                    target_compile_options(${name}_${backend}_static PRIVATE -DUSE_OPENMPI)
                elseif ("${backend}" STREQUAL "nccl")
                    target_compile_options(${name}_${backend}_static PRIVATE -DUSE_NCCL)
                endif()

                # Currently, we only support OpenMPI here for MPI but this might change
                if ("${backend}" STREQUAL "openmpi" OR CUDSS_USE_MPI)
                    target_include_directories(${name}_${backend}_static PUBLIC
                                               ${OPENMPI_INCLUDE_DIRECTORIES}
                    )
                endif()
                if ("${backend}" STREQUAL "nccl")
                    target_include_directories(${name}_${backend}_static PUBLIC
                                               ${NCCL_INCLUDE_DIRECTORIES}
                    )
                endif()

                # Note: for NCCL we still need the MPI header (for mpi.h) and library (for MPI_Init)
                if ("${backend}" STREQUAL "openmpi" OR CUDSS_USE_MPI)
                    target_link_directories(${name}_${backend}_static PUBLIC
                                            ${OPENMPI_LINK_DIRECTORIES}
                    )
                    target_link_libraries(${name}_${backend}_static PUBLIC mpi)
                endif()
                if ("${backend}" STREQUAL "nccl")
                    target_link_directories(${name}_${backend}_static PUBLIC
                                            ${NCCL_LINK_DIRECTORIES}
                    )
                    target_link_libraries(${name}_${backend}_static PUBLIC nccl)
                endif()

                #add_test(${name}_${backend}_static ${name}_${backend}_static)
                add_test(NAME ${name}_${backend}_static COMMAND ${name}_${backend}_static
                    ${backend} ${cudss_LIBRARY_DIR}/libcudss_commlayer_${backend}.so)

                test_mpi_launcher(${name}_${backend}_static ${name}_${backend}_static ${MGMN_NPROC})

                install(TARGETS ${name}_${backend}_static
                        RUNTIME
                            DESTINATION examples
                            COMPONENT ExampleTests
                        ARCHIVE
                            DESTINATION examples
                            COMPONENT ExampleTests
                )

            endif()

        endforeach()
    endforeach()
endif()

if (BUILD_MT)

    set(CUDSS_MT_EXAMPLE_SOURCES
        cudss_multithreaded_mode.cpp
    )

    set(CUDSS_MT_BACKENDS "")
    if (BUILD_MT_WITH_OPENMP)
        # Define the flavor of OpenMP picked up by find_package()
        if (OpenMP_CXX_LIBRARIES OR OpenMP_C_LIBRARIES) # should work on Linux
            if (OpenMP_CXX_LIBRARIES)
                string(REGEX MATCH ".*lib([A-Za-z0-9]*omp[A-Za-z0-9]*)[.](so|dll).*" dummy ${OpenMP_CXX_LIBRARIES})
            else()
                string(REGEX MATCH ".*lib([A-Za-z0-9]*omp[A-Za-z0-9]*)[.](so|dll).*" dummy ${OpenMP_C_LIBRARIES})
            endif()
            set(omp_backend_kind ${CMAKE_MATCH_1})
        else() # fallback for Windows
            set(omp_backend_kind "omp")
        endif()
        message(STATUS "omp_backend_kind = ${omp_backend_kind}")
        set(CUDSS_MT_BACKENDS ${CUDSS_MT_BACKENDS} ${omp_backend_kind})
    endif()

    message(STATUS "CUDSS_MT_BACKENDS = ${CUDSS_MT_BACKENDS}")

    foreach(src ${CUDSS_MT_EXAMPLE_SOURCES})
        set_source_files_properties(${src} PROPERTIES LANGUAGE CUDA)

        get_filename_component(name "${src}" NAME_WE)

        foreach(backend ${CUDSS_MT_BACKENDS})

            add_executable(${name}_${backend} ${src})

            target_include_directories(${name}_${backend} PUBLIC
                ${cudss_INCLUDE_DIR}
            )

            target_link_directories(${name}_${backend} PUBLIC
                ${cudss_LIBRARY_DIR}
                ${CUBLAS_LIB_PATH}
            )

            target_compile_options(${name}_${backend} PRIVATE
                "$<$<CONFIG:Debug>:-lineinfo -g>"
                "$<$<CONFIG:RelWithDebInfo>:-lineinfo -g>"
            )

            target_link_libraries(${name}_${backend} PUBLIC
                cudss
                cublas
            )

            if ("${backend}" MATCHES "omp")
                target_link_libraries(${name}_${backend} PUBLIC
                    OpenMP::OpenMP_CXX
                )
            endif()

            target_compile_options(${name}_${backend} PRIVATE -DUSE_OPENMP)

            # optional extra argument for the multithreaded examples, the threading layer library name
            set(extra_mt_input_arg ${cudss_LIBRARY_DIR}/libcudss_mtlayer_${backend}${CMAKE_SHARED_LIBRARY_SUFFIX})

            add_test(NAME ${name}_${backend} COMMAND ${name}_${backend} ${extra_mt_input_arg})

            install(TARGETS ${name}_${backend}
                    RUNTIME
                        DESTINATION examples
                        COMPONENT ExampleTests
                    ARCHIVE
                        DESTINATION examples
                        COMPONENT ExampleTests
            )

            if (BUILD_STATIC)
                add_executable(${name}_${backend}_static ${src})

                target_link_directories(${name}_${backend}_static PUBLIC ${CUBLAS_LIB_PATH} )

                target_link_libraries(${name}_${backend}_static PUBLIC
                    cudss_static
                    cublas
                )

                if ("${backend}" MATCHES "omp")
                    target_link_libraries(${name}_${backend}_static PUBLIC
                        OpenMP::OpenMP_CXX
                    )
                endif()

                target_compile_options(${name}_${backend}_static PRIVATE -DUSE_OPENMP)

                add_test(NAME ${name}_${backend}_static COMMAND ${name}_${backend}_static ${extra_mt_input_arg})

                install(TARGETS ${name}_${backend}_static
                        RUNTIME
                            DESTINATION examples
                            COMPONENT ExampleTests
                        ARCHIVE
                            DESTINATION examples
                            COMPONENT ExampleTests
                )

            endif()

        endforeach()
    endforeach()
endif()
