# Precompiler options
CPP_OPTIONS= -DHOST=\"DARWIN-Intel-OpenMPI-CUDA\"\
             -DMPI -DMPI_BLOCK=8000 -Duse_collective \
             -DscaLAPACK \
             -DCACHE_SIZE=4000 \
             -Davoidalloc \
             -Dvasp6 \
             -Duse_bse_te \
             -Dtbdyn \
             -Dfock_dblbuf

CPP        = fpp -f_com=no -free -w0  $*$(FUFFIX) $*$(SUFFIX) $(CPP_OPTIONS) $(CPPFLAGS)

FC         = mpifort
FCL        = mpifort -mkl=sequential

FREE       = -free -names lowercase

FFLAGS     = -assume byterecl -w -xCORE-AVX2
OFLAG      = -O2
OFLAG_IN   = $(OFLAG)
DEBUG      = -O0

MKL_PATH   = $(MKLROOT)/lib/intel64
BLAS       =
LAPACK     =
BLACS      = -lmkl_blacs_intelmpi_lp64
SCALAPACK  = -lmkl_scalapack_lp64 $(BLACS)

OBJECTS    = fftmpiw.o fftmpi_map.o fft3dlib.o fftw3d.o

INCS       =-I$(MKLROOT)/include/fftw

LLIBS      = $(LDFLAGS) $(SCALAPACK) $(LAPACK) $(BLAS)


OBJECTS_O1 += fftw3d.o fftmpi.o fftmpiw.o
OBJECTS_O2 += fft3dlib.o

# For what used to be vasp.5.lib
CPP_LIB    = $(CPP)
FC_LIB     = $(FC)
CC_LIB     = icc $(CPPFLAGS)
CFLAGS_LIB = -O
FFLAGS_LIB = -O1
FREE_LIB   = $(FREE)

OBJECTS_LIB= linpack_double.o getshmem.o

# For the parser library
CXX_PARS   = icpc
LLIBS      += -lstdc++

# Normally no need to change this
SRCDIR     = ../../src
BINDIR     = ../../bin

#================================================
# GPU Stuff

CPP_GPU    = -DCUDA_GPU -DRPROMU_CPROJ_OVERLAP -DUSE_PINNED_MEMORY -DCUFFT_MIN=28 -UscaLAPACK -Ufock_dblbuf

OBJECTS_GPU= fftmpiw.o fftmpi_map.o fft3dlib.o fftw3d_gpu.o fftmpiw_gpu.o

CC         = icc
CXX        = icpc
CFLAGS     = $(CPPFLAGS) -fPIC -DADD_ -Wall -qopenmp -DMAGMA_WITH_MKL -DMAGMA_SETAFFINITY -DGPUSHMEM=300 -DHAVE_CUBLAS

##
## vpkg_devrequire cuda/<version> will have setup everything in the
## environment for us (CUDA_PREFIX, PATH, and LD_LIBRARY_PATH)
##
CUDA_ROOT  ?= $(CUDA_PREFIX)
export CUDA_ROOT
NVCC       := nvcc -ccbin=icc $(CPPFLAGS)
CUDA_LIB   := $(LDFLAGS) -lnvToolsExt -lcudart -lcuda -lcufft -lcublas

##
## compute_30,35 dropped from CUDA 11
## compute_60 was Pascal (present on Caviness, not DARWIN)
## compute_70,72 are Volta
## compute_75 is Turing
##
##GENCODE_ARCH    := -gencode=arch=compute_30,code=\"sm_30,compute_30\" \
##                   -gencode=arch=compute_35,code=\"sm_35,compute_35\" \
GENCODE_ARCH    := -gencode=arch=compute_60,code=\"sm_60,compute_60\" \
                   -gencode=arch=compute_70,code=\"sm_70,compute_70\" \
                   -gencode=arch=compute_72,code=\"sm_72,compute_72\" \
                   -gencode=arch=compute_75,code=\"sm_75,compute_75\"
export GENCODE_ARCH

##
## vpkg_require openmpi/<version> will have the pkg-config path
## setup for us to query this argument
##
MPI_INC    = $(shell pkg-config --cflags-only-I ompi-c | sed 's/-I//')
export MPI_INC