basic_mod.F90

MODULE basic
  !   Basic module for time dependent problems
  use, intrinsic :: iso_c_binding
  use prec_const
  IMPLICIT none

  ! INCLUDE 'fftw3-mpi.f03'

  INTEGER  :: nrun   = 1           ! Number of time steps to run
  real(dp) :: tmax   = 100000.0    ! Maximum simulation time
  real(dp) :: dt     = 1.0         ! Time step
  real(dp) :: time   = 0           ! Current simulation time (Init from restart file)

  INTEGER :: comm0                 ! Default communicator with a topology
  INTEGER :: group0                ! Default group with a topology
  INTEGER :: rank_0                ! Ranks in comm0
  ! Communicators for 1-dim cartesian subgrids of comm0
  INTEGER :: comm_p, comm_ky, comm_z
  INTEGER :: rank_p, rank_ky, rank_z! Ranks
  INTEGER :: comm_pz,  rank_pz      ! 2D comm for N_a(p,j,z) output (mspfile)
  INTEGER :: comm_kyz, rank_kyz     ! 2D comm for N_a(p,j,z) output (mspfile)
  INTEGER :: comm_ky0, rank_ky0     ! comm along ky with p=0
  INTEGER :: comm_z0,  rank_z0      ! comm along z  with p=0

  INTEGER :: group_ky0, group_z0

  INTEGER :: jobnum  = 0           ! Job number
  INTEGER :: step    = 0           ! Calculation step of this run
  INTEGER :: cstep   = 0           ! Current step number (Init from restart file)
  LOGICAL :: nlend   = .FALSE.     ! Signal end of run
  LOGICAL :: crashed = .FALSE.     ! Signal end of crashed run

  INTEGER :: ierr                  ! flag for MPI error
  INTEGER :: my_id                 ! Rank in COMM_WORLD
  INTEGER :: num_procs             ! number of MPI processes
  INTEGER :: num_procs_p           ! Number of processes in p
  INTEGER :: num_procs_ky          ! Number of processes in r
  INTEGER :: num_procs_z           ! Number of processes in z
  INTEGER :: num_procs_pz          ! Number of processes in pz comm
  INTEGER :: num_procs_kyz         ! Number of processes in kyz comm
  INTEGER :: nbr_L, nbr_R          ! Left and right neighbours (along p)
  INTEGER :: nbr_T, nbr_B          ! Top and bottom neighbours (along kx)
  INTEGER :: nbr_U, nbr_D          ! Upstream and downstream neighbours (along z)

  INTEGER :: iframe0d              ! counting the number of times 0d datasets are outputed (for diagnose)
  INTEGER :: iframe1d              ! counting the number of times 1d datasets are outputed (for diagnose)
  INTEGER :: iframe2d              ! counting the number of times 2d datasets are outputed (for diagnose)
  INTEGER :: iframe3d              ! counting the number of times 3d datasets are outputed (for diagnose)
  INTEGER :: iframe5d              ! counting the number of times 5d datasets are outputed (for diagnose)

  !  List of logical file units
  INTEGER :: lu_in   = 90              ! File duplicated from STDIN
  INTEGER :: lu_stop = 91              ! stop file, see subroutine TESEND

  ! To measure computation time
  real     :: start, finish
  real(dp) :: t0_rhs, t0_adv_field, t0_poisson, t0_Sapj, t0_diag, t0_checkfield,&
              t0_step, t0_clos, t0_ghost, t0_coll, t0_process
  real(dp) :: t1_rhs, t1_adv_field, t1_poisson, t1_Sapj, t1_diag, t1_checkfield,&
              t1_step, t1_clos, t1_ghost, t1_coll, t1_process
  real(dp) :: tc_rhs, tc_adv_field, tc_poisson, tc_Sapj, tc_diag, tc_checkfield,&
              tc_step, tc_clos, tc_ghost, tc_coll, tc_process
  real(dp) :: maxruntime = 1e9 ! Maximum simulation CPU time

  INTERFACE allocate_array
    MODULE PROCEDURE allocate_array_dp1,allocate_array_dp2,allocate_array_dp3,allocate_array_dp4, allocate_array_dp5, allocate_array_dp6
    MODULE PROCEDURE allocate_array_dc1,allocate_array_dc2,allocate_array_dc3,allocate_array_dc4, allocate_array_dc5, allocate_array_dc6
    MODULE PROCEDURE allocate_array_i1,allocate_array_i2,allocate_array_i3,allocate_array_i4
    MODULE PROCEDURE allocate_array_l1,allocate_array_l2,allocate_array_l3,allocate_array_l4
  END INTERFACE allocate_array

CONTAINS
  !================================================================================
  SUBROUTINE basic_data
    !   Read basic data for input file

    use prec_const
    IMPLICIT NONE

    CALL find_input_file

    NAMELIST /BASIC/  nrun, dt, tmax, maxruntime

    READ(lu_in,basic)

    !Init cumulative timers
    tc_rhs       = 0.;tc_adv_field = 0.; tc_poisson  = 0.
    tc_Sapj      = 0.; tc_diag     = 0.; tc_checkfield = 0.

  END SUBROUTINE basic_data


  SUBROUTINE basic_outputinputs(fid, str)
    !
    !    Write the input parameters to the results_xx.h5 file
    !
    USE prec_const
    USE futils, ONLY: attach
    IMPLICIT NONE
    INTEGER, INTENT(in) :: fid
    CHARACTER(len=256), INTENT(in) :: str
    CALL attach(fid, TRIM(str), "start_iframe0d", iframe0d)
    CALL attach(fid, TRIM(str), "start_iframe2d", iframe2d)
    CALL attach(fid, TRIM(str), "start_iframe3d", iframe3d)
    CALL attach(fid, TRIM(str), "start_iframe5d", iframe5d)
    CALL attach(fid, TRIM(str),  "start_time",     time)
    CALL attach(fid, TRIM(str), "start_cstep",    cstep-1)
    CALL attach(fid, TRIM(str),          "dt",       dt)
    CALL attach(fid, TRIM(str),        "tmax",     tmax)
    CALL attach(fid, TRIM(str),        "nrun",     nrun)
    CALL attach(fid, TRIM(str),    "cpu_time",       -1)
    CALL attach(fid, TRIM(str),       "Nproc",   num_procs)
    CALL attach(fid, TRIM(str),       "Np_p" , num_procs_p)
    CALL attach(fid, TRIM(str),       "Np_kx",num_procs_ky)
    CALL attach(fid, TRIM(str),        "Np_z", num_procs_z)
  END SUBROUTINE basic_outputinputs
  !================================================================================
  SUBROUTINE find_input_file
    IMPLICIT NONE
    CHARACTER(len=32) :: str, input_file
    INTEGER :: nargs, fileid, l
    LOGICAL :: mlexist
    nargs = COMMAND_ARGUMENT_COUNT()
    IF((nargs .EQ. 1) .OR. (nargs .EQ. 4)) THEN
      CALL GET_COMMAND_ARGUMENT(nargs, str, l, ierr)
      READ(str(1:l),'(i3)')  fileid
      WRITE(input_file,'(a,a1,i2.2,a3)') 'fort','_',fileid,'.90'

      INQUIRE(file=input_file, exist=mlexist)
      IF( mlexist ) THEN
        IF(my_id.EQ.0) write(*,*) 'Reading input ', input_file,'...'
        OPEN(lu_in, file=input_file)
      ELSE
        IF(my_id.EQ.0) write(*,*) 'Reading input fort.90...'
        OPEN(lu_in, file='fort.90')
      ENDIF
    ENDIF
  END SUBROUTINE find_input_file
  !================================================================================
  SUBROUTINE daytim(str)
    !   Print date and time

    use prec_const
    IMPLICIT NONE

    CHARACTER(len=*) , INTENT(in) :: str
    CHARACTER(len=16) :: d, t, dat, time
    !________________________________________________________________________________
    !
    CALL DATE_AND_TIME(d,t)
    dat=d(7:8) // '/' // d(5:6) // '/' // d(1:4)
    time=t(1:2) // ':' // t(3:4) // ':' // t(5:10)
    WRITE(*,'(a,1x,a,1x,a)') str, dat(1:10), time(1:12)
    !
  END SUBROUTINE daytim
  !================================================================================
  SUBROUTINE display_h_min_s(time)
    real :: time
    integer :: days, hours, mins, secs
    days = FLOOR(time/24./3600.);
    hours= FLOOR(time/3600.);
    mins = FLOOR(time/60.);
    secs = FLOOR(time);

    IF ( days .GT. 0 ) THEN !display day h min s
      hours = (time/3600./24. - days) * 24
      mins  = (time/3600. - days*24. - hours) * 60
      secs  = (time/60. - days*24.*60 - hours*60 - mins) * 60
      WRITE(*,*) 'CPU Time = ', days, '[day]', hours, '[h]', mins, '[min]', secs, '[s]'
      WRITE(*,*) '(',time,'[s])'

    ELSEIF ( hours .GT. 0 ) THEN !display h min s
      mins  = (time/3600. - hours) * 60
      secs  = (time/60. - hours*60 - mins) * 60
      WRITE(*,*) 'CPU Time = ', hours, '[h]', mins, '[min]', secs, '[s]'
      WRITE(*,*) '(',time,'[s])'

    ELSEIF ( mins .GT. 0 ) THEN !display min s
      secs  = (time/60. - mins) * 60
      WRITE(*,*) 'CPU Time = ', mins, '[min]', secs, '[s]'
      WRITE(*,*) '(',time,'[s])'

    ELSE ! display s
      WRITE(*,*) 'CPU Time = ', FLOOR(time), '[s]'

    ENDIF
  END SUBROUTINE display_h_min_s
!================================================================================

! To allocate arrays of doubles, integers, etc. at run time
  SUBROUTINE allocate_array_dp1(a,is1,ie1)
    IMPLICIT NONE
    real(dp), DIMENSION(:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1
    ALLOCATE(a(is1:ie1))
    a=0.0_dp
  END SUBROUTINE allocate_array_dp1

  SUBROUTINE allocate_array_dp2(a,is1,ie1,is2,ie2)
    IMPLICIT NONE
    real(dp), DIMENSION(:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2
    ALLOCATE(a(is1:ie1,is2:ie2))
    a=0.0_dp
  END SUBROUTINE allocate_array_dp2

  SUBROUTINE allocate_array_dp3(a,is1,ie1,is2,ie2,is3,ie3)
    IMPLICIT NONE
    real(dp), DIMENSION(:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3))
    a=0.0_dp
  END SUBROUTINE allocate_array_dp3

  SUBROUTINE allocate_array_dp4(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4)
    IMPLICIT NONE
    real(dp), DIMENSION(:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4))
    a=0.0_dp
  END SUBROUTINE allocate_array_dp4

  SUBROUTINE allocate_array_dp5(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5)
    IMPLICIT NONE
    real(dp), DIMENSION(:,:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4,is5:ie5))
    a=0.0_dp
  END SUBROUTINE allocate_array_dp5

  SUBROUTINE allocate_array_dp6(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5,is6,ie6)
    IMPLICIT NONE
    real(dp), DIMENSION(:,:,:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5,is6,ie6
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4,is5:ie5,is6:ie6))
    a=0.0_dp
  END SUBROUTINE allocate_array_dp6
  !========================================

  SUBROUTINE allocate_array_dc1(a,is1,ie1)
    IMPLICIT NONE
    DOUBLE COMPLEX, DIMENSION(:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1
    ALLOCATE(a(is1:ie1))
    a=CMPLX(0.0_dp,0.0_dp)
  END SUBROUTINE allocate_array_dc1

  SUBROUTINE allocate_array_dc2(a,is1,ie1,is2,ie2)
    IMPLICIT NONE
    DOUBLE COMPLEX, DIMENSION(:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2
    ALLOCATE(a(is1:ie1,is2:ie2))
    a=CMPLX(0.0_dp,0.0_dp)
  END SUBROUTINE allocate_array_dc2

  SUBROUTINE allocate_array_dc3(a,is1,ie1,is2,ie2,is3,ie3)
    IMPLICIT NONE
    DOUBLE COMPLEX, DIMENSION(:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3))
    a=CMPLX(0.0_dp,0.0_dp)
  END SUBROUTINE allocate_array_dc3

  SUBROUTINE allocate_array_dc4(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4)
    IMPLICIT NONE
    DOUBLE COMPLEX, DIMENSION(:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4))
    a=CMPLX(0.0_dp,0.0_dp)
  END SUBROUTINE allocate_array_dc4

  SUBROUTINE allocate_array_dc5(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5)
    IMPLICIT NONE
    DOUBLE COMPLEX, DIMENSION(:,:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4,is5:ie5))
    a=CMPLX(0.0_dp,0.0_dp)
  END SUBROUTINE allocate_array_dc5

  SUBROUTINE allocate_array_dc6(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5,is6,ie6)
    IMPLICIT NONE
    DOUBLE COMPLEX, DIMENSION(:,:,:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5,is6,ie6
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4,is5:ie5,is6:ie6))
    a=CMPLX(0.0_dp,0.0_dp)
  END SUBROUTINE allocate_array_dc6
  !========================================

  SUBROUTINE allocate_array_i1(a,is1,ie1)
    IMPLICIT NONE
    INTEGER, DIMENSION(:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1
    ALLOCATE(a(is1:ie1))
    a=0
  END SUBROUTINE allocate_array_i1

  SUBROUTINE allocate_array_i2(a,is1,ie1,is2,ie2)
    IMPLICIT NONE
    INTEGER, DIMENSION(:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2
    ALLOCATE(a(is1:ie1,is2:ie2))
    a=0
  END SUBROUTINE allocate_array_i2

  SUBROUTINE allocate_array_i3(a,is1,ie1,is2,ie2,is3,ie3)
    IMPLICIT NONE
    INTEGER, DIMENSION(:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3))
    a=0
  END SUBROUTINE allocate_array_i3

  SUBROUTINE allocate_array_i4(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4)
    IMPLICIT NONE
    INTEGER, DIMENSION(:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4))
    a=0
  END SUBROUTINE allocate_array_i4

  SUBROUTINE allocate_array_i5(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5)
    IMPLICIT NONE
    real(dp), DIMENSION(:,:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4,is5:ie5))
    a=0
  END SUBROUTINE allocate_array_i5

  !========================================

  SUBROUTINE allocate_array_l1(a,is1,ie1)
    IMPLICIT NONE
    LOGICAL, DIMENSION(:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1
    ALLOCATE(a(is1:ie1))
    a=.false.
  END SUBROUTINE allocate_array_l1

  SUBROUTINE allocate_array_l2(a,is1,ie1,is2,ie2)
    IMPLICIT NONE
    LOGICAL, DIMENSION(:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2
    ALLOCATE(a(is1:ie1,is2:ie2))
    a=.false.
  END SUBROUTINE allocate_array_l2

  SUBROUTINE allocate_array_l3(a,is1,ie1,is2,ie2,is3,ie3)
    IMPLICIT NONE
    LOGICAL, DIMENSION(:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3))
    a=.false.
  END SUBROUTINE allocate_array_l3

  SUBROUTINE allocate_array_l4(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4)
    IMPLICIT NONE
    LOGICAL, DIMENSION(:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4))
    a=.false.
  END SUBROUTINE allocate_array_l4

  SUBROUTINE allocate_array_l5(a,is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5)
    IMPLICIT NONE
    LOGICAL, DIMENSION(:,:,:,:,:), ALLOCATABLE, INTENT(INOUT) :: a
    INTEGER, INTENT(IN) :: is1,ie1,is2,ie2,is3,ie3,is4,ie4,is5,ie5
    ALLOCATE(a(is1:ie1,is2:ie2,is3:ie3,is4:ie4,is5:ie5))
    a=.false.
  END SUBROUTINE allocate_array_l5

END MODULE basic