! Get a block cyclic decomposition
! If force_rearrange is FALSE, the decomposition is such that
! # All even procs have VEC_LOCAL_SZ elements
! # All odd procs have VEC_LOCAL_SZ + 1 elements
! e.g. For VEC_LOCAL_SZ = 2,
! e.g. 1)    [1,2] [3,4,5] [6,7]
! e.g. 2)    [1,2] [3,4,5] [6,7] [8,9,10]
! e.g. 3)    [1,2] [3,4,5] [6,7] [8,9,10] [11,12]
! If force_rearrange is TRUE, the decomposition is such that,
! If possible, the even rank "exchanges" elements with the next
! higher ranked odd proc.
! This for example can be used to force rearrangement when reading
! or writing data.
! e.g. For VEC_LOCAL_SZ = 2,
! e.g. 1)    [3,4,5] [1,2] [6,7]
! e.g. 2)    [3,4,5] [1,2] [8,9,10] [6,7]
! e.g. 3)    [3,4,5] [1,2] [8,9,10] [6,7] [11,12]
SUBROUTINE get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
  integer, parameter :: VEC_LOCAL_SZ = 7
  integer, parameter :: NUM_REGIONS = 1
  integer, intent(in) :: rank
  integer, intent(in) :: sz
  integer, dimension(1), intent(out) :: dims
  integer, dimension(:), allocatable, intent(out) :: start
  integer, dimension(:), allocatable, intent(out) :: count
  logical, intent(in) :: force_rearrange

  logical :: is_even_rank
  integer :: num_odd_procs, num_even_procs
  integer :: iodd, ieven

  allocate(start(NUM_REGIONS))
  allocate(count(NUM_REGIONS))

  is_even_rank = .false.
  if (mod(rank, 2) == 0) then
    is_even_rank = .true.
  end if
  num_odd_procs = sz / 2
  num_even_procs = sz - num_odd_procs
  dims(1) = num_even_procs * VEC_LOCAL_SZ + num_odd_procs * (VEC_LOCAL_SZ + 1)
  ! Number of odd and even procs before this rank
  iodd = rank / 2
  ieven = (rank + 1) / 2
  if(force_rearrange) then
    ! Make sure that we force rearrangement
    if (is_even_rank) then
      if(rank + 1 < sz) then
        ! Force rearrangement
        count(1) = VEC_LOCAL_SZ + 1
        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + (VEC_LOCAL_SZ) + 1
      else
        count(1) = VEC_LOCAL_SZ
        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
      end if
    else
      ! For all odd procs there is an even lower ranked, rank-1, proc
      ! So force rearrangement
      count(1) = VEC_LOCAL_SZ
      start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) - (VEC_LOCAL_SZ) + 1
    end if
  else
    if (is_even_rank) then
      count(1) = VEC_LOCAL_SZ
    else
      count(1) = VEC_LOCAL_SZ + 1
    end if
    start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
  end if

END SUBROUTINE

! Get a block cyclic decomposition with multiple regions to write
! from a single proc
! # All procs have VEC_LOCAL_SZ elements starting at offset rank+1
!   and also starting at offset (size * VEC_LOCAL_SZ)
! e.g. For VEC_LOCAL_SZ = 2, NUM_REGIONS=2
! e.g. 1)    [1,2,7,8] [3,4,9,10] [5,6,11,12]
SUBROUTINE get_1d_bc_mreg_info(rank, sz, dims, start, count)
  integer, parameter :: VEC_LOCAL_SZ = 7
  integer, parameter :: NUM_REGIONS = 3
  integer, intent(in) :: rank
  integer, intent(in) :: sz
  integer, dimension(1), intent(out) :: dims
  integer, dimension(:), allocatable, intent(out) :: start
  integer, dimension(:), allocatable, intent(out) :: count
  integer :: i

  dims(1) = sz * VEC_LOCAL_SZ * NUM_REGIONS
  allocate(start(NUM_REGIONS))
  allocate(count(NUM_REGIONS))

  
  do i=1,NUM_REGIONS
    count(i) = VEC_LOCAL_SZ
    start(i) = rank * VEC_LOCAL_SZ + (i-1) * (sz * VEC_LOCAL_SZ) + 1
  end do

END SUBROUTINE get_1d_bc_mreg_info

! Test writes multiple variables before syncing and reading them back
PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE>
PIO_TF_AUTO_TEST_SUB_BEGIN nc_mvar_wr_rd_1d_bc
  implicit none
  interface
    subroutine get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
      integer, intent(in) :: rank
      integer, intent(in) :: sz
      integer, dimension(1), intent(out) :: dims
      integer, dimension(:), allocatable, intent(out) :: start
      integer, dimension(:), allocatable, intent(out) :: count
      logical, intent(in) :: force_rearrange
    end subroutine get_1d_bc_info
  end interface
  type(var_desc_t)  :: pio_var1, pio_var2, pio_var3
  type(file_desc_t) :: pio_file
  character(len=PIO_TF_MAX_STR_LEN) :: filename
  character(len=*), parameter :: PIO_VAR1_NAME = 'PIO_TF_test_var1'
  character(len=*), parameter :: PIO_VAR2_NAME = 'PIO_TF_test_var2'
  character(len=*), parameter :: PIO_VAR3_NAME = 'PIO_TF_test_var3'
  type(io_desc_t) :: wr_iodesc, rd_iodesc
  integer, dimension(:), allocatable :: compdof
  integer, dimension(:), allocatable :: start, count
  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf, exp_val
  integer, dimension(1) :: dims
  integer :: pio_dim
  integer :: i, ierr
  ! iotypes = valid io types
  integer, dimension(:), allocatable :: iotypes
  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
  integer :: num_iotypes

  ! Set the decomposition for writing data - forcing rearrangement
  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
         start, count, .true.)
  allocate(wbuf(count(1)))
  allocate(compdof(count(1)))
  do i=1,count(1)
    wbuf(i) = start(1) + i - 1
    compdof(i) = start(1) + i - 1
  end do
  if(allocated(start))then
    deallocate(start)
  end if

  if(allocated(count))then
    deallocate(count)
  end if

  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
  deallocate(compdof)

  ! Set the decomposition for reading data - different from the write decomp
  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
         start, count, .false.)
  allocate(rbuf(count(1)))
  allocate(compdof(count(1)))
  allocate(exp_val(count(1)))
  do i=1,count(1)
    compdof(i) = start(1) + i -1
    ! Expected value, after reading, is the same as the compdof
    exp_val(i) = compdof(i)
  end do

  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
  deallocate(compdof)

  num_iotypes = 0
  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
  filename = "test_pio_decomp_simple_tests.testfile"
  do i=1,num_iotypes
    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))

    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR1_NAME, PIO_TF_DATA_TYPE, (/pio_dim/), pio_var1)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR2_NAME, PIO_TF_DATA_TYPE, (/pio_dim/), pio_var2)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR3_NAME, PIO_TF_DATA_TYPE, (/pio_dim/), pio_var3)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))

    ierr = PIO_enddef(pio_file)
    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))

    ! Write the variables out
    call PIO_write_darray(pio_file, pio_var1, wr_iodesc, wbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))

    call PIO_write_darray(pio_file, pio_var2, wr_iodesc, wbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))

    call PIO_write_darray(pio_file, pio_var3, wr_iodesc, wbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))

#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC
    call PIO_closefile(pio_file)

    ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite)
    PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR1_NAME, pio_var1)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var1 :" // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR2_NAME, pio_var2)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var2 :" // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR3_NAME, pio_var3)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var3 :" // trim(filename))
#else
    call PIO_syncfile(pio_file)
#endif

    rbuf = 0
    call PIO_read_darray(pio_file, pio_var1, rd_iodesc, rbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var1) : " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val (var1)")

    rbuf = 0
    call PIO_read_darray(pio_file, pio_var2, rd_iodesc, rbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var2) : " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val (var2)")

    rbuf = 0
    call PIO_read_darray(pio_file, pio_var3, rd_iodesc, rbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var3) : " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val (var3)")

    call PIO_closefile(pio_file)
    
    call PIO_deletefile(pio_tf_iosystem_, filename);
  end do

  if(allocated(iotypes)) then
    deallocate(iotypes)
    deallocate(iotype_descs)
  end if

  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
  deallocate(exp_val)
  deallocate(rbuf)
  deallocate(wbuf)
PIO_TF_AUTO_TEST_SUB_END nc_mvar_wr_rd_1d_bc

! Test writes multiple variables before syncing and reading them back
! This test adds multiple syncs between writes
PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE>
PIO_TF_AUTO_TEST_SUB_BEGIN nc_mvar_msync_1d_bc
  implicit none
  interface
    subroutine get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
      integer, intent(in) :: rank
      integer, intent(in) :: sz
      integer, dimension(1), intent(out) :: dims
      integer, dimension(:), allocatable, intent(out) :: start
      integer, dimension(:), allocatable, intent(out) :: count
      logical, intent(in) :: force_rearrange
    end subroutine get_1d_bc_info
  end interface
  integer, parameter :: NVARS = 8
  integer, parameter :: SYNC_STRIDE_START = 1
  ! The strides get incremented by SYNC_STRIDE_INC after each sync call
  ! With NVARS = 10, SYNC_STRIDE_START = 1, SYNC_STRIDE_INC = 1, the sync
  ! call is invoked after
  ! 1st (stride = 1), 3rd (stride = 2), 6th (stride = 3), 10th (stride = 4)
  ! pio_write_darray() calls
  integer, parameter :: SYNC_STRIDE_INC = 1
  type(var_desc_t)  :: pio_vars(NVARS)
  type(file_desc_t) :: pio_file
  character(len=PIO_TF_MAX_STR_LEN) :: filename
  character(len=PIO_TF_MAX_STR_LEN) :: vnames(NVARS)
  type(io_desc_t) :: wr_iodesc, rd_iodesc
  integer, dimension(:), allocatable :: compdof
  integer, dimension(:), allocatable :: start, count
  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf, exp_val
  integer, dimension(1) :: dims
  integer :: pio_dim
  integer :: i, j, k, cur_sync_stride, isync, prev_isync, ierr
  ! iotypes = valid io types
  integer, dimension(:), allocatable :: iotypes
  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
  integer :: num_iotypes

  ! Set the decomposition for writing data - forcing rearrangement
  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
         start, count, .true.)
  allocate(wbuf(count(1)))
  allocate(compdof(count(1)))
  do i=1,count(1)
    wbuf(i) = start(1) + i - 1
    compdof(i) = start(1) + i - 1
  end do
  if(allocated(start))then
    deallocate(start)
  end if

  if(allocated(count))then
    deallocate(count)
  end if

  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
  deallocate(compdof)

  ! Set the decomposition for reading data - different from the write decomp
  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
         start, count, .false.)
  allocate(rbuf(count(1)))
  allocate(compdof(count(1)))
  allocate(exp_val(count(1)))
  do i=1,count(1)
    compdof(i) = start(1) + i -1
    ! Expected value, after reading, is the same as the compdof
    exp_val(i) = compdof(i)
  end do

  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
  deallocate(compdof)

  num_iotypes = 0
  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
  filename = "test_pio_decomp_simple_tests.testfile"
  do i=1,num_iotypes
    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))

    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))

    do j=1,NVARS
      vnames(j) = ""
      write(vnames(j), "(A15,I6.6)") "PIO_TF_test_var", j
      ierr = PIO_def_var(pio_file, trim(vnames(j)), PIO_TF_DATA_TYPE, (/pio_dim/), pio_vars(j))
      PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
    end do

    ierr = PIO_enddef(pio_file)
    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))

    ! Write the variables out - all the variables write out the same
    ! data.
    ! The stride determines the pio_syncfile calls, stride starts
    ! at SYNC_STRIDE_START and increases by SYNC_STRIDE_INC
    isync = SYNC_STRIDE_START
    prev_isync = 0
    cur_sync_stride = SYNC_STRIDE_START
    do j=1,NVARS
      call PIO_write_darray(pio_file, pio_vars(j), wr_iodesc, wbuf, ierr)
      PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
      if(j == isync) then
#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC
        call PIO_closefile(pio_file)

        ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_write)
        PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename))

        do k=1,NVARS
          ierr = PIO_inq_varid(pio_file, vnames(k), pio_vars(k))
          PIO_TF_CHECK_ERR(ierr, "Could not inq var :" // trim(filename))
        end do
#else
        call PIO_syncfile(pio_file)
#endif
        do k=prev_isync+1, isync
          rbuf = 0
          call PIO_read_darray(pio_file, pio_vars(k), rd_iodesc, rbuf, ierr)
          PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))

          PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
        end do
        cur_sync_stride = cur_sync_stride + SYNC_STRIDE_INC
        prev_isync = isync
        isync = isync + cur_sync_stride
      end if
    end do

#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC
    call PIO_closefile(pio_file)

    ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite)
    PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename))

    do k=1,NVARS
      ierr = PIO_inq_varid(pio_file, vnames(k), pio_vars(k))
      PIO_TF_CHECK_ERR(ierr, "Could not inq var :" // trim(filename))
    end do
#else
    call PIO_syncfile(pio_file)
#endif

    if(prev_isync < NVARS) then
      do k=prev_isync+1,NVARS
          rbuf = 0
          call PIO_read_darray(pio_file, pio_vars(k), rd_iodesc, rbuf, ierr)
          PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))

          PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
      end do
    end if

    call PIO_closefile(pio_file)

    call PIO_deletefile(pio_tf_iosystem_, filename);
  end do

  if(allocated(iotypes)) then
    deallocate(iotypes)
    deallocate(iotype_descs)
  end if

  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
  deallocate(exp_val)
  deallocate(rbuf)
  deallocate(wbuf)
PIO_TF_AUTO_TEST_SUB_END nc_mvar_msync_1d_bc

! Test write/read of multiple variables before syncing
! * Variables 1 & 3 - each proc writes/reads multiple regions
!   of the data
! * Variables 2 & 4 - each proc writes/reads 1 region
!   (with data rearrangement)
PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE>
PIO_TF_AUTO_TEST_SUB_BEGIN nc_mvar_mreg_wr_rd_1d_bc
  implicit none
  interface
    subroutine get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
      integer, intent(in) :: rank
      integer, intent(in) :: sz
      integer, dimension(1), intent(out) :: dims
      integer, dimension(:), allocatable, intent(out) :: start
      integer, dimension(:), allocatable, intent(out) :: count
      logical, intent(in) :: force_rearrange
    end subroutine get_1d_bc_info
    subroutine get_1d_bc_mreg_info(rank, sz, dims, start, count)
      integer, intent(in) :: rank
      integer, intent(in) :: sz
      integer, dimension(1), intent(out) :: dims
      integer, dimension(:), allocatable, intent(out) :: start
      integer, dimension(:), allocatable, intent(out) :: count
    end subroutine get_1d_bc_mreg_info
  end interface
  type(var_desc_t)  :: pio_var1, pio_var2, pio_var3, pio_var4
  type(file_desc_t) :: pio_file
  character(len=PIO_TF_MAX_STR_LEN) :: filename
  character(len=*), parameter :: PIO_VAR1_NAME = 'PIO_TF_test_var1_mreg'
  character(len=*), parameter :: PIO_VAR2_NAME = 'PIO_TF_test_var2'
  character(len=*), parameter :: PIO_VAR3_NAME = 'PIO_TF_test_var3_mreg'
  character(len=*), parameter :: PIO_VAR4_NAME = 'PIO_TF_test_var4'
  type(io_desc_t) :: iodesc_mreg, iodesc
  integer, dimension(:), allocatable :: compdof
  integer, dimension(:), allocatable :: start, count
  integer :: total_count
  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf
  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf_mreg, wbuf_mreg
  integer, dimension(1) :: dims, dims_mreg
  integer :: pio_dim, pio_dim_mreg
  integer :: i, j, idx, ierr
  ! iotypes = valid io types
  integer, dimension(:), allocatable :: iotypes
  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
  integer :: num_iotypes

  ! Set the decomposition for writing data - forcing rearrangement
  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
         start, count, .true.)
  allocate(wbuf(count(1)))
  allocate(rbuf(count(1)))
  allocate(compdof(count(1)))
  do i=1,count(1)
    wbuf(i) = start(1) + i - 1
    compdof(i) = start(1) + i - 1
  end do
  rbuf = wbuf
  if(allocated(start)) then
    deallocate(start)
  end if
  if(allocated(count)) then
    deallocate(count)
  end if

  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
  deallocate(compdof)

  call get_1d_bc_mreg_info(pio_tf_world_rank_, pio_tf_world_sz_,&
         dims_mreg, start, count)
  total_count = 0
  do i=1,size(count)
    total_count = total_count + count(i) 
  end do
  allocate(wbuf_mreg(total_count))
  allocate(rbuf_mreg(total_count))
  allocate(compdof(total_count))
  do i=1,size(count)
    do j=1,count(i)
      idx = (i-1)*count(i) + j
      compdof(idx) = start(i) + j - 1
      wbuf_mreg(idx) = compdof(idx)
      rbuf_mreg(idx) = compdof(idx)
    end do
  end do
  if(allocated(start)) then
    deallocate(start)
  end if
  if(allocated(count)) then
    deallocate(count)
  end if

  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims_mreg, compdof, iodesc_mreg)
  deallocate(compdof)

  num_iotypes = 0
  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
  filename = "test_pio_decomp_simple_tests.testfile"
  do i=1,num_iotypes
    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))

    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))

    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_mreg', dims_mreg(1), pio_dim_mreg)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim (mreg) : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR1_NAME, PIO_TF_DATA_TYPE, (/pio_dim_mreg/), pio_var1)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var (var1) : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR2_NAME, PIO_TF_DATA_TYPE, (/pio_dim/), pio_var2)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var (var2) : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR3_NAME, PIO_TF_DATA_TYPE, (/pio_dim_mreg/), pio_var3)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var (var3) : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR4_NAME, PIO_TF_DATA_TYPE, (/pio_dim/), pio_var4)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var (var4) : " // trim(filename))

    ierr = PIO_enddef(pio_file)
    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))

    ! Write the variables out
    call PIO_write_darray(pio_file, pio_var1, iodesc_mreg, wbuf_mreg, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray (var1) : " // trim(filename))

    call PIO_write_darray(pio_file, pio_var2, iodesc, wbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray (var2) : " // trim(filename))

    call PIO_write_darray(pio_file, pio_var3, iodesc_mreg, wbuf_mreg, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray (var3) : " // trim(filename))

    call PIO_write_darray(pio_file, pio_var4, iodesc, wbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray (var4) : " // trim(filename))

#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC
    call PIO_closefile(pio_file)

    ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite)
    PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR1_NAME, pio_var1)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var1 :" // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR2_NAME, pio_var2)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var2 :" // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR3_NAME, pio_var3)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var3 :" // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR4_NAME, pio_var4)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var4 :" // trim(filename))
#else
    call PIO_syncfile(pio_file)
#endif

    rbuf_mreg = 0
    call PIO_read_darray(pio_file, pio_var1, iodesc_mreg, rbuf_mreg, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var1): " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf_mreg, wbuf_mreg), "Got wrong val (var1)")

    rbuf_mreg = 0
    call PIO_read_darray(pio_file, pio_var3, iodesc_mreg, rbuf_mreg, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var3): " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf_mreg, wbuf_mreg), "Got wrong val (var2)")

    rbuf = 0
    call PIO_read_darray(pio_file, pio_var2, iodesc, rbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var2): " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val (var2)")

    rbuf = 0
    call PIO_read_darray(pio_file, pio_var4, iodesc, rbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var4): " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val (var4)")

    call PIO_closefile(pio_file)
    
    call PIO_deletefile(pio_tf_iosystem_, filename);
  end do

  if(allocated(iotypes)) then
    deallocate(iotypes)
    deallocate(iotype_descs)
  end if

  call PIO_freedecomp(pio_tf_iosystem_, iodesc)
  call PIO_freedecomp(pio_tf_iosystem_, iodesc_mreg)
  deallocate(rbuf)
  deallocate(rbuf_mreg)
  deallocate(wbuf)
  deallocate(wbuf_mreg)
PIO_TF_AUTO_TEST_SUB_END nc_mvar_mreg_wr_rd_1d_bc

! Test reading data from a file while writes are pending
PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE>
PIO_TF_AUTO_TEST_SUB_BEGIN nc_rd_while_pend_wr_1d
  implicit none
  interface
    subroutine get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
      integer, intent(in) :: rank
      integer, intent(in) :: sz
      integer, dimension(1), intent(out) :: dims
      integer, dimension(:), allocatable, intent(out) :: start
      integer, dimension(:), allocatable, intent(out) :: count
      logical, intent(in) :: force_rearrange
    end subroutine get_1d_bc_info
  end interface
  type(var_desc_t)  :: pio_var1, pio_var2, pio_var3
  type(file_desc_t) :: pio_file
  character(len=PIO_TF_MAX_STR_LEN) :: filename
  character(len=*), parameter :: PIO_VAR1_NAME = 'PIO_TF_test_var1'
  character(len=*), parameter :: PIO_VAR2_NAME = 'PIO_TF_test_var2'
  character(len=*), parameter :: PIO_VAR3_NAME = 'PIO_TF_test_var3'
  type(io_desc_t) :: wr_iodesc, rd_iodesc
  integer, dimension(:), allocatable :: compdof
  integer, dimension(:), allocatable :: start, count
  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf, exp_val
  integer, dimension(1) :: dims
  integer :: pio_dim
  integer :: i, ierr
  ! iotypes = valid io types
  integer, dimension(:), allocatable :: iotypes
  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
  integer :: num_iotypes

  ! Set the decomposition for writing data - forcing rearrangement
  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
         start, count, .true.)
  allocate(wbuf(count(1)))
  allocate(compdof(count(1)))
  do i=1,count(1)
    wbuf(i) = start(1) + i - 1
    compdof(i) = start(1) + i - 1
  end do
  if(allocated(start))then
    deallocate(start)
  end if

  if(allocated(count))then
    deallocate(count)
  end if

  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
  deallocate(compdof)

  ! Set the decomposition for reading data - different from the write decomp
  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
         start, count, .false.)
  allocate(rbuf(count(1)))
  allocate(compdof(count(1)))
  allocate(exp_val(count(1)))
  do i=1,count(1)
    compdof(i) = start(1) + i -1
    ! Expected value, after reading, is the same as the compdof
    exp_val(i) = compdof(i)
  end do

  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
  deallocate(compdof)

  num_iotypes = 0
  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
  filename = "test_pio_decomp_simple_tests.testfile"
  do i=1,num_iotypes
    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))

    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR1_NAME, PIO_TF_DATA_TYPE, (/pio_dim/), pio_var1)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR2_NAME, PIO_TF_DATA_TYPE, (/pio_dim/), pio_var2)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))

    ierr = PIO_def_var(pio_file, PIO_VAR3_NAME, PIO_TF_DATA_TYPE, (/pio_dim/), pio_var3)
    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))

    ierr = PIO_enddef(pio_file)
    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))

    ! Write the variables out
    call PIO_write_darray(pio_file, pio_var1, wr_iodesc, wbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))

#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC
    call PIO_closefile(pio_file)

    ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_write)
    PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR1_NAME, pio_var1)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var1 :" // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR2_NAME, pio_var2)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var2 :" // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR3_NAME, pio_var3)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var3 :" // trim(filename))
#else
    call PIO_syncfile(pio_file)
#endif

    call PIO_write_darray(pio_file, pio_var2, wr_iodesc, wbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))

    call PIO_write_darray(pio_file, pio_var3, wr_iodesc, wbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))


    ! The data written out for pio_var1 is already synced and should be
    ! available to read. Note that two writes above are potentially
    ! pending since we haven't synced it yet

    rbuf = 0
    call PIO_read_darray(pio_file, pio_var1, rd_iodesc, rbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var1) : " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val (var1)")

    ! Sync pio_var2 and pio_var3
#ifdef PIO_TEST_CLOSE_OPEN_FOR_SYNC
    call PIO_closefile(pio_file)

    ierr = PIO_openfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_nowrite)
    PIO_TF_CHECK_ERR(ierr, "Could not reopen file " // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR1_NAME, pio_var1)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var1 :" // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR2_NAME, pio_var2)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var2 :" // trim(filename))

    ierr = PIO_inq_varid(pio_file, PIO_VAR3_NAME, pio_var3)
    PIO_TF_CHECK_ERR(ierr, "Could not inq var3 :" // trim(filename))
#else
    call PIO_syncfile(pio_file)
#endif

    rbuf = 0
    call PIO_read_darray(pio_file, pio_var2, rd_iodesc, rbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var2) : " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val (var2)")

    rbuf = 0
    call PIO_read_darray(pio_file, pio_var3, rd_iodesc, rbuf, ierr)
    PIO_TF_CHECK_ERR(ierr, "Failed to read darray (var3) : " // trim(filename))

    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val (var3)")

    call PIO_closefile(pio_file)

    call PIO_deletefile(pio_tf_iosystem_, filename);
  end do

  if(allocated(iotypes)) then
    deallocate(iotypes)
    deallocate(iotype_descs)
  end if

  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
  deallocate(exp_val)
  deallocate(rbuf)
  deallocate(wbuf)
PIO_TF_AUTO_TEST_SUB_END nc_rd_while_pend_wr_1d

