Skip to content

Add PnetCDF Non-blocking APIs to Perform I/O #2058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: develop
Choose a base branch
from
Open

Add PnetCDF Non-blocking APIs to Perform I/O #2058

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
c2759ba
Add enable_pnetcdf_bput option to registry
yzanhua May 28, 2024
9b48954
Subroutine for PnetCDF non-blocking buffersize calc
yzanhua May 28, 2024
aa76138
Subroutines to set buffersize for PNC-nb APIs.
yzanhua May 28, 2024
bd84fb2
Subroutines to flush pending PNC-nb write reqs
yzanhua May 28, 2024
981cb65
PNC-nb: cal and set buffersize, wait, and detach
yzanhua May 28, 2024
9a29d44
Implement posting write reqs for PNC-nb.
yzanhua May 28, 2024
46f7089
PnetCDF: enter define mode only once
yzanhua May 28, 2024
cad47a0
PnetCDF: only rank 0 write non-partitioned var
yzanhua May 28, 2024
77f88ad
Add new subroutine BputCalcBufferSize to da_name_space.pl
wkliao Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Implement posting write reqs for PNC-nb. 
PNC-nb refers to PnetCDF non-blocking APIs.
In this commit, we call NFMPI_BPUT instead of NFMPI_PUT if non-blocking
APIs is enabled.
  • Loading branch information
@yzanhua
yzanhua committed May 28, 2024
commit 9a29d4400f3c7ec9a40a116f78df08030dc49477
36 changes: 28 additions & 8 deletions external/io_pnetcdf/field_routines.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
!* Date: October 6, 2000
!*
!*----------------------------------------------------------------------------
subroutine ext_pnc_RealFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
subroutine ext_pnc_RealFieldIO(Coll,IO,NCID,VarID,VStart,VCount,EnableBput,Data,Status)
use wrf_data_pnc
use ext_pnc_support_routines
implicit none
Expand All @@ -45,16 +45,21 @@ subroutine ext_pnc_RealFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
integer ,intent(in) :: VarID
integer ,dimension(NVarDims),intent(in) :: VStart
integer ,dimension(NVarDims),intent(in) :: VCount
logical ,intent(in) :: EnableBput
real, dimension(*) ,intent(inout) :: Data
integer ,intent(out) :: Status
integer :: stat
!local
integer(KIND=MPI_OFFSET_KIND), dimension(NVarDims) :: VStart_mpi, VCount_mpi
integer :: BputReqID
VStart_mpi = VStart
VCount_mpi = VCount

if(IO == 'write') then
if(Coll)then
if(EnableBput)then
! Calling non-blocking buffered-version API
stat = NFMPI_BPUT_VARA_REAL(NCID,VarID,VStart_mpi,VCount_mpi,Data,BputReqID)
else if(Coll)then
stat = NFMPI_PUT_VARA_REAL_ALL(NCID,VarID,VStart_mpi,VCount_mpi,Data)
else
stat = NFMPI_PUT_VARA_REAL(NCID,VarID,VStart_mpi,VCount_mpi,Data)
Expand All @@ -74,7 +79,7 @@ subroutine ext_pnc_RealFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
return
end subroutine ext_pnc_RealFieldIO

subroutine ext_pnc_DoubleFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
subroutine ext_pnc_DoubleFieldIO(Coll,IO,NCID,VarID,VStart,VCount,EnableBput,Data,Status)
use wrf_data_pnc
use ext_pnc_support_routines
implicit none
Expand All @@ -86,16 +91,21 @@ subroutine ext_pnc_DoubleFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
integer ,intent(in) :: VarID
integer ,dimension(NVarDims),intent(in) :: VStart
integer ,dimension(NVarDims),intent(in) :: VCount
logical ,intent(in) :: EnableBput
real*8 ,intent(inout) :: Data
integer ,intent(out) :: Status
integer :: stat
!local
integer(KIND=MPI_OFFSET_KIND), dimension(NVarDims) :: VStart_mpi, VCount_mpi
integer :: BputReqID
VStart_mpi = VStart
VCount_mpi = VCount

if(IO == 'write') then
if(Coll)then
if(EnableBput)then
! Calling non-blocking buffered-version API
stat = NFMPI_BPUT_VARA_DOUBLE(NCID,VarID,VStart_mpi,VCount_mpi,Data,BputReqID)
else if(Coll)then
stat = NFMPI_PUT_VARA_DOUBLE_ALL(NCID,VarID,VStart_mpi,VCount_mpi,Data)
else
stat = NFMPI_PUT_VARA_DOUBLE(NCID,VarID,VStart_mpi,VCount_mpi,Data)
Expand All @@ -115,7 +125,7 @@ subroutine ext_pnc_DoubleFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
return
end subroutine ext_pnc_DoubleFieldIO

subroutine ext_pnc_IntFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
subroutine ext_pnc_IntFieldIO(Coll,IO,NCID,VarID,VStart,VCount,EnableBput,Data,Status)
use wrf_data_pnc
use ext_pnc_support_routines
implicit none
Expand All @@ -127,16 +137,21 @@ subroutine ext_pnc_IntFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
integer ,intent(in) :: VarID
integer ,dimension(NVarDims),intent(in) :: VStart
integer ,dimension(NVarDims),intent(in) :: VCount
logical ,intent(in) :: EnableBput
integer ,intent(inout) :: Data
integer ,intent(out) :: Status
integer :: stat
!local
integer(KIND=MPI_OFFSET_KIND), dimension(NVarDims) :: VStart_mpi, VCount_mpi
integer :: BputReqID
VStart_mpi = VStart
VCount_mpi = VCount

if(IO == 'write') then
if(Coll)then
if(EnableBput)then
! Calling non-blocking buffered-version API
stat = NFMPI_BPUT_VARA_INT(NCID,VarID,VStart_mpi,VCount_mpi,Data,BputReqID)
else if(Coll)then
stat = NFMPI_PUT_VARA_INT_ALL(NCID,VarID,VStart_mpi,VCount_mpi,Data)
else
stat = NFMPI_PUT_VARA_INT(NCID,VarID,VStart_mpi,VCount_mpi,Data)
Expand All @@ -156,7 +171,7 @@ subroutine ext_pnc_IntFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
return
end subroutine ext_pnc_IntFieldIO

subroutine ext_pnc_LogicalFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
subroutine ext_pnc_LogicalFieldIO(Coll,IO,NCID,VarID,VStart,VCount,EnableBput,Data,Status)
use wrf_data_pnc
use ext_pnc_support_routines
implicit none
Expand All @@ -168,13 +183,15 @@ subroutine ext_pnc_LogicalFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
integer ,intent(in) :: VarID
integer,dimension(NVarDims) ,intent(in) :: VStart
integer,dimension(NVarDims) ,intent(in) :: VCount
logical ,intent(in) :: EnableBput
logical,dimension(VCount(1),VCount(2),VCount(3)),intent(inout) :: Data
integer ,intent(out) :: Status
integer,dimension(:,:,:),allocatable :: Buffer
integer :: stat
integer :: i,j,k
!local
integer(KIND=MPI_OFFSET_KIND), dimension(NVarDims) :: VStart_mpi, VCount_mpi
integer :: BputReqID
VStart_mpi = VStart
VCount_mpi = VCount

Expand All @@ -197,7 +214,10 @@ subroutine ext_pnc_LogicalFieldIO(Coll,IO,NCID,VarID,VStart,VCount,Data,Status)
enddo
enddo
enddo
if(Coll)then
if(EnableBput)then
! Calling non-blocking buffered-version API
stat = NFMPI_BPUT_VARA_INT(NCID,VarID,VStart_mpi,VCount_mpi,Buffer,BputReqID)
else if(Coll)then
stat = NFMPI_PUT_VARA_INT_ALL(NCID,VarID,VStart_mpi,VCount_mpi,Buffer)
else
stat = NFMPI_PUT_VARA_INT(NCID,VarID,VStart_mpi,VCount_mpi,Buffer)
Expand Down
19 changes: 11 additions & 8 deletions external/io_pnetcdf/wrf_io.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -687,6 +687,7 @@ subroutine FieldIO(IO,DataHandle,DateStr,Starts,Length,MemoryOrder &
integer :: NDim
integer,dimension(NVarDims) :: VStart
integer,dimension(NVarDims) :: VCount
type(wrf_data_handle) ,pointer :: DH

call GetTimeIndex(IO,DataHandle,DateStr,TimeIndex,Status)
if(Status /= WRF_NO_ERR) then
Expand All @@ -704,19 +705,21 @@ subroutine FieldIO(IO,DataHandle,DateStr,Starts,Length,MemoryOrder &
VCount(1:NDim) = Length(1:NDim)
VStart(NDim+1) = TimeIndex
VCount(NDim+1) = 1
DH => WrfDataHandles(DataHandle)

select case (FieldType)
case (WRF_REAL)
call ext_pnc_RealFieldIO (WrfDataHandles(DataHandle)%Collective, &
IO,NCID,VarID,VStart,VCount,XField,Status)
call ext_pnc_RealFieldIO (DH%Collective,IO,NCID,VarID,&
VStart,VCount,DH%BputEnabled,XField,Status)
case (WRF_DOUBLE)
call ext_pnc_DoubleFieldIO (WrfDataHandles(DataHandle)%Collective, &
IO,NCID,VarID,VStart,VCount,XField,Status)
call ext_pnc_DoubleFieldIO (DH%Collective,IO,NCID,VarID,&
VStart,VCount,DH%BputEnabled,XField,Status)
case (WRF_INTEGER)
call ext_pnc_IntFieldIO (WrfDataHandles(DataHandle)%Collective, &
IO,NCID,VarID,VStart,VCount,XField,Status)
call ext_pnc_IntFieldIO (DH%Collective,IO,NCID,VarID,&
VStart,VCount,DH%BputEnabled,XField,Status)
case (WRF_LOGICAL)
call ext_pnc_LogicalFieldIO (WrfDataHandles(DataHandle)%Collective, &
IO,NCID,VarID,VStart,VCount,XField,Status)
call ext_pnc_LogicalFieldIO (DH%Collective,IO,NCID,VarID,&
VStart,VCount,DH%BputEnabled,XField,Status)
if(Status /= WRF_NO_ERR) return
case default
!for wrf_complex, double_complex
Expand Down