diff --git a/src_mtln/generators.F90 b/src_mtln/generators.F90 index 74b568ad..83c9cc9d 100644 --- a/src_mtln/generators.F90 +++ b/src_mtln/generators.F90 @@ -46,22 +46,26 @@ function generatorCtor(index, conductor, gen_type, resistance, path, layer_indic if (present(layer_indices)) then call MPI_COMM_SIZE(SUBCOMM_MPI, sizeof, ierr) if (sizeof > 1) then - res%in_layer = .false. - do i = 1, size(layer_indices,1) - if (index >= layer_indices(i, 1) .and. index <= layer_indices(i,2)+1) then - res%in_layer = .true. - slice = i - end if - end do - - layer_index = 0 - if (res%in_layer) then - do i = 1, slice - 1 - layer_index = layer_index + layer_indices(i,2) + 1 - (layer_indices(i,1) - 1) + if (size(layer_indices,1) == 0) then + res%in_layer = .true. + else + res%in_layer = .false. + do i = 1, size(layer_indices,1) + if (index >= layer_indices(i, 1) .and. index <= layer_indices(i,2)+1) then + res%in_layer = .true. + slice = i + end if end do - layer_index = layer_index + res%index - layer_indices(slice,1) + 1 + + layer_index = 0 + if (res%in_layer) then + do i = 1, slice - 1 + layer_index = layer_index + layer_indices(i,2) + 1 - (layer_indices(i,1) - 1) + end do + layer_index = layer_index + res%index - layer_indices(slice,1) + 1 + end if + res%index = layer_index end if - res%index = layer_index end if end if #endif diff --git a/src_mtln/mtl.F90 b/src_mtln/mtl.F90 index daf889ea..a3eddcc9 100644 --- a/src_mtln/mtl.F90 +++ b/src_mtln/mtl.F90 @@ -137,6 +137,8 @@ function mtl_shielded(lpul, cpul, rpul, gpul, & allocate(res%segments(0)) allocate(res%mpi_comm%comms(0)) end if + res%layer_indices = layer_indices + if (present(bundle_in_layer)) res%bundle_in_layer = bundle_in_layer else res%step_size = step_size allocate(res%layer_indices(0,0)) @@ -153,9 +155,13 @@ function mtl_shielded(lpul, cpul, rpul, gpul, & res%number_of_conductors = size(lpul, 1) call res%initDirections() call res%allocatePULMatrices() - call res%initLC(lpul, cpul) - call res%initRG(rpul, gpul) - call res%checkTimeStep(getMax = (lpul(1,1) /= 0.0), dt = dt) + if (size(res%step_size) /= 0) then + call res%initLC(lpul, cpul) + call res%initRG(rpul, gpul) + call res%checkTimeStep(getMax = (lpul(1,1) /= 0.0), dt = dt) + else + res%dt = dt + end if res%parent_name = parent_name res%conductor_in_parent = conductor_in_parent res%transfer_impedance = transfer_impedance @@ -192,7 +198,7 @@ function mtl_unshielded(lpul, cpul, rpul, gpul, & allocate(res%mpi_comm%comms(0)) end if res%layer_indices = layer_indices - res%bundle_in_layer = bundle_in_layer + if (present(bundle_in_layer)) res%bundle_in_layer = bundle_in_layer else res%step_size = step_size allocate(res%layer_indices(0,0)) @@ -217,9 +223,11 @@ function mtl_unshielded(lpul, cpul, rpul, gpul, & else call res%initLC(lpul, cpul) end if + call res%initRG(rpul, gpul) + call res%checkTimeStep(getMax = (lpul(1,1) /= 0.0), dt = dt) + else + res%dt = dt end if - call res%initRG(rpul, gpul) - call res%checkTimeStep(getMax = (lpul(1,1) /= 0.0), dt = dt) res%lumped_elements = dispersive_lumped_t(res%number_of_conductors, 0, size(res%step_size), res%dt) end function diff --git a/src_mtln/mtl_bundle.F90 b/src_mtln/mtl_bundle.F90 index d830ae9c..9286b5b8 100644 --- a/src_mtln/mtl_bundle.F90 +++ b/src_mtln/mtl_bundle.F90 @@ -1,12 +1,13 @@ module mtl_bundle_m use mtln_utils_m - use probes_m + use probes_m, only: probe_t, probeCtor, PROBE_TYPE_CURRENT use generators_m use dispersive_m use mtl_m #ifdef CompileWithMPI use FDETYPES_m, only: SUBCOMM_MPI, REALSIZE, INTEGERSIZE, MPI_STATUS_SIZE + use mpi #endif use mtln_types_m, only: SOURCE_TYPE_CURRENT, SOURCE_TYPE_VOLTAGE use FDETYPES_m, only: RKIND, RKIND_TIEMPO @@ -38,6 +39,8 @@ module mtl_bundle_m #ifdef CompileWithMPI integer(kind=4), allocatable, dimension(:,:) :: layer_indices type(comm_t) :: mpi_comm + ! Owner rank of each physical current segment. + integer(kind=4), allocatable :: segment_owner(:) #endif contains @@ -57,6 +60,8 @@ module mtl_bundle_m #ifdef CompileWithMPI procedure :: Comm_MPI_V procedure :: Comm_MPI_Fields + procedure :: Comm_MPI_I + procedure :: initMPIStateOwnership #endif @@ -70,6 +75,10 @@ module mtl_bundle_m integer, dimension(3) ::position integer :: direction = 0 real(kind=rkind) , pointer :: field => null() + ! True only on the rank that updates this FDTD field cell. + logical :: field_is_local = .false. + ! True on the geometric FDTD owner even when the cell is PEC/lossy. + logical :: field_is_owned = .false. end type contains @@ -401,10 +410,52 @@ subroutine bundle_updateGenerators(this, time, dt) end subroutine +#ifdef CompileWithMPI + subroutine initMPIStateOwnership(this) + class(mtl_bundle_t) :: this + integer :: i, ierr, rank, n + integer, allocatable :: candidate(:), owner(:) + + n = this%number_of_divisions + allocate(candidate(n), owner(n), source = -1) + call MPI_COMM_RANK(SUBCOMM_MPI, rank, ierr) + do i = 1, n + if (this%external_field_segments(i)%field_is_owned) candidate(i) = rank + end do + call MPI_Allreduce(candidate, owner, n, MPI_INTEGER, MPI_MAX, SUBCOMM_MPI, ierr) + if (any(owner < 0)) error stop "MTLN MPI ownership missing for a current segment" + if (allocated(this%segment_owner)) deallocate(this%segment_owner) + call move_alloc(owner, this%segment_owner) + do i = 1, n + this%external_field_segments(i)%field_is_local = this%external_field_segments(i)%field_is_local .and. & + this%segment_owner(i) == rank + end do + do i = 1, size(this%probes) + if (this%probes(i)%type == PROBE_TYPE_CURRENT) then + this%probes(i)%in_layer = this%segment_owner(this%probes(i)%index) == rank + else + this%probes(i)%in_layer = this%segment_owner(min(this%probes(i)%index, n)) == rank + end if + end do + do i = 1, size(this%generators) + this%generators(i)%in_layer = this%segment_owner(this%generators(i)%index) == rank + end do + end subroutine initMPIStateOwnership +#endif + subroutine bundle_advanceVoltage(this) class(mtl_bundle_t) ::this integer :: i +#ifdef CompileWithMPI + integer :: ierr, rank + call MPI_COMM_RANK(SUBCOMM_MPI, rank, ierr) +#endif do i = 2,this%number_of_divisions +#ifdef CompileWithMPI + if (allocated(this%segment_owner)) then + if (this%segment_owner(i) /= rank) cycle + end if +#endif this%v(:, i) = matmul(this%v_term(i,:,:), this%v(:,i)) - & matmul(this%i_diff(i,:,:), (this%i(:,i) - this%i(:,i-1)) + matmul(this%du(i,:,:), this%i_source(:,i))) end do @@ -417,17 +468,20 @@ subroutine bundle_advanceCurrent(this) integer :: i real(kind=rkind) :: eps_r #ifdef CompileWithMPI - integer(kind=4) :: sizeof, ierr - -#endif -#ifdef CompileWithMPI + integer(kind=4) :: sizeof, ierr, rank call MPI_COMM_SIZE(SUBCOMM_MPI, sizeof, ierr) + call MPI_COMM_RANK(SUBCOMM_MPI, rank, ierr) if (sizeof > 1) call this%Comm_MPI_V() #endif call this%transfer_impedance%updateQ3Phi() this%i_prev = this%i do i = 1, this%number_of_divisions +#ifdef CompileWithMPI + if (allocated(this%segment_owner)) then + if (this%segment_owner(i) /= rank) cycle + end if +#endif this%i(:,i) = matmul(this%i_term(i,:,:), this%i(:,i)) - & matmul(this%v_diff(i,:,:), (this%v(:,i+1) - this%v(:,i)) - & this%e_L(:,i) * this%step_size(i) - & @@ -435,57 +489,81 @@ subroutine bundle_advanceCurrent(this) matmul(this%v_diff(i,:,:), matmul(this%du(i,:,:), this%transfer_impedance%q3_phi(i,:))) enddo call this%transfer_impedance%updatePhi(this%i_prev, this%i) +#ifdef CompileWithMPI + if (sizeof > 1) call this%Comm_MPI_I() +#endif end subroutine subroutine bundle_setExternalLongitudinalField(this) class(mtl_bundle_t) :: this integer :: i, j #ifdef CompileWithMPI - integer :: sizeof, ierr - - call MPI_COMM_SIZE(SUBCOMM_MPI, sizeof, ierr) - if (sizeof > 1) call this%Comm_MPI_Fields() + integer :: ierr, rank + call MPI_COMM_RANK(SUBCOMM_MPI, rank, ierr) #endif - + this%e_L = 0.0_rkind do j = 1, this%conductors_in_level(1) do i = 1, size(this%e_L,2) - this%e_L(j,i) = this%external_field_segments(i)%field * & - this%external_field_segments(i)%direction/abs(this%external_field_segments(i)%direction) - +#ifdef CompileWithMPI + if (allocated(this%segment_owner)) then + if (this%segment_owner(i) /= rank) cycle + end if +#endif + this%e_L(j,i) = this%external_field_segments(i)%field * & + this%external_field_segments(i)%direction / abs(this%external_field_segments(i)%direction) end do end do - end subroutine #ifdef CompileWithMPI subroutine Comm_MPI_V(this) class(mtl_bundle_t) :: this - integer :: number_of_conductors, i, c - integer :: ierr, rank, status(MPI_STATUS_SIZE) + integer :: i, ierr, rank, nreq, source_rank, destination_rank + integer, allocatable :: requests(:), statuses(:,:) + + if (.not. allocated(this%segment_owner)) return call MPI_COMM_RANK(SUBCOMM_MPI, rank, ierr) - number_of_conductors = size(this%v,1) - do i = 1, size(this%mpi_comm%comms) - if (this%mpi_comm%comms(i)%comm_type == COMM_V .or. this%mpi_comm%comms(i)%comm_type == COMM_BOTH) then - if (this%mpi_comm%comms(i)%comm_task == COMM_SEND) then - do c = 1, number_of_conductors - call MPI_send(this%v(c, this%mpi_comm%comms(i)%v_index),1, REALSIZE, & - rank+this%mpi_comm%comms(i)%delta_rank, & - 200*(rank+this%mpi_comm%comms(i)%delta_rank+1)+c, & - SUBCOMM_MPI, ierr) - end do - end if - if (this%mpi_comm%comms(i)%comm_task == COMM_RECV) then - do c = 1, number_of_conductors - call MPI_recv(this%v(c, this%mpi_comm%comms(i)%v_index),1, REALSIZE, & - rank+this%mpi_comm%comms(i)%delta_rank, & - 200*(rank+1)+c, & - SUBCOMM_MPI, status, ierr) - end do - end if + allocate(requests(2 * max(0, this%number_of_divisions - 1)), & + statuses(MPI_STATUS_SIZE, 2 * max(0, this%number_of_divisions - 1))) + nreq = 0 + do i = 2, this%number_of_divisions + source_rank = this%segment_owner(i) + destination_rank = this%segment_owner(i-1) + if (source_rank == destination_rank) cycle + if (rank == source_rank) then + nreq = nreq + 1 + call MPI_Isend(this%v(:,i), size(this%v,1), REALSIZE, destination_rank, 501, SUBCOMM_MPI, requests(nreq), ierr) + else if (rank == destination_rank) then + nreq = nreq + 1 + call MPI_Irecv(this%v(:,i), size(this%v,1), REALSIZE, source_rank, 501, SUBCOMM_MPI, requests(nreq), ierr) end if end do + if (nreq > 0) call MPI_Waitall(nreq, requests, statuses, ierr) + end subroutine + subroutine Comm_MPI_I(this) + class(mtl_bundle_t) :: this + integer :: i, ierr, rank, nreq, source_rank, destination_rank + integer, allocatable :: requests(:), statuses(:,:) + if (.not. allocated(this%segment_owner)) return + call MPI_COMM_RANK(SUBCOMM_MPI, rank, ierr) + allocate(requests(2 * max(0, this%number_of_divisions - 1)), & + statuses(MPI_STATUS_SIZE, 2 * max(0, this%number_of_divisions - 1))) + nreq = 0 + do i = 2, this%number_of_divisions + source_rank = this%segment_owner(i-1) + destination_rank = this%segment_owner(i) + if (source_rank == destination_rank) cycle + if (rank == source_rank) then + nreq = nreq + 1 + call MPI_Isend(this%i(:,i-1), size(this%i,1), REALSIZE, destination_rank, 502, SUBCOMM_MPI, requests(nreq), ierr) + else if (rank == destination_rank) then + nreq = nreq + 1 + call MPI_Irecv(this%i(:,i-1), size(this%i,1), REALSIZE, source_rank, 502, SUBCOMM_MPI, requests(nreq), ierr) + end if + end do + if (nreq > 0) call MPI_Waitall(nreq, requests, statuses, ierr) end subroutine subroutine Comm_MPI_Fields(this) diff --git a/src_mtln/mtln_solver.F90 b/src_mtln/mtln_solver.F90 index 0b5526c8..034fc5d0 100644 --- a/src_mtln/mtln_solver.F90 +++ b/src_mtln/mtln_solver.F90 @@ -59,9 +59,10 @@ module mtln_solver_m contains - function mtlnCtor(parsed, alloc) result(res) + function mtlnCtor(parsed, alloc, sweep) result(res) type(parsed_mtln_t) :: parsed type(XYZlimit_t), dimension(1:6), intent(in), optional :: alloc + type(XYZlimit_t), dimension(1:6), intent(in), optional :: sweep type(mtln_t) :: res integer :: i type(preprocess_t) :: pre @@ -74,7 +75,11 @@ function mtlnCtor(parsed, alloc) result(res) call mpi_barrier(subcomm_mpi, ierr) #endif if (present(alloc)) then - pre = preprocess(parsed, alloc) + if (present(sweep)) then + pre = preprocess(parsed, alloc, sweep) + else + pre = preprocess(parsed, alloc) + end if else pre = preprocess(parsed) end if diff --git a/src_mtln/preprocess.F90 b/src_mtln/preprocess.F90 index 00c85cf5..224bfc81 100644 --- a/src_mtln/preprocess.F90 +++ b/src_mtln/preprocess.F90 @@ -53,9 +53,10 @@ module mtln_preprocess_m contains - function preprocess(parsed, alloc) result(res) + function preprocess(parsed, alloc, sweep) result(res) type(parsed_mtln_t), intent(in):: parsed type(XYZlimit_t), dimension(1:6), intent(in), optional :: alloc + type(XYZlimit_t), dimension(1:6), intent(in), optional :: sweep type(preprocess_t) :: res type(fhash_tbl_t) :: cable_name_to_bundle_id type(transmission_line_bundle_t), dimension(:), allocatable :: line_bundles @@ -81,7 +82,11 @@ function preprocess(parsed, alloc) result(res) #endif ! Create mtl objets from cables if (present(alloc)) then - line_bundles = buildLineBundles(cable_bundles, res%dt, alloc) + if (present(sweep)) then + line_bundles = buildLineBundles(cable_bundles, res%dt, alloc, sweep) + else + line_bundles = buildLineBundles(cable_bundles, res%dt, alloc) + end if else line_bundles = buildLineBundles(cable_bundles, res%dt) end if @@ -335,16 +340,17 @@ subroutine addEndConnector(line, connector) end function - function buildLineBundles(cable_bundles, dt, alloc) result(res) + function buildLineBundles(cable_bundles, dt, alloc, sweep) result(res) type(cable_bundle_t), dimension(:), allocatable :: cable_bundles type(transmission_line_bundle_t), dimension(:), allocatable :: res real(kind=RKIND_TIEMPO), intent(in) :: dt type(XYZlimit_t), dimension(1:6), intent(in), optional :: alloc + type(XYZlimit_t), dimension(1:6), intent(in), optional :: sweep integer :: i, j, k integer :: nb, nl, nc integer(kind=4), allocatable, dimension(:,:) :: layer_indices logical :: bundle_in_layer = .false. - integer(kind=4), dimension(2) :: alloc_z + integer(kind=4), dimension(2) :: alloc_z, sweep_z if (present(alloc)) then alloc_z(1) = alloc(3)%zi alloc_z(2) = alloc(3)%ze @@ -354,13 +360,16 @@ function buildLineBundles(cable_bundles, dt, alloc) result(res) do i = 1, nb if (present(alloc)) then if (allocated(layer_indices)) deallocate(layer_indices) - bundle_in_layer = isBundleInLayer(cable_bundles(i)%levels(1)%cables(1)%ptr, alloc_z) - if (bundle_in_layer) then - layer_indices = findIndicesInLayer(cable_bundles(i)%levels(1)%cables(1)%ptr, alloc_z) - else - allocate(layer_indices(0,2), source = 0) + if (present(sweep)) then + sweep_z(1) = sweep(iHz)%zi + sweep_z(2) = sweep(iHz)%ze + bundle_in_layer = isBundleInLayer(cable_bundles(i)%levels(1)%cables(1)%ptr, alloc_z, sweep_z) + if (bundle_in_layer) layer_indices = findIndicesInLayer(cable_bundles(i)%levels(1)%cables(1)%ptr, alloc_z, sweep_z) + else + bundle_in_layer = isBundleInLayer(cable_bundles(i)%levels(1)%cables(1)%ptr, alloc_z) + if (bundle_in_layer) layer_indices = findIndicesInLayer(cable_bundles(i)%levels(1)%cables(1)%ptr, alloc_z) end if - ! if (layer_indices(1,1) == layer_indices(1,2) ) bundle_in_layer = .false. + if (.not. bundle_in_layer) allocate(layer_indices(0,2), source = 0) end if nl = size(cable_bundles(i)%levels) allocate(res(i)%levels(nl)) @@ -378,15 +387,16 @@ function buildLineBundles(cable_bundles, dt, alloc) result(res) end do contains - logical function isBundleInLayer(cable, alloc_z) + logical function isBundleInLayer(cable, alloc_z, sweep_z) integer(kind=4), dimension(2), intent(in) :: alloc_z + integer(kind=4), dimension(2), intent(in), optional :: sweep_z class (cable_t), pointer, intent(in) :: cable integer :: n, i logical :: in_layer in_layer = .false. n = 0 do i = 1, size(cable%segments) - if (isSegmentWithinAllocBox(cable%segments, i, alloc_z)) then + if (isSegmentWithinAllocBox(cable%segments, i, alloc_z, sweep_z)) then if (.not. in_layer) then in_layer = .true. end if @@ -401,8 +411,9 @@ logical function isBundleInLayer(cable, alloc_z) isBundleInLayer = (n/=0) end function - function findIndicesInLayer(cable, alloc_z) result(res) + function findIndicesInLayer(cable, alloc_z, sweep_z) result(res) integer(kind=4), dimension(2), intent(in) :: alloc_z + integer(kind=4), dimension(2), intent(in), optional :: sweep_z class (cable_t), pointer, intent(in) :: cable integer(kind=4), allocatable, dimension(:,:) :: res integer :: n, i, direction, position(1:3) @@ -411,7 +422,7 @@ function findIndicesInLayer(cable, alloc_z) result(res) ! precount n = 0 do i = 1, size(cable%segments) - if (isSegmentWithinAllocBox(cable%segments, i, alloc_z)) then + if (isSegmentWithinAllocBox(cable%segments, i, alloc_z, sweep_z)) then if (.not. in_layer) then in_layer = .true. end if @@ -428,7 +439,7 @@ function findIndicesInLayer(cable, alloc_z) result(res) n = 1 in_layer = .false. do i = 1, size(cable%segments) - if (isSegmentWithinAllocBox(cable%segments, i, alloc_z)) then + if (isSegmentWithinAllocBox(cable%segments, i, alloc_z, sweep_z)) then if (.not. in_layer) then res(n,1) = i in_layer = .true. @@ -446,15 +457,29 @@ function findIndicesInLayer(cable, alloc_z) result(res) end if end function - logical function isSegmentWithinAllocBox(segs, i, z) + logical function isSegmentWithinAllocBox(segs, i, alloc_z, sweep_z) type(segment_t), intent(in), dimension(:), allocatable :: segs - type(segment_t) :: prev integer :: i - integer(kind=4), dimension(2), intent(in) :: z - isSegmentWithinAllocBox = (segs(i)%z >= z(1)) .and. (segs(i)%z <= z(2)) + integer(kind=4), dimension(2), intent(in) :: alloc_z + integer(kind=4), dimension(2), intent(in), optional :: sweep_z + if (present(sweep_z)) then + isSegmentWithinAllocBox = isSegmentOwnedByMTLNRank(segs(i), alloc_z, sweep_z) + else + isSegmentWithinAllocBox = (segs(i)%z >= alloc_z(1)) .and. (segs(i)%z <= alloc_z(2)) + end if end function end function + logical function isSegmentOwnedByMTLNRank(segment, alloc_z, sweep_z) + type(segment_t), intent(in) :: segment + integer(kind=4), dimension(2), intent(in) :: alloc_z, sweep_z + + if (abs(segment%orientation) == ZPOS) then + isSegmentOwnedByMTLNRank = (segment%z >= alloc_z(1)) .and. (segment%z <= alloc_z(2)) + else + isSegmentOwnedByMTLNRank = (segment%z > sweep_z(1)) .and. (segment%z <= sweep_z(2)) + end if + end function function buildCableBundleFromParent(parent, cables) result(res) type(cable_abstract_t), intent(in) :: parent diff --git a/src_mtln/probes.F90 b/src_mtln/probes.F90 index 3776ec6d..f5182172 100644 --- a/src_mtln/probes.F90 +++ b/src_mtln/probes.F90 @@ -67,22 +67,26 @@ function probeCtor(index, probe_type, dt, name, position, layer_indices) result( if (present(layer_indices)) then call MPI_COMM_SIZE(SUBCOMM_MPI, sizeof, ierr) if (sizeof > 1) then - res%in_layer = .false. - do i = 1, size(layer_indices,1) - if (index >= layer_indices(i, 1) .and. index <= layer_indices(i,2)+1) then - res%in_layer = .true. - slice = i - end if - end do - - layer_index = 0 - if (res%in_layer) then - do i = 1, slice - 1 - layer_index = layer_index + layer_indices(i,2) + 1 - (layer_indices(i,1) - 1) + if (size(layer_indices,1) == 0) then + res%in_layer = .true. + else + res%in_layer = .false. + do i = 1, size(layer_indices,1) + if (index >= layer_indices(i, 1) .and. index <= layer_indices(i,2)+1) then + res%in_layer = .true. + slice = i + end if end do - layer_index = layer_index + res%index - layer_indices(i,1) + 1 + + layer_index = 0 + if (res%in_layer) then + do i = 1, slice - 1 + layer_index = layer_index + layer_indices(i,2) + 1 - (layer_indices(i,1) - 1) + end do + layer_index = layer_index + res%index - layer_indices(i,1) + 1 + end if + res%index = layer_index end if - res%index = layer_index end if end if #endif diff --git a/src_wires_pub/wires_mtln.F90 b/src_wires_pub/wires_mtln.F90 index 6988fe7c..023368fb 100644 --- a/src_wires_pub/wires_mtln.F90 +++ b/src_wires_pub/wires_mtln.F90 @@ -53,13 +53,14 @@ subroutine InitWires_mtln(sgg,Ex,Ey,Ez, sggMiEx, sggMiEy, sggMiEz, sggMiHx, sggM logical :: thereAreMTLNbundles real(kind=rkind_tiempo), intent(inout) :: dtcritico #ifdef CompileWithMPI - integer(kind=4) :: ierr + integer(kind=4) :: ierr, m #endif eps0 = eps00 mu0 = mu00 #ifdef CompileWithMPI - mtln_solver = mtlnCtor(mtln_parsed, sgg%alloc) + ! Keep the MTLN state whole on every rank; field samples are synchronized in mtl_bundle_t. + mtln_solver = mtlnCtor(mtln_parsed) call mpi_barrier(subcomm_mpi,ierr) #else mtln_solver = mtlnCtor(mtln_parsed) @@ -73,6 +74,11 @@ subroutine InitWires_mtln(sgg,Ex,Ey,Ez, sggMiEx, sggMiEy, sggMiEz, sggMiHx, sggM end if if (mtln_solver%dt < dtcritico) dtcritico = mtln_solver%dt call pointSegmentsToFields() +#ifdef CompileWithMPI + do m = 1, mtln_solver%number_of_bundles + call mtln_solver%bundles(m)%initMPIStateOwnership() + end do +#endif call mtln_solver%updatePULTerms() contains @@ -88,24 +94,33 @@ subroutine pointSegmentsToFields() if (mtln_solver%bundles(m)%bundle_in_layer) then do n = 1, ubound(mtln_solver%bundles(m)%external_field_segments,1) call readGridIndices(i, j, k, mtln_solver%bundles(m)%external_field_segments(n)) + mtln_solver%bundles(m)%external_field_segments(n)%field_is_local = .false. + mtln_solver%bundles(m)%external_field_segments(n)%field_is_owned = segmentIsOwnedByThisRank(mtln_solver%bundles(m)%external_field_segments(n)) + if (.not. mtln_solver%bundles(m)%external_field_segments(n)%field_is_owned) then + mtln_solver%bundles(m)%external_field_segments(n)%field => mtln_solver%null_field + cycle + end if select case (abs(mtln_solver%bundles(m)%external_field_segments(n)%direction)) case(DIRECTION_X_POS) if (isEmbeddedInPECorLossy(sggmiEx(i,j,k))) then mtln_solver%bundles(m)%external_field_segments(n)%field => mtln_solver%null_field else mtln_solver%bundles(m)%external_field_segments(n)%field => Ex(i, j, k) + mtln_solver%bundles(m)%external_field_segments(n)%field_is_local = .true. end if case(DIRECTION_Y_POS) if (isEmbeddedInPECorLossy(sggmiEy(i,j,k))) then mtln_solver%bundles(m)%external_field_segments(n)%field => mtln_solver%null_field else mtln_solver%bundles(m)%external_field_segments(n)%field => Ey(i, j, k) + mtln_solver%bundles(m)%external_field_segments(n)%field_is_local = .true. end if case(DIRECTION_Z_POS) if (isEmbeddedInPECorLossy(sggmiEz(i,j,k))) then mtln_solver%bundles(m)%external_field_segments(n)%field => mtln_solver%null_field else mtln_solver%bundles(m)%external_field_segments(n)%field => Ez(i, j, k) + mtln_solver%bundles(m)%external_field_segments(n)%field_is_local = .true. end if end select end do @@ -113,6 +128,30 @@ subroutine pointSegmentsToFields() end do end subroutine + logical function segmentIsOwnedByThisRank(field_segment) + type(external_field_segment_t), intent(in) :: field_segment + integer :: field_component + + select case (abs(field_segment%direction)) + case (DIRECTION_X_POS) + field_component = iEx + case (DIRECTION_Y_POS) + field_component = iEy + case (DIRECTION_Z_POS) + field_component = iEz + case default + segmentIsOwnedByThisRank = .false. + return + end select + + segmentIsOwnedByThisRank = field_segment%position(1) >= sgg%sweep(field_component)%xi .and. & + field_segment%position(1) <= sgg%sweep(field_component)%xe .and. & + field_segment%position(2) >= sgg%sweep(field_component)%yi .and. & + field_segment%position(2) <= sgg%sweep(field_component)%ye .and. & + field_segment%position(3) >= sgg%sweep(field_component)%zi .and. & + field_segment%position(3) <= sgg%sweep(field_component)%ze + end function + logical function isEmbeddedInPECorLossy(media) integer(kind=INTEGERSIZEOFMEDIAMATRICES), intent(in) :: media isEmbeddedInPECorLossy = (media == 0 .or. sgg%med(media)%is%pec .or. sgg%med(media)%is%lossy) @@ -136,6 +175,7 @@ subroutine AdvanceWiresE_mtln(sgg,Idxh, Idyh, Idzh, eps00,mu00) do m = 1, mtln_solver%number_of_bundles if (mtln_solver%bundles(m)%bundle_in_layer) then do n = 1, ubound(mtln_solver%bundles(m)%external_field_segments,1) + if (.not. mtln_solver%bundles(m)%external_field_segments(n)%field_is_local) cycle punt => mtln_solver%bundles(m)%external_field_segments(n)%field punt = real(punt, kind=rkind_wires) - computeFieldFromCurrent(m,n) end do diff --git a/test/mtln/mtln_tests.h b/test/mtln/mtln_tests.h index e77b803d..5731195b 100644 --- a/test/mtln/mtln_tests.h +++ b/test/mtln/mtln_tests.h @@ -4,6 +4,8 @@ extern "C" int test_mtl_init_homogeneous(); extern "C" int test_mtl_time_step(); extern "C" int test_mtl_wrong_dt(); +extern "C" int test_mtl_inactive_mpi_slice(); +extern "C" int test_mtl_replicated_mpi_probe(); //test_mtl_bundle.F90 extern "C" int test_mtl_bundle_init(); extern "C" int test_mtl_bundle_generator(); @@ -44,6 +46,7 @@ extern "C" int test_preprocess_conductors_before_cable(); extern "C" int test_preprocess_conductors_in_level(); extern "C" int test_preprocess_zt_conductor_ranges_2(); extern "C" int test_preprocess_zt_conductor_ranges(); +extern "C" int test_preprocess_mpi_segment_ownership(); // extern "C" int test_coaxial_line_paul_8_6_square(); // extern "C" int test_coaxial_line_paul_8_6_triangle(); // extern "C" int test_2_conductor_line_paul_9_6(); @@ -61,6 +64,8 @@ extern "C" int test_preprocess_zt_conductor_ranges(); TEST(mtln, mtl_homogeneous) { EXPECT_EQ(0, test_mtl_init_homogeneous()); } TEST(mtln, mtl_time_step) { EXPECT_EQ(0, test_mtl_time_step()); } TEST(mtln, mtl_wrong_dt) { EXPECT_EQ(0, test_mtl_wrong_dt()); } +TEST(mtln, mtl_inactive_mpi_slice) { EXPECT_EQ(0, test_mtl_inactive_mpi_slice()); } +TEST(mtln, mtl_replicated_mpi_probe) { EXPECT_EQ(0, test_mtl_replicated_mpi_probe()); } TEST(mtln, mtl_bundle_init) { EXPECT_EQ(0, test_mtl_bundle_init()); } TEST(mtln, mtl_bundle_generator) { EXPECT_EQ(0, test_mtl_bundle_generator()); } @@ -73,6 +78,7 @@ TEST(mtln, preprocess_conductors_before_cable) { EXPECT_EQ(0, test_preprocess_co TEST(mtln, preprocess_conductors_in_level) { EXPECT_EQ(0, test_preprocess_conductors_in_level()); } TEST(mtln, preprocess_zt_conductor_ranges_2) { EXPECT_EQ(0, test_preprocess_zt_conductor_ranges_2()); } TEST(mtln, preprocess_zt_conductor_ranges) { EXPECT_EQ(0, test_preprocess_zt_conductor_ranges()); } +TEST(mtln, preprocess_mpi_segment_ownership) { EXPECT_EQ(0, test_preprocess_mpi_segment_ownership()); } TEST(mtln, math_eigvals) { EXPECT_EQ(0, test_math_eigvals()); } TEST(mtln, math_matmul_broadcast) { EXPECT_EQ(0, test_math_matmul_broadcast()); } diff --git a/test/mtln/test_mtl.F90 b/test/mtln/test_mtl.F90 index 30dc56d5..c182d0ce 100644 --- a/test/mtln/test_mtl.F90 +++ b/test/mtln/test_mtl.F90 @@ -96,4 +96,75 @@ integer function test_mtl_time_step() bind(C) result(error_cnt) error_cnt = error_cnt +1 end if -end function \ No newline at end of file +end function + +integer function test_mtl_inactive_mpi_slice() bind(C) result(error_cnt) + use mtl_m + use mtln_types_m, only: DIRECTION_X_POS + implicit none + + type(mtl_t) :: shielded_line, unshielded_line + type(transfer_impedance_per_meter_t) :: zt + type(multipolar_expansion_t), dimension(:), allocatable :: multipolar_expansion + real(kind=rkind), dimension(1,1) :: lpul, cpul, rpul, gpul + real(kind=rkind), dimension(2) :: step_size + type(segment_t), dimension(:), allocatable :: segments +#ifdef CompileWithMPI + integer(kind=4), dimension(:,:), allocatable :: layer_indices +#endif + integer :: i + real(kind=rkind_tiempo), parameter :: dt = 1e-11_rkind_tiempo + + error_cnt = 0 +#ifdef CompileWithMPI + lpul = reshape([4.4712610e-7_rkind], [1,1]) + cpul = reshape([2.242e-10_rkind], [1,1]) + rpul = 0.0_rkind + gpul = 0.0_rkind + step_size = [20.0_rkind, 20.0_rkind] + zt%inductive_term = 0.0_rkind + zt%resistive_term = 0.0_rkind + allocate(zt%poles(0), zt%residues(0), multipolar_expansion(0), segments(2), layer_indices(0,2)) + do i = 1, size(segments) + segments(i)%x = i + segments(i)%y = 1 + segments(i)%z = 1 + segments(i)%orientation = DIRECTION_X_POS + end do + + shielded_line = mtl_shielded(lpul, cpul, rpul, gpul, step_size, 'inactive', segments, dt, & + 'parent', 1, zt, layer_indices, .false.) + unshielded_line = mtl_unshielded(lpul, cpul, rpul, gpul, step_size, 'inactive', segments, dt, & + multipolar_expansion, 0.0_rkind, layer_indices, .false.) + + if (shielded_line%bundle_in_layer) error_cnt = error_cnt + 1 + if (size(shielded_line%step_size) /= 0) error_cnt = error_cnt + 1 + if (shielded_line%dt /= dt) error_cnt = error_cnt + 1 + if (unshielded_line%bundle_in_layer) error_cnt = error_cnt + 1 + if (size(unshielded_line%step_size) /= 0) error_cnt = error_cnt + 1 + if (unshielded_line%dt /= dt) error_cnt = error_cnt + 1 +#endif +end function + +integer function test_mtl_replicated_mpi_probe() bind(C) result(error_cnt) + use probes_m, only: probe_t, probeCtor + use mtln_types_m, only: PROBE_TYPE_CURRENT + use FDETYPES_m, only: RKIND, RKIND_TIEMPO + implicit none + + type(probe_t) :: probe + character(len=:), allocatable :: name +#ifdef CompileWithMPI + integer(kind=4), dimension(:,:), allocatable :: layer_indices +#endif + + error_cnt = 0 +#ifdef CompileWithMPI + allocate(layer_indices(0,0)) + name = "replicated" + probe = probeCtor(2, PROBE_TYPE_CURRENT, 1.0e-12_RKIND_TIEMPO, name, & + [0.0_RKIND, 0.0_RKIND, 0.0_RKIND], layer_indices) + if (.not. probe%in_layer) error_cnt = error_cnt + 1 + if (probe%index /= 2) error_cnt = error_cnt + 1 +#endif +end function diff --git a/test/mtln/test_preprocess.F90 b/test/mtln/test_preprocess.F90 index 8f10e1f4..42f9f489 100644 --- a/test/mtln/test_preprocess.F90 +++ b/test/mtln/test_preprocess.F90 @@ -216,3 +216,29 @@ integer function test_preprocess_zt_conductor_ranges_2() bind(C) result(error_cn end function + +integer function test_preprocess_mpi_segment_ownership() bind(C) result(error_cnt) + use mtln_preprocess_m, only: isSegmentOwnedByMTLNRank + use mtln_types_m, only: segment_t, DIRECTION_X_POS, DIRECTION_Z_POS + implicit none + + type(segment_t) :: segment + integer(kind=4), dimension(2) :: rank0_alloc, rank0_sweep, rank1_alloc, rank1_sweep + + error_cnt = 0 + rank0_alloc = [-1_4, 34_4] + rank0_sweep = [0_4, 34_4] + rank1_alloc = [33_4, 68_4] + rank1_sweep = [34_4, 67_4] + + segment%z = 33 + segment%orientation = DIRECTION_X_POS + if (.not. isSegmentOwnedByMTLNRank(segment, rank0_alloc, rank0_sweep)) error_cnt = error_cnt + 1 + if (isSegmentOwnedByMTLNRank(segment, rank1_alloc, rank1_sweep)) error_cnt = error_cnt + 1 + + segment%z = 34 + segment%orientation = DIRECTION_Z_POS + if (.not. isSegmentOwnedByMTLNRank(segment, rank0_alloc, rank0_sweep)) error_cnt = error_cnt + 1 + segment%z = 33 + if (.not. isSegmentOwnedByMTLNRank(segment, rank1_alloc, rank1_sweep)) error_cnt = error_cnt + 1 +end function