@@ -73,10 +73,10 @@ def lower_petsc(iet, **kwargs):
7373 # Protect PETSc solve targets from being dropped by `_drop_if_unwritten`.
7474 # `lower_petsc` runs before `mpiize`, replacing `PetscMetaData` (an
7575 # `Expression` subclass whose `.write` reveals the target function) with
76- # `Call` nodes to run the PETSc solver. Once that happens, `_drop_if_unwritten` can no
77- # longer see the target as written and incorrectly discards its `HaloSpot`. So we
78- # compose `dist-drop-unwritten` with a guard that always returns
79- # False for PETSc targets.
76+ # `Call` nodes to run the PETSc solver. Once that happens,
77+ # `_drop_if_unwritten` can no longer see the target as written and
78+ # incorrectly discards its `HaloSpot`. So we compose `dist-drop-unwritten`
79+ # with a guard that always returns False for PETSc targets.
8080 options = kwargs ['options' ]
8181 petsc_targets = {n .write for n in data if n .write is not None }
8282 if petsc_targets :
@@ -128,12 +128,6 @@ def lower_petsc(iet, **kwargs):
128128 populate_matrix_context (efuncs )
129129
130130 # Strip HaloSpots from PETSc callback efuncs before returning them.
131- # The callbacks are built via rcompile(..., mpi=False), so HaloSpots
132- # survive in their IETs but are NOT converted to haloupdate calls there.
133- # When the main mpiize pass (mpi=True) later processes these callbacks,
134- # it would convert those HaloSpots into haloupdate calls — which is wrong,
135- # since halo exchanges must only happen in the main kernel. Strip them here
136- # before they reach mpiize.
137131 for name , efunc in list (efuncs .items ()):
138132 if isinstance (efunc , PETScCallable ):
139133 halos = FindNodes (HaloSpot ).visit (efunc )
@@ -157,10 +151,7 @@ def strip_petsc_callback_halos(iet, **kwargs):
157151 Remove any HaloSpot nodes that `mpiize` may have injected into PETSc
158152 callback functions (FormFunction, SetPointBCs, FormRHS, etc.).
159153
160- HaloSpots should only appear in the main kernel, never inside PETSc
161- callbacks which run as part of the PETSc solver internals. All
162- PETSc callbacks are instances of `PETScCallable`; the main kernel is
163- not, so we use that to distinguish the two.
154+ HaloSpots should only appear in the main kernel.
164155 """
165156 if not isinstance (iet , PETScCallable ):
166157 return iet , {}
@@ -205,10 +196,9 @@ def linear_indices(iet, **kwargs):
205196
206197 tracker = Tracker ('basic' , dtype , kwargs ['sregistry' ])
207198
208- # Exclude SubDomainSet backing functions from linearization: they must
209- # remain as 2D array reads (border[n0][col]), not flat-indexed via a macro.
210- # SubDomainSet subfunctions are identified by having a DefaultDimension
211- # (sds_dim) among their dimensions.
199+ # TODO: CLEAN UP this is a hack
200+ # Exclude SubDomainSet backing functions from linearization - in SETPOINTBCS
201+ # I don't want to linearize the accesses to the SubDomainSet
212202 indexeds = [
213203 i for i in FindSymbols ('indexeds' ).visit (iet )
214204 if not isinstance (i .function , LocalType )
0 commit comments