Eliminate redundant array copies and vectorise marker scanning in binary event processing

.gitignore

@@ -0,0 +1 @@
+__pycache__/

Acqu.py

@@ -182,8 +182,8 @@ def runFunction(function,minEvents=0,maxEvents=0):
             print(i, 'Bad data format or file end')
             return
 
-        # Set event limits
-        eventBuffers = np.append([1],np.where(dataBuffer==df.EEndEvent)[0]+1)
+        # Set event limits — np.concatenate avoids the extra copy that np.append makes
+        eventBuffers = np.concatenate(([1], np.where(dataBuffer==df.EEndEvent)[0]+1))
 
         # Loop over events in buffer
         for j, (start,stop) in enumerate(zip(eventBuffers[0:-1],eventBuffers[1:])):
@@ -260,25 +260,25 @@ def processEvent(eventData):
     # Get Epics Data in event
     if(df.EPICSExist):
         epicsBuffers, epicsIndices, epicsInfo = df.FillEPICSArray(eventArray)
-        eventArray = np.delete(eventArray,epicsIndices)
+        if len(epicsIndices):
+            eventArray = np.delete(eventArray, epicsIndices)
         if(len(epicsBuffers)):
             epicsEvent=1
 
     scalerEvent=0
     #Separate scaler data out
     if(df.ScalersExist):
         scalerBuffers, scalerIndices = df.FillScalerArray(eventArray)
-        #if(len(scalerBuffers)):
-        eventArray = np.delete(eventArray,scalerIndices)
+        if len(scalerIndices):
+            eventArray = np.delete(eventArray, scalerIndices)
         if(len(scalerBuffers)):
             scalerEvent=1
 
 
     # Check for errors
     errorIndices = df.CheckErrors(eventArray)
-    eventArray   = np.delete(eventArray,errorIndices)
     if len(errorIndices):
-        #print(eventArray.view(np.uint16).reshape(-1,2)
+        eventArray = np.delete(eventArray, errorIndices)
         return 0
 
     #adcArray   = eventArray.view(np.uint16).reshape(-1,2)

Mk1Format.py

@@ -89,26 +89,27 @@ def MakeScalerArray(moduleList,scalerList):
 
 def FillScalerArray(dataArray):
 
-    #np.set_printoptions(threshold=np.nan)
-    scalerHeaders = []
-    scalerIndices = []
     scalerLocations = np.where(dataArray==EScalerBuffer)[0]
-    if(len(scalerLocations)):
-        if(len(scalerLocations)!=len(NScalerBlock)):
-            print('Bad scaler block')
-            return [], []
-        for i, index in enumerate(scalerLocations):
-            scalerHeaders += [index,index+1]
-            scalerIndices += range(index+2,index+2+NScalerBlock[i])
-        scalerArray = np.column_stack((np.arange(NScaler),np.take(dataArray,scalerIndices)))
-        dataArray
-        return scalerArray, scalerIndices+scalerHeaders
-    return [], []
-
+    if not len(scalerLocations):
+        return [], []
+    if len(scalerLocations) != len(NScalerBlock):
+        print('Bad scaler block')
+        return [], []
+
+    block_sizes = np.asarray(NScalerBlock, dtype=np.intp)
+    # Fully vectorised ragged range: replaces Python for-loop list builds
+    bases   = np.repeat(scalerLocations + 2, block_sizes)
+    cum     = np.concatenate([[0], np.cumsum(block_sizes[:-1])])
+    offsets = np.arange(NScaler, dtype=np.intp) - np.repeat(cum, block_sizes)
+    scalerIndices = bases + offsets
+
+    scalerHeaders = np.ravel(np.column_stack([scalerLocations, scalerLocations + 1]))
+    scalerArray = np.column_stack((np.arange(NScaler), dataArray[scalerIndices]))
+    return scalerArray, np.concatenate([scalerIndices, scalerHeaders])
+
 def CheckErrors(dataArray):
-    errorIndices = []
-    for errorMark in np.where(dataArray==EReadError)[0]:
-        #print errorMark
-        #print np.frombuffer(dataArray[errorMark:], dtype=readError, count=1)
-        errorIndices += range(errorMark,errorMark+4)
-    return errorIndices
+    errorMarks = np.where(dataArray==EReadError)[0]
+    if not len(errorMarks):
+        return []
+    # Vectorised: broadcast each error mark across offsets 0..3
+    return (errorMarks[:, np.newaxis] + np.arange(4, dtype=np.intp)).ravel()

Mk2Format.py

@@ -78,18 +78,26 @@
                              ('fTrailer', '<u4') ])     #end of error block marker
 
 def FillScalerArray(dataArray):
-
-    scalerIndices = []
-    scalerHeaders = []
+
     scalerLocations = np.where(dataArray==EScalerBuffer)[0]
-    if(len(scalerLocations)):        
-        scalerLocations = scalerLocations.reshape((-1,2),order='C')
-        for indeces in scalerLocations:
-            scalerHeaders += [indeces[0],indeces[0]+1]
-            scalerIndices += range(indeces[0]+2,indeces[1])
-        scalerArray = np.take(dataArray,scalerIndices).reshape((-1,2))
-        return scalerArray, scalerIndices+scalerHeaders
-    return [], []
+    if not len(scalerLocations):
+        return [], []
+
+    scalerLocations = scalerLocations.reshape((-1, 2), order='C')
+    starts = scalerLocations[:, 0]
+    ends   = scalerLocations[:, 1]
+    counts = (ends - starts - 2).astype(np.intp)
+    total  = int(np.sum(counts))
+
+    # Fully vectorised ragged range: replaces Python for-loop list builds
+    bases   = np.repeat(starts + 2, counts)
+    cum     = np.concatenate([[0], np.cumsum(counts[:-1])])
+    offsets = np.arange(total, dtype=np.intp) - np.repeat(cum, counts)
+    scalerIndices = bases + offsets
+
+    scalerHeaders = np.ravel(np.column_stack([starts, starts + 1]))
+    scalerArray = dataArray[scalerIndices].reshape((-1, 2))
+    return scalerArray, np.concatenate([scalerIndices, scalerHeaders])
 
 def FillEPICSArray(dataArray):
 
@@ -112,13 +120,13 @@ def FillEPICSArray(dataArray):
         epicsEnd     = int((index+6)/4)
         epicsBuffer  = epicsBuffer[:epicsEnd]
         epicsBuffers += [epicsBuffer]
-        epicsIndices += range(epicsBufferStart,epicsBufferStart+epicsEnd)
+        epicsIndices.append(np.arange(epicsBufferStart, epicsBufferStart+epicsEnd, dtype=np.intp))
+    epicsIndices = np.concatenate(epicsIndices) if epicsIndices else np.array([], dtype=np.intp)
     return epicsBuffers, epicsIndices, epicsInfo
 
 def CheckErrors(dataArray):
-    errorIndices = []
-    for errorMark in np.where(dataArray==EReadError)[0]:
-        #print errorMark
-        #print np.frombuffer(dataArray[errorMark:], dtype=readError, count=1)
-        errorIndices += range(errorMark,errorMark+5)
-    return errorIndices
+    errorMarks = np.where(dataArray==EReadError)[0]
+    if not len(errorMarks):
+        return []
+    # Vectorised: broadcast each error mark across offsets 0..4
+    return (errorMarks[:, np.newaxis] + np.arange(5, dtype=np.intp)).ravel()