Improve serialization/deserialization for raw bytes

cyclonedds/idl/_machinery.py

@@ -663,21 +663,21 @@ def default_initialize(self):
 
 class PlainCdrV2ArrayOfPrimitiveMachine(Machine):
     def __init__(self, type, length):
-        code, self.alignment, size, default = types._type_code_align_size_default_mapping[type]
+        self.elem_code, self.alignment, size, default = types._type_code_align_size_default_mapping[type]
         self.length = length
         self.size = size * length
-        self.code = str(length) + code
+        self.code = str(length) + self.elem_code
         self.default = [default] * length
         self.subtype = type
 
     def serialize(self, buffer, value, for_key=False):
         assert len(value) == self.length
         buffer.align(self.alignment)
-        buffer.write_multi(self.code, self.size, *value)
+        buffer.write_sequence(self.elem_code, value)
 
     def deserialize(self, buffer):
         buffer.align(self.alignment)
-        return list(buffer.read_multi(self.code, self.size))
+        return buffer.read_sequence(self.elem_code, self.length)
 
     def key_scan(self) -> KeyScanner:
         return KeyScanner.simple(self.alignment, self.size)
@@ -703,14 +703,14 @@ def serialize(self, buffer, value, for_key=False):
         buffer.write('I', 4, len(value))
         if value:
             buffer.align(self.alignment)
-            buffer.write_multi(f"{len(value)}{self.code}", self.size * len(value), *value)
+            buffer.write_sequence(self.code, value)
 
     def deserialize(self, buffer):
         buffer.align(4)
         length = buffer.read('I', 4)
         if length:
             buffer.align(self.alignment)
-            return list(buffer.read_multi(f"{length}{self.code}", self.size * length))
+            return buffer.read_sequence(self.code, length)
         else:
             return []
 

cyclonedds/idl/_support.py

@@ -10,6 +10,7 @@
  * SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
 """
 
+import array as _array
 import sys
 import struct
 
@@ -143,6 +144,40 @@ def read_multi(self, pack: str, size: int) -> Tuple[Any, ...]:
         self._pos += size
         return v
 
+    def write_sequence(self, code: str, values) -> 'Buffer':
+        """Serialize a homogeneous sequence of primitives in one shot.
+
+        Using array.array instead of struct.pack with a length-prefixed format
+        string (e.g. "100f") avoids building that string on every call and lets
+        the C array machinery do the copy in bulk.  byteswap() is applied when
+        the buffer's wire endianness differs from the host's native order.
+        'B' (uint8) is special-cased because array.array('B').tobytes() and
+        bytes() are equivalent but bytes() is marginally faster for that type.
+        """
+        if code == 'B':
+            return self.write_bytes(bytes(values))
+        a = _array.array(code, values)
+        if (self.endianness == Endianness.Little) != (sys.byteorder == 'little'):
+            a.byteswap()
+        return self.write_bytes(a.tobytes())
+
+    def read_sequence(self, code: str, length: int) -> list:
+        """Deserialize a homogeneous sequence of primitives in one shot.
+
+        Mirrors write_sequence: frombytes() fills the array directly from the
+        buffer slice without per-element struct.unpack calls, then byteswap()
+        fixes up the byte order when wire and host endianness differ.
+        """
+        if code == 'B':
+            return list(self.read_bytes(length))
+        a = _array.array(code)
+        nbytes = a.itemsize * length
+        a.frombytes(self._bytes[self._pos:self._pos + nbytes])
+        self._pos += nbytes
+        if (self.endianness == Endianness.Little) != (sys.byteorder == 'little'):
+            a.byteswap()
+        return a.tolist()
+
     def asbytes(self) -> bytes:
         return bytes(self._bytes[0:self._pos])
 

tests/test_encoding_machinery.py

@@ -1,3 +1,5 @@
+import array
+import struct
 import pytest
 
 from dataclasses import dataclass
@@ -212,5 +214,115 @@ def test_all_machine_serializers():
     assert b.asbytes() == b"\x04\x00\x00\x00\x01\x00\x34\x12\x02\x00\x00\x00\x00\x77"
 
 
+# All array.array-compatible primitive typecodes used by the IDL machinery.
+_SEQUENCE_CASES = [
+    ('b', [0, -1, 127, -128]),
+    ('B', [0, 1, 127, 255]),
+    ('h', [0, -1, 32767, -32768]),
+    ('H', [0, 1, 32767, 65535]),
+    ('i', [0, -1, 2**31 - 1, -(2**31)]),
+    ('I', [0, 1, 2**31 - 1, 2**32 - 1]),
+    ('q', [0, -1, 2**63 - 1, -(2**63)]),
+    ('Q', [0, 1, 2**63 - 1, 2**64 - 1]),
+    ('f', [0.0, 1.5, -1.5, 3.14]),
+    ('d', [0.0, 1.5, -1.5, 3.14159265358979]),
+]
+
+
+@pytest.mark.parametrize("code,values", _SEQUENCE_CASES)
+def test_write_read_sequence_roundtrip_native(code, values):
+    """write_sequence followed by read_sequence returns the original values."""
+    b = Buffer()
+    b.write_sequence(code, values)
+    b.seek(0)
+    result = b.read_sequence(code, len(values))
+    if code in ('f',):
+        assert result == pytest.approx(values, rel=1e-6)
+    else:
+        assert result == values
+
+
+@pytest.mark.parametrize("code,values", _SEQUENCE_CASES)
+def test_write_read_sequence_roundtrip_non_native(code, values):
+    """Round-trip is correct when buffer endianness is opposite to the host."""
+    non_native = Endianness.Big if Endianness.native() == Endianness.Little else Endianness.Little
+    b = Buffer()
+    b.set_endianness(non_native)
+    b.write_sequence(code, values)
+    b.seek(0)
+    result = b.read_sequence(code, len(values))
+    if code in ('f',):
+        assert result == pytest.approx(values, rel=1e-6)
+    else:
+        assert result == values
 
 
+@pytest.mark.parametrize("code,values", _SEQUENCE_CASES)
+def test_write_sequence_matches_struct_little_endian(code, values):
+    """write_sequence produces the same bytes as struct.pack with '<' prefix."""
+    b = Buffer()
+    b.set_endianness(Endianness.Little)
+    b.write_sequence(code, values)
+    expected = struct.pack(f"<{len(values)}{code}", *values)
+    assert b.asbytes() == expected
+
+
+@pytest.mark.parametrize("code,values", _SEQUENCE_CASES)
+def test_write_sequence_matches_struct_big_endian(code, values):
+    """write_sequence produces the same bytes as struct.pack with '>' prefix."""
+    b = Buffer()
+    b.set_endianness(Endianness.Big)
+    b.write_sequence(code, values)
+    expected = struct.pack(f">{len(values)}{code}", *values)
+    assert b.asbytes() == expected
+
+
+def test_write_sequence_uint8_special_case():
+    """'B' (uint8) fast path produces identical bytes to the generic array path."""
+    values = list(range(256))
+    b_special = Buffer()
+    b_special.write_sequence('B', values)
+
+    b_generic = Buffer()
+    a = array.array('B', values)
+    b_generic.write_bytes(a.tobytes())
+
+    assert b_special.asbytes() == b_generic.asbytes()
+
+
+@pytest.mark.parametrize("idl_type,code,values", [
+    (tp.uint16, 'H', [0x0102, 0x0304, 0x0506]),
+    (tp.int32,  'i', [1, -1, 2**16]),
+    (tp.float64,'d', [1.0, -1.0, 0.5]),
+])
+def test_plain_cdr_v2_array_roundtrip(idl_type, code, values):
+    """PlainCdrV2ArrayOfPrimitiveMachine serializes and deserializes correctly."""
+    m = mc.PlainCdrV2ArrayOfPrimitiveMachine(idl_type, len(values))
+    b = Buffer()
+    b.set_endianness(Endianness.Little)
+    m.serialize(b, values)
+    b.seek(0)
+    result = m.deserialize(b)
+    if code == 'd':
+        assert result == pytest.approx(values)
+    else:
+        assert result == values
+
+
+@pytest.mark.parametrize("idl_type,code,values", [
+    (tp.uint16, 'H', [0x0102, 0x0304, 0x0506]),
+    (tp.int32,  'i', [1, -1, 2**16]),
+    (tp.float64,'d', [1.0, -1.0, 0.5]),
+])
+def test_plain_cdr_v2_sequence_roundtrip(idl_type, code, values):
+    """PlainCdrV2SequenceOfPrimitiveMachine serializes and deserializes correctly."""
+    m = mc.PlainCdrV2SequenceOfPrimitiveMachine(idl_type)
+    b = Buffer()
+    b.set_endianness(Endianness.Little)
+    m.serialize(b, values)
+    b.seek(0)
+    result = m.deserialize(b)
+    if code == 'd':
+        assert result == pytest.approx(values)
+    else:
+        assert result == values