Skip to content

ensure_contiguous_memoryview has trouble with numpy M data type in a structured array. #755

@d-v-b

Description

@d-v-b
# /// script
# requires-python = ">=3.11"
# dependencies = [
#   "zarr @ git+https://github.com/zarr-developers/zarr-python.git@27615fd",
#   "numcodecs==0.16.1"
# ]
# ///
import zarr
import numpy as np
import numcodecs
arr = np.array([('19000101', 123), ('19021201', 321)], dtype=[('date', 'datetime64[D]'), ('num', 'i8')])
z = zarr.create_array(store={}, data=arr, compressors=numcodecs.Zstd(), zarr_format=2)
print(z)
# <zarr.core.Array (2,) [('date', '<M8[D]'), ('num', '<i8')]>

this errors with

  File "/home/bennettd/dev/zarr-python/test.py", line 12, in <module>
    z = zarr.create_array(store={}, data=arr, compressors=numcodecs.Zstd(), zarr_format=2)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/api/synchronous.py", line 894, in create_array
    sync(
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/sync.py", line 163, in sync
    raise return_result
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/sync.py", line 119, in _runner
    return await coro
           ^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/array.py", line 4449, in create_array
    return await from_array(
           ^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/array.py", line 4083, in from_array
    await concurrent_map(
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/common.py", line 78, in concurrent_map
    return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/common.py", line 76, in run
    return await func(*item)
           ^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/array.py", line 4080, in _copy_arraylike_region
    await result.setitem(chunk_coords, _data[chunk_coords])
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/array.py", line 1507, in setitem
    return await self._set_selection(indexer, value, prototype=prototype)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/array.py", line 1446, in _set_selection
    await self.codec_pipeline.write(
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/codec_pipeline.py", line 481, in write
    await concurrent_map(
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/common.py", line 78, in concurrent_map
    return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/common.py", line 76, in run
    return await func(*item)
           ^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/codec_pipeline.py", line 416, in write_batch
    chunk_bytes_batch = await self.encode_batch(
                        ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/codec_pipeline.py", line 223, in encode_batch
    chunk_bytes_batch = await self.array_bytes_codec.encode(
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/abc/codec.py", line 157, in encode
    return await _batching_helper(self._encode_single, chunks_and_specs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/abc/codec.py", line 414, in _batching_helper
    return await concurrent_map(
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/common.py", line 78, in concurrent_map
    return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/core/common.py", line 76, in run
    return await func(*item)
           ^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/abc/codec.py", line 427, in wrap
    return await func(chunk, chunk_spec)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/uv/archive-v0/Egb4S2utEbE0hxDA6XufL/lib/python3.11/site-packages/zarr/codecs/_v2.py", line 96, in _encode_single
    cdata = await asyncio.to_thread(self.compressor.encode, chunk)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/pyapp/distributions/_11600656464259912759/python/lib/python3.11/asyncio/threads.py", line 25, in to_thread
    return await loop.run_in_executor(None, func_call)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/bennettd/.cache/pyapp/distributions/_11600656464259912759/python/lib/python3.11/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "numcodecs/zstd.pyx", line 257, in numcodecs.zstd.Zstd.encode
  File "numcodecs/zstd.pyx", line 106, in numcodecs.zstd.compress
  File "numcodecs/compat_ext.pyx", line 16, in numcodecs.compat_ext.ensure_continguous_memoryview
ValueError: cannot include dtype 'M' in a buffer

older versions of numcodecs are OK, so this is likely a side effect of the recent memoryview changes

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type
    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions