[Python] Move the TTree branch pythonization to Python

bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx

@@ -806,6 +806,60 @@ static PyObject* BindObject(PyObject*, PyObject* args, PyObject* kwds)
     return BindCppObjectNoCast(addr, cast_type);
 }
 
+//----------------------------------------------------------------------------
+static PyObject* ValueFromMemory(PyObject*, PyObject* args, PyObject* kwds)
+{
+// Build a Python object from the memory at the given address, using the
+// Converter for the given type name. See cppyy.ll.value_from_memory for the
+// user-facing documentation.
+    static const char* kwlist[] = {(char*)"type_name", (char*)"address", (char*)"dims", nullptr};
+
+    const char* type_name = nullptr;
+    PyObject* pyaddress = nullptr;
+    PyObject* pydims = nullptr;
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "sO|O:value_from_memory",
+            (char**)kwlist, &type_name, &pyaddress, &pydims))
+        return nullptr;
+
+    void* address = PyLong_AsVoidPtr(pyaddress);
+    if (PyErr_Occurred())
+        return nullptr;
+
+    std::vector<dim_t> dims;
+    if (pydims && pydims != Py_None) {
+        PyObject* seq = PySequence_Fast(pydims, "dims must be a sequence of integers");
+        if (!seq)
+            return nullptr;
+        Py_ssize_t ndim = PySequence_Fast_GET_SIZE(seq);
+        dims.reserve(ndim);
+        for (Py_ssize_t i = 0; i < ndim; ++i) {
+            dim_t d = (dim_t)PyLong_AsSsize_t(PySequence_Fast_GET_ITEM(seq, i));
+            if (d == (dim_t)-1 && PyErr_Occurred()) {
+                Py_DECREF(seq);
+                return nullptr;
+            }
+            dims.push_back(d);
+        }
+        Py_DECREF(seq);
+    }
+
+    Converter* cnv = CreateConverter(type_name, Dimensions((dim_t)dims.size(), dims.data()));
+    if (!cnv) {
+        PyErr_Format(PyExc_TypeError, "no converter available for type \'%s\'", type_name);
+        return nullptr;
+    }
+
+// an array converter reads through the data pointer, so it wants the address
+// of that pointer; a scalar converter wants the address of the value itself
+    PyObject* result = dims.empty() ? cnv->FromMemory(address) : cnv->FromMemory(&address);
+    DestroyConverter(cnv);
+
+    if (!result && !PyErr_Occurred())
+        PyErr_Format(PyExc_TypeError, "failed to convert a value of type \'%s\' from memory", type_name);
+
+    return result;
+}
+
 //----------------------------------------------------------------------------
 static PyObject* Move(PyObject*, PyObject* pyobject)
 {
@@ -986,6 +1040,8 @@ static PyMethodDef gCPyCppyyMethods[] = {
       METH_O, (char*)"Represent an array of objects as raw memory."},
     {(char*)"bind_object", (PyCFunction)BindObject,
       METH_VARARGS | METH_KEYWORDS, (char*) "Create an object of given type, from given address."},
+    {(char*)"value_from_memory", (PyCFunction)ValueFromMemory,
+      METH_VARARGS | METH_KEYWORDS, (char*) "Read a value of given type, from given address."},
     {(char*) "move", (PyCFunction)Move,
       METH_O, (char*)"Cast the C++ object to become movable."},
     {(char*) "add_pythonization", (PyCFunction)AddPythonization,

bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx

@@ -7,6 +7,7 @@
 
 // Standard
 #include <map>
+#include <vector>
 #include <assert.h>
 #include <string.h>
 #include <limits.h>
@@ -765,28 +766,89 @@ static PyObject* ll_reshape(CPyCppyy::LowLevelView* self, PyObject* shape)
 
     Py_buffer& view = self->fBufInfo;
 
-// verify size match
-    Py_ssize_t oldsz = 0;
+// verify size match (the number of elements is the product of the dimensions)
+    Py_ssize_t oldsz = 1;
     for (Py_ssize_t idim = 0; idim < view.ndim; ++idim) {
         Py_ssize_t nlen = view.shape[idim];
         if (nlen == CPyCppyy::UNKNOWN_SIZE || nlen == INT_MAX/view.itemsize /* fake 'max' */) {
             oldsz = -1;      // meaning, unable to check size match
             break;
         }
-        oldsz += view.shape[idim];
+        oldsz *= view.shape[idim];
     }
 
-    if (0 < oldsz) {
-        Py_ssize_t newsz = 0;
-        for (Py_ssize_t idim = 0; idim < PyTuple_GET_SIZE(shape); ++idim)
-            newsz += PyInt_AsSsize_t(PyTuple_GET_ITEM(shape, idim));
-        if (oldsz != newsz) {
-            PyObject* tas = PyObject_Str(shape);
-            PyErr_Format(PyExc_ValueError,
-                "cannot reshape array of size %ld into shape %s", (long)oldsz, CPyCppyy_PyText_AsString(tas));
-            Py_DECREF(tas);
+    Py_ssize_t newsz = 1;
+    for (Py_ssize_t idim = 0; idim < PyTuple_GET_SIZE(shape); ++idim) {
+        Py_ssize_t nlen = PyInt_AsSsize_t(PyTuple_GET_ITEM(shape, idim));
+        if (nlen == -1 && PyErr_Occurred())
+            return nullptr;
+        newsz *= nlen;
+    }
+
+    if (0 < oldsz && oldsz != newsz) {
+        PyObject* tas = PyObject_Str(shape);
+        PyErr_Format(PyExc_ValueError,
+            "cannot reshape array of size %ld into shape %s", (long)oldsz, CPyCppyy_PyText_AsString(tas));
+        Py_DECREF(tas);
+        return nullptr;
+    }
+
+// A multi-dimensional view is not simply a Py_buffer with more entries in its
+// shape: element access projects sub-views through a dedicated converter that
+// is chosen when the view is created. Adding dimensions to a one-dimensional
+// view therefore requires re-creating it through the same creator that made
+// it, rather than patching up the Py_buffer in place.
+//
+// Only rank-1 views can be grown this way: their data is by construction a
+// flat block. A view that is already multi-dimensional carries a layout that
+// the shape alone does not describe (T** data, for example, is an array of
+// row pointers, not a contiguous block), so for those the dimensions are
+// merely filled in below, leaving the layout as it was.
+    if (view.ndim == 1 && 1 < PyTuple_GET_SIZE(shape)) {
+        if (!self->fCreator) {
+            PyErr_SetString(PyExc_TypeError,
+                "this low level view does not support multi-dimensional reshaping");
             return nullptr;
         }
+
+        std::vector<CPyCppyy::dim_t> dims;
+        dims.reserve(PyTuple_GET_SIZE(shape));
+        for (Py_ssize_t idim = 0; idim < PyTuple_GET_SIZE(shape); ++idim)
+            dims.push_back((CPyCppyy::dim_t)PyInt_AsSsize_t(PyTuple_GET_ITEM(shape, idim)));
+
+    // the creator takes the address the view was originally created from: the
+    // address of the data pointer if there is one, the data itself otherwise
+        CPyCppyy::LowLevelView* llnew = self->fCreator(
+            self->fBuf ? (void*)self->fBuf : view.buf,
+            CPyCppyy::Dimensions((CPyCppyy::dim_t)dims.size(), dims.data()));
+        if (!llnew)
+            return nullptr;
+
+    // ownership of the data (if any) stays with self, so keep that flag and
+    // make sure llnew does not free anything that is now ours
+        intptr_t owner = (intptr_t)view.internal & CPyCppyy::LowLevelView::kIsOwner;
+
+        PyMem_Free(view.shape);
+        PyMem_Free(view.strides);
+        if (self->fElemCnv != self->fConverter && self->fElemCnv && self->fElemCnv->HasState())
+            delete self->fElemCnv;
+        if (self->fConverter && self->fConverter->HasState())
+            delete self->fConverter;
+
+        self->fBufInfo   = llnew->fBufInfo;
+        self->fBuf       = llnew->fBuf;
+        self->fConverter = llnew->fConverter;
+        self->fElemCnv   = llnew->fElemCnv;
+        (intptr_t&)self->fBufInfo.internal |= owner;
+
+        llnew->fBufInfo.shape   = nullptr;
+        llnew->fBufInfo.strides = nullptr;
+        (intptr_t&)llnew->fBufInfo.internal &= ~CPyCppyy::LowLevelView::kIsOwner;
+        llnew->fConverter = nullptr;
+        llnew->fElemCnv   = nullptr;
+        Py_DECREF((PyObject*)llnew);
+
+        Py_RETURN_NONE;
     }
 
 // reshape
@@ -812,6 +874,14 @@ static PyObject* ll_reshape(CPyCppyy::LowLevelView* self, PyObject* shape)
 
     set_strides(view, itemsize, false /* by definition not fixed */);
 
+// a rank-1 view hands out elements rather than sub-views, so drop any
+// projecting converter that a previous, higher-rank shape installed
+    if (view.ndim == 1 && self->fConverter != self->fElemCnv) {
+        if (self->fConverter && self->fConverter->HasState())
+            delete self->fConverter;
+        self->fConverter = self->fElemCnv;
+    }
+
     Py_RETURN_NONE;
 }
 

bindings/pyroot/cppyy/cppyy/python/cppyy/ll.py

@@ -1,10 +1,10 @@
 """ Low-level utilities, to be used for "emergencies only".
 """
 
 import cppyy
 import ctypes
 import sys
 import warnings
 
 try:
     import __pypy__
@@ -24,6 +24,7 @@
     'free',
     'array_new',
     'array_delete',
+    'value_from_memory',
     'signals_as_exception',
     'set_signals_as_exception',
     'FatalError',
@@ -44,6 +45,34 @@
     """Return C's argc for use with cppyy/ctypes."""
     return len(sys.argv)
 
+def value_from_memory(type_name, address, dims=None):
+    """Read a value of the C++ type `type_name` from `address`.
+
+    This is the reading half of the conversion that cppyy performs when a
+    function returns, or a data member is read: given a type name and a raw
+    address, it hands back the Python object that cppyy would have produced.
+    It is meant for code that has an address and a type name in hand, but no
+    C++ entity to read them from, such as a framework exposing its own data
+    description.
+
+    `address` is an integer, as returned by `addressof`. `type_name` is any
+    C++ type name that cppyy can resolve, including typedefs. For a class
+    type a bound proxy is returned, for a builtin type a Python value.
+
+    If `dims` is given, it is a sequence of integers describing the shape of
+    an array, `address` is taken to be the start of the array data, and a
+    `LowLevelView` of that shape is returned. The type name should then name
+    the element type followed by `[]`, e.g. `"double[]"`. Pass a single-entry
+    sequence for a one-dimensional array.
+
+        v = ll.value_from_memory('double', addr)             # a float
+        a = ll.value_from_memory('double[]', addr, (2, 3))   # a 2x3 view
+
+    Note that no lifetime or bounds checking is or can be done: the caller
+    vouches for the address, the type and the shape.
+    """
+    return cppyy._backend.value_from_memory(type_name, address, dims)
+
 # import low-level python converters
 for _name in ['addressof', 'as_cobject', 'as_capsule', 'as_ctypes', 'as_memoryview']:
     try:

bindings/pyroot/pythonizations/CMakeLists.txt

@@ -13,7 +13,6 @@ set(cpp_sources
     src/RPyROOTApplication.cxx
     src/GenericPyz.cxx
     src/TClassPyz.cxx
-    src/TTreePyz.cxx
     src/CPPInstancePyz.cxx
 )