[geom] Complete TLS ownership and cleanup

README/ReleaseNotes/v642/index.md

@@ -36,6 +36,7 @@ The following people have contributed to this new version:
  Devajith Valaparambil Sreeramaswamy, CERN/EP-SFT,\
  Vassil Vassilev, Princeton,\
  Sandro Wenzel, CERN/EP-ALICE,\
+ Tristan Wenzel, ETHZ,\
 
 ## Deprecation and Removal
 
@@ -207,6 +208,15 @@ Such file can be loaded locally in any web browser or send as attachment in emai
 
 ## Geometry
 
+### Improved multithreaded `TGeo` navigation
+
+Multithreaded `TGeo` navigation is now faster and more scalable, with improved thread-local state management that avoids
+false sharing, releases temporary memory during geometry cleanup, and correctly supports concurrent navigation of
+multiple geometries.
+
+For ALICE material-budget lookup-table generation on 28 cores, these changes reduced the runtime from 139 s to 72 s
+and improved scaling from 12x to 23x.
+
 The [TGeometry](https://root.cern/doc/master/classTGeometry.html) classes (Geant 3 shapes) have been moved out of Graf3D into their own library.
 To link to these classes, use the cmake target `TGeometry` (preferred), `root-config --libs`, or link with `-lTGeometry`.
 When ROOT is configured with `-Dgeom=Off`, these classes are now off as well.

geom/geom/inc/TGeoBoolNode.h

@@ -24,8 +24,8 @@ class TGeoMatrix;
 class TGeoHMatrix;
 
 class TGeoBoolNode : public TObject {
-   static std::atomic<UInt_t> fgInstanceCount; //! source of dense per-object indices
-   UInt_t fIndex{fgInstanceCount++};           //! dense index of this node into the per-thread vector
+   static std::atomic<UInt_t> fgInstanceCount; //! source of monotonic per-object indices
+   UInt_t fIndex{fgInstanceCount++};           //! non-reused index of this node into the per-thread vector
 
 public:
    enum EGeoBoolType {
@@ -39,6 +39,7 @@ class TGeoBoolNode : public TObject {
 
    /// Per-thread scratch state, owned by the calling thread and indexed by this node.
    /// Each thread owns its whole vector, so no two threads ever write the same cache line.
+   /// The vector retains its high-water size until the owning thread exits.
    ThreadData_t &GetThreadData() const
    {
       thread_local std::vector<ThreadData_t> tdata;
@@ -47,6 +48,7 @@ class TGeoBoolNode : public TObject {
       return tdata[fIndex];
    }
    void ClearThreadData() const {}
+   /// No-op: this node allocates its scratch state lazily for every calling thread.
    void CreateThreadData(Int_t) {}
 
 private:

geom/geom/inc/TGeoPatternFinder.h

@@ -24,8 +24,8 @@ class TGeoMatrix;
 
 /// base finder class for patterns. A pattern is specifying a division type
 class TGeoPatternFinder : public TObject {
-   static std::atomic<UInt_t> fgInstanceCount; //! source of dense per-object indices
-   UInt_t fIndex{fgInstanceCount++};           //! dense index of this finder into the per-thread vector
+   static std::atomic<UInt_t> fgInstanceCount; //! source of monotonic per-object indices
+   UInt_t fIndex{fgInstanceCount++};           //! non-reused index of this finder into the per-thread vector
    mutable std::atomic<Int_t> fGeneration{0};  //! bumped whenever the per-thread state must be rebuilt
 
 public:
@@ -41,6 +41,7 @@ class TGeoPatternFinder : public TObject {
 
    /// Per-thread scratch state, owned by the calling thread and indexed by this finder.
    /// Hot path: a TLS read plus an indexed load; the cold rebuild lives in InitThreadSlot().
+   /// The vector retains its high-water size until the owning thread exits.
    ThreadData_t &GetThreadData() const
    {
       thread_local std::vector<ThreadData_t> tdata;
@@ -54,14 +55,19 @@ class TGeoPatternFinder : public TObject {
    /// Invalidate the per-thread data. Each thread rebuilds its own slot lazily on next access,
    /// so no cross-thread reach-in is needed.
    void ClearThreadData() const { fGeneration.fetch_add(1, std::memory_order_release); }
-   /// No-op: per-thread data is allocated lazily, so no provisioning for a fixed thread count
-   /// is required and any number of threads works.
+   /// No-op: this finder allocates its scratch state lazily for every calling thread.
    void CreateThreadData(Int_t) {}
 
 protected:
    void InitThreadSlot(ThreadData_t &td) const;
+   TGeoManager *GetOwnerManager() const { return fVolume ? fVolume->GetGeoManager() : nullptr; }
+   TGeoMatrix *GetOwnerIdentity() const;
+   void RegisterMatrix(TGeoMatrix *matrix) const;
 
-   enum EGeoPatternFlags { kPatternReflected = BIT(14), kPatternSpacedOut = BIT(15) };
+   enum EGeoPatternFlags {
+      kPatternReflected = BIT(14),
+      kPatternSpacedOut = BIT(15)
+   };
    Double_t fStep;      // division step length
    Double_t fStart;     // starting point on divided axis
    Double_t fEnd;       // ending point

geom/geom/inc/TGeoPgon.h

@@ -16,31 +16,25 @@
 
 #include <algorithm>
 #include <atomic>
+#include <memory>
+#include <mutex>
 #include <vector>
 
 class TGeoPgon : public TGeoPcon {
-   static std::atomic<UInt_t> fgInstanceCount; //! source of dense per-object indices
-   UInt_t fIndex{fgInstanceCount++};           //! dense index of this shape into the per-thread vector
+   static std::atomic<UInt_t> fgInstanceCount; //! source of monotonic per-object indices
+   UInt_t fIndex{fgInstanceCount++};           //! non-reused index of this shape into the per-thread vector
    mutable std::atomic<Int_t> fGeneration{0};  //! bumped whenever the per-thread state must be rebuilt
 
 public:
    struct ThreadData_t {
       Int_t *fIntBuffer{nullptr};    //![fNedges+4] temporary int buffer array
       Double_t *fDblBuffer{nullptr}; //![fNedges+4] temporary double buffer array
       Int_t fInitGen{-1};            //! generation this slot was last initialized for
-
-      ThreadData_t() = default;
-      ~ThreadData_t();
-      // Owns the two scratch buffers: movable so the slot can live in a resizable vector,
-      // but not copyable.
-      ThreadData_t(ThreadData_t &&other) noexcept;
-      ThreadData_t &operator=(ThreadData_t &&other) noexcept;
-      ThreadData_t(const ThreadData_t &) = delete;
-      ThreadData_t &operator=(const ThreadData_t &) = delete;
    };
 
-   /// Per-thread scratch buffers, owned by the calling thread and indexed by this shape.
+   /// Per-thread non-owning cache of scratch buffers indexed by this shape.
    /// Hot path: a TLS read plus an indexed load; the cold rebuild lives in InitThreadSlot().
+   /// The vector retains its high-water size until the owning thread exits.
    ThreadData_t &GetThreadData() const
    {
       thread_local std::vector<ThreadData_t> tdata;
@@ -51,17 +45,20 @@ class TGeoPgon : public TGeoPcon {
          InitThreadSlot(td);
       return td;
    }
-   /// Invalidate the per-thread data. Each thread rebuilds its own slot lazily on next access.
-   void ClearThreadData() const override { fGeneration.fetch_add(1, std::memory_order_release); }
-   /// No-op: per-thread data is allocated lazily, so no provisioning for a fixed thread count
-   /// is required and any number of threads works.
+   /// Release object-owned scratch buffers and invalidate the non-owning TLS slots.
+   /// Navigation using this shape must not be active when this method is called.
+   void ClearThreadData() const override;
+   /// No-op: this shape allocates scratch data lazily for every calling thread.
    void CreateThreadData(Int_t) override {}
 
 protected:
+   struct OwnedThreadData_t;
    void InitThreadSlot(ThreadData_t &td) const;
 
    // data members
-   Int_t fNedges; // number of edges (at least one)
+   Int_t fNedges;                                                      // number of edges (at least one)
+   mutable std::vector<std::unique_ptr<OwnedThreadData_t>> fOwnedData; ///<! Object-owned per-thread buffers
+   mutable std::mutex fOwnedDataMutex;                                 ///<! Protects cold allocation and cleanup
 
    // internal utility methods
    Int_t GetPhiCrossList(const Double_t *point, const Double_t *dir, Int_t istart, Double_t *sphi, Int_t *iphi,

geom/geom/inc/TGeoVolume.h

@@ -317,8 +317,8 @@ class TGeoVolumeMulti : public TGeoVolume {
 ////////////////////////////////////////////////////////////////////////////
 
 class TGeoVolumeAssembly : public TGeoVolume {
-   static std::atomic<UInt_t> fgInstanceCount; //! source of dense per-object indices
-   UInt_t fIndex{fgInstanceCount++};           //! dense index of this assembly into the per-thread vector
+   static std::atomic<UInt_t> fgInstanceCount; //! source of monotonic per-object indices
+   UInt_t fIndex{fgInstanceCount++};           //! non-reused index of this assembly into the per-thread vector
 
 public:
    struct ThreadData_t {
@@ -328,6 +328,7 @@ class TGeoVolumeAssembly : public TGeoVolume {
 
    /// Per-thread scratch state, owned by the calling thread and indexed by this assembly.
    /// Each thread owns its whole vector, so no two threads ever write the same cache line.
+   /// The vector retains its high-water size until the owning thread exits.
    ThreadData_t &GetThreadData() const
    {
       thread_local std::vector<ThreadData_t> tdata;

geom/geom/inc/TGeoXtru.h

@@ -16,13 +16,15 @@
 
 #include <algorithm>
 #include <atomic>
+#include <memory>
+#include <mutex>
 #include <vector>
 
 class TGeoPolygon;
 
 class TGeoXtru : public TGeoBBox {
-   static std::atomic<UInt_t> fgInstanceCount;          //! source of dense per-object indices
-   UInt_t fIndex{fgInstanceCount++};                    //! dense index of this shape into the per-thread vector
+   static std::atomic<UInt_t> fgInstanceCount;          //! source of monotonic per-object indices
+   UInt_t fIndex{fgInstanceCount++};                    //! non-reused index of this shape into the per-thread vector
    mutable std::atomic<Int_t> fGeneration{0};           //! bumped whenever the per-thread state must be rebuilt
    mutable std::atomic<Bool_t> fIllegalChecked{kFALSE}; //! illegal-polygon warning already emitted
 
@@ -34,18 +36,11 @@ class TGeoXtru : public TGeoBBox {
       Double_t *fYc{nullptr};      //![fNvert] current Y positions for polygon vertices
       TGeoPolygon *fPoly{nullptr}; //! polygon defining section shape
       Int_t fInitGen{-1};          //! generation this slot was last initialized for
-
-      ThreadData_t() = default;
-      ~ThreadData_t();
-      // Owns fXc/fYc/fPoly: movable so the slot can live in a resizable vector, not copyable.
-      ThreadData_t(ThreadData_t &&other) noexcept;
-      ThreadData_t &operator=(ThreadData_t &&other) noexcept;
-      ThreadData_t(const ThreadData_t &) = delete;
-      ThreadData_t &operator=(const ThreadData_t &) = delete;
    };
 
-   /// Per-thread scratch state, owned by the calling thread and indexed by this shape.
+   /// Per-thread non-owning cache of scratch state indexed by this shape.
    /// Hot path: a TLS read plus an indexed load; the cold rebuild lives in InitThreadSlot().
+   /// The vector retains its high-water size until the owning thread exits.
    ThreadData_t &GetThreadData() const
    {
       thread_local std::vector<ThreadData_t> tdata;
@@ -56,17 +51,14 @@ class TGeoXtru : public TGeoBBox {
          InitThreadSlot(td);
       return td;
    }
-   /// Invalidate the per-thread data. Each thread rebuilds its own slot lazily on next access.
-   void ClearThreadData() const override
-   {
-      fGeneration.fetch_add(1, std::memory_order_release);
-      fIllegalChecked.store(kFALSE, std::memory_order_relaxed);
-   }
-   /// No-op: per-thread data is allocated lazily, so no provisioning for a fixed thread count
-   /// is required and any number of threads works.
+   /// Release object-owned scratch buffers and invalidate the non-owning TLS slots.
+   /// Navigation using this shape must not be active when this method is called.
+   void ClearThreadData() const override;
+   /// No-op: this shape allocates scratch data lazily for every calling thread.
    void CreateThreadData(Int_t) override {}
 
 protected:
+   struct OwnedThreadData_t;
    void InitThreadSlot(ThreadData_t &td) const;
 
    // data members
@@ -79,6 +71,8 @@ class TGeoXtru : public TGeoBBox {
    Double_t *fScale;   //[fNz] array of scale factors (for each Z)
    Double_t *fX0;      //[fNz] array of X offsets (for each Z)
    Double_t *fY0;      //[fNz] array of Y offsets (for each Z)
+   mutable std::vector<std::unique_ptr<OwnedThreadData_t>> fOwnedData; ///<! Object-owned per-thread buffers
+   mutable std::mutex fOwnedDataMutex;                                 ///<! Protects cold allocation and cleanup
 
    TGeoXtru(const TGeoXtru &) = delete;
    TGeoXtru &operator=(const TGeoXtru &) = delete;

geom/geom/src/TGeoManager.cxx

@@ -976,7 +976,9 @@ void TGeoManager::RemoveNavigator(const TGeoNavigator *nav)
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-/// Set maximum number of threads for navigation.
+/// Enable multi-threaded navigation for at most `nthreads` worker threads.
+/// The geometry must be closed and navigation must not be active when this method is called.
+/// This enables ROOT thread safety and prepares the manager and geometry objects for concurrent navigation.
 
 void TGeoManager::SetMaxThreads(Int_t nthreads)
 {