From be282c161900749c2b9ad55ba66d9d6c435ef09e Mon Sep 17 00:00:00 2001
From: Hui Zhou <hzhou321@anl.gov>
Date: Wed, 25 Mar 2020 09:17:09 -0500
Subject: [PATCH 1/2] thread: add internal thread_id

It is often needed to track threads during debugging and the system
level thread_id is difficult to use. Create a 0-based internal thread_id
to facilitate thread-tracking and debugging.

The thread_id potentially can also be used for occasional hashing based
performance experiments.
---
 src/include/mpir_thread.h       |  3 +++
 src/mpi/init/local_proc_attrs.c | 20 ++++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/src/include/mpir_thread.h b/src/include/mpir_thread.h
index 4c66c5de744..a9304dd22bc 100644
--- a/src/include/mpir_thread.h
+++ b/src/include/mpir_thread.h
@@ -45,6 +45,9 @@ extern MPIR_Thread_info_t MPIR_ThreadInfo;
 #define MPIR_IS_THREADED    0
 #endif
 
+/* We keep an internal 0-based thread_id to facilitate thread tracking */
+int MPIR_thread_id(void);
+
 /* ------------------------------------------------------------ */
 /* Global thread model, used for non-performance-critical paths */
 /* CONSIDER:
diff --git a/src/mpi/init/local_proc_attrs.c b/src/mpi/init/local_proc_attrs.c
index 1bd45a93f6d..0c2440709b2 100644
--- a/src/mpi/init/local_proc_attrs.c
+++ b/src/mpi/init/local_proc_attrs.c
@@ -25,6 +25,23 @@
 === END_MPI_T_CVAR_INFO_BLOCK ===
 */
 
+static MPL_atomic_int_t next_thread_id = MPL_ATOMIC_INT_T_INITIALIZER(0);
+#if defined(MPICH_IS_THREADED) && defined(MPL_TLS)
+static MPL_TLS int thread_id;
+#else
+static int thread_id;
+#endif
+
+int MPIR_thread_id(void)
+{
+    /* internally thread_id is 1-based so we can tell whether it is valid */
+    if (thread_id == 0) {
+        thread_id = MPL_atomic_fetch_add_int(&next_thread_id, 1) + 1;
+    }
+    /* externally thread_id is 0-based, so it could be used as an index */
+    return (thread_id - 1);
+}
+
 int MPII_init_local_proc_attrs(int *p_thread_required)
 {
     int mpi_errno = MPI_SUCCESS;
@@ -38,6 +55,9 @@ int MPII_init_local_proc_attrs(int *p_thread_required)
     /* We need this inorder to implement IS_THREAD_MAIN */
 #if (MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED)
     MPID_Thread_self(&MPIR_ThreadInfo.main_thread);
+
+    int main_id = MPIR_thread_id();
+    MPIR_Assert(main_id == 0);
 #endif
 #endif /* MPICH_IS_THREADED */
 

From cf7322d2eb4e3568587881db7ea595f177895dd7 Mon Sep 17 00:00:00 2001
From: Hui Zhou <hzhou321@anl.gov>
Date: Wed, 26 May 2021 17:35:24 -0500
Subject: [PATCH 2/2] test: demonstrate how to use MPIR_thread_id()

---
 src/mpid/ch4/netmod/ofi/ofi_events.h     | 1 +
 test/mpi/threads/perf/mt_pt2pt_msgrate.c | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/mpid/ch4/netmod/ofi/ofi_events.h b/src/mpid/ch4/netmod/ofi/ofi_events.h
index 79f31916591..d96adb24c42 100644
--- a/src/mpid/ch4/netmod/ofi/ofi_events.h
+++ b/src/mpid/ch4/netmod/ofi/ofi_events.h
@@ -67,6 +67,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_recv_event(int vni, struct fi_cq_tagged_e
         MPIR_T_PVAR_COUNTER_INC(MULTINIC, nic_recvd_bytes_count[MPIDI_OFI_REQUEST(rreq, nic_num)],
                                 wc->len);
     }
+    printf("OFI_recv_event: thread_id = %d, count = %ld bytes\n", MPIR_thread_id(), count);
 #ifndef MPIDI_CH4_DIRECT_NETMOD
     int is_cancelled;
     MPIDI_anysrc_try_cancel_partner(rreq, &is_cancelled);
diff --git a/test/mpi/threads/perf/mt_pt2pt_msgrate.c b/test/mpi/threads/perf/mt_pt2pt_msgrate.c
index eaec050608b..c86462f69bd 100644
--- a/test/mpi/threads/perf/mt_pt2pt_msgrate.c
+++ b/test/mpi/threads/perf/mt_pt2pt_msgrate.c
@@ -21,8 +21,8 @@
 #define BUFFER_ALIGNMENT 4096
 
 #define MESSAGE_SIZE 8
-#define NUM_MESSAGES 64000
-#define WINDOW_SIZE 64
+#define NUM_MESSAGES 4
+#define WINDOW_SIZE 1
 
 #define ERROR_MARGIN 0.05       /* FIXME: a better margin? */