From 69b3180b4f48d0d61e3cdd2e4c1ea75846082d84 Mon Sep 17 00:00:00 2001
From: Edward Tremel <high.order.bits@gmail.com>
Date: Wed, 15 Oct 2025 14:26:33 -0400
Subject: [PATCH] Moved ServiceClient to its own set of files

The service.hpp and service_impl.hpp files were getting very long and
unwieldy due to the large number of different classes they contained.
The ServiceClient class in particular is the longest, and most often
needs to be consulted by users since it defines the API clients interact
with, so I moved it to its own set of header/impl files.

Also, in a minor fix, removed a reference to the nonexistant
service_server_api.hpp from the documentation comments at the top of
service.hpp. This seems to refer to a much older design for Casacade
(where there was something called an 'onData()' function on servers) and
is confusing for users.
---
 .../detail/_user_defined_logic_interface.hpp  |    5 +-
 .../cascade/detail/service_client_impl.hpp    | 2141 +++++++++++++++++
 include/cascade/detail/service_impl.hpp       | 2117 +---------------
 include/cascade/service.hpp                   | 1419 +----------
 include/cascade/service_client.hpp            | 1386 +++++++++++
 include/cascade/service_client_api.hpp        |   43 +-
 src/service/server.cpp                        |    1 +
 src/service/server.hpp                        |    9 +-
 8 files changed, 3588 insertions(+), 3533 deletions(-)
 create mode 100644 include/cascade/detail/service_client_impl.hpp
 create mode 100644 include/cascade/service_client.hpp

diff --git a/include/cascade/detail/_user_defined_logic_interface.hpp b/include/cascade/detail/_user_defined_logic_interface.hpp
index ed7d7cf0..ac638a7b 100644
--- a/include/cascade/detail/_user_defined_logic_interface.hpp
+++ b/include/cascade/detail/_user_defined_logic_interface.hpp
@@ -1,6 +1,7 @@
 #pragma once
 #include <cascade/config.h>
 #include "../service.hpp"
+#include "../service_client.hpp"
 #include "../service_types.hpp"
 #include <unordered_set>
 #include <memory>
@@ -72,7 +73,7 @@ void release(ICascadeContext* ctxt);
  *             |      +------------------------------+
  *             |      |
  * DefaultOffCriticalDataPathObserver
- * 
+ *
  * Please derive your own ocdpo from DefaultOffCriticalDataPathObserver, and override the virtual methods defined in
  * IDefaultOffCriticalDataPathObserver
  */
@@ -94,7 +95,7 @@ using emit_func_t = std::function<void(const std::string&,
 
 class IDefaultOffCriticalDataPathObserver {
 public:
-    /** 
+    /**
      * Typed ocdpo handler derived from the Cascade service types defined in service_types.hpp
      * @param sender                The sender id
      * @param object_pool_pathname  The object pool pathname
diff --git a/include/cascade/detail/service_client_impl.hpp b/include/cascade/detail/service_client_impl.hpp
new file mode 100644
index 00000000..4c0269e0
--- /dev/null
+++ b/include/cascade/detail/service_client_impl.hpp
@@ -0,0 +1,2141 @@
+#include "../service_client.hpp"
+
+#include <cascade/config.h>
+#include <cascade/utils.hpp>
+#include <derecho/core/derecho.hpp>
+#include <mutex>
+#include <typeindex>
+
+namespace derecho {
+namespace cascade {
+
+#ifdef ENABLE_EVALUATION
+#define LOG_SERVICE_CLIENT_TIMESTAMP(tag, msgid) \
+    TimestampLogger::log(tag, this->get_my_id(), msgid);
+#else
+#define LOG_SERVICE_CLIENT_TIMESTAMP(tag, msgid)
+#endif
+
+template <typename SubgroupType>
+template <typename T>
+void SubgroupNotificationHandler<SubgroupType>::initialize(derecho::ExternalClientCaller<SubgroupType, T>& subgroup_caller) {
+    dbg_default_trace("SubgroupNotificationHandler(this={:x}) is initialized for SubgroupType:{}",
+                      reinterpret_cast<uint64_t>(this), typeid(SubgroupType).name());
+    subgroup_caller.register_notification_handler(
+            [this](const derecho::NotificationMessage& msg) {
+                dbg_default_trace("subgroup notification handler is triggered with this={:x}, msg type={}, size={} bytes",
+                                  reinterpret_cast<uint64_t>(this), msg.message_type, msg.size);
+                (*this)(msg);
+            });
+}
+
+template <typename SubgroupType>
+void SubgroupNotificationHandler<SubgroupType>::operator()(const derecho::NotificationMessage& msg) {
+    dbg_default_trace("SubgroupNotificationHandler(this={:x}) is triggered with message_type={:x}, size={} bytes",
+                      reinterpret_cast<uint64_t>(this), msg.message_type, msg.size);
+    if(msg.message_type != CASCADE_NOTIFICATION_MESSAGE_TYPE) {
+        return;
+    }
+    mutils::deserialize_and_run(nullptr, msg.body,
+                                [this](const CascadeNotificationMessage& cascade_message) -> void {
+                                    dbg_default_trace("Handling cascade_message: {}. size={} bytes",
+                                                      cascade_message.object_pool_pathname, cascade_message.blob.size);
+                                    std::lock_guard<std::mutex> lck(*object_pool_notification_handlers_mutex);
+                                    // call default handler
+                                    if(object_pool_notification_handlers.find("") != object_pool_notification_handlers.cend()) {
+                                        if(object_pool_notification_handlers.at("").has_value()) {
+                                            (*object_pool_notification_handlers.at(""))(cascade_message.blob);
+                                        }
+                                    }
+                                    // call object pool handler
+                                    if(object_pool_notification_handlers.find(cascade_message.object_pool_pathname) != object_pool_notification_handlers.cend()) {
+                                        if(object_pool_notification_handlers.at(cascade_message.object_pool_pathname).has_value()) {
+                                            (*object_pool_notification_handlers.at(cascade_message.object_pool_pathname))(cascade_message.blob);
+                                        }
+                                    }
+                                });
+}
+
+template <typename CascadeType>
+std::unique_ptr<CascadeType> client_stub_factory() {
+    return std::make_unique<CascadeType>();
+}
+
+template <typename... CascadeTypes>
+ServiceClient<CascadeTypes...>::ServiceClient(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr)
+        : external_group_ptr(nullptr),
+          group_ptr(_group_ptr) {
+    if(group_ptr == nullptr) {
+        this->external_group_ptr = std::make_unique<derecho::ExternalGroupClient<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>>(
+                client_stub_factory<CascadeMetadataService<CascadeTypes...>>,
+                client_stub_factory<CascadeTypes>...);
+    }
+}
+
+template <typename... CascadeTypes>
+bool ServiceClient<CascadeTypes...>::is_external_client() const {
+    return (group_ptr == nullptr) && (external_group_ptr != nullptr);
+}
+
+template <typename... CascadeTypes>
+node_id_t ServiceClient<CascadeTypes...>::get_my_id() const {
+    if(!is_external_client()) {
+        return group_ptr->get_my_id();
+    } else {
+        return external_group_ptr->get_my_id();
+    }
+}
+
+template <typename... CascadeTypes>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_members() const {
+    if(!is_external_client()) {
+        return group_ptr->get_members();
+    } else {
+        return external_group_ptr->get_members();
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_shard_members(uint32_t subgroup_index, uint32_t shard_index) const {
+    if(!is_external_client()) {
+        std::vector<std::vector<node_id_t>> subgroup_members = group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
+        if(subgroup_members.size() > shard_index) {
+            return subgroup_members[shard_index];
+        } else {
+            return {};
+        }
+    } else {
+        return external_group_ptr->template get_shard_members<SubgroupType>(subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::type_recursive_get_shard_members(
+        uint32_t type_index, uint32_t subgroup_index, uint32_t shard_index) const {
+    if(type_index == 0) {
+        return this->template get_shard_members<FirstType>(subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get_shard_members<SecondType, RestTypes...>(type_index - 1, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::type_recursive_get_shard_members(
+        uint32_t type_index, uint32_t subgroup_index, uint32_t shard_index) const {
+    if(type_index == 0) {
+        return this->template get_shard_members<LastType>(subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
+    }
+}
+
+template <typename... CascadeTypes>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_shard_members(
+        const std::string& object_pool_pathname, uint32_t shard_index) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    return this->template type_recursive_get_shard_members<CascadeTypes...>(opm.subgroup_type_index, opm.subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::get_subgroup_members(uint32_t subgroup_index) const {
+    if(!is_external_client()) {
+        return group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
+    } else {
+        return external_group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::type_recursive_get_subgroup_members(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_subgroup_members<FirstType>(subgroup_index);
+    } else {
+        return this->template type_recursive_get_subgroup_members<SecondType, RestTypes...>(type_index - 1, subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::type_recursive_get_subgroup_members(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_subgroup_members<LastType>(subgroup_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
+    }
+}
+
+template <typename... CascadeTypes>
+std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::get_subgroup_members(
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    return this->template type_recursive_get_subgroup_members<CascadeTypes...>(opm.subgroup_type_index, opm.subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+uint32_t ServiceClient<CascadeTypes...>::get_number_of_subgroups() const {
+    if(!is_external_client()) {
+        return group_ptr->template get_num_subgroups<SubgroupType>();
+    } else {
+        return external_group_ptr->template get_number_of_subgroups<SubgroupType>();
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards(uint32_t subgroup_index) const {
+    if(!is_external_client()) {
+        return group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index).size();
+    } else {
+        return external_group_ptr->template get_number_of_shards<SubgroupType>(subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+uint32_t ServiceClient<CascadeTypes...>::type_recursive_get_number_of_shards(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_number_of_shards<FirstType>(subgroup_index);
+    } else {
+        return this->template type_recursive_get_number_of_shards<SecondType, RestTypes...>(type_index - 1, subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+uint32_t ServiceClient<CascadeTypes...>::type_recursive_get_number_of_shards(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_number_of_shards<LastType>(subgroup_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
+    }
+}
+
+template <typename... CascadeTypes>
+uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards(
+        uint32_t subgroup_type_index, uint32_t subgroup_index) const {
+    return type_recursive_get_number_of_shards<CascadeTypes...>(subgroup_type_index, subgroup_index);
+}
+
+template <typename... CascadeTypes>
+uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards(
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    return get_number_of_shards(opm.subgroup_type_index, opm.subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+int32_t ServiceClient<CascadeTypes...>::get_my_shard(uint32_t subgroup_index) const {
+    if(!is_external_client()) {
+        return group_ptr->template get_my_shard<SubgroupType>(subgroup_index);
+    } else {
+        return -1;
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+int32_t ServiceClient<CascadeTypes...>::type_recursive_get_my_shard(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_my_shard<FirstType>(subgroup_index);
+    } else {
+        return this->template type_recursive_get_number_of_shards<SecondType, RestTypes...>(type_index - 1, subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+int32_t ServiceClient<CascadeTypes...>::type_recursive_get_my_shard(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_my_shard<LastType>(subgroup_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
+    }
+}
+
+template <typename... CascadeTypes>
+int32_t ServiceClient<CascadeTypes...>::get_my_shard(
+        uint32_t subgroup_type_index, uint32_t subgroup_index) const {
+    return this->template type_recursive_get_my_shard<CascadeTypes...>(subgroup_type_index, subgroup_index);
+}
+
+template <typename... CascadeTypes>
+int32_t ServiceClient<CascadeTypes...>::get_my_shard(
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    return get_my_shard(opm.subgroup_type_index, opm.subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::set_member_selection_policy(uint32_t subgroup_index, uint32_t shard_index,
+                                                                 ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id) {
+    // write lock policies
+    std::unique_lock wlck(this->member_selection_policies_mutex);
+    // update map
+    this->member_selection_policies[std::make_tuple(std::type_index(typeid(SubgroupType)), subgroup_index, shard_index)]
+            = std::make_tuple(policy, user_specified_node_id);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::tuple<ShardMemberSelectionPolicy, node_id_t> ServiceClient<CascadeTypes...>::get_member_selection_policy(
+        uint32_t subgroup_index, uint32_t shard_index) const {
+    // read lock policies
+    std::shared_lock rlck(this->member_selection_policies_mutex);
+    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)), subgroup_index, shard_index);
+    // read map
+    if(member_selection_policies.find(key) != member_selection_policies.end()) {
+        return member_selection_policies.at(key);
+    } else {
+        return std::make_tuple(DEFAULT_SHARD_MEMBER_SELECTION_POLICY, INVALID_NODE_ID);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::refresh_member_cache_entry(uint32_t subgroup_index,
+                                                                uint32_t shard_index) {
+    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)), subgroup_index, shard_index);
+    auto members = get_shard_members<SubgroupType>(subgroup_index, shard_index);
+    std::shared_lock rlck(member_cache_mutex);
+    if(member_cache.find(key) == member_cache.end()) {
+        rlck.unlock();
+        std::unique_lock wlck(member_cache_mutex);
+        member_cache[key] = members;
+    } else {
+        member_cache[key].swap(members);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+std::tuple<uint32_t, uint32_t, uint32_t> ServiceClient<CascadeTypes...>::key_to_shard(
+        const KeyType& key,
+        bool check_object_location) {
+    auto pair = find_object_pool_and_affinity_set_by_key(key);
+
+    auto& opm = std::get<0>(pair);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to identify the object_pool from key:" + key);
+    }
+    auto& affinity_set = std::get<1>(pair);
+
+    return std::tuple<uint32_t, uint32_t, uint32_t>{opm.subgroup_type_index, opm.subgroup_index,
+                                                    opm.key_to_shard_index(key, affinity_set,
+                                                                           get_number_of_shards(opm.subgroup_type_index, opm.subgroup_index),
+                                                                           check_object_location)};
+}
+
+template <typename... CascadeTypes>
+ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::ObjectPoolMetadataCacheEntry(
+        const ObjectPoolMetadata<CascadeTypes...>& _opm) : opm(_opm), database(nullptr) {
+    if(opm.affinity_set_regex.size() > 0) {
+        hs_compile_error_t* compile_err;
+        if(hs_compile(opm.affinity_set_regex.c_str(), HS_FLAG_DOTALL | HS_FLAG_SOM_LEFTMOST, HS_MODE_BLOCK, NULL, &database,
+                      &compile_err)
+           != HS_SUCCESS) {
+            hs_free_compile_error(compile_err);
+            dbg_default_error("Compilation of affinity set regex:" + opm.affinity_set_regex + " failed with message:"
+                              + compile_err->message);
+            throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__)
+                                             + ": compilation of affinity_set_regex:"
+                                             + opm.affinity_set_regex
+                                             + " failed with message:"
+                                             + compile_err->message);
+        }
+    }
+}
+
+template <typename... CascadeTypes>
+ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::~ObjectPoolMetadataCacheEntry() {
+    if(this->database != nullptr) {
+        hs_free_database(database);
+        this->database = nullptr;
+    }
+}
+
+template <typename... CascadeTypes>
+inline std::string ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::to_affinity_set(
+        const std::string& key_string) {
+    if(key_string.size() > 0 && this->opm.affinity_set_regex.size() > 0) {
+        if(scratch == nullptr) {
+            if(hs_alloc_scratch(database, &scratch) != HS_SUCCESS) {
+                hs_free_database(database);
+                dbg_default_error("failed to allocate hyperscan scratch space.");
+                throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__)
+                                                 + " failed to allocate hyperscan scratch space.");
+            }
+        }
+
+        struct hs_scan_ctxt {
+            unsigned long long from = 0;
+            unsigned long long to = 0;
+        } ctxt;
+
+        hs_scan(database, key_string.c_str(), key_string.size(), HS_FLAG_SOM_LEFTMOST, scratch, [](unsigned int /*id*/, unsigned long long from, unsigned long long to, unsigned int /*flags*/, void* ctxt) -> int {
+            struct hs_scan_ctxt* p_hs_ctxt = static_cast<struct hs_scan_ctxt*>(ctxt);
+            p_hs_ctxt->from = from;
+            p_hs_ctxt->to = to;
+            return 0;  // do the longest match
+        },
+                &ctxt);
+        if(ctxt.to > ctxt.from) {
+            return key_string.substr(ctxt.from, (ctxt.to - ctxt.from));
+        }
+    }
+
+    return key_string;
+}
+
+template <typename... CascadeTypes>
+thread_local hs_scratch_t* ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::scratch = nullptr;
+
+template <typename... CascadeTypes>
+template <typename SubgroupType, typename KeyTypeForHashing>
+node_id_t ServiceClient<CascadeTypes...>::pick_member_by_policy(uint32_t subgroup_index,
+                                                                uint32_t shard_index,
+                                                                const KeyTypeForHashing& key_for_hashing,
+                                                                bool retry) {
+    ShardMemberSelectionPolicy policy;
+    node_id_t last_specified_node_id_or_index;
+
+    std::tie(policy, last_specified_node_id_or_index) = get_member_selection_policy<SubgroupType>(subgroup_index, shard_index);
+
+    if(policy == ShardMemberSelectionPolicy::UserSpecified) {
+        return last_specified_node_id_or_index;
+    }
+
+    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)), subgroup_index, shard_index);
+
+    if(member_cache.find(key) == member_cache.end() || retry) {
+        refresh_member_cache_entry<SubgroupType>(subgroup_index, shard_index);
+    }
+
+    std::shared_lock rlck(member_cache_mutex);
+
+    node_id_t node_id = last_specified_node_id_or_index;
+
+    switch(policy) {
+        case ShardMemberSelectionPolicy::FirstMember:
+            node_id = member_cache.at(key).front();
+            break;
+        case ShardMemberSelectionPolicy::LastMember:
+            node_id = member_cache.at(key).back();
+            break;
+        case ShardMemberSelectionPolicy::Random:
+            node_id = member_cache.at(key)[get_time() % member_cache.at(key).size()];  // use time as random source.
+            break;
+        case ShardMemberSelectionPolicy::FixedRandom:
+            if(node_id == INVALID_NODE_ID || retry) {
+                node_id = member_cache.at(key)[get_time() % member_cache.at(key).size()];  // use time as random source.
+            }
+            break;
+        case ShardMemberSelectionPolicy::RoundRobin: {
+            std::shared_lock rlck(member_selection_policies_mutex);
+            node_id = static_cast<uint32_t>(node_id + 1) % member_cache.at(key).size();
+            auto new_policy = std::make_tuple(ShardMemberSelectionPolicy::RoundRobin, node_id);
+            member_selection_policies[key].swap(new_policy);
+        }
+            node_id = member_cache.at(key)[node_id];
+            break;
+        case ShardMemberSelectionPolicy::KeyHashing: {
+            uint64_t hash = 0;
+            if constexpr(std::is_integral_v<KeyTypeForHashing>) {
+                hash = static_cast<uint64_t>(key_for_hashing);
+            } else if constexpr(std::is_convertible_v<KeyTypeForHashing, std::string>) {
+                hash = std::hash<std::string>{}(std::string(key_for_hashing));
+            } else {
+                dbg_default_warn("Key type {} is neither integral nor string, falling back to FirstMember policy. {}:{}",
+                                 typeid(KeyTypeForHashing).name(), __FILE__, __LINE__);
+            }
+            node_id = member_cache.at(key)[hash % member_cache.at(key).size()];
+        } break;
+        default:
+            throw derecho::derecho_exception("Unknown member selection policy:" + std::to_string(static_cast<unsigned int>(policy)));
+    }
+
+    return node_id;
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::put(
+        const typename SubgroupType::ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_PUT_START,
+                                 (std::is_base_of<IHasMessageID, typename SubgroupType::ObjectType>::value ? value.get_message_id() : 0));
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            // ordered put as a shard member
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template ordered_send<RPC_NAME(ordered_put)>(value, as_trigger);
+        } else {
+            // p2p put
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+            try {
+                // as a subgroup member
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                return subgroup_handle.template p2p_send<RPC_NAME(put)>(node_id, value, as_trigger);
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                // as an external caller
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                return subgroup_handle.template p2p_send<RPC_NAME(put)>(node_id, value, as_trigger);
+            }
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+        return caller.template p2p_send<RPC_NAME(put)>(node_id, value, as_trigger);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_put(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    if(type_index == 0) {
+        return this->template put<FirstType>(value, subgroup_index, shard_index, as_trigger);
+    } else {
+        return this->template type_recursive_put<ObjectType, SecondType, RestTypes...>(type_index - 1, value, subgroup_index, shard_index, as_trigger);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename LastType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_put(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    if(type_index == 0) {
+        return this->template put<LastType>(value, subgroup_index, shard_index, as_trigger);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::put(
+        const ObjectType& value, bool as_trigger) {
+    // STEP 1 - get key
+    if constexpr(!std::is_base_of_v<ICascadeObject<std::string, ObjectType>, ObjectType>) {
+        throw derecho::derecho_exception(std::string("ServiceClient<>::put() only support object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(value.get_key_ref());
+
+    // STEP 3 - call recursive put
+    return this->template type_recursive_put<ObjectType, CascadeTypes...>(subgroup_type_index, value, subgroup_index, shard_index, as_trigger);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::put_and_forget(
+        const typename SubgroupType::ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_PUT_AND_FORGET_START,
+                                 (std::is_base_of<IHasMessageID, typename SubgroupType::ObjectType>::value ? value.get_message_id() : 0));
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            // do ordered put as a shard member (Replicated).
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            subgroup_handle.template ordered_send<RPC_NAME(ordered_put_and_forget)>(value, as_trigger);
+        } else {
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+            // do p2p put
+            try {
+                // as a subgroup member
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                subgroup_handle.template p2p_send<RPC_NAME(put_and_forget)>(node_id, value, as_trigger);
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                // as an external caller
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                subgroup_handle.template p2p_send<RPC_NAME(put_and_forget)>(node_id, value, as_trigger);
+            }
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+        caller.template p2p_send<RPC_NAME(put_and_forget)>(node_id, value, as_trigger);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+void ServiceClient<CascadeTypes...>::type_recursive_put_and_forget(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    if(type_index == 0) {
+        put_and_forget<FirstType>(value, subgroup_index, shard_index, as_trigger);
+    } else {
+        type_recursive_put_and_forget<ObjectType, SecondType, RestTypes...>(type_index - 1, value, subgroup_index, shard_index, as_trigger);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename LastType>
+void ServiceClient<CascadeTypes...>::type_recursive_put_and_forget(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    if(type_index == 0) {
+        put_and_forget<LastType>(value, subgroup_index, shard_index, as_trigger);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType>
+void ServiceClient<CascadeTypes...>::put_and_forget(const ObjectType& value, bool as_trigger) {
+    // STEP 1 - get key
+    if constexpr(!std::is_base_of_v<ICascadeObject<std::string, ObjectType>, ObjectType>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(value.get_key_ref());
+
+    // STEP 3 - call recursive put_and_forget
+    this->template type_recursive_put_and_forget<ObjectType, CascadeTypes...>(subgroup_type_index, value, subgroup_index, shard_index, as_trigger);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::trigger_put(
+        const typename SubgroupType::ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_TRIGGER_PUT_START,
+                                 (std::is_base_of<IHasMessageID, typename SubgroupType::ObjectType>::value ? value.get_message_id() : 0));
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+            dbg_default_trace("trigger_put to node {}", node_id);
+            return subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(node_id, value);
+        } else {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+            dbg_default_trace("trigger_put to node {}", node_id);
+            return subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(node_id, value);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+        dbg_default_trace("trigger_put to node {}", node_id);
+        return caller.template p2p_send<RPC_NAME(trigger_put)>(node_id, value);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::type_recursive_trigger_put(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return trigger_put<FirstType>(value, subgroup_index, shard_index);
+    } else {
+        return type_recursive_trigger_put<ObjectType, SecondType, RestTypes...>(type_index - 1, value, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename LastType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::type_recursive_trigger_put(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return trigger_put<LastType>(value, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::trigger_put(
+        const ObjectType& value) {
+    // STEP 1 - get key
+    if constexpr(!std::is_base_of_v<ICascadeObject<std::string, ObjectType>, ObjectType>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(value.get_key_ref());
+
+    // STEP 3 - call recursive trigger_put
+    return this->template type_recursive_trigger_put<ObjectType, CascadeTypes...>(subgroup_type_index, value, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::collective_trigger_put(
+        const typename SubgroupType::ObjectType& value,
+        uint32_t subgroup_index,
+        std::unordered_map<node_id_t, std::unique_ptr<derecho::rpc::QueryResults<void>>>& nodes_and_futures) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_COLLECTIVE_TRIGGER_PUT_START,
+                                 (std::is_base_of<IHasMessageID, typename SubgroupType::ObjectType>::value ? value.get_message_id() : 0));
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(group_ptr->template get_my_shard<SubgroupType>(subgroup_index) != -1) {
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            for(auto& kv : nodes_and_futures) {
+                nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
+                        std::move(subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(kv.first, value)));
+            }
+        } else {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            for(auto& kv : nodes_and_futures) {
+                nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
+                        std::move(subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(kv.first, value)));
+            }
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        for(auto& kv : nodes_and_futures) {
+            nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
+                    std::move(caller.template p2p_send<RPC_NAME(trigger_put)>(kv.first, value)));
+        }
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove(
+        const typename SubgroupType::KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_REMOVE_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            // do ordered remove as a member (Replicated).
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template ordered_send<RPC_NAME(ordered_remove)>(key);
+        } else {
+            // do p2p remove
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+            try {
+                // as a subgroup member
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                return subgroup_handle.template p2p_send<RPC_NAME(remove)>(node_id, key);
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                // as an external caller
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                return subgroup_handle.template p2p_send<RPC_NAME(remove)>(node_id, key);
+            }
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(remove)>(node_id, key);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_remove(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template remove<FirstType>(key, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_remove<KeyType, SecondType, RestTypes...>(type_index - 1, key, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_remove(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template remove<LastType>(key, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove(
+        const KeyType& key) {
+    // STEP 1 - get key
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive remove
+    return this->template type_recursive_remove<KeyType, CascadeTypes...>(subgroup_type_index, key, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::get(
+        const typename SubgroupType::KeyType& key,
+        const persistent::version_t& version,
+        bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_GET_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p get as a subgroup member
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+                // local get
+                auto obj = subgroup_handle.get_ref().get(key, version, stable);
+                auto pending_results = std::make_shared<PendingResults<const typename SubgroupType::ObjectType>>();
+                pending_results->fulfill_map({node_id});
+                pending_results->set_value(node_id, obj);
+                auto query_results = pending_results->get_future();
+                return std::move(*query_results);
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(get)>(node_id, key, version, stable, false);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(get)>(node_id, key, version, stable, false);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(get)>(node_id, key, version, stable, false);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::multi_get(
+        const typename SubgroupType::KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_GET_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p multi_get as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_get)>(node_id, key);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p multi_get as an external caller.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_get)>(node_id, key);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(multi_get)>(node_id, key);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_get(
+        uint32_t type_index,
+        const KeyType& key,
+        const persistent::version_t& version,
+        bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get<FirstType>(key, version, stable, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get<KeyType, SecondType, RestTypes...>(type_index - 1, key, version, stable, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_get(
+        uint32_t type_index,
+        const KeyType& key,
+        const persistent::version_t& version,
+        bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get<LastType>(key, version, stable, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+auto ServiceClient<CascadeTypes...>::get(
+        // const std::decay_t<typename std::result_of_t<decltype(&ObjectType::get_key_ref())>>& key,
+        const KeyType& key,
+        const persistent::version_t& version,
+        bool stable) {
+    // STEP 1 - get key
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_get<KeyType, CascadeTypes...>(subgroup_type_index, key, version, stable, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_multi_get(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template multi_get<FirstType>(key, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_multi_get<KeyType, SecondType, RestTypes...>(type_index - 1, key, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_multi_get(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template multi_get<LastType>(key, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+auto ServiceClient<CascadeTypes...>::multi_get(
+        const KeyType& key) {
+    // STEP 1 - get key
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_multi_get<KeyType, CascadeTypes...>(subgroup_type_index, key, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::get_by_time(
+        const typename SubgroupType::KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p get_by_time
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                // as a shard member.
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(get_by_time)>(node_id, key, ts_us, stable);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p get_by_time as an external caller
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(get_by_time)>(node_id, key, ts_us, stable);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>();
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(get_by_time)>(node_id, key, ts_us, stable);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_get_by_time(
+        uint32_t type_index,
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_by_time<FirstType>(key, ts_us, stable, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get_by_time<KeyType, SecondType, RestTypes...>(type_index - 1, key, ts_us, stable, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_get_by_time(
+        uint32_t type_index,
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_by_time<LastType>(key, ts_us, stable, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+auto ServiceClient<CascadeTypes...>::get_by_time(
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable) {
+    // STEP 1 - get key
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get_by_time
+    return this->template type_recursive_get_by_time<KeyType, CascadeTypes...>(subgroup_type_index, key, ts_us, stable, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size(
+        const typename SubgroupType::KeyType& key,
+        const persistent::version_t& version,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_GET_SIZE_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p get_size as a subgroup_member
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                // as a shard member.
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(get_size)>(node_id, key, version, stable, false);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p get_size as an external caller
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(get_size)>(node_id, key, version, stable, false);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(get_size)>(node_id, key, version, stable, false);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size(
+        uint32_t type_index,
+        const KeyType& key,
+        const persistent::version_t& version,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_size<FirstType>(key, version, stable, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get_size<KeyType, SecondType, RestTypes...>(type_index - 1, key, version, stable, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size(
+        uint32_t type_index,
+        const KeyType& key,
+        const persistent::version_t& version,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_size<LastType>(key, version, stable, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size(
+        const KeyType& key,
+        const persistent::version_t& version,
+        const bool stable) {
+    // STEP 1 - verify the keys
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_get_size<KeyType, CascadeTypes...>(subgroup_type_index, key, version, stable, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::multi_get_size(
+        const typename SubgroupType::KeyType& key,
+        uint32_t subgroup_index, uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_GET_SIZE_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p multi_get_size as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_get_size)>(node_id, key);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p multi_get_size as an external caller.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_get_size)>(node_id, key);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(multi_get_size)>(node_id, key);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_multi_get_size(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template multi_get_size<FirstType>(key, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_multi_get_size<KeyType, SecondType, RestTypes...>(type_index - 1, key, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_multi_get_size(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template multi_get_size<LastType>(key, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::multi_get_size(const KeyType& key) {
+    // STEP 1 - verify the keys
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_multi_get_size<KeyType, CascadeTypes...>(subgroup_type_index, key, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size_by_time(
+        const typename SubgroupType::KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p get_size_by_time as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(get_size_by_time)>(node_id, key, ts_us, stable);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p get_size_by_time as an external caller.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(get_size_by_time)>(node_id, key, ts_us, stable);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(get_size_by_time)>(node_id, key, ts_us, stable);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size_by_time(
+        uint32_t type_index,
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_size_by_time<FirstType>(key, ts_us, stable, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get_size_by_time<KeyType, SecondType, RestTypes...>(type_index - 1, key, ts_us, stable, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size_by_time(
+        uint32_t type_index,
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_size_by_time<LastType>(key, ts_us, stable, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size_by_time(
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable) {
+    // STEP 1 - verify the keys
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_get_size_by_time<KeyType, CascadeTypes...>(subgroup_type_index, key, ts_us, stable, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::list_keys(
+        const persistent::version_t& version,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_LIST_KEYS_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        try {
+            // do p2p list_keys as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id, "", version, stable);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p list_keys as an external client.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id, "", version, stable);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        return caller.template p2p_send<RPC_NAME(list_keys)>(node_id, "", version, stable);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_list_keys(
+        uint32_t type_index,
+        const persistent::version_t& version,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __list_keys<FirstType>(version, stable, object_pool_pathname);
+    } else {
+        return this->template type_recursive_list_keys<SecondType, RestTypes...>(type_index - 1, version, stable, object_pool_pathname);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_list_keys(
+        uint32_t type_index,
+        const persistent::version_t& version,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __list_keys<LastType>(version, stable, object_pool_pathname);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__list_keys(
+        const persistent::version_t& version,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    uint32_t subgroup_index = opm.subgroup_index;
+    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
+    for(uint32_t shard_index = 0; shard_index < shards; shard_index++) {
+        if(!is_external_client()) {
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+            try {
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                    node_id = group_ptr->get_my_id();
+                }
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id, object_pool_pathname, version, stable);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id, object_pool_pathname, version, stable);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            }
+        } else {
+            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+            auto shard_keys = caller.template p2p_send<RPC_NAME(list_keys)>(node_id, object_pool_pathname, version, stable);
+            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+        }
+    }
+    return result;
+}
+
+template <typename... CascadeTypes>
+auto ServiceClient<CascadeTypes...>::list_keys(
+        const persistent::version_t& version,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    volatile uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(object_pool_pathname + "/_");
+    return this->template type_recursive_list_keys<CascadeTypes...>(subgroup_type_index, version, stable, object_pool_pathname);
+}
+
+template <typename ReturnType>
+inline ReturnType wait_for_future(derecho::rpc::QueryResults<ReturnType>& result) {
+    // iterate through ReplyMap
+    for(auto& reply_future : result.get()) {
+        ReturnType reply = static_cast<ReturnType>(reply_future.second.get());
+        return reply;
+    }
+    return ReturnType();
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+std::vector<KeyType> ServiceClient<CascadeTypes...>::wait_list_keys(
+        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<KeyType>>>>& future) {
+    std::vector<KeyType> result;
+    // iterate over each shard's Query result
+    for(auto& query_result : future) {
+        std::vector<KeyType> reply = wait_for_future<std::vector<KeyType>>(*(query_result.get()));
+        std::move(reply.begin(), reply.end(), std::back_inserter(result));
+    }
+    return result;
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::multi_list_keys(
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_LIST_KEYS_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        try {
+            // do p2p multi_list_keys as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, "");
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p multi_list_keys as an external client.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, "");
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        return caller.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, "");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_multi_list_keys(
+        uint32_t type_index,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __multi_list_keys<FirstType>(object_pool_pathname);
+    } else {
+        return this->template type_recursive_multi_list_keys<SecondType, RestTypes...>(type_index - 1, object_pool_pathname);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_multi_list_keys(
+        uint32_t type_index,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __multi_list_keys<LastType>(object_pool_pathname);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__multi_list_keys(const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    uint32_t subgroup_index = opm.subgroup_index;
+    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
+    for(uint32_t shard_index = 0; shard_index < shards; shard_index++) {
+        if(!is_external_client()) {
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, object_pool_pathname);
+            try {
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                    node_id = group_ptr->get_my_id();
+                }
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, object_pool_pathname);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, object_pool_pathname);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            }
+        } else {
+            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, object_pool_pathname);
+            auto shard_keys = caller.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, object_pool_pathname);
+            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+        }
+    }
+    return result;
+}
+
+template <typename... CascadeTypes>
+auto ServiceClient<CascadeTypes...>::multi_list_keys(const std::string& object_pool_pathname) {
+    volatile uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(object_pool_pathname + "/_");
+    return this->template type_recursive_multi_list_keys<CascadeTypes...>(subgroup_type_index, object_pool_pathname);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::list_keys_by_time(
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        try {
+            // do p2p list_keys_by_time as a subgroup member
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, "", ts_us, stable);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p list_keys_by_time as an external client.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, "", ts_us, stable);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        return caller.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, "", ts_us, stable);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_list_keys_by_time(
+        uint32_t type_index,
+        const uint64_t& ts_us,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __list_keys_by_time<FirstType>(ts_us, stable, object_pool_pathname);
+    } else {
+        return this->template type_recursive_list_keys_by_time<SecondType, RestTypes...>(type_index - 1, ts_us, stable, object_pool_pathname);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_list_keys_by_time(
+        uint32_t type_index,
+        const uint64_t& ts_us,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __list_keys_by_time<LastType>(ts_us, stable, object_pool_pathname);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__list_keys_by_time(
+        const uint64_t& ts_us,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    uint32_t subgroup_index = opm.subgroup_index;
+    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
+    for(uint32_t shard_index = 0; shard_index < shards; shard_index++) {
+        if(!is_external_client()) {
+            std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, object_pool_pathname);
+            try {
+                // do p2p list_keys_by_time as a subgroup member.
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                    node_id = group_ptr->get_my_id();
+                }
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, object_pool_pathname, ts_us, stable);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                // do p2p list_keys_by_time as an external client.
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, object_pool_pathname, ts_us, stable);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            }
+        } else {
+            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+            // call as an external client (ExternalClientCaller).
+            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, object_pool_pathname);
+            auto shard_keys = caller.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, object_pool_pathname, ts_us, stable);
+            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+        }
+    }
+    return result;
+}
+
+template <typename... CascadeTypes>
+auto ServiceClient<CascadeTypes...>::list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname) {
+    volatile uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(object_pool_pathname + "/_");
+    return this->template type_recursive_list_keys_by_time<CascadeTypes...>(subgroup_type_index, ts_us, stable, object_pool_pathname);
+}
+
+template <typename... CascadeTypes>
+void ServiceClient<CascadeTypes...>::refresh_object_pool_metadata_cache() {
+    std::unordered_map<std::string, ObjectPoolMetadataCacheEntry> refreshed_metadata;
+    uint32_t num_shards = this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
+    for(uint32_t shard = 0; shard < num_shards; shard++) {
+        auto results = this->template multi_list_keys<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX, shard);
+        for(auto& reply : results.get()) {         // only once
+            for(auto& key : reply.second.get()) {  // iterate over keys
+                // we only read the stable version.
+                auto opm_result = this->template get<CascadeMetadataService<CascadeTypes...>>(key, CURRENT_VERSION, false, METADATA_SERVICE_SUBGROUP_INDEX, shard);
+                for(auto& opm_reply : opm_result.get()) {  // only once
+                    refreshed_metadata.emplace(key, opm_reply.second.get());
+                    break;
+                }
+            }
+            break;
+        }
+    }
+
+    std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
+    this->object_pool_metadata_cache = std::move(refreshed_metadata);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::create_object_pool(
+        const std::string& pathname, const uint32_t subgroup_index,
+        const sharding_policy_t sharding_policy, const std::unordered_map<std::string, uint32_t>& object_locations,
+        const std::string& affinity_set_regex) {
+    uint32_t subgroup_type_index = ObjectPoolMetadata<CascadeTypes...>::template get_subgroup_type_index<SubgroupType>();
+    if(subgroup_type_index == ObjectPoolMetadata<CascadeTypes...>::invalid_subgroup_type_index) {
+        dbg_default_crit("Create object pool failed because of invalid SubgroupType:{}", typeid(SubgroupType).name());
+        throw derecho::derecho_exception(std::string("Create object pool failed because SubgroupType is invalid:") + typeid(SubgroupType).name());
+    }
+    ObjectPoolMetadata<CascadeTypes...> opm(pathname, subgroup_type_index, subgroup_index, sharding_policy, object_locations, affinity_set_regex, false);
+    // clear local cache entry.
+    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
+    if(object_pool_metadata_cache.find(pathname) == object_pool_metadata_cache.end()) {
+        rlck.unlock();
+    } else {
+        rlck.unlock();
+        std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
+        object_pool_metadata_cache.erase(pathname);
+    }
+    // determine the shard index by hashing
+    uint32_t metadata_service_shard_index = std::hash<std::string>{}(pathname) % this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
+
+    return this->template put<CascadeMetadataService<CascadeTypes...>>(opm, METADATA_SERVICE_SUBGROUP_INDEX, metadata_service_shard_index);
+}
+
+template <typename... CascadeTypes>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove_object_pool(const std::string& pathname) {
+    // determine the shard index by hashing
+    uint32_t metadata_service_shard_index = std::hash<std::string>{}(pathname) % this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
+
+    // check if this object pool exist in metadata service.
+    auto opm = find_object_pool(pathname);
+    // remove it from local cache.
+    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
+    if(object_pool_metadata_cache.find(pathname) == object_pool_metadata_cache.end()) {
+        // no entry in cache
+        rlck.unlock();
+    } else {
+        // remove from cache
+        rlck.unlock();
+        std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
+        object_pool_metadata_cache.erase(pathname);
+        wlck.unlock();
+    }
+    if(opm.is_valid() && !opm.is_null()) {
+        if(opm.deleted) {
+            throw derecho::derecho_exception(std::string("object pool:") + pathname + " has been deleted already.");
+        }
+        opm.deleted = true;
+        opm.set_previous_version(CURRENT_VERSION, opm.version);  // only check previous_version_by_key
+        return this->template put<CascadeMetadataService<CascadeTypes...>>(opm, METADATA_SERVICE_SUBGROUP_INDEX, metadata_service_shard_index);
+    }
+
+    // we didn't find the object pool, but we do the normal 'remove', which has no effect but return a version.
+    dbg_default_warn("deleteing a non-existing objectpool:{}.", pathname);
+    return this->template remove<CascadeMetadataService<CascadeTypes...>>(pathname, METADATA_SERVICE_SUBGROUP_INDEX, metadata_service_shard_index);
+}
+
+template <typename... CascadeTypes>
+ObjectPoolMetadata<CascadeTypes...> ServiceClient<CascadeTypes...>::internal_find_object_pool(
+        const std::string& pathname,
+        std::shared_lock<std::shared_mutex>& rlck) {
+    auto components = str_tokenizer(pathname);
+    std::string prefix;
+    for(const auto& comp : components) {
+        prefix = prefix + PATH_SEPARATOR + comp;
+        if(object_pool_metadata_cache.find(prefix) != object_pool_metadata_cache.end()) {
+            return object_pool_metadata_cache.at(prefix).opm;
+        }
+    }
+    rlck.unlock();
+
+    // refresh and try again.
+    refresh_object_pool_metadata_cache();
+    prefix = "";
+    rlck.lock();
+    for(const auto& comp : components) {
+        prefix = prefix + PATH_SEPARATOR + comp;
+        if(object_pool_metadata_cache.find(prefix) != object_pool_metadata_cache.end()) {
+            return object_pool_metadata_cache.at(prefix).opm;
+        }
+    }
+    return ObjectPoolMetadata<CascadeTypes...>::IV;
+}
+
+template <typename... CascadeTypes>
+ObjectPoolMetadata<CascadeTypes...> ServiceClient<CascadeTypes...>::find_object_pool(const std::string& pathname) {
+    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
+    return this->internal_find_object_pool(pathname, rlck);
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+std::pair<ObjectPoolMetadata<CascadeTypes...>, std::string> ServiceClient<CascadeTypes...>::find_object_pool_and_affinity_set_by_key(
+        const KeyType& key) {
+    std::string object_pool_pathname = get_pathname<KeyType>(key);
+    if(object_pool_pathname.empty()) {
+        throw derecho::derecho_exception(std::string("Key:") + key + " does not belong to any object pool.");
+    }
+
+    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
+    auto opm = this->internal_find_object_pool(object_pool_pathname, rlck);
+
+    std::string affinity_set = "";
+    if(opm.is_valid() && !opm.is_null() && !opm.deleted) {
+        affinity_set = object_pool_metadata_cache.at(opm.pathname).to_affinity_set(key);
+    }
+
+    return {opm, affinity_set};
+}
+
+template <typename... CascadeTypes>
+std::vector<std::string> ServiceClient<CascadeTypes...>::list_object_pools(bool include_deleted, bool refresh) {
+    if(refresh) {
+        this->refresh_object_pool_metadata_cache();
+    }
+
+    std::vector<std::string> ret;
+    std::shared_lock rlck(this->object_pool_metadata_cache_mutex);
+    for(auto& op : this->object_pool_metadata_cache) {
+        if(op.second.opm.deleted) {
+            if(include_deleted) {
+                ret.emplace_back(op.first + "(!)");
+            }
+        } else {
+            ret.emplace_back(op.first);
+        }
+    }
+
+    return ret;
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+bool ServiceClient<CascadeTypes...>::register_notification_handler(
+        const cascade_notification_handler_t& handler,
+        const uint32_t subgroup_index) {
+    return register_notification_handler<SubgroupType>(handler, std::string{}, subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+bool ServiceClient<CascadeTypes...>::register_notification_handler(
+        const cascade_notification_handler_t& handler,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index) {
+    if(!is_external_client()) {
+        throw derecho_exception(std::string(__PRETTY_FUNCTION__) + "Cannot register notification handler because external_group_ptr is null.");
+    }
+
+    std::unique_lock<std::mutex> type_registry_lock(this->notification_handler_registry_mutex);
+    auto& per_type_registry = notification_handler_registry.template get<SubgroupType>();
+    // Register Cascade's root handler:
+    // if subgroup_index exists in the per_type_registry, Cascade's root handler is registered already.
+    if(per_type_registry.find(subgroup_index) == per_type_registry.cend()) {
+        per_type_registry.emplace(subgroup_index, SubgroupNotificationHandler<SubgroupType>{});
+        // register to subgroup_caller
+        auto& subgroup_caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        // to do ... register it.
+        per_type_registry.at(subgroup_index).initialize(subgroup_caller);
+    }
+    auto& subgroup_handlers = per_type_registry.at(subgroup_index);
+
+    // Register the handler
+    std::lock_guard<std::mutex> subgroup_handlers_lock(*subgroup_handlers.object_pool_notification_handlers_mutex);
+    bool ret = (subgroup_handlers.object_pool_notification_handlers.find(object_pool_pathname) != subgroup_handlers.object_pool_notification_handlers.cend());
+
+    if(handler) {
+        subgroup_handlers.object_pool_notification_handlers[object_pool_pathname] = handler;
+    } else {
+        subgroup_handlers.object_pool_notification_handlers[object_pool_pathname].reset();
+    }
+    return ret;
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+bool ServiceClient<CascadeTypes...>::type_recursive_register_notification_handler(
+        uint32_t type_index,
+        const cascade_notification_handler_t& handler,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index) {
+    if(type_index == 0) {
+        return this->template register_notification_handler<FirstType>(handler, object_pool_pathname, subgroup_index);
+    } else {
+        return this->template type_recursive_register_notification_handler<SecondType, RestTypes...>(
+                type_index - 1, handler, object_pool_pathname, subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+bool ServiceClient<CascadeTypes...>::type_recursive_register_notification_handler(
+        uint32_t type_index,
+        const cascade_notification_handler_t& handler,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index) {
+    if(type_index == 0) {
+        return this->template register_notification_handler<LastType>(handler, object_pool_pathname, subgroup_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+bool ServiceClient<CascadeTypes...>::register_notification_handler(
+        const cascade_notification_handler_t& handler,
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+
+    return this->template type_recursive_register_notification_handler<CascadeTypes...>(
+            opm.subgroup_type_index, handler, object_pool_pathname, opm.subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::notify(
+        const Blob& msg,
+        const uint32_t subgroup_index,
+        const node_id_t client_id) const {
+    notify<SubgroupType>(msg, "", subgroup_index, client_id);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::notify(
+        const Blob& msg,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index,
+        const node_id_t client_id) const {
+    if(is_external_client()) {
+        throw derecho_exception(std::string(__PRETTY_FUNCTION__) + "Cannot notify an external client from an external client.");
+    }
+
+    auto& client_handle = group_ptr->template get_client_callback<SubgroupType>(subgroup_index);
+
+    // TODO: redesign to avoid memory copies.
+    CascadeNotificationMessage cascade_notification_message(object_pool_pathname, msg);
+    derecho::NotificationMessage derecho_notification_message(CASCADE_NOTIFICATION_MESSAGE_TYPE, mutils::bytes_size(cascade_notification_message));
+    mutils::to_bytes(cascade_notification_message, derecho_notification_message.body);
+
+    client_handle.template p2p_send<RPC_NAME(notify)>(client_id, derecho_notification_message);
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+void ServiceClient<CascadeTypes...>::type_recursive_notify(
+        uint32_t type_index,
+        const Blob& msg,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index,
+        const node_id_t client_id) const {
+    if(type_index == 0) {
+        this->template notify<FirstType>(msg, object_pool_pathname, subgroup_index, client_id);
+    } else {
+        this->template type_recursive_notify<SecondType, RestTypes...>(type_index - 1, msg, object_pool_pathname, subgroup_index, client_id);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+void ServiceClient<CascadeTypes...>::type_recursive_notify(
+        uint32_t type_index,
+        const Blob& msg,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index,
+        const node_id_t client_id) const {
+    if(type_index == 0) {
+        this->template notify<LastType>(msg, object_pool_pathname, subgroup_index, client_id);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+void ServiceClient<CascadeTypes...>::notify(
+        const Blob& msg,
+        const std::string& object_pool_pathname,
+        const node_id_t client_id) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    this->template type_recursive_notify<CascadeTypes...>(opm.subgroup_type_index, msg, object_pool_pathname, opm.subgroup_index, client_id);
+}
+
+#ifdef ENABLE_EVALUATION
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::dump_timestamp(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template ordered_send<RPC_NAME(ordered_dump_timestamp_log)>(filename);
+        } else {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, filename);
+            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log)>(node_id, filename);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, filename);
+        return caller.template p2p_send<RPC_NAME(dump_timestamp_log)>(node_id, filename);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+void ServiceClient<CascadeTypes...>::type_recursive_dump(
+        uint32_t type_index,
+        uint32_t subgroup_index,
+        const std::string& filename) {
+    if(type_index == 0) {
+        this->template dump_timestamp<FirstType>(subgroup_index, filename);
+    } else {
+        this->template type_recursive_dump<SecondType, RestTypes...>(type_index - 1, subgroup_index, filename);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+void ServiceClient<CascadeTypes...>::type_recursive_dump(
+        uint32_t type_index,
+        uint32_t subgroup_index,
+        const std::string& filename) {
+    if(type_index == 0) {
+        this->template dump_timestamp<LastType>(subgroup_index, filename);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+void ServiceClient<CascadeTypes...>::dump_timestamp(const std::string& filename, const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+
+    this->template type_recursive_dump<CascadeTypes...>(opm.subgroup_type_index, opm.subgroup_index, filename);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::dump_timestamp(const uint32_t subgroup_index, const std::string& filename) {
+    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
+    for(uint32_t shard_index = 0; shard_index < shards; shard_index++) {
+        auto result = this->template dump_timestamp<SubgroupType>(filename, subgroup_index, shard_index);
+        result.get();
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::dump_timestamp_workaround(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index, const node_id_t node_id) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id, filename);
+        } else {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id, filename);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        return caller.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id, filename);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<double> ServiceClient<CascadeTypes...>::perf_put(const uint32_t message_size, const uint64_t duration_sec, const uint32_t subgroup_index, const uint32_t shard_index) {
+    if(!is_external_client()) {
+        // 'perf_put' must be issued from an external client.
+        throw derecho::derecho_exception{"perf_put must be issued from an external client."};
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        return caller.template p2p_send<RPC_NAME(perf_put)>(node_id, message_size, duration_sec);
+    }
+}
+#endif  // ENABLE_EVALUATION
+
+#ifndef __WITHOUT_SERVICE_SINGLETONS__
+template <typename... CascadeTypes>
+const std::vector<std::type_index> ServiceClient<CascadeTypes...>::subgroup_type_order{typeid(CascadeTypes)...};
+
+template <typename... CascadeTypes>
+const uint32_t ServiceClient<CascadeTypes...>::invalid_subgroup_type_index = 0xffffffff;
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+uint32_t ServiceClient<CascadeTypes...>::get_subgroup_type_index() {
+    uint32_t index = 0;
+    while(index < subgroup_type_order.size()) {
+        if(std::type_index(typeid(SubgroupType)) == subgroup_type_order.at(index)) {
+            return index;
+        }
+        index++;
+    }
+    return invalid_subgroup_type_index;
+}
+
+template <typename... CascadeTypes>
+std::unique_ptr<ServiceClient<CascadeTypes...>> ServiceClient<CascadeTypes...>::service_client_singleton_ptr;
+
+template <typename... CascadeTypes>
+std::mutex ServiceClient<CascadeTypes...>::singleton_mutex;
+
+template <typename... CascadeTypes>
+void ServiceClient<CascadeTypes...>::initialize(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr) {
+    std::lock_guard<std::mutex> lock_guard(singleton_mutex);
+    if(!service_client_singleton_ptr) {
+        dbg_default_trace("initializing ServiceClient singleton as cascade member, group pointer={:p}", static_cast<void*>(_group_ptr));
+        service_client_singleton_ptr = std::unique_ptr<ServiceClient<CascadeTypes...>>(new ServiceClient<CascadeTypes...>(_group_ptr));
+    }
+}
+
+template <typename... CascadeTypes>
+ServiceClient<CascadeTypes...>& ServiceClient<CascadeTypes...>::get_service_client() {
+    if(!service_client_singleton_ptr) {
+        std::lock_guard<std::mutex> lock_guard(singleton_mutex);
+        // test again in case another thread has initialized it already.
+        if(!service_client_singleton_ptr) {
+            dbg_default_trace("initializing ServiceClient singleton as external client");
+            service_client_singleton_ptr = std::unique_ptr<ServiceClient<CascadeTypes...>>(new ServiceClient<CascadeTypes...>(nullptr));
+        }
+    }
+    return *service_client_singleton_ptr;
+}
+#endif  //__WITHOUT_SERVICE_SINGLETONS__
+
+}  // namespace cascade
+}  // namespace derecho
diff --git a/include/cascade/detail/service_impl.hpp b/include/cascade/detail/service_impl.hpp
index 7d992e54..ef407301 100644
--- a/include/cascade/detail/service_impl.hpp
+++ b/include/cascade/detail/service_impl.hpp
@@ -1,14 +1,14 @@
-#include <derecho/core/derecho_exception.hpp>
-#include <derecho/core/detail/rpc_utils.hpp>
-#include <derecho/core/notification.hpp>
-#include <vector>
-#include <map>
-#include <typeindex>
-#include <variant>
-#include <derecho/core/derecho.hpp>
 #include <cascade/config.h>
 #include <cascade/data_flow_graph.hpp>
+#include <cascade/service_client.hpp>
+#include <cascade/utils.hpp>
 #include <chrono>
+#include <derecho/core/derecho.hpp>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <variant>
+#include <vector>
 
 using namespace std::chrono_literals;
 
@@ -139,2107 +139,6 @@ void Service<CascadeTypes...>::wait() {
 }
 #endif//__WITHOUT_SERVICE_SINGLETONS__
 
-template <typename CascadeType>
-std::unique_ptr<CascadeType> client_stub_factory() {
-    return std::make_unique<CascadeType>();
-}
-
-
-#ifdef ENABLE_EVALUATION
-#define LOG_SERVICE_CLIENT_TIMESTAMP(tag,msgid) \
-    TimestampLogger::log(tag,this->get_my_id(),msgid);
-#else
-#define LOG_SERVICE_CLIENT_TIMESTAMP(tag,msgid)
-#endif
-
-
-template <typename... CascadeTypes>
-ServiceClient<CascadeTypes...>::ServiceClient(derecho::Group<CascadeMetadataService<CascadeTypes...>,CascadeTypes...>* _group_ptr):
-    external_group_ptr(nullptr),
-    group_ptr(_group_ptr) {
-    if (group_ptr == nullptr) {
-        this->external_group_ptr =
-            std::make_unique<derecho::ExternalGroupClient<CascadeMetadataService<CascadeTypes...>,CascadeTypes...>>(
-                    client_stub_factory<CascadeMetadataService<CascadeTypes...>>,
-                    client_stub_factory<CascadeTypes>...);
-    }
-}
-
-template <typename... CascadeTypes>
-bool ServiceClient<CascadeTypes...>::is_external_client() const {
-    return (group_ptr == nullptr) && (external_group_ptr != nullptr);
-}
-
-template <typename... CascadeTypes>
-node_id_t ServiceClient<CascadeTypes...>::get_my_id() const {
-    if (!is_external_client()) {
-        return group_ptr->get_my_id();
-    } else {
-        return external_group_ptr->get_my_id();
-    }
-}
-
-template <typename... CascadeTypes>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_members() const {
-    if (!is_external_client()) {
-        return group_ptr->get_members();
-    } else {
-        return external_group_ptr->get_members();
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_shard_members(uint32_t subgroup_index, uint32_t shard_index) const {
-    if (!is_external_client()) {
-        std::vector<std::vector<node_id_t>> subgroup_members = group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
-        if (subgroup_members.size() > shard_index) {
-            return subgroup_members[shard_index];
-        } else {
-            return {};
-        }
-    } else {
-        return external_group_ptr->template get_shard_members<SubgroupType>(subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::type_recursive_get_shard_members(uint32_t type_index,
-        uint32_t subgroup_index, uint32_t shard_index) const {
-    if (type_index == 0) {
-        return this->template get_shard_members<FirstType>(subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get_shard_members<SecondType, RestTypes...>(type_index-1,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::type_recursive_get_shard_members(uint32_t type_index,
-        uint32_t subgroup_index, uint32_t shard_index) const {
-    if (type_index == 0) {
-        return this->template get_shard_members<LastType>(subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
-    }
-}
-
-template <typename... CascadeTypes>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_shard_members(
-        const std::string& object_pool_pathname,uint32_t shard_index) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    return this->template type_recursive_get_shard_members<CascadeTypes...>(opm.subgroup_type_index,opm.subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::get_subgroup_members(uint32_t subgroup_index) const {
-    if (!is_external_client()) {
-        return group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
-    } else {
-        return external_group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType,typename...RestTypes>
-std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::type_recursive_get_subgroup_members(
-        uint32_t type_index, uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_subgroup_members<FirstType>(subgroup_index);
-    } else {
-        return this->template type_recursive_get_subgroup_members<SecondType,RestTypes...>(type_index-1,subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::type_recursive_get_subgroup_members(
-        uint32_t type_index, uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_subgroup_members<LastType>(subgroup_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
-    }
-}
-
-template <typename... CascadeTypes>
-std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::get_subgroup_members(
-        const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    return this->template type_recursive_get_subgroup_members<CascadeTypes...>(opm.subgroup_type_index,opm.subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-uint32_t ServiceClient<CascadeTypes...>::get_number_of_subgroups() const {
-    if (!is_external_client()) {
-        return group_ptr->template get_num_subgroups<SubgroupType>();
-    } else {
-        return external_group_ptr->template get_number_of_subgroups<SubgroupType>();
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards(uint32_t subgroup_index) const {
-    if (!is_external_client()) {
-        return group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index).size();
-    } else {
-        return external_group_ptr->template get_number_of_shards<SubgroupType>(subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-uint32_t ServiceClient<CascadeTypes...>::type_recursive_get_number_of_shards (
-        uint32_t type_index,uint32_t subgroup_index) const {
-    if (type_index == 0 ) {
-        return this->template get_number_of_shards<FirstType>(subgroup_index);
-    } else {
-        return this->template type_recursive_get_number_of_shards<SecondType,RestTypes...>(type_index-1,subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-uint32_t ServiceClient<CascadeTypes...>::type_recursive_get_number_of_shards (
-        uint32_t type_index,uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_number_of_shards<LastType>(subgroup_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
-    }
-}
-
-template <typename... CascadeTypes>
-uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards (
-        uint32_t subgroup_type_index,uint32_t subgroup_index) const {
-    return type_recursive_get_number_of_shards<CascadeTypes...>(subgroup_type_index,subgroup_index);
-}
-
-template <typename... CascadeTypes>
-uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards (
-        const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    return get_number_of_shards(opm.subgroup_type_index,opm.subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-int32_t ServiceClient<CascadeTypes...>::get_my_shard(uint32_t subgroup_index) const {
-    if (!is_external_client()) {
-        return group_ptr->template get_my_shard<SubgroupType>(subgroup_index);
-    } else {
-        return -1;
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-int32_t ServiceClient<CascadeTypes...>::type_recursive_get_my_shard (
-        uint32_t type_index,uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_my_shard<FirstType>(subgroup_index);
-    } else {
-        return this->template type_recursive_get_number_of_shards<SecondType,RestTypes...>(type_index-1,subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-int32_t ServiceClient<CascadeTypes...>::type_recursive_get_my_shard (
-        uint32_t type_index, uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_my_shard<LastType>(subgroup_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
-    }
-}
-
-template <typename... CascadeTypes>
-int32_t ServiceClient<CascadeTypes...>::get_my_shard (
-        uint32_t subgroup_type_index, uint32_t subgroup_index) const {
-    return this->template type_recursive_get_my_shard<CascadeTypes...>(subgroup_type_index,subgroup_index);
-}
-
-template <typename... CascadeTypes>
-int32_t ServiceClient<CascadeTypes...>::get_my_shard (
-        const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    return get_my_shard(opm.subgroup_type_index,opm.subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::set_member_selection_policy(uint32_t subgroup_index,uint32_t shard_index,
-        ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id) {
-    // write lock policies
-    std::unique_lock wlck(this->member_selection_policies_mutex);
-    // update map
-    this->member_selection_policies[std::make_tuple(std::type_index(typeid(SubgroupType)),subgroup_index,shard_index)] =
-            std::make_tuple(policy,user_specified_node_id);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::tuple<ShardMemberSelectionPolicy,node_id_t> ServiceClient<CascadeTypes...>::get_member_selection_policy(
-        uint32_t subgroup_index, uint32_t shard_index) const {
-    // read lock policies
-    std::shared_lock rlck(this->member_selection_policies_mutex);
-    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)),subgroup_index,shard_index);
-    // read map
-    if (member_selection_policies.find(key) != member_selection_policies.end()) {
-        return member_selection_policies.at(key);
-    } else {
-        return std::make_tuple(DEFAULT_SHARD_MEMBER_SELECTION_POLICY,INVALID_NODE_ID);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::refresh_member_cache_entry(uint32_t subgroup_index,
-                                                          uint32_t shard_index) {
-    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)),subgroup_index,shard_index);
-    auto members = get_shard_members<SubgroupType>(subgroup_index,shard_index);
-    std::shared_lock rlck(member_cache_mutex);
-    if (member_cache.find(key) == member_cache.end()) {
-        rlck.unlock();
-        std::unique_lock wlck(member_cache_mutex);
-        member_cache[key] = members;
-    } else {
-        member_cache[key].swap(members);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-std::tuple<uint32_t,uint32_t,uint32_t> ServiceClient<CascadeTypes...>::key_to_shard(
-        const KeyType& key,
-        bool check_object_location) {
-
-    auto pair = find_object_pool_and_affinity_set_by_key(key);
-
-    auto& opm = std::get<0>(pair);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to identify the object_pool from key:" + key);
-    }
-    auto& affinity_set = std::get<1>(pair);
-
-    return std::tuple<uint32_t,uint32_t,uint32_t>{opm.subgroup_type_index,opm.subgroup_index,
-        opm.key_to_shard_index(key,affinity_set,get_number_of_shards(opm.subgroup_type_index,opm.subgroup_index),check_object_location)};
-}
-
-template <typename... CascadeTypes>
-ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::ObjectPoolMetadataCacheEntry(
-        const ObjectPoolMetadata<CascadeTypes...>& _opm): opm(_opm), database(nullptr) {
-    if (opm.affinity_set_regex.size() > 0) {
-        hs_compile_error_t* compile_err;
-        if (hs_compile(opm.affinity_set_regex.c_str(), HS_FLAG_DOTALL|HS_FLAG_SOM_LEFTMOST, HS_MODE_BLOCK, NULL, &database,
-                       &compile_err) != HS_SUCCESS) {
-            hs_free_compile_error(compile_err);
-            dbg_default_error("Compilation of affinity set regex:" + opm.affinity_set_regex + " failed with message:" +
-                    compile_err->message);
-            throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) +
-                    ": compilation of affinity_set_regex:" +
-                    opm.affinity_set_regex +
-                    " failed with message:" +
-                    compile_err->message);
-        }
-    }
-}
-
-template <typename... CascadeTypes>
-ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::~ObjectPoolMetadataCacheEntry() {
-    if (this->database != nullptr) {
-        hs_free_database(database);
-        this->database = nullptr;
-    }
-}
-
-template <typename... CascadeTypes>
-inline std::string ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::to_affinity_set(
-        const std::string& key_string) {
-    if (key_string.size() > 0 && this->opm.affinity_set_regex.size() > 0) {
-        if (scratch == nullptr) {
-            if (hs_alloc_scratch(database, &scratch) != HS_SUCCESS) {
-                hs_free_database(database);
-                dbg_default_error("failed to allocate hyperscan scratch space.");
-                throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) +
-                        " failed to allocate hyperscan scratch space.");
-            }
-        }
-
-        struct hs_scan_ctxt {
-            unsigned long long from = 0;
-            unsigned long long to = 0;
-        } ctxt;
-
-        hs_scan(database, key_string.c_str(), key_string.size(), HS_FLAG_SOM_LEFTMOST, scratch,
-                [](unsigned int /*id*/, unsigned long long from,
-                   unsigned long long to, unsigned int /*flags*/,
-                   void* ctxt)->int {
-                    struct hs_scan_ctxt* p_hs_ctxt = static_cast<struct hs_scan_ctxt*>(ctxt);
-                    p_hs_ctxt->from = from;
-                    p_hs_ctxt->to = to;
-                    return 0; // do the longest match
-                },
-                &ctxt);
-        if (ctxt.to > ctxt.from) {
-            return key_string.substr(ctxt.from,(ctxt.to-ctxt.from));
-        }
-    }
-
-    return key_string;
-}
-
-template <typename... CascadeTypes>
-thread_local hs_scratch_t* ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::scratch = nullptr;
-
-template <typename... CascadeTypes>
-template <typename SubgroupType,typename KeyTypeForHashing>
-node_id_t ServiceClient<CascadeTypes...>::pick_member_by_policy(uint32_t subgroup_index,
-                                                                uint32_t shard_index,
-                                                                const KeyTypeForHashing& key_for_hashing,
-                                                                bool retry) {
-    ShardMemberSelectionPolicy policy;
-    node_id_t last_specified_node_id_or_index;
-
-    std::tie(policy,last_specified_node_id_or_index) = get_member_selection_policy<SubgroupType>(subgroup_index,shard_index);
-
-    if (policy == ShardMemberSelectionPolicy::UserSpecified) {
-        return last_specified_node_id_or_index;
-    }
-
-    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)),subgroup_index,shard_index);
-
-    if (member_cache.find(key) == member_cache.end() || retry) {
-        refresh_member_cache_entry<SubgroupType>(subgroup_index,shard_index);
-    }
-
-    std::shared_lock rlck(member_cache_mutex);
-
-    node_id_t node_id = last_specified_node_id_or_index;
-
-    switch(policy) {
-    case ShardMemberSelectionPolicy::FirstMember:
-        node_id = member_cache.at(key).front();
-        break;
-    case ShardMemberSelectionPolicy::LastMember:
-        node_id = member_cache.at(key).back();
-        break;
-    case ShardMemberSelectionPolicy::Random:
-        node_id = member_cache.at(key)[get_time()%member_cache.at(key).size()]; // use time as random source.
-        break;
-    case ShardMemberSelectionPolicy::FixedRandom:
-        if (node_id == INVALID_NODE_ID || retry) {
-            node_id = member_cache.at(key)[get_time()%member_cache.at(key).size()]; // use time as random source.
-        }
-        break;
-    case ShardMemberSelectionPolicy::RoundRobin:
-        {
-            std::shared_lock rlck(member_selection_policies_mutex);
-            node_id = static_cast<uint32_t>(node_id+1)%member_cache.at(key).size();
-            auto new_policy = std::make_tuple(ShardMemberSelectionPolicy::RoundRobin,node_id);
-            member_selection_policies[key].swap(new_policy);
-        }
-        node_id = member_cache.at(key)[node_id];
-        break;
-    case ShardMemberSelectionPolicy::KeyHashing:
-        {
-            uint64_t hash = 0;
-            if constexpr (std::is_integral_v<KeyTypeForHashing>) {
-                hash = static_cast<uint64_t>(key_for_hashing);
-            } else if constexpr (std::is_convertible_v<KeyTypeForHashing,std::string>) {
-                hash = std::hash<std::string>{}(std::string(key_for_hashing));
-            } else {
-                dbg_default_warn("Key type {} is neither integral nor string, falling back to FirstMember policy. {}:{}",
-                        typeid(KeyTypeForHashing).name(), __FILE__, __LINE__);
-            }
-            node_id = member_cache.at(key)[hash % member_cache.at(key).size()];
-        }
-        break;
-    default:
-        throw derecho::derecho_exception("Unknown member selection policy:" + std::to_string(static_cast<unsigned int>(policy)) );
-    }
-
-    return node_id;
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::put(
-        const typename SubgroupType::ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_PUT_START,
-            (std::is_base_of<IHasMessageID,typename SubgroupType::ObjectType>::value?value.get_message_id():0));
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            // ordered put as a shard member
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template ordered_send<RPC_NAME(ordered_put)>(value,as_trigger);
-        } else {
-            // p2p put
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-            try {
-                // as a subgroup member
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                return subgroup_handle.template p2p_send<RPC_NAME(put)>(node_id,value,as_trigger);
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                // as an external caller
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                return subgroup_handle.template p2p_send<RPC_NAME(put)>(node_id,value,as_trigger);
-            }
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-        return caller.template p2p_send<RPC_NAME(put)>(node_id,value,as_trigger);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_put(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    if (type_index == 0) {
-        return this->template put<FirstType>(value,subgroup_index,shard_index,as_trigger);
-    } else {
-        return this->template type_recursive_put<ObjectType, SecondType, RestTypes...>(type_index-1,value,subgroup_index,shard_index,as_trigger);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename LastType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_put(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    if (type_index == 0) {
-        return this->template put<LastType>(value,subgroup_index,shard_index,as_trigger);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::put(
-        const ObjectType& value, bool as_trigger) {
-
-    // STEP 1 - get key
-    if constexpr (!std::is_base_of_v<ICascadeObject<std::string,ObjectType>,ObjectType>) {
-        throw derecho::derecho_exception(std::string("ServiceClient<>::put() only support object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(value.get_key_ref());
-
-    // STEP 3 - call recursive put
-    return this->template type_recursive_put<ObjectType,CascadeTypes...>(subgroup_type_index,value,subgroup_index,shard_index,as_trigger);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::put_and_forget(
-        const typename SubgroupType::ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_PUT_AND_FORGET_START,
-            (std::is_base_of<IHasMessageID,typename SubgroupType::ObjectType>::value?value.get_message_id():0));
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            // do ordered put as a shard member (Replicated).
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            subgroup_handle.template ordered_send<RPC_NAME(ordered_put_and_forget)>(value,as_trigger);
-        } else {
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-            // do p2p put
-            try{
-                // as a subgroup member
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                subgroup_handle.template p2p_send<RPC_NAME(put_and_forget)>(node_id,value,as_trigger);
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                // as an external caller
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                subgroup_handle.template p2p_send<RPC_NAME(put_and_forget)>(node_id,value,as_trigger);
-            }
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-        caller.template p2p_send<RPC_NAME(put_and_forget)>(node_id,value,as_trigger);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-void ServiceClient<CascadeTypes...>::type_recursive_put_and_forget(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    if (type_index == 0) {
-        put_and_forget<FirstType>(value,subgroup_index,shard_index,as_trigger);
-    } else {
-        type_recursive_put_and_forget<ObjectType,SecondType,RestTypes...>(type_index-1,value,subgroup_index,shard_index,as_trigger);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename LastType>
-void ServiceClient<CascadeTypes...>::type_recursive_put_and_forget(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    if (type_index == 0) {
-        put_and_forget<LastType>(value,subgroup_index,shard_index,as_trigger);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType>
-void ServiceClient<CascadeTypes...>::put_and_forget(const ObjectType& value,bool as_trigger) {
-    // STEP 1 - get key
-    if constexpr (!std::is_base_of_v<ICascadeObject<std::string,ObjectType>,ObjectType>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(value.get_key_ref());
-
-    // STEP 3 - call recursive put_and_forget
-    this->template type_recursive_put_and_forget<ObjectType,CascadeTypes...>(subgroup_type_index,value,subgroup_index,shard_index,as_trigger);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::trigger_put(
-        const typename SubgroupType::ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_TRIGGER_PUT_START,
-            (std::is_base_of<IHasMessageID,typename SubgroupType::ObjectType>::value?value.get_message_id():0));
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index){
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-            dbg_default_trace("trigger_put to node {}",node_id);
-            return subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(node_id,value);
-        } else {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-            dbg_default_trace("trigger_put to node {}",node_id);
-            return subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(node_id,value);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-        dbg_default_trace("trigger_put to node {}",node_id);
-        return caller.template p2p_send<RPC_NAME(trigger_put)>(node_id,value);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::type_recursive_trigger_put(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return trigger_put<FirstType>(value,subgroup_index,shard_index);
-    } else {
-        return type_recursive_trigger_put<ObjectType,SecondType,RestTypes...>(type_index-1,value,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename LastType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::type_recursive_trigger_put(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return trigger_put<LastType>(value,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::trigger_put(
-        const ObjectType& value) {
-    // STEP 1 - get key
-    if constexpr (!std::is_base_of_v<ICascadeObject<std::string,ObjectType>,ObjectType>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(value.get_key_ref());
-
-    // STEP 3 - call recursive trigger_put
-    return this->template type_recursive_trigger_put<ObjectType,CascadeTypes...>(subgroup_type_index,value,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::collective_trigger_put(
-        const typename SubgroupType::ObjectType& value,
-        uint32_t subgroup_index,
-        std::unordered_map<node_id_t,std::unique_ptr<derecho::rpc::QueryResults<void>>>& nodes_and_futures) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_COLLECTIVE_TRIGGER_PUT_START,
-            (std::is_base_of<IHasMessageID,typename SubgroupType::ObjectType>::value?value.get_message_id():0));
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (group_ptr->template get_my_shard<SubgroupType>(subgroup_index) != -1) {
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            for (auto& kv: nodes_and_futures) {
-                nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
-                        std::move(subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(kv.first,value)));
-            }
-        } else {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            for (auto& kv: nodes_and_futures) {
-                nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
-                        std::move(subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(kv.first,value)));
-            }
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        for (auto& kv: nodes_and_futures) {
-            nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
-                    std::move(caller.template p2p_send<RPC_NAME(trigger_put)>(kv.first,value)));
-        }
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove(
-        const typename SubgroupType::KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_REMOVE_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            // do ordered remove as a member (Replicated).
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template ordered_send<RPC_NAME(ordered_remove)>(key);
-        } else {
-            // do p2p remove
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-            try {
-                // as a subgroup member
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                return subgroup_handle.template p2p_send<RPC_NAME(remove)>(node_id,key);
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                // as an external caller
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                return subgroup_handle.template p2p_send<RPC_NAME(remove)>(node_id,key);
-            }
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(remove)>(node_id,key);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_remove(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template remove<FirstType>(key,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_remove<KeyType,SecondType,RestTypes...>(type_index-1,key,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_remove(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template remove<LastType>(key,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove(
-        const KeyType& key) {
-    // STEP 1 - get key
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive remove
-    return this->template type_recursive_remove<KeyType,CascadeTypes...>(subgroup_type_index,key,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::get(
-        const typename SubgroupType::KeyType& key,
-        const persistent::version_t& version,
-        bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_GET_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p get as a subgroup member
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-                // local get
-                auto obj = subgroup_handle.get_ref().get(key,version,stable);
-                auto pending_results = std::make_shared<PendingResults<const typename SubgroupType::ObjectType>>();
-                pending_results->fulfill_map({node_id});
-                pending_results->set_value(node_id,obj);
-                auto query_results = pending_results->get_future();
-                return std::move(*query_results);
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(get)>(node_id,key,version,stable,false);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(get)>(node_id,key,version,stable,false);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(get)>(node_id,key,version,stable,false);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::multi_get(
-        const typename SubgroupType::KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_GET_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p multi_get as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_get)>(node_id,key);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p multi_get as an external caller.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_get)>(node_id,key);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(multi_get)>(node_id,key);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_get(
-        uint32_t type_index,
-        const KeyType& key,
-        const persistent::version_t& version,
-        bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get<FirstType>(key,version,stable,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get<KeyType,SecondType,RestTypes...>(type_index-1,key,version,stable,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_get(
-        uint32_t type_index,
-        const KeyType& key,
-        const persistent::version_t& version,
-        bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get<LastType>(key,version,stable,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-auto ServiceClient<CascadeTypes...>::get(
-        // const std::decay_t<typename std::result_of_t<decltype(&ObjectType::get_key_ref())>>& key,
-        const KeyType& key,
-        const persistent::version_t& version,
-        bool stable) {
-    // STEP 1 - get key
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_get<KeyType,CascadeTypes...>(subgroup_type_index,key,version,stable,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_multi_get(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template multi_get<FirstType>(key,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_multi_get<KeyType,SecondType,RestTypes...>(type_index-1,key,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_multi_get(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template multi_get<LastType>(key,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-auto ServiceClient<CascadeTypes...>::multi_get(
-        const KeyType& key) {
-    // STEP 1 - get key
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_multi_get<KeyType,CascadeTypes...>(subgroup_type_index,key,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::get_by_time(
-        const typename SubgroupType::KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p get_by_time
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                // as a shard member.
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(get_by_time)>(node_id,key,ts_us,stable);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p get_by_time as an external caller
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(get_by_time)>(node_id,key,ts_us,stable);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>();
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(get_by_time)>(node_id,key,ts_us,stable);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_get_by_time(
-        uint32_t type_index,
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_by_time<FirstType>(key,ts_us,stable,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get_by_time<KeyType,SecondType,RestTypes...>(type_index-1,key,ts_us,stable,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_get_by_time(
-        uint32_t type_index,
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_by_time<LastType>(key,ts_us,stable,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-auto ServiceClient<CascadeTypes...>::get_by_time(
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable) {
-    // STEP 1 - get key
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get_by_time
-    return this->template type_recursive_get_by_time<KeyType,CascadeTypes...>(subgroup_type_index,key,ts_us,stable,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size(
-        const typename SubgroupType::KeyType& key,
-        const persistent::version_t& version,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_GET_SIZE_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p get_size as a subgroup_member
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                // as a shard member.
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(get_size)>(node_id,key,version,stable,false);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p get_size as an external caller
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(get_size)>(node_id,key,version,stable,false);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(get_size)>(node_id,key,version,stable,false);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size(
-        uint32_t type_index,
-        const KeyType& key,
-        const persistent::version_t& version,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_size<FirstType>(key,version,stable,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get_size<KeyType,SecondType,RestTypes...>(type_index-1,key,version,stable,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size(
-        uint32_t type_index,
-        const KeyType& key,
-        const persistent::version_t& version,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_size<LastType>(key,version,stable,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size(
-        const KeyType& key,
-        const persistent::version_t& version,
-        const bool stable) {
-    // STEP 1 - verify the keys
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_get_size<KeyType,CascadeTypes...>(subgroup_type_index,key,version,stable,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::multi_get_size(
-        const typename SubgroupType::KeyType& key,
-        uint32_t subgroup_index, uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_GET_SIZE_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p multi_get_size as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_get_size)>(node_id,key);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p multi_get_size as an external caller.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_get_size)>(node_id,key);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(multi_get_size)>(node_id,key);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_multi_get_size(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template multi_get_size<FirstType>(key,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_multi_get_size<KeyType,SecondType,RestTypes...>(type_index-1,key,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_multi_get_size(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template multi_get_size<LastType>(key,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::multi_get_size(const KeyType& key) {
-    // STEP 1 - verify the keys
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_multi_get_size<KeyType,CascadeTypes...>(subgroup_type_index,key,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size_by_time(
-        const typename SubgroupType::KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p get_size_by_time as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(get_size_by_time)>(node_id,key,ts_us,stable);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p get_size_by_time as an external caller.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(get_size_by_time)>(node_id,key,ts_us,stable);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(get_size_by_time)>(node_id,key,ts_us,stable);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size_by_time(
-        uint32_t type_index,
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_size_by_time<FirstType>(key,ts_us,stable,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get_size_by_time<KeyType,SecondType,RestTypes...>(type_index-1,key,ts_us,stable,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size_by_time(
-        uint32_t type_index,
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_size_by_time<LastType>(key,ts_us,stable,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size_by_time(
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable) {
-    // STEP 1 - verify the keys
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_get_size_by_time<KeyType,CascadeTypes...>(subgroup_type_index,key,ts_us,stable,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::list_keys(
-        const persistent::version_t& version,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_LIST_KEYS_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        try {
-            // do p2p list_keys as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id,"",version,stable);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p list_keys as an external client.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id,"",version,stable);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        return caller.template p2p_send<RPC_NAME(list_keys)>(node_id,"",version,stable);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_list_keys(
-        uint32_t type_index,
-        const persistent::version_t& version,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __list_keys<FirstType>(version,stable,object_pool_pathname);
-    } else {
-        return this->template type_recursive_list_keys<SecondType, RestTypes...>(type_index-1, version, stable, object_pool_pathname);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_list_keys(
-        uint32_t type_index,
-        const persistent::version_t& version,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __list_keys<LastType>(version,stable,object_pool_pathname);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__list_keys(
-        const persistent::version_t& version,
-        const bool stable,
-        const std::string& object_pool_pathname){
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    uint32_t subgroup_index = opm.subgroup_index;
-    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
-    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
-    for (uint32_t shard_index = 0; shard_index < shards; shard_index ++){
-        if (!is_external_client()) {
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-            try {
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                    node_id = group_ptr->get_my_id();
-                }
-                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id,object_pool_pathname,version,stable);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                auto shard_keys= subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id,object_pool_pathname,version,stable);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            }
-        } else {
-            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-            auto shard_keys = caller.template p2p_send<RPC_NAME(list_keys)>(node_id,object_pool_pathname,version,stable);
-            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-        }
-    }
-    return result;
-}
-
-template <typename... CascadeTypes>
-auto ServiceClient<CascadeTypes...>::list_keys(
-        const persistent::version_t& version,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    volatile uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(object_pool_pathname+"/_");
-    return this->template type_recursive_list_keys<CascadeTypes...>(subgroup_type_index,version,stable,object_pool_pathname);
-}
-
-template <typename ReturnType>
-inline ReturnType wait_for_future(derecho::rpc::QueryResults<ReturnType>& result){
-    // iterate through ReplyMap
-    for (auto& reply_future: result.get()) {
-        ReturnType reply = static_cast<ReturnType>(reply_future.second.get());
-        return reply;
-    }
-    return ReturnType();
-}
-
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-std::vector<KeyType> ServiceClient<CascadeTypes...>::wait_list_keys(
-        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<KeyType>>>>& future){
-    std::vector<KeyType> result;
-    // iterate over each shard's Query result
-    for(auto& query_result: future){
-        std::vector<KeyType> reply = wait_for_future<std::vector<KeyType>>(*(query_result.get()));
-        std::move(reply.begin(), reply.end(), std::back_inserter(result));
-    }
-    return result;
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::multi_list_keys(
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_LIST_KEYS_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        try {
-            // do p2p multi_list_keys as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,"");
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p multi_list_keys as an external client.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,"");
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        return caller.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,"");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_multi_list_keys (
-        uint32_t type_index,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __multi_list_keys<FirstType>(object_pool_pathname);
-    } else {
-        return this->template type_recursive_multi_list_keys<SecondType, RestTypes...>(type_index-1,object_pool_pathname);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_multi_list_keys (
-        uint32_t type_index,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __multi_list_keys<LastType>(object_pool_pathname);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__multi_list_keys(const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    uint32_t subgroup_index = opm.subgroup_index;
-    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
-    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
-    for (uint32_t shard_index = 0; shard_index < shards; shard_index ++){
-        if (!is_external_client()) {
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,object_pool_pathname);
-            try {
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                    node_id = group_ptr->get_my_id();
-                }
-                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,object_pool_pathname);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                auto shard_keys= subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,object_pool_pathname);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            }
-        } else {
-            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,object_pool_pathname);
-            auto shard_keys = caller.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,object_pool_pathname);
-            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-        }
-    }
-    return result;
-}
-
-template <typename... CascadeTypes>
-auto ServiceClient<CascadeTypes...>::multi_list_keys(const std::string& object_pool_pathname) {
-    volatile uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(object_pool_pathname+"/_");
-    return this->template type_recursive_multi_list_keys<CascadeTypes...>(subgroup_type_index,object_pool_pathname);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::list_keys_by_time(
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        try {
-            // do p2p list_keys_by_time as a subgroup member
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,"",ts_us,stable);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p list_keys_by_time as an external client.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,"",ts_us,stable);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        return caller.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,"",ts_us,stable);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_list_keys_by_time(
-        uint32_t type_index,
-        const uint64_t& ts_us,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __list_keys_by_time<FirstType>(ts_us,stable,object_pool_pathname);
-    } else {
-        return this->template type_recursive_list_keys_by_time<SecondType, RestTypes...>(type_index-1,ts_us,stable,object_pool_pathname);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_list_keys_by_time(
-        uint32_t type_index,
-        const uint64_t& ts_us,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __list_keys_by_time<LastType>(ts_us,stable,object_pool_pathname);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__list_keys_by_time(
-        const uint64_t& ts_us,
-        const bool stable,
-        const std::string& object_pool_pathname){
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    uint32_t subgroup_index = opm.subgroup_index;
-    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
-    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
-    for (uint32_t shard_index = 0; shard_index < shards; shard_index ++){
-        if (!is_external_client()) {
-            std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,object_pool_pathname);
-            try {
-                // do p2p list_keys_by_time as a subgroup member.
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                    node_id = group_ptr->get_my_id();
-                }
-                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,object_pool_pathname,ts_us,stable);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                // do p2p list_keys_by_time as an external client.
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,object_pool_pathname,ts_us,stable);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            }
-        } else {
-            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-            // call as an external client (ExternalClientCaller).
-            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,object_pool_pathname);
-            auto shard_keys = caller.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,object_pool_pathname,ts_us,stable);
-            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-        }
-    }
-    return result;
-}
-
-template <typename... CascadeTypes>
-auto ServiceClient<CascadeTypes...>::list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname) {
-    volatile uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(object_pool_pathname+"/_");
-    return this->template type_recursive_list_keys_by_time<CascadeTypes...>(subgroup_type_index,ts_us,stable,object_pool_pathname);
-}
-
-template <typename... CascadeTypes>
-void ServiceClient<CascadeTypes...>::refresh_object_pool_metadata_cache() {
-    std::unordered_map<std::string,ObjectPoolMetadataCacheEntry> refreshed_metadata;
-    uint32_t num_shards = this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
-    for(uint32_t shard=0;shard<num_shards;shard++) {
-        auto results = this->template multi_list_keys<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX,shard);
-        for (auto& reply : results.get()) { // only once
-            for(auto& key: reply.second.get()) { // iterate over keys
-                // we only read the stable version.
-                auto opm_result = this->template get<CascadeMetadataService<CascadeTypes...>>(key,CURRENT_VERSION,false,METADATA_SERVICE_SUBGROUP_INDEX,shard);
-                for (auto& opm_reply:opm_result.get()) { // only once
-                    refreshed_metadata.emplace(key,opm_reply.second.get());
-                    break;
-                }
-            }
-            break;
-        }
-    }
-
-    std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
-    this->object_pool_metadata_cache = std::move(refreshed_metadata);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::create_object_pool(
-        const std::string& pathname, const uint32_t subgroup_index,
-        const sharding_policy_t sharding_policy, const std::unordered_map<std::string,uint32_t>& object_locations,
-        const std::string& affinity_set_regex) {
-    uint32_t subgroup_type_index = ObjectPoolMetadata<CascadeTypes...>::template get_subgroup_type_index<SubgroupType>();
-    if (subgroup_type_index == ObjectPoolMetadata<CascadeTypes...>::invalid_subgroup_type_index) {
-        dbg_default_crit("Create object pool failed because of invalid SubgroupType:{}", typeid(SubgroupType).name());
-        throw derecho::derecho_exception(std::string("Create object pool failed because SubgroupType is invalid:")+typeid(SubgroupType).name());
-    }
-    ObjectPoolMetadata<CascadeTypes...> opm(pathname,subgroup_type_index,subgroup_index,sharding_policy,object_locations,affinity_set_regex,false);
-    // clear local cache entry.
-    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
-    if (object_pool_metadata_cache.find(pathname)==object_pool_metadata_cache.end()) {
-        rlck.unlock();
-    } else {
-        rlck.unlock();
-        std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
-        object_pool_metadata_cache.erase(pathname);
-    }
-    // determine the shard index by hashing
-    uint32_t metadata_service_shard_index = std::hash<std::string>{}(pathname) % this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
-
-    return this->template put<CascadeMetadataService<CascadeTypes...>>(opm,METADATA_SERVICE_SUBGROUP_INDEX,metadata_service_shard_index);
-}
-
-template <typename... CascadeTypes>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove_object_pool(const std::string& pathname) {
-    // determine the shard index by hashing
-    uint32_t metadata_service_shard_index = std::hash<std::string>{}(pathname) % this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
-
-
-    // check if this object pool exist in metadata service.
-    auto opm = find_object_pool(pathname);
-    // remove it from local cache.
-    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
-    if (object_pool_metadata_cache.find(pathname) == object_pool_metadata_cache.end()) {
-        // no entry in cache
-        rlck.unlock();
-    } else {
-        // remove from cache
-        rlck.unlock();
-        std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
-        object_pool_metadata_cache.erase(pathname);
-        wlck.unlock();
-    }
-    if (opm.is_valid() && !opm.is_null()) {
-        if (opm.deleted) {
-            throw derecho::derecho_exception(std::string("object pool:")+pathname+" has been deleted already.");
-        }
-        opm.deleted = true;
-        opm.set_previous_version(CURRENT_VERSION,opm.version); // only check previous_version_by_key
-        return this->template put<CascadeMetadataService<CascadeTypes...>>(opm,METADATA_SERVICE_SUBGROUP_INDEX,metadata_service_shard_index);
-    }
-
-    // we didn't find the object pool, but we do the normal 'remove', which has no effect but return a version.
-    dbg_default_warn("deleteing a non-existing objectpool:{}.", pathname);
-    return this->template remove<CascadeMetadataService<CascadeTypes...>>(pathname,METADATA_SERVICE_SUBGROUP_INDEX,metadata_service_shard_index);
-}
-
-template <typename... CascadeTypes>
-ObjectPoolMetadata<CascadeTypes...> ServiceClient<CascadeTypes...>::internal_find_object_pool(
-        const std::string& pathname,
-        std::shared_lock<std::shared_mutex>& rlck) {
-    auto components = str_tokenizer(pathname);
-    std::string prefix;
-    for (const auto& comp: components) {
-        prefix = prefix + PATH_SEPARATOR + comp;
-        if (object_pool_metadata_cache.find(prefix) != object_pool_metadata_cache.end()) {
-            return object_pool_metadata_cache.at(prefix).opm;
-        }
-    }
-    rlck.unlock();
-
-    // refresh and try again.
-    refresh_object_pool_metadata_cache();
-    prefix = "";
-    rlck.lock();
-    for (const auto& comp: components) {
-        prefix = prefix + PATH_SEPARATOR + comp;
-        if (object_pool_metadata_cache.find(prefix) != object_pool_metadata_cache.end()) {
-            return object_pool_metadata_cache.at(prefix).opm;
-        }
-    }
-    return ObjectPoolMetadata<CascadeTypes...>::IV;
-}
-
-template <typename... CascadeTypes>
-ObjectPoolMetadata<CascadeTypes...> ServiceClient<CascadeTypes...>::find_object_pool(const std::string& pathname) {
-    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
-    return this->internal_find_object_pool(pathname,rlck);
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-std::pair<ObjectPoolMetadata<CascadeTypes...>,std::string> ServiceClient<CascadeTypes...>::find_object_pool_and_affinity_set_by_key(
-        const KeyType& key) {
-    std::string object_pool_pathname = get_pathname<KeyType>(key);
-    if (object_pool_pathname.empty()) {
-        throw derecho::derecho_exception(std::string("Key:") + key + " does not belong to any object pool.");
-    }
-
-    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
-    auto opm = this->internal_find_object_pool(object_pool_pathname,rlck);
-
-    std::string affinity_set = "";
-    if (opm.is_valid() && !opm.is_null() && !opm.deleted) {
-        affinity_set = object_pool_metadata_cache.at(opm.pathname).to_affinity_set(key);
-    }
-
-    return {opm,affinity_set};
-}
-
-template <typename... CascadeTypes>
-std::vector<std::string> ServiceClient<CascadeTypes...>::list_object_pools(bool include_deleted, bool refresh) {
-    if (refresh) {
-        this->refresh_object_pool_metadata_cache();
-    }
-
-    std::vector<std::string> ret;
-    std::shared_lock rlck(this->object_pool_metadata_cache_mutex);
-    for (auto& op:this->object_pool_metadata_cache) {
-        if (op.second.opm.deleted) {
-            if (include_deleted) {
-                ret.emplace_back(op.first+"(!)");
-            }
-        } else {
-            ret.emplace_back(op.first);
-        }
-    }
-
-    return ret;
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-bool ServiceClient<CascadeTypes...>::register_notification_handler(
-        const cascade_notification_handler_t& handler,
-        const uint32_t subgroup_index) {
-    return register_notification_handler<SubgroupType>(handler,std::string{},subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-bool ServiceClient<CascadeTypes...>::register_notification_handler(
-        const cascade_notification_handler_t& handler,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index) {
-    if (!is_external_client()) {
-        throw derecho_exception(std::string(__PRETTY_FUNCTION__) +
-            "Cannot register notification handler because external_group_ptr is null.");
-    }
-
-    std::unique_lock<std::mutex> type_registry_lock(this->notification_handler_registry_mutex);
-    auto& per_type_registry = notification_handler_registry.template get<SubgroupType>();
-    // Register Cascade's root handler:
-    // if subgroup_index exists in the per_type_registry, Cascade's root handler is registered already.
-    if (per_type_registry.find(subgroup_index) == per_type_registry.cend()) {
-        per_type_registry.emplace(subgroup_index,SubgroupNotificationHandler<SubgroupType>{});
-        // register to subgroup_caller
-        auto& subgroup_caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        // to do ... register it.
-        per_type_registry.at(subgroup_index).initialize(subgroup_caller);
-    }
-    auto& subgroup_handlers = per_type_registry.at(subgroup_index);
-
-    // Register the handler
-    std::lock_guard<std::mutex> subgroup_handlers_lock(*subgroup_handlers.object_pool_notification_handlers_mutex);
-    bool ret = (subgroup_handlers.object_pool_notification_handlers.find(object_pool_pathname) !=
-                subgroup_handlers.object_pool_notification_handlers.cend());
-
-    if (handler) {
-        subgroup_handlers.object_pool_notification_handlers[object_pool_pathname] = handler;
-    } else {
-        subgroup_handlers.object_pool_notification_handlers[object_pool_pathname].reset();
-    }
-    return ret;
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-bool ServiceClient<CascadeTypes...>::type_recursive_register_notification_handler(
-        uint32_t type_index,
-        const cascade_notification_handler_t& handler,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index) {
-    if (type_index == 0) {
-        return this->template register_notification_handler<FirstType>(handler,object_pool_pathname,subgroup_index);
-    } else {
-        return this->template type_recursive_register_notification_handler<SecondType,RestTypes...>(
-            type_index-1,handler,object_pool_pathname,subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-bool ServiceClient<CascadeTypes...>::type_recursive_register_notification_handler(
-        uint32_t type_index,
-        const cascade_notification_handler_t& handler,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index) {
-    if (type_index == 0) {
-        return this->template register_notification_handler<LastType>(handler,object_pool_pathname,subgroup_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-bool ServiceClient<CascadeTypes...>::register_notification_handler(
-        const cascade_notification_handler_t& handler,
-        const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-
-    return this->template type_recursive_register_notification_handler<CascadeTypes...>(
-        opm.subgroup_type_index,handler,object_pool_pathname,opm.subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::notify(
-        const Blob& msg,
-        const uint32_t subgroup_index,
-        const node_id_t client_id) const {
-    notify<SubgroupType>(msg,"",subgroup_index,client_id);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::notify(
-        const Blob& msg,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index,
-        const node_id_t client_id) const {
-    if (is_external_client()) {
-        throw derecho_exception(std::string(__PRETTY_FUNCTION__) +
-                "Cannot notify an external client from an external client.");
-    }
-
-    auto& client_handle = group_ptr->template get_client_callback<SubgroupType>(subgroup_index);
-
-    //TODO: redesign to avoid memory copies.
-    CascadeNotificationMessage cascade_notification_message(object_pool_pathname,msg);
-    derecho::NotificationMessage derecho_notification_message(CASCADE_NOTIFICATION_MESSAGE_TYPE, mutils::bytes_size(cascade_notification_message));
-    mutils::to_bytes(cascade_notification_message,derecho_notification_message.body);
-
-    client_handle.template p2p_send<RPC_NAME(notify)>(client_id,derecho_notification_message);
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-void ServiceClient<CascadeTypes...>::type_recursive_notify(
-        uint32_t type_index,
-        const Blob& msg,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index,
-        const node_id_t client_id) const {
-    if (type_index == 0) {
-        this->template notify<FirstType>(msg,object_pool_pathname,subgroup_index,client_id);
-    } else {
-        this->template type_recursive_notify<SecondType,RestTypes...>(type_index-1,msg,object_pool_pathname,subgroup_index,client_id);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-void ServiceClient<CascadeTypes...>::type_recursive_notify(
-        uint32_t type_index,
-        const Blob& msg,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index,
-        const node_id_t client_id) const {
-    if (type_index == 0) {
-        this->template notify<LastType>(msg,object_pool_pathname,subgroup_index,client_id);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-void ServiceClient<CascadeTypes...>::notify(
-        const Blob& msg,
-        const std::string& object_pool_pathname,
-        const node_id_t client_id) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    this->template type_recursive_notify<CascadeTypes...>(opm.subgroup_type_index,msg,object_pool_pathname,opm.subgroup_index,client_id);
-}
-
-#ifdef ENABLE_EVALUATION
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::dump_timestamp(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index) {
-
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template ordered_send<RPC_NAME(ordered_dump_timestamp_log)>(filename);
-        } else {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,filename);
-            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log)>(node_id,filename);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,filename);
-        return caller.template p2p_send<RPC_NAME(dump_timestamp_log)>(node_id,filename);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType, typename SecondType, typename... RestTypes>
-void ServiceClient<CascadeTypes...>::type_recursive_dump(
-        uint32_t type_index,
-        uint32_t subgroup_index,
-        const std::string& filename) {
-    if (type_index == 0) {
-        this->template dump_timestamp<FirstType>(subgroup_index,filename);
-    } else {
-        this->template type_recursive_dump<SecondType, RestTypes...>(type_index-1,subgroup_index,filename);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-void ServiceClient<CascadeTypes...>::type_recursive_dump(
-        uint32_t type_index,
-        uint32_t subgroup_index,
-        const std::string& filename) {
-    if (type_index == 0) {
-        this->template dump_timestamp<LastType>(subgroup_index,filename);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-void ServiceClient<CascadeTypes...>::dump_timestamp(const std::string& filename, const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-
-    this->template type_recursive_dump<CascadeTypes...>(opm.subgroup_type_index,opm.subgroup_index,filename);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::dump_timestamp(const uint32_t subgroup_index,const std::string& filename){
-    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
-    for (uint32_t shard_index = 0; shard_index < shards; shard_index ++){
-        auto result = this->template dump_timestamp<SubgroupType>(filename,subgroup_index,shard_index);
-        result.get();
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::dump_timestamp_workaround(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index, const node_id_t node_id) {
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id, filename);
-        } else {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id,filename);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        return caller.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id,filename);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<double> ServiceClient<CascadeTypes...>::perf_put(const uint32_t message_size, const uint64_t duration_sec, const uint32_t subgroup_index, const uint32_t shard_index) {
-    if (!is_external_client()) {
-        // 'perf_put' must be issued from an external client.
-        throw derecho::derecho_exception{"perf_put must be issued from an external client."};
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        return caller.template p2p_send<RPC_NAME(perf_put)>(node_id,message_size,duration_sec);
-    }
-}
-#endif//ENABLE_EVALUATION
-
-#ifndef __WITHOUT_SERVICE_SINGLETONS__
-template <typename... CascadeTypes>
-const std::vector<std::type_index> ServiceClient<CascadeTypes...>::subgroup_type_order{typeid(CascadeTypes)...};
-
-template <typename... CascadeTypes>
-const uint32_t ServiceClient<CascadeTypes...>::invalid_subgroup_type_index = 0xffffffff;
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-uint32_t ServiceClient<CascadeTypes...>::get_subgroup_type_index() {
-    uint32_t index = 0;
-    while (index < subgroup_type_order.size()) {
-        if ( std::type_index(typeid(SubgroupType)) == subgroup_type_order.at(index)) {
-            return index;
-        }
-        index ++;
-    }
-    return invalid_subgroup_type_index;
-}
-
-template <typename... CascadeTypes>
-std::unique_ptr<ServiceClient<CascadeTypes...>> ServiceClient<CascadeTypes...>::service_client_singleton_ptr;
-
-template <typename... CascadeTypes>
-std::mutex ServiceClient<CascadeTypes...>::singleton_mutex;
-
-template <typename... CascadeTypes>
-void ServiceClient<CascadeTypes...>::initialize(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr) {
-    std::lock_guard<std::mutex> lock_guard(singleton_mutex);
-    if (!service_client_singleton_ptr) {
-        dbg_default_trace("initializing ServiceClient singleton as cascade member, group pointer={:p}",static_cast<void*>(_group_ptr));
-        service_client_singleton_ptr = std::unique_ptr<ServiceClient<CascadeTypes...>>(new ServiceClient<CascadeTypes...>(_group_ptr));
-    }
-}
-
-template <typename... CascadeTypes>
-ServiceClient<CascadeTypes...>& ServiceClient<CascadeTypes...>::get_service_client() {
-    if (!service_client_singleton_ptr) {
-        std::lock_guard<std::mutex> lock_guard(singleton_mutex);
-        // test again in case another thread has initialized it already.
-        if (!service_client_singleton_ptr) {
-            dbg_default_trace("initializing ServiceClient singleton as external client");
-            service_client_singleton_ptr = std::unique_ptr<ServiceClient<CascadeTypes...>>(new ServiceClient<CascadeTypes...>(nullptr));
-        }
-    }
-    return *service_client_singleton_ptr;
-}
-#endif//__WITHOUT_SERVICE_SINGLETONS__
 
 template <typename... CascadeTypes>
 ExecutionEngine<CascadeTypes...>::ExecutionEngine() {
diff --git a/include/cascade/service.hpp b/include/cascade/service.hpp
index 891d26f0..bd2d8fd1 100644
--- a/include/cascade/service.hpp
+++ b/include/cascade/service.hpp
@@ -6,40 +6,38 @@
  * @brief   This file includes the cascade service templates
  *
  * Type neutral templates components go here. Since the server binary and client library has to be type aware (because
- * they are pre-compiled), we separate the api and implementation of them in type-awared header files as follows:
+ * they are pre-compiled), we separate the api and implementation of them in type-aware header files as follows:
  * - service_types.hpp contains the predefined types for derecho Subgroups, which are specialized from
- *   derecho::cascade::VolatileCascadeStore/PersistentCascadeStore templates.
- * - service_client_api.hpp contains the client API definition.
- * - service_server_api.hpp contains the server API definition. Huh, Server API??? YES! because the application need to
- *   specify their 'onData()' behaviours by implementing the APIs in service_server_api.hpp as a shared library. The
- *   server will load them on restart.
+ *   derecho::cascade::VolatileCascadeStore/PersistentCascadeStore templates, and the server API definition,
+ *   which is a specialization of the Service template.
+ * - service_client_api.hpp contains the client API definition, which is a specialization of ServiceClient.
  */
 
+#include "cascade.hpp"
+#include "data_flow_graph.hpp"
+#include "detail/prefix_registry.hpp"
+#include "object_pool_metadata.hpp"
+#include "user_defined_logic_manager.hpp"
+#include "utils.hpp"
+
+#include <atomic>
+#include <condition_variable>
 #include <cstdint>
+#include <derecho/conf/conf.hpp>
 #include <derecho/core/notification.hpp>
 #include <derecho/mutils-serialization/SerializationSupport.hpp>
 #include <derecho/persistent/PersistentInterface.hpp>
+#include <functional>
+#include <iostream>
+#include <map>
 #include <memory>
 #include <mutex>
-#include <shared_mutex>
-#include <typeinfo>
-#include <tuple>
-#include <derecho/utils/time.h>
-#include <list>
-#include <condition_variable>
 #include <thread>
-#include <functional>
-#include <iostream>
+#include <tuple>
+#include <typeinfo>
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
-#include <derecho/conf/conf.hpp>
-#include "cascade.hpp"
-#include "utils.hpp"
-#include "object_pool_metadata.hpp"
-#include "user_defined_logic_manager.hpp"
-#include "data_flow_graph.hpp"
-#include "detail/prefix_registry.hpp"
 
 namespace derecho {
 namespace cascade {
@@ -81,15 +79,10 @@ namespace cascade {
 #define METADATA_SERVICE_SUBGROUP_INDEX (0)
 
 
-    /* The cascade context to be defined later */
-    template <typename... CascadeTypes>
-    class CascadeContext;
-
+    /* The cascade execution engine to be defined later */
     template <typename... CascadeTypes>
     class ExecutionEngine;
 
-    /* The Action to be defined later */
-    struct Action;
     /**
      * The off-critical data path handler API
      */
@@ -347,1377 +340,9 @@ namespace cascade {
         }
     };
 
-
-    /** The notification handler type */
-    using cascade_notification_handler_t = std::function<void(const Blob&)>;
-
-    /** The CascadeNotificationMessage type */
-#define CASCADE_NOTIFICATION_MESSAGE_TYPE   (0x100000000ull)
-    struct CascadeNotificationMessage: public mutils::ByteRepresentable {
-        /** The object pool pathname, empty string for raw cascade notification message */
-        std::string object_pool_pathname;
-        /** data */
-        Blob blob;
-
-        /** TODO: the default serialization support macro might contain unnecessary copies. Check it!!! */
-        DEFAULT_SERIALIZATION_SUPPORT(CascadeNotificationMessage,object_pool_pathname,blob);
-
-        /** constructors */
-        CascadeNotificationMessage():
-            object_pool_pathname(),
-            blob() {}
-        CascadeNotificationMessage(CascadeNotificationMessage&& other):
-            object_pool_pathname(other.object_pool_pathname),
-            blob(std::move(other.blob)) {}
-        CascadeNotificationMessage(const CascadeNotificationMessage& other):
-            object_pool_pathname(other.object_pool_pathname),
-            blob(other.blob) {}
-        CascadeNotificationMessage(const std::string& _object_pool_pathname,
-                const Blob& _blob) :
-            object_pool_pathname(_object_pool_pathname),
-            blob(_blob) {}
-    };
-
-    /**
-     * This is the structure for the server side notification handlers
-     */
-    template <typename SubgroupType>
-    struct SubgroupNotificationHandler {
-        // key: object_pool_pathname
-        // value: an option for the handler
-        // The handler for "" key is the default handler, which will always be triggered.
-        std::unordered_map<std::string, std::optional<cascade_notification_handler_t>> object_pool_notification_handlers;
-        mutable std::unique_ptr<std::mutex> object_pool_notification_handlers_mutex;
-
-        SubgroupNotificationHandler():
-            object_pool_notification_handlers_mutex(std::make_unique<std::mutex>()) {}
-
-        template <typename T>
-        inline void initialize(derecho::ExternalClientCaller<SubgroupType,T>& subgroup_caller) {
-            dbg_default_trace("SubgroupNotificationHandler(this={:x}) is initialized for SubgroupType:{}",
-                    reinterpret_cast<uint64_t>(this),typeid(SubgroupType).name());
-            subgroup_caller.register_notification_handler(
-                    [this](const derecho::NotificationMessage& msg){
-                        dbg_default_trace("subgroup notification handler is triggered with this={:x}, msg type={}, size={} bytes",
-                                reinterpret_cast<uint64_t>(this), msg.message_type, msg.size);
-                        (*this)(msg);
-                    });
-        }
-
-        inline void operator ()(const derecho::NotificationMessage& msg) {
-            dbg_default_trace("SubgroupNotificationHandler(this={:x}) is triggered with message_type={:x}, size={} bytes",
-                    reinterpret_cast<uint64_t>(this),msg.message_type, msg.size);
-            if (msg.message_type != CASCADE_NOTIFICATION_MESSAGE_TYPE) {
-                return;
-            }
-            // mutils::deserialize_and_run<CascadeNotificationMessage>(nullptr, msg.body,
-            mutils::deserialize_and_run(nullptr, msg.body,
-                    [this](const CascadeNotificationMessage& cascade_message)->void {
-                        dbg_default_trace("Handling cascade_message: {}. size={} bytes",
-                                cascade_message.object_pool_pathname,cascade_message.blob.size);
-                        std::lock_guard<std::mutex> lck(*object_pool_notification_handlers_mutex);
-                        // call default handler
-                        if (object_pool_notification_handlers.find("") !=
-                            object_pool_notification_handlers.cend()) {
-                            if (object_pool_notification_handlers.at("").has_value()) {
-                                (*object_pool_notification_handlers.at(""))(cascade_message.blob);
-                            }
-                        }
-                        // call object pool handler
-                        if (object_pool_notification_handlers.find(cascade_message.object_pool_pathname) !=
-                            object_pool_notification_handlers.cend()) {
-                            if (object_pool_notification_handlers.at(cascade_message.object_pool_pathname).has_value()) {
-                                (*object_pool_notification_handlers.at(cascade_message.object_pool_pathname))(cascade_message.blob);
-                            }
-                        }
-                    });
-        }
-    };
-
-    template <typename SubgroupType>
-    using per_type_notification_handler_registry_t =
-        std::unordered_map<uint32_t,SubgroupNotificationHandler<SubgroupType>>;
-    /**
-     * The ServiceClient template class contains all APIs needed to read/write data. The four core APIs are put, remove,
-     * get, and get_by_time. We also provide a set of helper APIs for the client to get the group topology. The core APIs
-     * target a specific subgroup and shard of the service, or a specific object pool (which maps to a subgroup).
-     * The client uses a ShardMemberSelectionPolicy to determine which member of that subgroup/shard to communicate with.
-     */
+    /* Forward declaration of the ServiceClient, to be defined in service_client.hpp */
     template <typename... CascadeTypes>
-    class ServiceClient {
-        static_assert(have_same_object_type<CascadeTypes...>());
-    private:
-        // default caller as an external client.
-        std::unique_ptr<derecho::ExternalGroupClient<CascadeMetadataService<CascadeTypes...>,CascadeTypes...>> external_group_ptr;
-        mutable std::mutex external_group_ptr_mutex;
-        // caller as a group member.
-        derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* group_ptr;
-        mutable std::mutex group_ptr_mutex;
-        // cascade server side notification handler registry.
-        mutable mutils::KindMap<per_type_notification_handler_registry_t,CascadeTypes...> notification_handler_registry;
-        mutable std::mutex notification_handler_registry_mutex;
-        /**
-         * 'member_selection_policies' is a map from derecho shard to its member selection policy.
-         * We use a 3-tuple consisting of subgroup type index, subgroup index, and shard index to identify a shard. And
-         * the policy is defined by a 2-tuple with the ShardMemberSelectionPolicy enum and a user specified node id, in
-         * case of ShardMemorySelectionPolicy::UserSpecified. The user specified node id is used as member index if the
-         * policy is ShardMemberSelectionPolicy::RoundRobin
-         *
-         * The default member selection policy is defined as DEFAULT_SHARD_MEMBER_SELECTION_POLICY.
-         */
-        std::unordered_map<
-            std::tuple<std::type_index,uint32_t,uint32_t>,
-            std::tuple<ShardMemberSelectionPolicy,node_id_t>,
-            do_hash<std::tuple<std::type_index,uint32_t,uint32_t>>> member_selection_policies;
-        mutable std::shared_mutex member_selection_policies_mutex;
-        /**
-         * 'member_cache' is a map from derecho shard to its member list. This cache is used to accelerate the member
-         * choices process. If the client cannot connect to the cached member (after a couple of retries), it will refresh
-         * the corresponding cache entry.
-         */
-        std::unordered_map<
-            std::tuple<std::type_index,uint32_t,uint32_t>,
-            std::vector<node_id_t>,
-            do_hash<std::tuple<std::type_index,uint32_t,uint32_t>>> member_cache;
-        mutable std::shared_mutex member_cache_mutex;
-        /**
-         * 'object_pool_info_cache' is a local cache for object pool metadata. This cache is used to accelerate the
-         * object access process. If an object pool does not exists, it will be loaded from metadata service.
-         *
-         * Each entry of the object_pool_info_cache is an object of type ObjectPoolMetadataCacheEntry. Such an object
-         * caches an object pool metadata object (opm) along with the affinity set regex processing data structures.
-         */
-        class ObjectPoolMetadataCacheEntry {
-        public:
-            ObjectPoolMetadata<CascadeTypes...> opm;
-            /**
-             * The constructor
-             * @param[in] _opm object pool metadata
-             */
-            ObjectPoolMetadataCacheEntry(const ObjectPoolMetadata<CascadeTypes...>& _opm);
-
-            /**
-             * The destructor
-             */
-            virtual ~ObjectPoolMetadataCacheEntry();
-
-            /**
-             * Convert a key string to corresponding affinity set string.
-             * @param[in] key_string
-             *
-             * @return affinity set string
-             */
-            inline std::string to_affinity_set(const std::string& key_string);
-        private:
-            /* the database storing compiled regex */
-            hs_database_t*                      database;
-            /* the scratch for the regex */
-            thread_local static hs_scratch_t*   scratch;
-        };
-
-        std::unordered_map<
-            std::string,
-            ObjectPoolMetadataCacheEntry> object_pool_metadata_cache;
-        mutable std::shared_mutex object_pool_metadata_cache_mutex;
-
-        /**
-         * Pick a member by a given a policy.
-         * @param[in] subgroup_index
-         * @param[in] shard_index
-         * @param[in] key_for_hashing   - only for KeyHashing policy, ignored otherwise.
-         * @param[in] retry             - if true, refresh the member_cache.
-         */
-        template <typename SubgroupType, typename KeyTypeForHashing>
-        node_id_t pick_member_by_policy(uint32_t subgroup_index,
-                                        uint32_t shard_index,
-                                        const KeyTypeForHashing& key_for_hashing,
-                                        bool retry = false);
-
-        /**
-         * Refresh(or fill) a member cache entry.
-         * @param[in] subgroup_index
-         * @param[in] shard_index
-         */
-        template <typename SubgroupType>
-        void refresh_member_cache_entry(uint32_t subgroup_index, uint32_t shard_index);
-
-        /**
-         * Metadata API Helper: turn a string key to subgroup type index, subgroup index, and shard index.
-         */
-        template <typename KeyType>
-        std::tuple<uint32_t,uint32_t,uint32_t> key_to_shard(
-                const KeyType& key, bool check_object_location = true);
-
-        /**
-         * The Constructor
-         * We prevent calling the constructor explicitly, because the ServiceClient is a singleton.
-         * @param[in] _group_ptr The caller can pass a pointer pointing to a derecho group object. If the pointer is
-         *                   valid, the implementation will reply on the group object instead of creating an external
-         *                   client to communicate with group members.
-         */
-        ServiceClient(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr=nullptr);
-
-    public:
-        /**
-         * ServiceClient can be an external client or a cascade server. is_external_client() test this condition.
-         * The external client implementation is based on ExternalGroupClient<> while the cascade node implementation is
-         * based on Group<>.
-         *
-         * @return true for external client; other wise false.
-         */
-        inline bool is_external_client() const;
-
-        /**
-         * Derecho group helpers: They derive the API in derecho::ExternalClient.
-         * - get_my_id                  return my local node id.
-         * - get_members                returns all members in the top-level Derecho group.
-         * - get_subgroup_members       returns a vector of vectors of node ids: [[node ids in shard 0],[node ids in shard 1],...]
-         * - get_shard_members          returns the members in a shard specified by subgroup id(or subgroup type/index pair) and
-         *                              shard index.
-         * - get_number_of_subgroups    returns the number of subgroups of a given type
-         * - get_number_of_shards       returns the number of shards of a given subgroup
-         * - get_my_shard               returns the shard number that this node is a member of in the specific
-         *                              subgroup (by subgroup type and index), or -1 if this node is not a member
-         *                              of any shard in the specified subgroup.
-         * During view change, the Client might experience failure if the member is gone. In such a case, the client needs
-         * refresh its local member cache by calling get_shard_members.
-         */
-        node_id_t get_my_id() const;
-
-        std::vector<node_id_t> get_members() const;
-
-        template <typename SubgroupType>
-        std::vector<std::vector<node_id_t>> get_subgroup_members(uint32_t subgroup_index) const;
-    protected:
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        std::vector<std::vector<node_id_t>> type_recursive_get_subgroup_members(uint32_t type_index, uint32_t subgroup_index) const;
-        template <typename LastType>
-        std::vector<std::vector<node_id_t>> type_recursive_get_subgroup_members(uint32_t type_index, uint32_t subgroup_index) const;
-    public:
-        std::vector<std::vector<node_id_t>> get_subgroup_members(const std::string& object_pool_pathname);
-
-        template <typename SubgroupType>
-        std::vector<node_id_t> get_shard_members(uint32_t subgroup_index,uint32_t shard_index) const;
-    protected:
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        std::vector<node_id_t> type_recursive_get_shard_members(uint32_t type_index,
-                uint32_t subgroup_index, uint32_t shard_index) const;
-        template <typename LastType>
-        std::vector<node_id_t> type_recursive_get_shard_members(uint32_t type_index,
-                uint32_t subgroup_index, uint32_t shard_index) const;
-    public:
-        std::vector<node_id_t> get_shard_members(const std::string& object_pool_pathname,uint32_t shard_index);
-
-        template <typename SubgroupType>
-        uint32_t get_number_of_subgroups() const;
-
-        template <typename SubgroupType>
-        uint32_t get_number_of_shards(uint32_t subgroup_index) const;
-
-        // type recursive helpers for get_number_of_shards
-    protected:
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        uint32_t type_recursive_get_number_of_shards(uint32_t type_index, uint32_t subgroup_index) const;
-        template <typename LastType>
-        uint32_t type_recursive_get_number_of_shards(uint32_t type_index, uint32_t subgroup_index) const;
-    public:
-        /**
-         * This get_number_of_shards() overload the typed version.
-         * @param[in] subgroup_type_index   - the type index of the subrgoup type.
-         * @param[in] subgroup_index        - the subgroup index in the given type.
-         */
-        uint32_t get_number_of_shards(uint32_t subgroup_type_index, uint32_t subgroup_index) const;
-
-        /**
-         * This get_number_of_shards(), pick subgroup using object pool pathname.
-         * @param[in] object_pool_pathname  - the object pool name
-         */
-        uint32_t get_number_of_shards(const std::string& object_pool_pathname);
-
-        template <typename SubgroupType>
-        int32_t get_my_shard(uint32_t subgroup_index) const;
-    protected:
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        int32_t type_recursive_get_my_shard(uint32_t type_index, uint32_t subgroup_index) const;
-        template <typename LastType>
-        int32_t type_recursive_get_my_shard(uint32_t type_index, uint32_t subgroup_index) const;
-    public:
-        /**
-         * @fn int32_t get_my_shard(uint32_t subgroup_type_index, uint32_t subgroup_index) const
-         * @brief find the shard I belong to, given the subgroup specified by type and index.
-         * @param[in]   subgroup_type_index     - the type index of the subgroup type.
-         * @param[in]   subgroup_index          - the subgroup index in the given type.
-         * @return  The number of the shard, or -1 if current node is not in the specified subgroup.
-         */
-        int32_t get_my_shard(uint32_t subgroup_type_index, uint32_t subgroup_index) const;
-
-        /**
-         * @fn int32_t get_my_shard(const std::string& object_pool_pathname)
-         * @brief find the shard I belong to, given the object pool specified by object pool path name.
-         * @param[in]   object_pool_pathname    - the object pool path name.
-         * @return  The number of the shard, or -1 if current node is not in the specified subgroup.
-         */
-        int32_t get_my_shard(const std::string& object_pool_pathname);
-
-        /**
-         * Updates the member selection policy for a shard.
-         *
-         * @tparam SubgroupType The Cascade subgroup type the shard is in
-         * @param[in] subgroup_index
-         * @param[in] shard_index
-         * @param[in] policy - the new policy
-         * @param[in] user_specified_node_id - optional, the node ID to contact if the policy is UserSpecified
-         */
-        template <typename SubgroupType>
-        void set_member_selection_policy(uint32_t subgroup_index,uint32_t shard_index,
-                ShardMemberSelectionPolicy policy,node_id_t user_specified_node_id=INVALID_NODE_ID);
-
-        /**
-         * Reads the member selection policy for a shard.
-         *
-         * @tparam SubgroupType The Cascade subgroup type the shard is in
-         * @param[in] subgroup_index
-         * @param[in] shard_index
-         * @return a 2-tuple of policy and user_specified_node_id.
-         */
-        template <typename SubgroupType>
-        std::tuple<ShardMemberSelectionPolicy,node_id_t> get_member_selection_policy(
-                uint32_t subgroup_index, uint32_t shard_index) const;
-
-        /**
-         * "put" writes an object to a given subgroup/shard.
-         *
-         * @param[in] object            the object to write.
-         *                          User provided SubgroupType::ObjectType must have the following two members:
-         *                          - SubgroupType::ObjectType::key of SubgroupType::KeyType, which must be set to a
-         *                            valid key.
-         *                          - SubgroupType::ObjectType::ver of std::tuple<persistent::version_t, uint64_t>.
-         *                            Similar to the return object, this member is a two tuple with the first member
-         *                            for a version and the second for a timestamp. A caller of put can specify either
-         *                            of the version and timestamp meaning what is the latest version/timestamp the caller
-         *                            has seen. Cascade will reject the write if the corresponding key has been updated
-         *                            already. TODO: should we make it an optional feature?
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
-         *                              used to update the state.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<version_tuple> put(const typename SubgroupType::ObjectType& object,
-                uint32_t subgroup_index, uint32_t shard_index, bool as_trigger = false);
-        /**
-         * "type_recursive_put" is a helper function for internal use only.
-         * @param[in]   type_index  the index of the subgroup type in the CascadeTypes... list. And the FirstType,
-         *                          SecondType, ..., RestTypes should be in the same order.
-         * @param[in]   object      the object to write
-         * @param[in]   subgroup_index
-         *                          the subgroup index in the subgroup type designated by type_index
-         * @param[in]   shard_index the shard index
-         * @param[in]   as_trigger  If true, the object will NOT apply to the K/V store. The object will only be
-         *                          used to update the state.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-    protected:
-        template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<version_tuple> type_recursive_put(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index,
-                bool as_trigger = false);
-
-        template <typename ObjectType, typename LastType>
-        derecho::rpc::QueryResults<version_tuple> type_recursive_put(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index,
-                bool as_trigger = false);
-    public:
-        /**
-         * object pool version of "put"
-         * @param[in] object            the object to write, the object pool is extracted from the object key.
-         * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
-         *                              used to update the state.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-        template <typename ObjectType>
-        derecho::rpc::QueryResults<version_tuple> put(const ObjectType& object, bool as_trigger = false);
-
-        /**
-         * "put_and_forget" writes an object to a given subgroup/shard, but no return value.
-         *
-         * @param[in] object            the object to write.
-         *                          User provided SubgroupType::ObjectType must have the following two members:
-         *                          - SubgroupType::ObjectType::key of SubgroupType::KeyType, which must be set to a
-         *                            valid key.
-         *                          - SubgroupType::ObjectType::ver of std::tuple<persistent::version_t, uint64_t>.
-         *                            Similar to the return object, this member is a two tuple with the first member
-         *                            for a version and the second for a timestamp. A caller of put can specify either
-         *                            of the version and timestamp meaning what is the latest version/timestamp the caller
-         *                            has seen. Cascade will reject the write if the corresponding key has been updated
-         *                            already. TODO: should we make it an optional feature?
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
-         *                              used to update the state.
-         */
-        template <typename SubgroupType>
-        void put_and_forget(const typename SubgroupType::ObjectType& object,
-                uint32_t subgroup_index, uint32_t shard_index, bool as_trigger = false);
-
-    protected:
-        /**
-         * "type_recursive_put_and_forget" is a helper function for internal use only.
-         * @param[in] type_index    the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] object        the object to write
-         * @param[in] subgroup_index
-         *                          the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index   the shard index
-         * @param[in] as_trigger    If true, the object will NOT apply to the K/V store. The object will only be
-         *                          used to update the state.
-         */
-        template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-        void type_recursive_put_and_forget(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index,
-                bool as_trigger = false);
-
-        template <typename ObjectType, typename LastType>
-        void type_recursive_put_and_forget(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index,
-                bool as_trigger = false);
-    public:
-        /**
-         * object pool version of "put_and_forget"
-         * @param[in] object        the object to write, the object pool is extracted from the object key.
-         * @param[in] as_trigger    If true, the object will NOT apply to the K/V store. The object will only be
-         *                          used to update the state.
-         */
-        template <typename ObjectType>
-        void put_and_forget(const ObjectType& object, bool as_trigger = false);
-
-        /**
-         * "trigger_put" writes an object to a given subgroup/shard.
-         *
-         * @param[in] object            the object to write.
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a void future.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<void> trigger_put(const typename SubgroupType::ObjectType& object,
-                uint32_t subgroup_index, uint32_t shard_index);
-    protected:
-        /**
-         * "type_recursive_trigger_put" is a helper function for internal use only.
-         * @param[in]   type_index  the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in]   object      the object to write
-         * @param[in]   subgroup_index
-         *                          the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index   the shard index
-         *
-         * @return future
-         */
-        template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<void> type_recursive_trigger_put(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename ObjectType, typename LastType>
-        derecho::rpc::QueryResults<void> type_recursive_trigger_put(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-        /**
-         * object pool version of "trigger_put"
-         * @param[in] object    the object to write, the object pool is extracted from the object key.
-         */
-        template <typename ObjectType>
-        derecho::rpc::QueryResults<void> trigger_put(const ObjectType& object);
-        /**
-         * "collective_trigger_put" writes an object to a set of nodes.
-         *
-         * Please notice that returning from QueryResults<void>::get() only means that the message has been sent by the
-         * sender. It does NOT guarantee that the message is/will be successfully processed by the remote side. However,
-         * we agree that QueryResults<void> should reflect exceptions or errors either on local or remote side, which is
-         * not enabled so far. TODO: Track exception in derecho::rpc::QueryResults<void>
-         *
-         * @param[in] object            the object to write.
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] nodes_and_futures map from node ids to futures.
-         */
-        template <typename SubgroupType>
-        void collective_trigger_put(const typename SubgroupType::ObjectType& object,
-                uint32_t subgroup_index,
-                std::unordered_map<node_id_t,std::unique_ptr<derecho::rpc::QueryResults<void>>>& nodes_and_futures);
-
-        /**
-         * "remove" deletes an object with the given key.
-         *
-         * @param[in] key               the object key
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<version_tuple> remove(const typename SubgroupType::KeyType& key,
-                uint32_t subgroup_index, uint32_t shard_index);
-
-    protected:
-        /**
-         * "type_recursive_remove" is a helper function for internal use only.
-         * @param[in]   type_index              the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in]   key                     the key
-         * @param[in]   subgroup_index          the subgroup index in the subgroup type designated by type_index
-         * @param[in]   shard_index             the shard index
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<version_tuple> type_recursive_remove(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        derecho::rpc::QueryResults<version_tuple> type_recursive_remove(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-        /**
-         * object pool version of "remove"
-         * @param[in]   key             the object key
-         *
-         * @return  returns a future
-         */
-        template <typename KeyType>
-        derecho::rpc::QueryResults<version_tuple> remove(const KeyType& key);
-
-        /**
-         * "get" retrieve the object of a given key
-         *
-         * @param[in] key               the object key
-         * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, get will either
-         *                              read the current object from memory of the replica that handles the get request
-         *                              (if stable is false), or read the latest stable version that is persisted (if
-         *                              stable is true). Note that in any case "get" will contact only a single replica;
-         *                              to use an atomic multicast to get the latest version that is present on all replicas,
-         *                              use multi_get.
-         * @param[in] stable            if true, get will wait until the requested version's persistent data is safe, meaning the
-         *                              persistent data is persisted on all replicas.
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved object.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> get(
-                const typename SubgroupType::KeyType& key,
-                const persistent::version_t& version = CURRENT_VERSION,
-                bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-    protected:
-        /**
-         * "type_recursive_get" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] version           the version
-         * @param[in] stable            stable or not?
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the object.
-         */
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_get(
-                uint32_t type_index,
-                const KeyType& key,
-                const persistent::version_t& version,
-                bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        auto type_recursive_get(
-                uint32_t type_index,
-                const KeyType& key,
-                const persistent::version_t& version,
-                bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-        /**
-         * object pool version of "get"
-         *
-         * @param[in] key               the object key; the object pool is extracted from this key
-         * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, get will either
-         *                              read the current object from memory of the replica that handles the get request
-         *                              (if stable is false), or read the latest stable version that is persisted (if
-         *                              stable is true). Note that in any case "get" will contact only a single replica;
-         *                              to use an atomic multicast to get the latest version that is present on all replicas,
-         *                              use multi_get.
-         * @param[in] stable            if true, get will wait until the requested version's persistent data is safe, meaning the
-         *                              persistent data is persisted on all replicas.
-         * @return a future to the retrieved object.
-         */
-        template <typename KeyType>
-        auto get(
-                const KeyType& key,
-                const persistent::version_t& version = CURRENT_VERSION,
-                bool stable = true);
-
-        /**
-         * "multi_get" retrieves the latest version of the object for a given key using an atomic broadcast.
-         * This ensures that the get request is mutually exclusive and linearizable with any concurrent put
-         * requests to the same key.
-         *
-         * @param[in] key               the object key
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved object, including a response from each replica in the shard.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> multi_get(const typename SubgroupType::KeyType& key,
-                uint32_t subgroup_index, uint32_t shard_index);
-
-        /**
-         * "type_recursive_multi_get"   is a helper function for internal use only.
-         * @param[in]   type_index      the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                              SecondType, .../ RestTypes should be in the same order.
-         * @param[in]   key             the key
-         * @param[in]   subgroup_index  the subgroup index in the subgroup type designated by type_index
-         * @param[in]   shard_index     the shard index
-         *
-         * @return a future for the object.
-         */
-    protected:
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_multi_get(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        auto type_recursive_multi_get(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-
-        /**
-         * object pool version of "multi_get"
-         *
-         * @param[in] key               the object key; the object pool is extracted from this key
-         *
-         * @return a future for the retrieved object, including a response from each replica in the object pool
-         */
-        template <typename KeyType>
-        auto multi_get(const KeyType& key);
-
-        /**
-         * "get_by_time" retrieve the object of a given key
-         *
-         * @param[in] key               the object key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved object.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> get_by_time(
-                const typename SubgroupType::KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-        /**
-         * "type_recursive_get_by_time" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the object.
-         */
-    protected:
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_get_by_time(
-                uint32_t type_index,
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        auto type_recursive_get_by_time(
-                uint32_t type_index,
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-
-        /**
-         * object pool version of "get_by_time"
-         *
-         * @param[in] key               the object key; the object pool is extracted from this key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         *
-         * @return a future for the retrieved object.
-         */
-        template <typename KeyType>
-        auto get_by_time(
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable = true);
-
-        /**
-         * "get_size" retrieve size of the object of a given key
-         *
-         * @param[in] key               the object key
-         * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, this will either
-         *                              get the size of the current object from memory of the replica that handles the request
-         *                              (if stable is false), or get the size of the latest stable version that is persisted
-         *                              (if stable is true). Note that in any case "get_size" will contact only a single replica;
-         *                              to use an atomic multicast to check the latest version that is present on all replicas,
-         *                              use multi_get_size.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved size.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<uint64_t> get_size(
-                const typename SubgroupType::KeyType& key,
-                const persistent::version_t& version,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-    protected:
-        /**
-         * "type_recursive_get_size" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] version           version
-         * @param[in] stable            stable get size or not
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the retrieved size.
-         */
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_get_size(
-                uint32_t type_index,
-                const KeyType& key,
-                const persistent::version_t& version,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_get_size(
-                uint32_t type_index,
-                const KeyType& key,
-                const persistent::version_t& version,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-    public:
-        /**
-         * object pool version of "get_size"
-         * @param[in] key               the object key
-         * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, this will either
-         *                              get the size of the current object from memory of the replica that handles the request
-         *                              (if stable is false), or get the size of the latest stable version that is persisted
-         *                              (if stable is true). Note that in any case "get_size" will contact only a single replica;
-         *                              to use an atomic multicast to check the latest version that is present on all replicas,
-         *                              use multi_get_size.
-         * @param[in] stable            stable get or not
-         * @return a future for the retrieved size.
-         */
-        template <typename KeyType>
-        derecho::rpc::QueryResults<uint64_t> get_size(
-                const KeyType& key,
-                const persistent::version_t& version,
-                const bool stable = true);
-
-        /**
-         * "multi_get_size" retrieves the size of the current version of the object with a given key, using an atomic broadcast.
-         *
-         * @param[in] key               the object key
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved size.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<uint64_t> multi_get_size(
-                const typename SubgroupType::KeyType& key,
-                uint32_t subgroup_index, uint32_t shard_index);
-
-        /**
-         * "type_recursive_multi_get_size" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the object.
-         */
-    protected:
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_multi_get_size(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_multi_get_size(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-    public:
-        /**
-         * object pool version of "multi_get_size"
-         *
-         * @param[in] key               the object key; the object pool is extracted from this key
-         *
-         * @return a future for the retrieved size.
-         */
-        template <typename KeyType>
-        derecho::rpc::QueryResults<uint64_t> multi_get_size(const KeyType& key);
-
-        /**
-         * "get_size_by_time" retrieve size of the object of a given key
-         *
-         * @param[in] key               the object key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved size.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<uint64_t> get_size_by_time(
-                const typename SubgroupType::KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-    protected:
-        /**
-         * "type_recursive_get_size" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the object.
-         */
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_get_size_by_time(
-                uint32_t type_index,
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_get_size_by_time(
-                uint32_t type_index,
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-
-        /**
-         * object pool version of "get_size_by_time"
-         *
-         * @param[in] key               the object key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         *
-         * @return a future for the retrieved size.
-         */
-        template <typename KeyType>
-        derecho::rpc::QueryResults<uint64_t> get_size_by_time(
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable = true);
-
-        /**
-         * "list_keys" retrieve the list of keys in a shard
-         *
-         * @param[in] version           the version at which to list the keys; all keys that existed at or before this version
-         *                              will be included. If equal to CURRENT_VERSION, list_keys will list all keys currently
-         *                              in memory at the replica that handles the request.
-         * @param[in] stable            if true, list_keys will wait until the requested version's persistent data is safe, meaning the
-         *                              persistent data is persisted on all replicas, before listing the keys present at that version.
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future for the vector of keys
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> list_keys(
-                const persistent::version_t& version,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-    protected:
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_list_keys(
-                uint32_t type_index,
-                const persistent::version_t& version,
-                const bool stable,
-                const std::string& object_pool_pathname);
-        template <typename LastType>
-        auto type_recursive_list_keys(
-                uint32_t type_index,
-                const persistent::version_t& version,
-                const bool stable,
-                const std::string& object_pool_pathname);
-        template <typename SubgroupType>
-        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
-            __list_keys(const persistent::version_t& version, const bool stable, const std::string& object_pool_pathname);
-    public:
-        /**
-         * @brief object pool version of "list_keys"; lists all keys in an object pool
-         *
-         * @param[in] version               the version at which to list the keys; all keys that existed at or before this version
-         *                                  will be included. If equal to CURRENT_VERSION, list_keys will list all keys currently
-         *                                  in memory at the replica that handles the request.
-         * @param[in] stable                if true, list_keys will wait until the requested version's persistent data is safe, meaning the
-         *                                  persistent data is persisted on all replicas, before listing the keys present at that version.
-         * @param[in] object_pool_pathname  the object pathname
-         *
-         * @return a vector of futures for key lists, with one key list for each shard in the object pool.
-         * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
-         */
-        auto list_keys(const persistent::version_t& version, const bool stable, const std::string& object_pool_pathname);
-
-        /**
-         * A function that helps unpack the return value of list_keys (object-pool version). Iterates through the
-         * vector of QueryResults and waits for each one to complete, then combines the resulting vectors of keys
-         * into a single vector of keys (across all shards in the object pool).
-         *
-         * @tparam KeyType The type of a key in this object pool
-         * @param future The vector-of-futures returned by list_keys or multi_list_keys
-         * @return a vector of keys in the object pool
-         */
-        template <typename KeyType>
-        std::vector<KeyType> wait_list_keys(
-                                std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<KeyType>>>>& future);
-
-        /**
-         * "multi_list_keys" retrieves the list of keys in a shard at the current version, using an atomic multicast
-         *
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future for the list of keys.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> multi_list_keys(
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-    protected:
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_multi_list_keys(
-                uint32_t type_index,
-                const std::string& object_pool_pathname);
-        template <typename LastType>
-        auto type_recursive_multi_list_keys(
-                uint32_t type_index,
-                const std::string& object_pool_pathname);
-        template <typename SubgroupType>
-        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
-            __multi_list_keys(const std::string& object_pool_pathname);
-    public:
-        /**
-         * object pool version of "multi_list_keys"
-         *
-         * @param[in] object_pool_pathname  the object pathname
-         *
-         * @return a vector of futures for key lists, with one key list for each shard in the object pool.
-         * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
-         */
-        auto multi_list_keys(const std::string& object_pool_pathname);
-
-        /**
-         * "list_keys_by_time" retrieves the list of keys in a shard at a specific time
-         *
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future for the list of keys
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> list_keys_by_time(
-                const uint64_t& ts_us,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-    protected:
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_list_keys_by_time(
-                uint32_t type_index,
-                const uint64_t& ts_us,
-                const bool stable,
-                const std::string& object_pool_pathname);
-        template <typename LastType>
-        auto type_recursive_list_keys_by_time(
-                uint32_t type_index,
-                const uint64_t& ts_us,
-                const bool stable,
-                const std::string& object_pool_pathname);
-        template <typename SubgroupType>
-        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
-            __list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname);
-    public:
-        /**
-        * object pool version of "list_keys_by_time"
-        *
-        * @param[in] ts_us                  timestamp
-        * @param[in] stable                 stable flag
-        * @param[in] object_pool_pathname   the object pathname
-        *
-        * @return a vector of futures for key lists, with one key list for each shard in the object pool.
-        * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
-        */
-        auto list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname);
-
-        /**
-         * Object Pool Management API: refresh object pool cache
-         * We load 'unstable' (committed but maybe not persisted) metadata here.
-         */
-        void refresh_object_pool_metadata_cache();
-
-        /**
-         * Object Pool Management API: create object pool
-         *
-         * @tparam SubgroupType     Type of the subgroup for the created object pool
-         * @param[in]  pathname         Object pool's pathname as identifier.
-         * @param[in]  subgroup_index   Index of the subgroup
-         * @param[in]  sharding_policy  The default sharding policy for this object pool
-         * @param[in]  object_locations The set of special object locations.
-         * @param[in]  affinity_set_regex
-         *                          The affinity set regex.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<version_tuple> create_object_pool(
-                const std::string& pathname, const uint32_t subgroup_index,
-                const sharding_policy_t sharding_policy = HASH,
-                const std::unordered_map<std::string,uint32_t>& object_locations = {},
-                const std::string& affinity_set_regex = "");
-
-        /**
-         * ObjectPoolManagement API: remove object pool
-         *
-         * @param[in]  pathname         Object pool pathname
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-        derecho::rpc::QueryResults<version_tuple> remove_object_pool(const std::string& pathname);
-    private:
-        /**
-         * ObjectPoolManagement API: find object pool
-         *
-         * @param[in]  pathname         Object pool pathname
-         * @param[in]  rlck             shared lock, which needs to be hold.
-         *
-         * @return the object pool metadata
-         */
-        ObjectPoolMetadata<CascadeTypes...> internal_find_object_pool(const std::string& pathname,
-                                                                      std::shared_lock<std::shared_mutex>& rlck);
-    public:
-        /**
-         * ObjectPoolManagement API: find object pool
-         *
-         * @param[in]  pathname         Object pool pathname
-         *
-         * @return the object pool metadata
-         */
-        ObjectPoolMetadata<CascadeTypes...> find_object_pool(const std::string& pathname);
-
-        /**
-         * ObjectPoolManagement API: find object pool and affinity_set from key
-         *
-         * @param[in]  key              The key of an object.
-         *
-         * @return the object pool metadata along with the affinity set string
-         */
-        template <typename KeyType>
-        std::pair<ObjectPoolMetadata<CascadeTypes...>,std::string>
-            find_object_pool_and_affinity_set_by_key(const KeyType& key);
-
-        /**
-         * ObjectPoolManagement API: list all the object pools by pathnames
-         *
-         * @param[in] include_deleted   show deleted pools with an exclaimation point(!).
-         * @param[in] refresh           false for cached object ids, true for refreshed ids.
-         *
-         * @return the pool ids.
-         */
-        std::vector<std::string> list_object_pools(bool include_deleted, bool refresh = false);
-
-        /**
-         * Register an notification handler to a subgroup. If such a handler has been registered, it will be replaced
-         * by the new one.
-         *
-         * @tparam SubgroupType     The Subgroup Type
-         * @param[in] handler           The handler to reigster
-         * @param[in] subgroup_index    Index of the subgroup
-         *
-         * @return true if a previous notification handler is replaced.
-         */
-        template <typename SubgroupType>
-        bool register_notification_handler(
-                const cascade_notification_handler_t& handler,
-                const uint32_t subgroup_index = 0);
-
-    protected:
-        template <typename SubgroupType>
-        bool register_notification_handler(
-                const cascade_notification_handler_t& handler,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index);
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        bool type_recursive_register_notification_handler(
-                uint32_t type_index,
-                const cascade_notification_handler_t& handler,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index);
-        template <typename LastType>
-        bool type_recursive_register_notification_handler(
-                uint32_t type_index,
-                const cascade_notification_handler_t& handler,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index);
-
-    public:
-        /**
-         * Register notification handler(object pool version). If such a handler has been registered, it will be
-         * replaced by the new one.
-         *
-         * @tparam SubgroupType         The Subgroup Type
-         * @param[in] handler               The handler to reigster
-         * @param[in] object_pool_pathname  To with object pool is this handler registered.
-         *
-         * @return true if a previous notification handler is replaced.
-         */
-        bool register_notification_handler(
-                const cascade_notification_handler_t& handler,
-                const std::string& object_pool_pathname);
-
-        /**
-         * Send a notification message to an external client.
-         *
-         * @tparam SubgroupType     The Subgroup Type
-         * @param[in] msg               The message to send
-         * @param[in] subgroup_index    The subgroup index
-         * @param[in] client_id         The node id of the external client to be notified
-         */
-        template <typename SubgroupType>
-        void notify(const Blob& msg,
-                const uint32_t subgroup_index,
-                const node_id_t client_id) const;
-    protected:
-        template <typename SubgroupType>
-        void notify(const Blob& msg,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index,
-                const node_id_t client_id) const;
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        void type_recursive_notify(
-                uint32_t type_index,
-                const Blob& msg,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index,
-                const node_id_t client_id) const;
-        template <typename LastType>
-        void type_recursive_notify(
-                uint32_t type_index,
-                const Blob& msg,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index,
-                const node_id_t client_id) const;
-    public:
-        /**
-         * Send a notification message to an external client.
-         *
-         * @param[in] msg                   The messgae to send
-         * @param[in] object_pool_pathname  In which object_pool the notification is in.
-         * @param[in] client_id             The client id
-         */
-        void notify(const Blob& msg,
-                const std::string& object_pool_pathname,
-                const node_id_t client_id);
-
-#ifdef ENABLE_EVALUATION
-        /**
-         * Dump the timestamp log entries into a file on each of the nodes in a shard.
-         *
-         * @param[in] filename         - the output filename
-         * @param[in] subgroup_index   - the subgroup index
-         * @param[in] shard_index      - the shard index
-         *
-         * @return query results
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<void> dump_timestamp(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index);
-
-        /**
-         * The object store version:
-         *
-         * @param[in] filename             -   the filename
-         * @param[in] object_pool_pathname -   the object pool pathname
-         */
-        void dump_timestamp(const std::string& filename, const std::string& object_pool_pathname);
-
-        /**
-         * Dump the timestamp log entries into a file on each of the nodes in a subgroup.
-         *
-         * @param[in] filename         - the output filename
-         * @param[in] subgroup_index   - the subgroup index
-         */
-        template <typename SubgroupType>
-        void dump_timestamp(const uint32_t subgroup_index, const std::string& filename);
-
-    protected:
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        void type_recursive_dump(uint32_t type_index, uint32_t subgroup_index, const std::string& filename);
-
-        template <typename LastType>
-        void type_recursive_dump(uint32_t type_index, uint32_t subgroup_index, const std::string& filename);
-
-    public:
-#ifdef DUMP_TIMESTAMP_WORKAROUND
-        /**
-         * Dump the timestamp log entries into a file on a specific node.
-         *
-         * @param[in] filename         - the output filename
-         * @param[in] subgroup_index   - the subgroup index
-         * @param[in] shard_index      - the shard index
-         * @param[in] node_id          - the given node id.
-         *
-         * @return a vector of query results.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<void> dump_timestamp_workaround(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index, const node_id_t node_id);
-#endif
-
-        /**
-         * Evaluate the ordered put performance inside a shard. Please note that those put does not involve the
-         * external client data path.
-         *
-         * @param[in] message_size      - the message size for the shard. TODO: we should be able to retrieve the maximum
-         *                            message size from SubgroupType, subgroup_index and shard_index. How?
-         * @param[in] duration_sec      - the duration of the test in seconds.
-         * @param[in] subgroup_index    - the subgroup index
-         * @param[in] shard_index       - the shard index
-         *
-         * @return the value in ops.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<double> perf_put(const uint32_t message_size, const uint64_t duration_sec, const uint32_t subgroup_index, const uint32_t shard_index);
-#endif//ENABLE_EVALUATION
-
-        const static std::vector<std::type_index> subgroup_type_order;
-        const static uint32_t invalid_subgroup_type_index;
-        /**
-         * Get type index
-         * @return the the subgroup type index
-         */
-        template <typename SubgroupType>
-        static uint32_t get_subgroup_type_index();
-
-        /* singleton */
-    private:
-        static std::unique_ptr<ServiceClient> service_client_singleton_ptr;
-        static std::mutex                     singleton_mutex;
-    public:
-        /**
-         * Initialize the service_client_single_ptr singleton with a cascade service. This can only be called once
-         * before any get_service_client() is called.
-         * @param[in] _group_ptr The caller can pass a pointer pointing to a derecho group object. If the pointer is
-         *                   valid, the implementation will reply on the group object instead of creating an external
-         *                   client to communicate with group members.
-         */
-        static void initialize(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr);
-
-        /**
-         * Get the singleton ServiceClient API. If it does not exists, initialize it as an external client.
-         */
-        static ServiceClient& get_service_client();
-    }; // ServiceClient
-
+    class ServiceClient;
 
     /**
      * configuration keys
diff --git a/include/cascade/service_client.hpp b/include/cascade/service_client.hpp
new file mode 100644
index 00000000..ce6e015a
--- /dev/null
+++ b/include/cascade/service_client.hpp
@@ -0,0 +1,1386 @@
+#pragma once
+
+#include "object.hpp"
+#include "object_pool_metadata.hpp"
+#include "service.hpp"
+#include <cascade/config.h>
+
+#include <cstdint>
+#include <derecho/core/notification.hpp>
+#include <derecho/mutils-serialization/SerializationSupport.hpp>
+#include <derecho/persistent/PersistentInterface.hpp>
+#include <functional>
+#include <hs/hs.h>
+#include <memory>
+#include <mutex>
+#include <mutils-containers/KindMap.hpp>
+#include <optional>
+#include <shared_mutex>
+#include <string>
+#include <tuple>
+#include <unordered_map>
+#include <vector>
+
+namespace derecho {
+namespace cascade {
+/** The notification handler type */
+using cascade_notification_handler_t = std::function<void(const Blob&)>;
+
+/** The CascadeNotificationMessage type */
+#define CASCADE_NOTIFICATION_MESSAGE_TYPE (0x100000000ull)
+struct CascadeNotificationMessage : public mutils::ByteRepresentable {
+    /** The object pool pathname, empty string for raw cascade notification message */
+    std::string object_pool_pathname;
+    /** data */
+    Blob blob;
+
+    /** TODO: the default serialization support macro might contain unnecessary copies. Check it!!! */
+    DEFAULT_SERIALIZATION_SUPPORT(CascadeNotificationMessage, object_pool_pathname, blob);
+
+    /** constructors */
+    CascadeNotificationMessage() : object_pool_pathname(),
+                                   blob() {}
+    CascadeNotificationMessage(CascadeNotificationMessage&& other) : object_pool_pathname(other.object_pool_pathname),
+                                                                     blob(std::move(other.blob)) {}
+    CascadeNotificationMessage(const CascadeNotificationMessage& other) : object_pool_pathname(other.object_pool_pathname),
+                                                                          blob(other.blob) {}
+    CascadeNotificationMessage(const std::string& _object_pool_pathname,
+                               const Blob& _blob) : object_pool_pathname(_object_pool_pathname),
+                                                    blob(_blob) {}
+};
+
+/**
+ * This is the structure for the server side notification handlers
+ */
+template <typename SubgroupType>
+struct SubgroupNotificationHandler {
+    /**
+     * key: object_pool_pathname
+     * value: an optional std::function for the handler
+     * The handler for "" key is the default handler, which will always be triggered.
+     */
+    std::unordered_map<std::string, std::optional<cascade_notification_handler_t>> object_pool_notification_handlers;
+    mutable std::unique_ptr<std::mutex> object_pool_notification_handlers_mutex;
+
+    SubgroupNotificationHandler() : object_pool_notification_handlers_mutex(std::make_unique<std::mutex>()) {}
+
+    template <typename T>
+    void initialize(derecho::ExternalClientCaller<SubgroupType, T>& subgroup_caller);
+
+    void operator()(const derecho::NotificationMessage& msg);
+};
+
+template <typename SubgroupType>
+using per_type_notification_handler_registry_t = std::unordered_map<uint32_t, SubgroupNotificationHandler<SubgroupType>>;
+/**
+ * The ServiceClient template class contains all APIs needed to read/write data. The four core APIs are put, remove,
+ * get, and get_by_time. We also provide a set of helper APIs for the client to get the group topology. The core APIs
+ * target a specific subgroup and shard of the service, or a specific object pool (which maps to a subgroup).
+ * The client uses a ShardMemberSelectionPolicy to determine which member of that subgroup/shard to communicate with.
+ */
+template <typename... CascadeTypes>
+class ServiceClient {
+    static_assert(have_same_object_type<CascadeTypes...>());
+
+private:
+    // default caller as an external client.
+    std::unique_ptr<derecho::ExternalGroupClient<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>> external_group_ptr;
+    mutable std::mutex external_group_ptr_mutex;
+    // caller as a group member.
+    derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* group_ptr;
+    mutable std::mutex group_ptr_mutex;
+    // cascade server side notification handler registry.
+    mutable mutils::KindMap<per_type_notification_handler_registry_t, CascadeTypes...> notification_handler_registry;
+    mutable std::mutex notification_handler_registry_mutex;
+    /**
+     * 'member_selection_policies' is a map from derecho shard to its member selection policy.
+     * We use a 3-tuple consisting of subgroup type index, subgroup index, and shard index to identify a shard. And
+     * the policy is defined by a 2-tuple with the ShardMemberSelectionPolicy enum and a user specified node id, in
+     * case of ShardMemorySelectionPolicy::UserSpecified. The user specified node id is used as member index if the
+     * policy is ShardMemberSelectionPolicy::RoundRobin
+     *
+     * The default member selection policy is defined as DEFAULT_SHARD_MEMBER_SELECTION_POLICY.
+     */
+    std::unordered_map<
+            std::tuple<std::type_index, uint32_t, uint32_t>,
+            std::tuple<ShardMemberSelectionPolicy, node_id_t>,
+            do_hash<std::tuple<std::type_index, uint32_t, uint32_t>>>
+            member_selection_policies;
+    mutable std::shared_mutex member_selection_policies_mutex;
+    /**
+     * 'member_cache' is a map from derecho shard to its member list. This cache is used to accelerate the member
+     * choices process. If the client cannot connect to the cached member (after a couple of retries), it will refresh
+     * the corresponding cache entry.
+     */
+    std::unordered_map<
+            std::tuple<std::type_index, uint32_t, uint32_t>,
+            std::vector<node_id_t>,
+            do_hash<std::tuple<std::type_index, uint32_t, uint32_t>>>
+            member_cache;
+    mutable std::shared_mutex member_cache_mutex;
+    /**
+     * 'object_pool_info_cache' is a local cache for object pool metadata. This cache is used to accelerate the
+     * object access process. If an object pool does not exists, it will be loaded from metadata service.
+     *
+     * Each entry of the object_pool_info_cache is an object of type ObjectPoolMetadataCacheEntry. Such an object
+     * caches an object pool metadata object (opm) along with the affinity set regex processing data structures.
+     */
+    class ObjectPoolMetadataCacheEntry {
+    public:
+        ObjectPoolMetadata<CascadeTypes...> opm;
+        /**
+         * The constructor
+         * @param[in] _opm object pool metadata
+         */
+        ObjectPoolMetadataCacheEntry(const ObjectPoolMetadata<CascadeTypes...>& _opm);
+
+        /**
+         * The destructor
+         */
+        virtual ~ObjectPoolMetadataCacheEntry();
+
+        /**
+         * Convert a key string to corresponding affinity set string.
+         * @param[in] key_string
+         *
+         * @return affinity set string
+         */
+        inline std::string to_affinity_set(const std::string& key_string);
+
+    private:
+        /* the database storing compiled regex */
+        hs_database_t* database;
+        /* the scratch for the regex */
+        thread_local static hs_scratch_t* scratch;
+    };
+
+    std::unordered_map<
+            std::string,
+            ObjectPoolMetadataCacheEntry>
+            object_pool_metadata_cache;
+    mutable std::shared_mutex object_pool_metadata_cache_mutex;
+
+    /**
+     * Pick a member by a given a policy.
+     * @param[in] subgroup_index
+     * @param[in] shard_index
+     * @param[in] key_for_hashing   - only for KeyHashing policy, ignored otherwise.
+     * @param[in] retry             - if true, refresh the member_cache.
+     */
+    template <typename SubgroupType, typename KeyTypeForHashing>
+    node_id_t pick_member_by_policy(uint32_t subgroup_index,
+                                    uint32_t shard_index,
+                                    const KeyTypeForHashing& key_for_hashing,
+                                    bool retry = false);
+
+    /**
+     * Refresh(or fill) a member cache entry.
+     * @param[in] subgroup_index
+     * @param[in] shard_index
+     */
+    template <typename SubgroupType>
+    void refresh_member_cache_entry(uint32_t subgroup_index, uint32_t shard_index);
+
+    /**
+     * Metadata API Helper: turn a string key to subgroup type index, subgroup index, and shard index.
+     */
+    template <typename KeyType>
+    std::tuple<uint32_t, uint32_t, uint32_t> key_to_shard(
+            const KeyType& key, bool check_object_location = true);
+
+    /**
+     * The Constructor
+     * We prevent calling the constructor explicitly, because the ServiceClient is a singleton.
+     * @param[in] _group_ptr The caller can pass a pointer pointing to a derecho group object. If the pointer is
+     *                   valid, the implementation will reply on the group object instead of creating an external
+     *                   client to communicate with group members.
+     */
+    ServiceClient(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr = nullptr);
+
+public:
+    /**
+     * ServiceClient can be an external client or a cascade server. is_external_client() test this condition.
+     * The external client implementation is based on ExternalGroupClient<> while the cascade node implementation is
+     * based on Group<>.
+     *
+     * @return true for external client; other wise false.
+     */
+    inline bool is_external_client() const;
+
+    /**
+     * Derecho group helpers: They derive the API in derecho::ExternalClient.
+     * - get_my_id                  return my local node id.
+     * - get_members                returns all members in the top-level Derecho group.
+     * - get_subgroup_members       returns a vector of vectors of node ids: [[node ids in shard 0],[node ids in shard 1],...]
+     * - get_shard_members          returns the members in a shard specified by subgroup id(or subgroup type/index pair) and
+     *                              shard index.
+     * - get_number_of_subgroups    returns the number of subgroups of a given type
+     * - get_number_of_shards       returns the number of shards of a given subgroup
+     * - get_my_shard               returns the shard number that this node is a member of in the specific
+     *                              subgroup (by subgroup type and index), or -1 if this node is not a member
+     *                              of any shard in the specified subgroup.
+     * During view change, the Client might experience failure if the member is gone. In such a case, the client needs
+     * refresh its local member cache by calling get_shard_members.
+     */
+    node_id_t get_my_id() const;
+
+    std::vector<node_id_t> get_members() const;
+
+    template <typename SubgroupType>
+    std::vector<std::vector<node_id_t>> get_subgroup_members(uint32_t subgroup_index) const;
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    std::vector<std::vector<node_id_t>> type_recursive_get_subgroup_members(uint32_t type_index, uint32_t subgroup_index) const;
+    template <typename LastType>
+    std::vector<std::vector<node_id_t>> type_recursive_get_subgroup_members(uint32_t type_index, uint32_t subgroup_index) const;
+
+public:
+    std::vector<std::vector<node_id_t>> get_subgroup_members(const std::string& object_pool_pathname);
+
+    template <typename SubgroupType>
+    std::vector<node_id_t> get_shard_members(uint32_t subgroup_index, uint32_t shard_index) const;
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    std::vector<node_id_t> type_recursive_get_shard_members(uint32_t type_index,
+                                                            uint32_t subgroup_index, uint32_t shard_index) const;
+    template <typename LastType>
+    std::vector<node_id_t> type_recursive_get_shard_members(uint32_t type_index,
+                                                            uint32_t subgroup_index, uint32_t shard_index) const;
+
+public:
+    std::vector<node_id_t> get_shard_members(const std::string& object_pool_pathname, uint32_t shard_index);
+
+    template <typename SubgroupType>
+    uint32_t get_number_of_subgroups() const;
+
+    template <typename SubgroupType>
+    uint32_t get_number_of_shards(uint32_t subgroup_index) const;
+
+    // type recursive helpers for get_number_of_shards
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    uint32_t type_recursive_get_number_of_shards(uint32_t type_index, uint32_t subgroup_index) const;
+    template <typename LastType>
+    uint32_t type_recursive_get_number_of_shards(uint32_t type_index, uint32_t subgroup_index) const;
+
+public:
+    /**
+     * This get_number_of_shards() overload the typed version.
+     * @param[in] subgroup_type_index   - the type index of the subrgoup type.
+     * @param[in] subgroup_index        - the subgroup index in the given type.
+     */
+    uint32_t get_number_of_shards(uint32_t subgroup_type_index, uint32_t subgroup_index) const;
+
+    /**
+     * This get_number_of_shards(), pick subgroup using object pool pathname.
+     * @param[in] object_pool_pathname  - the object pool name
+     */
+    uint32_t get_number_of_shards(const std::string& object_pool_pathname);
+
+    template <typename SubgroupType>
+    int32_t get_my_shard(uint32_t subgroup_index) const;
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    int32_t type_recursive_get_my_shard(uint32_t type_index, uint32_t subgroup_index) const;
+    template <typename LastType>
+    int32_t type_recursive_get_my_shard(uint32_t type_index, uint32_t subgroup_index) const;
+
+public:
+    /**
+     * @fn int32_t get_my_shard(uint32_t subgroup_type_index, uint32_t subgroup_index) const
+     * @brief find the shard I belong to, given the subgroup specified by type and index.
+     * @param[in]   subgroup_type_index     - the type index of the subgroup type.
+     * @param[in]   subgroup_index          - the subgroup index in the given type.
+     * @return  The number of the shard, or -1 if current node is not in the specified subgroup.
+     */
+    int32_t get_my_shard(uint32_t subgroup_type_index, uint32_t subgroup_index) const;
+
+    /**
+     * @fn int32_t get_my_shard(const std::string& object_pool_pathname)
+     * @brief find the shard I belong to, given the object pool specified by object pool path name.
+     * @param[in]   object_pool_pathname    - the object pool path name.
+     * @return  The number of the shard, or -1 if current node is not in the specified subgroup.
+     */
+    int32_t get_my_shard(const std::string& object_pool_pathname);
+
+    /**
+     * Updates the member selection policy for a shard.
+     *
+     * @tparam SubgroupType The Cascade subgroup type the shard is in
+     * @param[in] subgroup_index
+     * @param[in] shard_index
+     * @param[in] policy - the new policy
+     * @param[in] user_specified_node_id - optional, the node ID to contact if the policy is UserSpecified
+     */
+    template <typename SubgroupType>
+    void set_member_selection_policy(uint32_t subgroup_index, uint32_t shard_index,
+                                     ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id = INVALID_NODE_ID);
+
+    /**
+     * Reads the member selection policy for a shard.
+     *
+     * @tparam SubgroupType The Cascade subgroup type the shard is in
+     * @param[in] subgroup_index
+     * @param[in] shard_index
+     * @return a 2-tuple of policy and user_specified_node_id.
+     */
+    template <typename SubgroupType>
+    std::tuple<ShardMemberSelectionPolicy, node_id_t> get_member_selection_policy(
+            uint32_t subgroup_index, uint32_t shard_index) const;
+
+    /**
+     * "put" writes an object to a given subgroup/shard.
+     *
+     * @param[in] object            the object to write.
+     *                          User provided SubgroupType::ObjectType must have the following two members:
+     *                          - SubgroupType::ObjectType::key of SubgroupType::KeyType, which must be set to a
+     *                            valid key.
+     *                          - SubgroupType::ObjectType::ver of std::tuple<persistent::version_t, uint64_t>.
+     *                            Similar to the return object, this member is a two tuple with the first member
+     *                            for a version and the second for a timestamp. A caller of put can specify either
+     *                            of the version and timestamp meaning what is the latest version/timestamp the caller
+     *                            has seen. Cascade will reject the write if the corresponding key has been updated
+     *                            already. TODO: should we make it an optional feature?
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
+     *                              used to update the state.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<version_tuple> put(const typename SubgroupType::ObjectType& object,
+                                                  uint32_t subgroup_index, uint32_t shard_index, bool as_trigger = false);
+    /**
+     * "type_recursive_put" is a helper function for internal use only.
+     * @param[in]   type_index  the index of the subgroup type in the CascadeTypes... list. And the FirstType,
+     *                          SecondType, ..., RestTypes should be in the same order.
+     * @param[in]   object      the object to write
+     * @param[in]   subgroup_index
+     *                          the subgroup index in the subgroup type designated by type_index
+     * @param[in]   shard_index the shard index
+     * @param[in]   as_trigger  If true, the object will NOT apply to the K/V store. The object will only be
+     *                          used to update the state.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+protected:
+    template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<version_tuple> type_recursive_put(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index,
+            bool as_trigger = false);
+
+    template <typename ObjectType, typename LastType>
+    derecho::rpc::QueryResults<version_tuple> type_recursive_put(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index,
+            bool as_trigger = false);
+
+public:
+    /**
+     * object pool version of "put"
+     * @param[in] object            the object to write, the object pool is extracted from the object key.
+     * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
+     *                              used to update the state.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+    template <typename ObjectType>
+    derecho::rpc::QueryResults<version_tuple> put(const ObjectType& object, bool as_trigger = false);
+
+    /**
+     * "put_and_forget" writes an object to a given subgroup/shard, but no return value.
+     *
+     * @param[in] object            the object to write.
+     *                          User provided SubgroupType::ObjectType must have the following two members:
+     *                          - SubgroupType::ObjectType::key of SubgroupType::KeyType, which must be set to a
+     *                            valid key.
+     *                          - SubgroupType::ObjectType::ver of std::tuple<persistent::version_t, uint64_t>.
+     *                            Similar to the return object, this member is a two tuple with the first member
+     *                            for a version and the second for a timestamp. A caller of put can specify either
+     *                            of the version and timestamp meaning what is the latest version/timestamp the caller
+     *                            has seen. Cascade will reject the write if the corresponding key has been updated
+     *                            already. TODO: should we make it an optional feature?
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
+     *                              used to update the state.
+     */
+    template <typename SubgroupType>
+    void put_and_forget(const typename SubgroupType::ObjectType& object,
+                        uint32_t subgroup_index, uint32_t shard_index, bool as_trigger = false);
+
+protected:
+    /**
+     * "type_recursive_put_and_forget" is a helper function for internal use only.
+     * @param[in] type_index    the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] object        the object to write
+     * @param[in] subgroup_index
+     *                          the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index   the shard index
+     * @param[in] as_trigger    If true, the object will NOT apply to the K/V store. The object will only be
+     *                          used to update the state.
+     */
+    template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+    void type_recursive_put_and_forget(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index,
+            bool as_trigger = false);
+
+    template <typename ObjectType, typename LastType>
+    void type_recursive_put_and_forget(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index,
+            bool as_trigger = false);
+
+public:
+    /**
+     * object pool version of "put_and_forget"
+     * @param[in] object        the object to write, the object pool is extracted from the object key.
+     * @param[in] as_trigger    If true, the object will NOT apply to the K/V store. The object will only be
+     *                          used to update the state.
+     */
+    template <typename ObjectType>
+    void put_and_forget(const ObjectType& object, bool as_trigger = false);
+
+    /**
+     * "trigger_put" writes an object to a given subgroup/shard.
+     *
+     * @param[in] object            the object to write.
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a void future.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<void> trigger_put(const typename SubgroupType::ObjectType& object,
+                                                 uint32_t subgroup_index, uint32_t shard_index);
+
+protected:
+    /**
+     * "type_recursive_trigger_put" is a helper function for internal use only.
+     * @param[in]   type_index  the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in]   object      the object to write
+     * @param[in]   subgroup_index
+     *                          the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index   the shard index
+     *
+     * @return future
+     */
+    template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<void> type_recursive_trigger_put(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename ObjectType, typename LastType>
+    derecho::rpc::QueryResults<void> type_recursive_trigger_put(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "trigger_put"
+     * @param[in] object    the object to write, the object pool is extracted from the object key.
+     */
+    template <typename ObjectType>
+    derecho::rpc::QueryResults<void> trigger_put(const ObjectType& object);
+    /**
+     * "collective_trigger_put" writes an object to a set of nodes.
+     *
+     * Please notice that returning from QueryResults<void>::get() only means that the message has been sent by the
+     * sender. It does NOT guarantee that the message is/will be successfully processed by the remote side. However,
+     * we agree that QueryResults<void> should reflect exceptions or errors either on local or remote side, which is
+     * not enabled so far. TODO: Track exception in derecho::rpc::QueryResults<void>
+     *
+     * @param[in] object            the object to write.
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] nodes_and_futures map from node ids to futures.
+     */
+    template <typename SubgroupType>
+    void collective_trigger_put(const typename SubgroupType::ObjectType& object,
+                                uint32_t subgroup_index,
+                                std::unordered_map<node_id_t, std::unique_ptr<derecho::rpc::QueryResults<void>>>& nodes_and_futures);
+
+    /**
+     * "remove" deletes an object with the given key.
+     *
+     * @param[in] key               the object key
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<version_tuple> remove(const typename SubgroupType::KeyType& key,
+                                                     uint32_t subgroup_index, uint32_t shard_index);
+
+protected:
+    /**
+     * "type_recursive_remove" is a helper function for internal use only.
+     * @param[in]   type_index              the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in]   key                     the key
+     * @param[in]   subgroup_index          the subgroup index in the subgroup type designated by type_index
+     * @param[in]   shard_index             the shard index
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<version_tuple> type_recursive_remove(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    derecho::rpc::QueryResults<version_tuple> type_recursive_remove(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "remove"
+     * @param[in]   key             the object key
+     *
+     * @return  returns a future
+     */
+    template <typename KeyType>
+    derecho::rpc::QueryResults<version_tuple> remove(const KeyType& key);
+
+    /**
+     * "get" retrieve the object of a given key
+     *
+     * @param[in] key               the object key
+     * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, get will either
+     *                              read the current object from memory of the replica that handles the get request
+     *                              (if stable is false), or read the latest stable version that is persisted (if
+     *                              stable is true). Note that in any case "get" will contact only a single replica;
+     *                              to use an atomic multicast to get the latest version that is present on all replicas,
+     *                              use multi_get.
+     * @param[in] stable            if true, get will wait until the requested version's persistent data is safe, meaning the
+     *                              persistent data is persisted on all replicas.
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved object.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> get(
+            const typename SubgroupType::KeyType& key,
+            const persistent::version_t& version = CURRENT_VERSION,
+            bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    /**
+     * "type_recursive_get" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] version           the version
+     * @param[in] stable            stable or not?
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the object.
+     */
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_get(
+            uint32_t type_index,
+            const KeyType& key,
+            const persistent::version_t& version,
+            bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    auto type_recursive_get(
+            uint32_t type_index,
+            const KeyType& key,
+            const persistent::version_t& version,
+            bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "get"
+     *
+     * @param[in] key               the object key; the object pool is extracted from this key
+     * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, get will either
+     *                              read the current object from memory of the replica that handles the get request
+     *                              (if stable is false), or read the latest stable version that is persisted (if
+     *                              stable is true). Note that in any case "get" will contact only a single replica;
+     *                              to use an atomic multicast to get the latest version that is present on all replicas,
+     *                              use multi_get.
+     * @param[in] stable            if true, get will wait until the requested version's persistent data is safe, meaning the
+     *                              persistent data is persisted on all replicas.
+     * @return a future to the retrieved object.
+     */
+    template <typename KeyType>
+    auto get(
+            const KeyType& key,
+            const persistent::version_t& version = CURRENT_VERSION,
+            bool stable = true);
+
+    /**
+     * "multi_get" retrieves the latest version of the object for a given key using an atomic broadcast.
+     * This ensures that the get request is mutually exclusive and linearizable with any concurrent put
+     * requests to the same key.
+     *
+     * @param[in] key               the object key
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved object, including a response from each replica in the shard.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> multi_get(const typename SubgroupType::KeyType& key,
+                                                                                  uint32_t subgroup_index, uint32_t shard_index);
+
+    /**
+     * "type_recursive_multi_get"   is a helper function for internal use only.
+     * @param[in]   type_index      the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                              SecondType, .../ RestTypes should be in the same order.
+     * @param[in]   key             the key
+     * @param[in]   subgroup_index  the subgroup index in the subgroup type designated by type_index
+     * @param[in]   shard_index     the shard index
+     *
+     * @return a future for the object.
+     */
+protected:
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_multi_get(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    auto type_recursive_multi_get(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "multi_get"
+     *
+     * @param[in] key               the object key; the object pool is extracted from this key
+     *
+     * @return a future for the retrieved object, including a response from each replica in the object pool
+     */
+    template <typename KeyType>
+    auto multi_get(const KeyType& key);
+
+    /**
+     * "get_by_time" retrieve the object of a given key
+     *
+     * @param[in] key               the object key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved object.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> get_by_time(
+            const typename SubgroupType::KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+    /**
+     * "type_recursive_get_by_time" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the object.
+     */
+protected:
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_get_by_time(
+            uint32_t type_index,
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    auto type_recursive_get_by_time(
+            uint32_t type_index,
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "get_by_time"
+     *
+     * @param[in] key               the object key; the object pool is extracted from this key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     *
+     * @return a future for the retrieved object.
+     */
+    template <typename KeyType>
+    auto get_by_time(
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable = true);
+
+    /**
+     * "get_size" retrieve size of the object of a given key
+     *
+     * @param[in] key               the object key
+     * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, this will either
+     *                              get the size of the current object from memory of the replica that handles the request
+     *                              (if stable is false), or get the size of the latest stable version that is persisted
+     *                              (if stable is true). Note that in any case "get_size" will contact only a single replica;
+     *                              to use an atomic multicast to check the latest version that is present on all replicas,
+     *                              use multi_get_size.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved size.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<uint64_t> get_size(
+            const typename SubgroupType::KeyType& key,
+            const persistent::version_t& version,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    /**
+     * "type_recursive_get_size" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] version           version
+     * @param[in] stable            stable get size or not
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the retrieved size.
+     */
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_get_size(
+            uint32_t type_index,
+            const KeyType& key,
+            const persistent::version_t& version,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_get_size(
+            uint32_t type_index,
+            const KeyType& key,
+            const persistent::version_t& version,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "get_size"
+     * @param[in] key               the object key
+     * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, this will either
+     *                              get the size of the current object from memory of the replica that handles the request
+     *                              (if stable is false), or get the size of the latest stable version that is persisted
+     *                              (if stable is true). Note that in any case "get_size" will contact only a single replica;
+     *                              to use an atomic multicast to check the latest version that is present on all replicas,
+     *                              use multi_get_size.
+     * @param[in] stable            stable get or not
+     * @return a future for the retrieved size.
+     */
+    template <typename KeyType>
+    derecho::rpc::QueryResults<uint64_t> get_size(
+            const KeyType& key,
+            const persistent::version_t& version,
+            const bool stable = true);
+
+    /**
+     * "multi_get_size" retrieves the size of the current version of the object with a given key, using an atomic broadcast.
+     *
+     * @param[in] key               the object key
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved size.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<uint64_t> multi_get_size(
+            const typename SubgroupType::KeyType& key,
+            uint32_t subgroup_index, uint32_t shard_index);
+
+    /**
+     * "type_recursive_multi_get_size" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the object.
+     */
+protected:
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_multi_get_size(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_multi_get_size(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "multi_get_size"
+     *
+     * @param[in] key               the object key; the object pool is extracted from this key
+     *
+     * @return a future for the retrieved size.
+     */
+    template <typename KeyType>
+    derecho::rpc::QueryResults<uint64_t> multi_get_size(const KeyType& key);
+
+    /**
+     * "get_size_by_time" retrieve size of the object of a given key
+     *
+     * @param[in] key               the object key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved size.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<uint64_t> get_size_by_time(
+            const typename SubgroupType::KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    /**
+     * "type_recursive_get_size" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the object.
+     */
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_get_size_by_time(
+            uint32_t type_index,
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_get_size_by_time(
+            uint32_t type_index,
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "get_size_by_time"
+     *
+     * @param[in] key               the object key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     *
+     * @return a future for the retrieved size.
+     */
+    template <typename KeyType>
+    derecho::rpc::QueryResults<uint64_t> get_size_by_time(
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable = true);
+
+    /**
+     * "list_keys" retrieve the list of keys in a shard
+     *
+     * @param[in] version           the version at which to list the keys; all keys that existed at or before this version
+     *                              will be included. If equal to CURRENT_VERSION, list_keys will list all keys currently
+     *                              in memory at the replica that handles the request.
+     * @param[in] stable            if true, list_keys will wait until the requested version's persistent data is safe, meaning the
+     *                              persistent data is persisted on all replicas, before listing the keys present at that version.
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future for the vector of keys
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> list_keys(
+            const persistent::version_t& version,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_list_keys(
+            uint32_t type_index,
+            const persistent::version_t& version,
+            const bool stable,
+            const std::string& object_pool_pathname);
+    template <typename LastType>
+    auto type_recursive_list_keys(
+            uint32_t type_index,
+            const persistent::version_t& version,
+            const bool stable,
+            const std::string& object_pool_pathname);
+    template <typename SubgroupType>
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
+    __list_keys(const persistent::version_t& version, const bool stable, const std::string& object_pool_pathname);
+
+public:
+    /**
+     * @brief object pool version of "list_keys"; lists all keys in an object pool
+     *
+     * @param[in] version               the version at which to list the keys; all keys that existed at or before this version
+     *                                  will be included. If equal to CURRENT_VERSION, list_keys will list all keys currently
+     *                                  in memory at the replica that handles the request.
+     * @param[in] stable                if true, list_keys will wait until the requested version's persistent data is safe, meaning the
+     *                                  persistent data is persisted on all replicas, before listing the keys present at that version.
+     * @param[in] object_pool_pathname  the object pathname
+     *
+     * @return a vector of futures for key lists, with one key list for each shard in the object pool.
+     * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
+     */
+    auto list_keys(const persistent::version_t& version, const bool stable, const std::string& object_pool_pathname);
+
+    /**
+     * A function that helps unpack the return value of list_keys (object-pool version). Iterates through the
+     * vector of QueryResults and waits for each one to complete, then combines the resulting vectors of keys
+     * into a single vector of keys (across all shards in the object pool).
+     *
+     * @tparam KeyType The type of a key in this object pool
+     * @param future The vector-of-futures returned by list_keys or multi_list_keys
+     * @return a vector of keys in the object pool
+     */
+    template <typename KeyType>
+    std::vector<KeyType> wait_list_keys(
+            std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<KeyType>>>>& future);
+
+    /**
+     * "multi_list_keys" retrieves the list of keys in a shard at the current version, using an atomic multicast
+     *
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future for the list of keys.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> multi_list_keys(
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_multi_list_keys(
+            uint32_t type_index,
+            const std::string& object_pool_pathname);
+    template <typename LastType>
+    auto type_recursive_multi_list_keys(
+            uint32_t type_index,
+            const std::string& object_pool_pathname);
+    template <typename SubgroupType>
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
+    __multi_list_keys(const std::string& object_pool_pathname);
+
+public:
+    /**
+     * object pool version of "multi_list_keys"
+     *
+     * @param[in] object_pool_pathname  the object pathname
+     *
+     * @return a vector of futures for key lists, with one key list for each shard in the object pool.
+     * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
+     */
+    auto multi_list_keys(const std::string& object_pool_pathname);
+
+    /**
+     * "list_keys_by_time" retrieves the list of keys in a shard at a specific time
+     *
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future for the list of keys
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> list_keys_by_time(
+            const uint64_t& ts_us,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_list_keys_by_time(
+            uint32_t type_index,
+            const uint64_t& ts_us,
+            const bool stable,
+            const std::string& object_pool_pathname);
+    template <typename LastType>
+    auto type_recursive_list_keys_by_time(
+            uint32_t type_index,
+            const uint64_t& ts_us,
+            const bool stable,
+            const std::string& object_pool_pathname);
+    template <typename SubgroupType>
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
+    __list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname);
+
+public:
+    /**
+     * object pool version of "list_keys_by_time"
+     *
+     * @param[in] ts_us                  timestamp
+     * @param[in] stable                 stable flag
+     * @param[in] object_pool_pathname   the object pathname
+     *
+     * @return a vector of futures for key lists, with one key list for each shard in the object pool.
+     * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
+     */
+    auto list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname);
+
+    /**
+     * Object Pool Management API: refresh object pool cache
+     * We load 'unstable' (committed but maybe not persisted) metadata here.
+     */
+    void refresh_object_pool_metadata_cache();
+
+    /**
+     * Object Pool Management API: create object pool
+     *
+     * @tparam SubgroupType     Type of the subgroup for the created object pool
+     * @param[in]  pathname         Object pool's pathname as identifier.
+     * @param[in]  subgroup_index   Index of the subgroup
+     * @param[in]  sharding_policy  The default sharding policy for this object pool
+     * @param[in]  object_locations The set of special object locations.
+     * @param[in]  affinity_set_regex
+     *                          The affinity set regex.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<version_tuple> create_object_pool(
+            const std::string& pathname, const uint32_t subgroup_index,
+            const sharding_policy_t sharding_policy = HASH,
+            const std::unordered_map<std::string, uint32_t>& object_locations = {},
+            const std::string& affinity_set_regex = "");
+
+    /**
+     * ObjectPoolManagement API: remove object pool
+     *
+     * @param[in]  pathname         Object pool pathname
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+    derecho::rpc::QueryResults<version_tuple> remove_object_pool(const std::string& pathname);
+
+private:
+    /**
+     * ObjectPoolManagement API: find object pool
+     *
+     * @param[in]  pathname         Object pool pathname
+     * @param[in]  rlck             shared lock, which needs to be hold.
+     *
+     * @return the object pool metadata
+     */
+    ObjectPoolMetadata<CascadeTypes...> internal_find_object_pool(const std::string& pathname,
+                                                                  std::shared_lock<std::shared_mutex>& rlck);
+
+public:
+    /**
+     * ObjectPoolManagement API: find object pool
+     *
+     * @param[in]  pathname         Object pool pathname
+     *
+     * @return the object pool metadata
+     */
+    ObjectPoolMetadata<CascadeTypes...> find_object_pool(const std::string& pathname);
+
+    /**
+     * ObjectPoolManagement API: find object pool and affinity_set from key
+     *
+     * @param[in]  key              The key of an object.
+     *
+     * @return the object pool metadata along with the affinity set string
+     */
+    template <typename KeyType>
+    std::pair<ObjectPoolMetadata<CascadeTypes...>, std::string>
+    find_object_pool_and_affinity_set_by_key(const KeyType& key);
+
+    /**
+     * ObjectPoolManagement API: list all the object pools by pathnames
+     *
+     * @param[in] include_deleted   show deleted pools with an exclaimation point(!).
+     * @param[in] refresh           false for cached object ids, true for refreshed ids.
+     *
+     * @return the pool ids.
+     */
+    std::vector<std::string> list_object_pools(bool include_deleted, bool refresh = false);
+
+    /**
+     * Register an notification handler to a subgroup. If such a handler has been registered, it will be replaced
+     * by the new one.
+     *
+     * @tparam SubgroupType     The Subgroup Type
+     * @param[in] handler           The handler to reigster
+     * @param[in] subgroup_index    Index of the subgroup
+     *
+     * @return true if a previous notification handler is replaced.
+     */
+    template <typename SubgroupType>
+    bool register_notification_handler(
+            const cascade_notification_handler_t& handler,
+            const uint32_t subgroup_index = 0);
+
+protected:
+    template <typename SubgroupType>
+    bool register_notification_handler(
+            const cascade_notification_handler_t& handler,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index);
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    bool type_recursive_register_notification_handler(
+            uint32_t type_index,
+            const cascade_notification_handler_t& handler,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index);
+    template <typename LastType>
+    bool type_recursive_register_notification_handler(
+            uint32_t type_index,
+            const cascade_notification_handler_t& handler,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index);
+
+public:
+    /**
+     * Register notification handler(object pool version). If such a handler has been registered, it will be
+     * replaced by the new one.
+     *
+     * @tparam SubgroupType         The Subgroup Type
+     * @param[in] handler               The handler to reigster
+     * @param[in] object_pool_pathname  To with object pool is this handler registered.
+     *
+     * @return true if a previous notification handler is replaced.
+     */
+    bool register_notification_handler(
+            const cascade_notification_handler_t& handler,
+            const std::string& object_pool_pathname);
+
+    /**
+     * Send a notification message to an external client.
+     *
+     * @tparam SubgroupType     The Subgroup Type
+     * @param[in] msg               The message to send
+     * @param[in] subgroup_index    The subgroup index
+     * @param[in] client_id         The node id of the external client to be notified
+     */
+    template <typename SubgroupType>
+    void notify(const Blob& msg,
+                const uint32_t subgroup_index,
+                const node_id_t client_id) const;
+
+protected:
+    template <typename SubgroupType>
+    void notify(const Blob& msg,
+                const std::string& object_pool_pathname,
+                const uint32_t subgroup_index,
+                const node_id_t client_id) const;
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    void type_recursive_notify(
+            uint32_t type_index,
+            const Blob& msg,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index,
+            const node_id_t client_id) const;
+    template <typename LastType>
+    void type_recursive_notify(
+            uint32_t type_index,
+            const Blob& msg,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index,
+            const node_id_t client_id) const;
+
+public:
+    /**
+     * Send a notification message to an external client.
+     *
+     * @param[in] msg                   The messgae to send
+     * @param[in] object_pool_pathname  In which object_pool the notification is in.
+     * @param[in] client_id             The client id
+     */
+    void notify(const Blob& msg,
+                const std::string& object_pool_pathname,
+                const node_id_t client_id);
+
+#ifdef ENABLE_EVALUATION
+    /**
+     * Dump the timestamp log entries into a file on each of the nodes in a shard.
+     *
+     * @param[in] filename         - the output filename
+     * @param[in] subgroup_index   - the subgroup index
+     * @param[in] shard_index      - the shard index
+     *
+     * @return query results
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<void> dump_timestamp(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index);
+
+    /**
+     * The object store version:
+     *
+     * @param[in] filename             -   the filename
+     * @param[in] object_pool_pathname -   the object pool pathname
+     */
+    void dump_timestamp(const std::string& filename, const std::string& object_pool_pathname);
+
+    /**
+     * Dump the timestamp log entries into a file on each of the nodes in a subgroup.
+     *
+     * @param[in] filename         - the output filename
+     * @param[in] subgroup_index   - the subgroup index
+     */
+    template <typename SubgroupType>
+    void dump_timestamp(const uint32_t subgroup_index, const std::string& filename);
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    void type_recursive_dump(uint32_t type_index, uint32_t subgroup_index, const std::string& filename);
+
+    template <typename LastType>
+    void type_recursive_dump(uint32_t type_index, uint32_t subgroup_index, const std::string& filename);
+
+public:
+#ifdef DUMP_TIMESTAMP_WORKAROUND
+    /**
+     * Dump the timestamp log entries into a file on a specific node.
+     *
+     * @param[in] filename         - the output filename
+     * @param[in] subgroup_index   - the subgroup index
+     * @param[in] shard_index      - the shard index
+     * @param[in] node_id          - the given node id.
+     *
+     * @return a vector of query results.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<void> dump_timestamp_workaround(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index, const node_id_t node_id);
+#endif
+
+    /**
+     * Evaluate the ordered put performance inside a shard. Please note that those put does not involve the
+     * external client data path.
+     *
+     * @param[in] message_size      - the message size for the shard. TODO: we should be able to retrieve the maximum
+     *                            message size from SubgroupType, subgroup_index and shard_index. How?
+     * @param[in] duration_sec      - the duration of the test in seconds.
+     * @param[in] subgroup_index    - the subgroup index
+     * @param[in] shard_index       - the shard index
+     *
+     * @return the value in ops.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<double> perf_put(const uint32_t message_size, const uint64_t duration_sec, const uint32_t subgroup_index, const uint32_t shard_index);
+#endif  // ENABLE_EVALUATION
+
+    const static std::vector<std::type_index> subgroup_type_order;
+    const static uint32_t invalid_subgroup_type_index;
+    /**
+     * Get type index
+     * @return the the subgroup type index
+     */
+    template <typename SubgroupType>
+    static uint32_t get_subgroup_type_index();
+
+    /* singleton */
+private:
+    static std::unique_ptr<ServiceClient> service_client_singleton_ptr;
+    static std::mutex singleton_mutex;
+
+public:
+    /**
+     * Initialize the service_client_single_ptr singleton with a cascade service. This can only be called once
+     * before any get_service_client() is called.
+     * @param[in] _group_ptr The caller can pass a pointer pointing to a derecho group object. If the pointer is
+     *                   valid, the implementation will reply on the group object instead of creating an external
+     *                   client to communicate with group members.
+     */
+    static void initialize(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr);
+
+    /**
+     * Get the singleton ServiceClient API. If it does not exists, initialize it as an external client.
+     */
+    static ServiceClient& get_service_client();
+};  // ServiceClient
+
+}  // namespace cascade
+}  // namespace derecho
+
+#include "detail/service_client_impl.hpp"
diff --git a/include/cascade/service_client_api.hpp b/include/cascade/service_client_api.hpp
index 04d7bc9d..cbaa224d 100644
--- a/include/cascade/service_client_api.hpp
+++ b/include/cascade/service_client_api.hpp
@@ -1,6 +1,7 @@
 #pragma once
 #include "service_types.hpp"
 #include "service.hpp"
+#include "service_client.hpp"
 
 #ifdef HAS_BOOLINQ
 #include <boolinq/boolinq.h>
@@ -60,7 +61,7 @@ class CascadeShardLinq : public boolinq::Linq<CascadeShardLinqStorageType<Cascad
 template <typename CascadeType, typename ServiceClientType>
 CascadeShardLinq<CascadeType,ServiceClientType> from_shard(
         std::vector<typename CascadeType::KeyType>& key_list,
-        ServiceClientType& capi, uint32_t subgroup_index, 
+        ServiceClientType& capi, uint32_t subgroup_index,
         uint32_t shard_index, persistent::version_t version) {
     /* load keys. */
     auto result = capi.template list_keys<CascadeType>(version, true, subgroup_index, shard_index);
@@ -95,7 +96,7 @@ CascadeShardLinq<CascadeType,ServiceClientType> from_shard(
  * @param capi      The cascade client.
  * @param subgroup_index
  * @param shard_index
- * @param ts_us     The unix epoch time in microsecond. 
+ * @param ts_us     The unix epoch time in microsecond.
  * @return a Linq object
  */
 template <typename CascadeType, typename ServiceClientType>
@@ -138,11 +139,11 @@ class CascadeVersionLinq : public boolinq::Linq<persistent::version_t, typename
 
 public:
     CascadeVersionLinq() : boolinq::Linq<persistent::version_t,typename CascadeType::ObjectType>() {};
-    
-    CascadeVersionLinq(ServiceClientType& capi, 
-        uint32_t sgidx, 
-        uint32_t shidx, 
-        const typename CascadeType::KeyType& objkey, 
+
+    CascadeVersionLinq(ServiceClientType& capi,
+        uint32_t sgidx,
+        uint32_t shidx,
+        const typename CascadeType::KeyType& objkey,
         persistent::version_t ver,
                        std::function<typename CascadeType::ObjectType(persistent::version_t&)> nextFunc) :
 
@@ -165,7 +166,7 @@ class CascadeVersionLinq : public boolinq::Linq<persistent::version_t, typename
  */
 template <typename CascadeType, typename ServiceClientType>
 CascadeVersionLinq<CascadeType,ServiceClientType> from_versions(
-    const typename CascadeType::KeyType& key, 
+    const typename CascadeType::KeyType& key,
  ServiceClientType &capi, uint32_t subgroup_index,
     uint32_t shard_index, persistent::version_t version) {
 
@@ -174,12 +175,12 @@ CascadeVersionLinq<CascadeType,ServiceClientType> from_versions(
    if (ver == INVALID_VERSION) {
        throw boolinq::LinqEndException();
    }
-  
+
             do {
                 auto result = capi.template get<CascadeType>(key,ver,true/*always use stable data*/,subgroup_index,shard_index);
                 for (auto& reply_future:result.get()) {
                     auto object = reply_future.second.get();
-                    ver = object.previous_version_by_key; 
+                    ver = object.previous_version_by_key;
                     if (!object.is_null())
                         return object;
                 }
@@ -210,7 +211,7 @@ class CascadeSubgroupLinq : public boolinq::Linq<CascadeSubgroupLinqStorageType<
       persistent::version_t ver,
       std::vector<CascadeShardLinq<CascadeType,ServiceClientType>>& shard_linq_list,
       std::function<typename CascadeType::ObjectType(CascadeSubgroupLinqStorageType<CascadeType,ServiceClientType>&)> nextFunc) :
-  
+
   boolinq::Linq<CascadeSubgroupLinqStorageType<CascadeType,ServiceClientType>, typename CascadeType::ObjectType>(std::make_pair(shard_linq_list.begin(),shard_linq_list.end()), nextFunc),
   client_api(capi),
   subgroup_index(sgidx),
@@ -219,10 +220,10 @@ class CascadeSubgroupLinq : public boolinq::Linq<CascadeSubgroupLinqStorageType<
 
 /**
  * Create a Linq iterating the objects in a subgroup.
- * @param shardidx_to_keys      The map that contains mappings between shard index and the corresponding 
- *                              key lists. The key lists are used to store the generated key_list in [from_shard] 
+ * @param shardidx_to_keys      The map that contains mappings between shard index and the corresponding
+ *                              key lists. The key lists are used to store the generated key_list in [from_shard]
  *                              function call. Please keep it alive throughout the life time of the Link object.
- * @param shard_linq_list       This is an output argument to keep the generated ShardLinq for each shard in the 
+ * @param shard_linq_list       This is an output argument to keep the generated ShardLinq for each shard in the
  *                              in the subgroup. Please keep it alive throughout the life time of the Link object.
  * @param capi                  The cascade client.
  * @param sgidx
@@ -231,10 +232,10 @@ class CascadeSubgroupLinq : public boolinq::Linq<CascadeSubgroupLinqStorageType<
  */
 template <typename CascadeType, typename ServiceClientType>
 CascadeSubgroupLinq<CascadeType,ServiceClientType> from_subgroup(
-    std::unordered_map<uint32_t, std::vector<typename CascadeType::KeyType>>& shardidx_to_keys, 
+    std::unordered_map<uint32_t, std::vector<typename CascadeType::KeyType>>& shardidx_to_keys,
  std::vector<CascadeShardLinq<CascadeType,ServiceClientType>>& shard_linq_list,
  ServiceClientType& capi, uint32_t sgidx, persistent::version_t ver) {
- 
+
     uint32_t num_shards = capi.template get_number_of_shards<CascadeType>(sgidx);
     std::cout << "num_shards=" << num_shards << std::endl;
     for (uint32_t shidx=0;shidx<num_shards;shidx++) {
@@ -242,7 +243,7 @@ CascadeSubgroupLinq<CascadeType,ServiceClientType> from_subgroup(
         shard_linq_list.emplace_back(from_shard<CascadeType, ServiceClientType>(shardidx_to_keys[shidx],capi,sgidx,shidx,ver));
     }
     std::cout << "done prepare shard iterators." << std::endl;
- 
+
  return CascadeSubgroupLinq<CascadeType,ServiceClientType>(capi,sgidx,ver,shard_linq_list,
      [&shard_linq_list](CascadeSubgroupLinqStorageType<CascadeType,ServiceClientType>& _storage) {
             if (_storage.first == _storage.second) {
@@ -253,12 +254,12 @@ CascadeSubgroupLinq<CascadeType,ServiceClientType> from_subgroup(
           try {
            auto obj = _storage.first->next();
      return obj;
-       } 
+       }
     catch(boolinq::LinqEndException &) {
             _storage.first++;
-       } 
+       }
    } while (_storage.first != _storage.second);
-         
+
    throw boolinq::LinqEndException();
      });
 }
@@ -301,7 +302,7 @@ class CascadeObjpoolLinq : public boolinq::Linq<CascadeObjectpoolLinqStorageType
 template <typename CascadeType, typename ServiceClientType>
 CascadeObjpoolLinq<CascadeType,ServiceClientType> from_objectpool(
         ServiceClientType& capi,
-        std::vector<typename CascadeType::KeyType>& key_list, 
+        std::vector<typename CascadeType::KeyType>& key_list,
         persistent::version_t version,
         const std::string objpool_path) {
     /* load keys. */
diff --git a/src/service/server.cpp b/src/service/server.cpp
index c7850d45..0f29d25b 100644
--- a/src/service/server.cpp
+++ b/src/service/server.cpp
@@ -2,6 +2,7 @@
 
 #include "cascade/cascade.hpp"
 #include "cascade/service.hpp"
+#include "cascade/service_client.hpp"
 #include "cascade/service_types.hpp"
 #include "cascade/object.hpp"
 
diff --git a/src/service/server.hpp b/src/service/server.hpp
index ee348cb5..1c1e31ad 100644
--- a/src/service/server.hpp
+++ b/src/service/server.hpp
@@ -3,6 +3,7 @@
 #include <cascade/cascade.hpp>
 #include <cascade/object.hpp>
 #include <cascade/service.hpp>
+#include <cascade/service_client.hpp>
 #include <cascade/service_types.hpp>
 #include <cascade/utils.hpp>
 
@@ -73,13 +74,13 @@ class CascadeServiceCDPO : public CriticalDataPathObserver<CascadeType> {
                         // dfg_ocdpos.second is a set of ocdpo info object of type prefix_ocdpo_info_t.
                         for(auto oiit = dfg_ocdpos.second.begin(); oiit != dfg_ocdpos.second.end();) {
                             if((oiit->hook != DataFlowGraph::VertexHook::BOTH) && (
-                                (oiit->hook == DataFlowGraph::VertexHook::ORDERED_PUT && is_trigger) || 
+                                (oiit->hook == DataFlowGraph::VertexHook::ORDERED_PUT && is_trigger) ||
                                 (oiit->hook == DataFlowGraph::VertexHook::TRIGGER_PUT && !is_trigger))) {
                                 // not my hook, skip it.
                                 oiit = dfg_ocdpos.second.erase(oiit);
                             } else if (is_trigger) {
                                 new_actions = true;
-                                if (oiit->execution_environment != 
+                                if (oiit->execution_environment !=
                                     DataFlowGraph::VertexExecutionEnvironment::PTHREAD) {
                                     has_mproc_udl = true;
                                 }
@@ -133,13 +134,13 @@ class CascadeServiceCDPO : public CriticalDataPathObserver<CascadeType> {
                                 value_ptr,
                                 oi.output_map  // outputs
                         );
-    
+
 #ifdef ENABLE_EVALUATION
                         ActionPostExtraInfo apei;
                         apei.uint64_val = 0;
                         apei.info.is_trigger = is_trigger;
 #endif
-    
+
 #ifdef ENABLE_EVALUATION
                         apei.info.stateful = oi.statefulness;
 #endif