diff --git a/include/cascade/detail/_user_defined_logic_interface.hpp b/include/cascade/detail/_user_defined_logic_interface.hpp
index ed7d7cf0..ac638a7b 100644
--- a/include/cascade/detail/_user_defined_logic_interface.hpp
+++ b/include/cascade/detail/_user_defined_logic_interface.hpp
@@ -1,6 +1,7 @@
 #pragma once
 #include <cascade/config.h>
 #include "../service.hpp"
+#include "../service_client.hpp"
 #include "../service_types.hpp"
 #include <unordered_set>
 #include <memory>
@@ -72,7 +73,7 @@ void release(ICascadeContext* ctxt);
  *             |      +------------------------------+
  *             |      |
  * DefaultOffCriticalDataPathObserver
- * 
+ *
  * Please derive your own ocdpo from DefaultOffCriticalDataPathObserver, and override the virtual methods defined in
  * IDefaultOffCriticalDataPathObserver
  */
@@ -94,7 +95,7 @@ using emit_func_t = std::function<void(const std::string&,
 
 class IDefaultOffCriticalDataPathObserver {
 public:
-    /** 
+    /**
      * Typed ocdpo handler derived from the Cascade service types defined in service_types.hpp
      * @param sender                The sender id
      * @param object_pool_pathname  The object pool pathname
diff --git a/include/cascade/detail/service_client_impl.hpp b/include/cascade/detail/service_client_impl.hpp
new file mode 100644
index 00000000..4c0269e0
--- /dev/null
+++ b/include/cascade/detail/service_client_impl.hpp
@@ -0,0 +1,2141 @@
+#include "../service_client.hpp"
+
+#include <cascade/config.h>
+#include <cascade/utils.hpp>
+#include <derecho/core/derecho.hpp>
+#include <mutex>
+#include <typeindex>
+
+namespace derecho {
+namespace cascade {
+
+#ifdef ENABLE_EVALUATION
+#define LOG_SERVICE_CLIENT_TIMESTAMP(tag, msgid) \
+    TimestampLogger::log(tag, this->get_my_id(), msgid);
+#else
+#define LOG_SERVICE_CLIENT_TIMESTAMP(tag, msgid)
+#endif
+
+template <typename SubgroupType>
+template <typename T>
+void SubgroupNotificationHandler<SubgroupType>::initialize(derecho::ExternalClientCaller<SubgroupType, T>& subgroup_caller) {
+    dbg_default_trace("SubgroupNotificationHandler(this={:x}) is initialized for SubgroupType:{}",
+                      reinterpret_cast<uint64_t>(this), typeid(SubgroupType).name());
+    subgroup_caller.register_notification_handler(
+            [this](const derecho::NotificationMessage& msg) {
+                dbg_default_trace("subgroup notification handler is triggered with this={:x}, msg type={}, size={} bytes",
+                                  reinterpret_cast<uint64_t>(this), msg.message_type, msg.size);
+                (*this)(msg);
+            });
+}
+
+template <typename SubgroupType>
+void SubgroupNotificationHandler<SubgroupType>::operator()(const derecho::NotificationMessage& msg) {
+    dbg_default_trace("SubgroupNotificationHandler(this={:x}) is triggered with message_type={:x}, size={} bytes",
+                      reinterpret_cast<uint64_t>(this), msg.message_type, msg.size);
+    if(msg.message_type != CASCADE_NOTIFICATION_MESSAGE_TYPE) {
+        return;
+    }
+    mutils::deserialize_and_run(nullptr, msg.body,
+                                [this](const CascadeNotificationMessage& cascade_message) -> void {
+                                    dbg_default_trace("Handling cascade_message: {}. size={} bytes",
+                                                      cascade_message.object_pool_pathname, cascade_message.blob.size);
+                                    std::lock_guard<std::mutex> lck(*object_pool_notification_handlers_mutex);
+                                    // call default handler
+                                    if(object_pool_notification_handlers.find("") != object_pool_notification_handlers.cend()) {
+                                        if(object_pool_notification_handlers.at("").has_value()) {
+                                            (*object_pool_notification_handlers.at(""))(cascade_message.blob);
+                                        }
+                                    }
+                                    // call object pool handler
+                                    if(object_pool_notification_handlers.find(cascade_message.object_pool_pathname) != object_pool_notification_handlers.cend()) {
+                                        if(object_pool_notification_handlers.at(cascade_message.object_pool_pathname).has_value()) {
+                                            (*object_pool_notification_handlers.at(cascade_message.object_pool_pathname))(cascade_message.blob);
+                                        }
+                                    }
+                                });
+}
+
+template <typename CascadeType>
+std::unique_ptr<CascadeType> client_stub_factory() {
+    return std::make_unique<CascadeType>();
+}
+
+template <typename... CascadeTypes>
+ServiceClient<CascadeTypes...>::ServiceClient(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr)
+        : external_group_ptr(nullptr),
+          group_ptr(_group_ptr) {
+    if(group_ptr == nullptr) {
+        this->external_group_ptr = std::make_unique<derecho::ExternalGroupClient<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>>(
+                client_stub_factory<CascadeMetadataService<CascadeTypes...>>,
+                client_stub_factory<CascadeTypes>...);
+    }
+}
+
+template <typename... CascadeTypes>
+bool ServiceClient<CascadeTypes...>::is_external_client() const {
+    return (group_ptr == nullptr) && (external_group_ptr != nullptr);
+}
+
+template <typename... CascadeTypes>
+node_id_t ServiceClient<CascadeTypes...>::get_my_id() const {
+    if(!is_external_client()) {
+        return group_ptr->get_my_id();
+    } else {
+        return external_group_ptr->get_my_id();
+    }
+}
+
+template <typename... CascadeTypes>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_members() const {
+    if(!is_external_client()) {
+        return group_ptr->get_members();
+    } else {
+        return external_group_ptr->get_members();
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_shard_members(uint32_t subgroup_index, uint32_t shard_index) const {
+    if(!is_external_client()) {
+        std::vector<std::vector<node_id_t>> subgroup_members = group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
+        if(subgroup_members.size() > shard_index) {
+            return subgroup_members[shard_index];
+        } else {
+            return {};
+        }
+    } else {
+        return external_group_ptr->template get_shard_members<SubgroupType>(subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::type_recursive_get_shard_members(
+        uint32_t type_index, uint32_t subgroup_index, uint32_t shard_index) const {
+    if(type_index == 0) {
+        return this->template get_shard_members<FirstType>(subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get_shard_members<SecondType, RestTypes...>(type_index - 1, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::type_recursive_get_shard_members(
+        uint32_t type_index, uint32_t subgroup_index, uint32_t shard_index) const {
+    if(type_index == 0) {
+        return this->template get_shard_members<LastType>(subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
+    }
+}
+
+template <typename... CascadeTypes>
+std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_shard_members(
+        const std::string& object_pool_pathname, uint32_t shard_index) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    return this->template type_recursive_get_shard_members<CascadeTypes...>(opm.subgroup_type_index, opm.subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::get_subgroup_members(uint32_t subgroup_index) const {
+    if(!is_external_client()) {
+        return group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
+    } else {
+        return external_group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::type_recursive_get_subgroup_members(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_subgroup_members<FirstType>(subgroup_index);
+    } else {
+        return this->template type_recursive_get_subgroup_members<SecondType, RestTypes...>(type_index - 1, subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::type_recursive_get_subgroup_members(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_subgroup_members<LastType>(subgroup_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
+    }
+}
+
+template <typename... CascadeTypes>
+std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::get_subgroup_members(
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    return this->template type_recursive_get_subgroup_members<CascadeTypes...>(opm.subgroup_type_index, opm.subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+uint32_t ServiceClient<CascadeTypes...>::get_number_of_subgroups() const {
+    if(!is_external_client()) {
+        return group_ptr->template get_num_subgroups<SubgroupType>();
+    } else {
+        return external_group_ptr->template get_number_of_subgroups<SubgroupType>();
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards(uint32_t subgroup_index) const {
+    if(!is_external_client()) {
+        return group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index).size();
+    } else {
+        return external_group_ptr->template get_number_of_shards<SubgroupType>(subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+uint32_t ServiceClient<CascadeTypes...>::type_recursive_get_number_of_shards(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_number_of_shards<FirstType>(subgroup_index);
+    } else {
+        return this->template type_recursive_get_number_of_shards<SecondType, RestTypes...>(type_index - 1, subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+uint32_t ServiceClient<CascadeTypes...>::type_recursive_get_number_of_shards(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_number_of_shards<LastType>(subgroup_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
+    }
+}
+
+template <typename... CascadeTypes>
+uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards(
+        uint32_t subgroup_type_index, uint32_t subgroup_index) const {
+    return type_recursive_get_number_of_shards<CascadeTypes...>(subgroup_type_index, subgroup_index);
+}
+
+template <typename... CascadeTypes>
+uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards(
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    return get_number_of_shards(opm.subgroup_type_index, opm.subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+int32_t ServiceClient<CascadeTypes...>::get_my_shard(uint32_t subgroup_index) const {
+    if(!is_external_client()) {
+        return group_ptr->template get_my_shard<SubgroupType>(subgroup_index);
+    } else {
+        return -1;
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+int32_t ServiceClient<CascadeTypes...>::type_recursive_get_my_shard(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_my_shard<FirstType>(subgroup_index);
+    } else {
+        return this->template type_recursive_get_number_of_shards<SecondType, RestTypes...>(type_index - 1, subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+int32_t ServiceClient<CascadeTypes...>::type_recursive_get_my_shard(
+        uint32_t type_index, uint32_t subgroup_index) const {
+    if(type_index == 0) {
+        return this->template get_my_shard<LastType>(subgroup_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
+    }
+}
+
+template <typename... CascadeTypes>
+int32_t ServiceClient<CascadeTypes...>::get_my_shard(
+        uint32_t subgroup_type_index, uint32_t subgroup_index) const {
+    return this->template type_recursive_get_my_shard<CascadeTypes...>(subgroup_type_index, subgroup_index);
+}
+
+template <typename... CascadeTypes>
+int32_t ServiceClient<CascadeTypes...>::get_my_shard(
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    return get_my_shard(opm.subgroup_type_index, opm.subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::set_member_selection_policy(uint32_t subgroup_index, uint32_t shard_index,
+                                                                 ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id) {
+    // write lock policies
+    std::unique_lock wlck(this->member_selection_policies_mutex);
+    // update map
+    this->member_selection_policies[std::make_tuple(std::type_index(typeid(SubgroupType)), subgroup_index, shard_index)]
+            = std::make_tuple(policy, user_specified_node_id);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::tuple<ShardMemberSelectionPolicy, node_id_t> ServiceClient<CascadeTypes...>::get_member_selection_policy(
+        uint32_t subgroup_index, uint32_t shard_index) const {
+    // read lock policies
+    std::shared_lock rlck(this->member_selection_policies_mutex);
+    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)), subgroup_index, shard_index);
+    // read map
+    if(member_selection_policies.find(key) != member_selection_policies.end()) {
+        return member_selection_policies.at(key);
+    } else {
+        return std::make_tuple(DEFAULT_SHARD_MEMBER_SELECTION_POLICY, INVALID_NODE_ID);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::refresh_member_cache_entry(uint32_t subgroup_index,
+                                                                uint32_t shard_index) {
+    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)), subgroup_index, shard_index);
+    auto members = get_shard_members<SubgroupType>(subgroup_index, shard_index);
+    std::shared_lock rlck(member_cache_mutex);
+    if(member_cache.find(key) == member_cache.end()) {
+        rlck.unlock();
+        std::unique_lock wlck(member_cache_mutex);
+        member_cache[key] = members;
+    } else {
+        member_cache[key].swap(members);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+std::tuple<uint32_t, uint32_t, uint32_t> ServiceClient<CascadeTypes...>::key_to_shard(
+        const KeyType& key,
+        bool check_object_location) {
+    auto pair = find_object_pool_and_affinity_set_by_key(key);
+
+    auto& opm = std::get<0>(pair);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to identify the object_pool from key:" + key);
+    }
+    auto& affinity_set = std::get<1>(pair);
+
+    return std::tuple<uint32_t, uint32_t, uint32_t>{opm.subgroup_type_index, opm.subgroup_index,
+                                                    opm.key_to_shard_index(key, affinity_set,
+                                                                           get_number_of_shards(opm.subgroup_type_index, opm.subgroup_index),
+                                                                           check_object_location)};
+}
+
+template <typename... CascadeTypes>
+ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::ObjectPoolMetadataCacheEntry(
+        const ObjectPoolMetadata<CascadeTypes...>& _opm) : opm(_opm), database(nullptr) {
+    if(opm.affinity_set_regex.size() > 0) {
+        hs_compile_error_t* compile_err;
+        if(hs_compile(opm.affinity_set_regex.c_str(), HS_FLAG_DOTALL | HS_FLAG_SOM_LEFTMOST, HS_MODE_BLOCK, NULL, &database,
+                      &compile_err)
+           != HS_SUCCESS) {
+            hs_free_compile_error(compile_err);
+            dbg_default_error("Compilation of affinity set regex:" + opm.affinity_set_regex + " failed with message:"
+                              + compile_err->message);
+            throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__)
+                                             + ": compilation of affinity_set_regex:"
+                                             + opm.affinity_set_regex
+                                             + " failed with message:"
+                                             + compile_err->message);
+        }
+    }
+}
+
+template <typename... CascadeTypes>
+ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::~ObjectPoolMetadataCacheEntry() {
+    if(this->database != nullptr) {
+        hs_free_database(database);
+        this->database = nullptr;
+    }
+}
+
+template <typename... CascadeTypes>
+inline std::string ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::to_affinity_set(
+        const std::string& key_string) {
+    if(key_string.size() > 0 && this->opm.affinity_set_regex.size() > 0) {
+        if(scratch == nullptr) {
+            if(hs_alloc_scratch(database, &scratch) != HS_SUCCESS) {
+                hs_free_database(database);
+                dbg_default_error("failed to allocate hyperscan scratch space.");
+                throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__)
+                                                 + " failed to allocate hyperscan scratch space.");
+            }
+        }
+
+        struct hs_scan_ctxt {
+            unsigned long long from = 0;
+            unsigned long long to = 0;
+        } ctxt;
+
+        hs_scan(database, key_string.c_str(), key_string.size(), HS_FLAG_SOM_LEFTMOST, scratch, [](unsigned int /*id*/, unsigned long long from, unsigned long long to, unsigned int /*flags*/, void* ctxt) -> int {
+            struct hs_scan_ctxt* p_hs_ctxt = static_cast<struct hs_scan_ctxt*>(ctxt);
+            p_hs_ctxt->from = from;
+            p_hs_ctxt->to = to;
+            return 0;  // do the longest match
+        },
+                &ctxt);
+        if(ctxt.to > ctxt.from) {
+            return key_string.substr(ctxt.from, (ctxt.to - ctxt.from));
+        }
+    }
+
+    return key_string;
+}
+
+template <typename... CascadeTypes>
+thread_local hs_scratch_t* ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::scratch = nullptr;
+
+template <typename... CascadeTypes>
+template <typename SubgroupType, typename KeyTypeForHashing>
+node_id_t ServiceClient<CascadeTypes...>::pick_member_by_policy(uint32_t subgroup_index,
+                                                                uint32_t shard_index,
+                                                                const KeyTypeForHashing& key_for_hashing,
+                                                                bool retry) {
+    ShardMemberSelectionPolicy policy;
+    node_id_t last_specified_node_id_or_index;
+
+    std::tie(policy, last_specified_node_id_or_index) = get_member_selection_policy<SubgroupType>(subgroup_index, shard_index);
+
+    if(policy == ShardMemberSelectionPolicy::UserSpecified) {
+        return last_specified_node_id_or_index;
+    }
+
+    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)), subgroup_index, shard_index);
+
+    if(member_cache.find(key) == member_cache.end() || retry) {
+        refresh_member_cache_entry<SubgroupType>(subgroup_index, shard_index);
+    }
+
+    std::shared_lock rlck(member_cache_mutex);
+
+    node_id_t node_id = last_specified_node_id_or_index;
+
+    switch(policy) {
+        case ShardMemberSelectionPolicy::FirstMember:
+            node_id = member_cache.at(key).front();
+            break;
+        case ShardMemberSelectionPolicy::LastMember:
+            node_id = member_cache.at(key).back();
+            break;
+        case ShardMemberSelectionPolicy::Random:
+            node_id = member_cache.at(key)[get_time() % member_cache.at(key).size()];  // use time as random source.
+            break;
+        case ShardMemberSelectionPolicy::FixedRandom:
+            if(node_id == INVALID_NODE_ID || retry) {
+                node_id = member_cache.at(key)[get_time() % member_cache.at(key).size()];  // use time as random source.
+            }
+            break;
+        case ShardMemberSelectionPolicy::RoundRobin: {
+            std::shared_lock rlck(member_selection_policies_mutex);
+            node_id = static_cast<uint32_t>(node_id + 1) % member_cache.at(key).size();
+            auto new_policy = std::make_tuple(ShardMemberSelectionPolicy::RoundRobin, node_id);
+            member_selection_policies[key].swap(new_policy);
+        }
+            node_id = member_cache.at(key)[node_id];
+            break;
+        case ShardMemberSelectionPolicy::KeyHashing: {
+            uint64_t hash = 0;
+            if constexpr(std::is_integral_v<KeyTypeForHashing>) {
+                hash = static_cast<uint64_t>(key_for_hashing);
+            } else if constexpr(std::is_convertible_v<KeyTypeForHashing, std::string>) {
+                hash = std::hash<std::string>{}(std::string(key_for_hashing));
+            } else {
+                dbg_default_warn("Key type {} is neither integral nor string, falling back to FirstMember policy. {}:{}",
+                                 typeid(KeyTypeForHashing).name(), __FILE__, __LINE__);
+            }
+            node_id = member_cache.at(key)[hash % member_cache.at(key).size()];
+        } break;
+        default:
+            throw derecho::derecho_exception("Unknown member selection policy:" + std::to_string(static_cast<unsigned int>(policy)));
+    }
+
+    return node_id;
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::put(
+        const typename SubgroupType::ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_PUT_START,
+                                 (std::is_base_of<IHasMessageID, typename SubgroupType::ObjectType>::value ? value.get_message_id() : 0));
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            // ordered put as a shard member
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template ordered_send<RPC_NAME(ordered_put)>(value, as_trigger);
+        } else {
+            // p2p put
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+            try {
+                // as a subgroup member
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                return subgroup_handle.template p2p_send<RPC_NAME(put)>(node_id, value, as_trigger);
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                // as an external caller
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                return subgroup_handle.template p2p_send<RPC_NAME(put)>(node_id, value, as_trigger);
+            }
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+        return caller.template p2p_send<RPC_NAME(put)>(node_id, value, as_trigger);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_put(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    if(type_index == 0) {
+        return this->template put<FirstType>(value, subgroup_index, shard_index, as_trigger);
+    } else {
+        return this->template type_recursive_put<ObjectType, SecondType, RestTypes...>(type_index - 1, value, subgroup_index, shard_index, as_trigger);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename LastType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_put(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    if(type_index == 0) {
+        return this->template put<LastType>(value, subgroup_index, shard_index, as_trigger);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::put(
+        const ObjectType& value, bool as_trigger) {
+    // STEP 1 - get key
+    if constexpr(!std::is_base_of_v<ICascadeObject<std::string, ObjectType>, ObjectType>) {
+        throw derecho::derecho_exception(std::string("ServiceClient<>::put() only support object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(value.get_key_ref());
+
+    // STEP 3 - call recursive put
+    return this->template type_recursive_put<ObjectType, CascadeTypes...>(subgroup_type_index, value, subgroup_index, shard_index, as_trigger);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::put_and_forget(
+        const typename SubgroupType::ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_PUT_AND_FORGET_START,
+                                 (std::is_base_of<IHasMessageID, typename SubgroupType::ObjectType>::value ? value.get_message_id() : 0));
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            // do ordered put as a shard member (Replicated).
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            subgroup_handle.template ordered_send<RPC_NAME(ordered_put_and_forget)>(value, as_trigger);
+        } else {
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+            // do p2p put
+            try {
+                // as a subgroup member
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                subgroup_handle.template p2p_send<RPC_NAME(put_and_forget)>(node_id, value, as_trigger);
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                // as an external caller
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                subgroup_handle.template p2p_send<RPC_NAME(put_and_forget)>(node_id, value, as_trigger);
+            }
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+        caller.template p2p_send<RPC_NAME(put_and_forget)>(node_id, value, as_trigger);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+void ServiceClient<CascadeTypes...>::type_recursive_put_and_forget(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    if(type_index == 0) {
+        put_and_forget<FirstType>(value, subgroup_index, shard_index, as_trigger);
+    } else {
+        type_recursive_put_and_forget<ObjectType, SecondType, RestTypes...>(type_index - 1, value, subgroup_index, shard_index, as_trigger);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename LastType>
+void ServiceClient<CascadeTypes...>::type_recursive_put_and_forget(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index,
+        bool as_trigger) {
+    if(type_index == 0) {
+        put_and_forget<LastType>(value, subgroup_index, shard_index, as_trigger);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType>
+void ServiceClient<CascadeTypes...>::put_and_forget(const ObjectType& value, bool as_trigger) {
+    // STEP 1 - get key
+    if constexpr(!std::is_base_of_v<ICascadeObject<std::string, ObjectType>, ObjectType>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(value.get_key_ref());
+
+    // STEP 3 - call recursive put_and_forget
+    this->template type_recursive_put_and_forget<ObjectType, CascadeTypes...>(subgroup_type_index, value, subgroup_index, shard_index, as_trigger);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::trigger_put(
+        const typename SubgroupType::ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_TRIGGER_PUT_START,
+                                 (std::is_base_of<IHasMessageID, typename SubgroupType::ObjectType>::value ? value.get_message_id() : 0));
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+            dbg_default_trace("trigger_put to node {}", node_id);
+            return subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(node_id, value);
+        } else {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+            dbg_default_trace("trigger_put to node {}", node_id);
+            return subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(node_id, value);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, value.get_key_ref());
+        dbg_default_trace("trigger_put to node {}", node_id);
+        return caller.template p2p_send<RPC_NAME(trigger_put)>(node_id, value);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::type_recursive_trigger_put(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return trigger_put<FirstType>(value, subgroup_index, shard_index);
+    } else {
+        return type_recursive_trigger_put<ObjectType, SecondType, RestTypes...>(type_index - 1, value, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType, typename LastType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::type_recursive_trigger_put(
+        uint32_t type_index,
+        const ObjectType& value,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return trigger_put<LastType>(value, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename ObjectType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::trigger_put(
+        const ObjectType& value) {
+    // STEP 1 - get key
+    if constexpr(!std::is_base_of_v<ICascadeObject<std::string, ObjectType>, ObjectType>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(value.get_key_ref());
+
+    // STEP 3 - call recursive trigger_put
+    return this->template type_recursive_trigger_put<ObjectType, CascadeTypes...>(subgroup_type_index, value, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::collective_trigger_put(
+        const typename SubgroupType::ObjectType& value,
+        uint32_t subgroup_index,
+        std::unordered_map<node_id_t, std::unique_ptr<derecho::rpc::QueryResults<void>>>& nodes_and_futures) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_COLLECTIVE_TRIGGER_PUT_START,
+                                 (std::is_base_of<IHasMessageID, typename SubgroupType::ObjectType>::value ? value.get_message_id() : 0));
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(group_ptr->template get_my_shard<SubgroupType>(subgroup_index) != -1) {
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            for(auto& kv : nodes_and_futures) {
+                nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
+                        std::move(subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(kv.first, value)));
+            }
+        } else {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            for(auto& kv : nodes_and_futures) {
+                nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
+                        std::move(subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(kv.first, value)));
+            }
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        for(auto& kv : nodes_and_futures) {
+            nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
+                    std::move(caller.template p2p_send<RPC_NAME(trigger_put)>(kv.first, value)));
+        }
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove(
+        const typename SubgroupType::KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_REMOVE_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            // do ordered remove as a member (Replicated).
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template ordered_send<RPC_NAME(ordered_remove)>(key);
+        } else {
+            // do p2p remove
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+            try {
+                // as a subgroup member
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                return subgroup_handle.template p2p_send<RPC_NAME(remove)>(node_id, key);
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                // as an external caller
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                return subgroup_handle.template p2p_send<RPC_NAME(remove)>(node_id, key);
+            }
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(remove)>(node_id, key);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_remove(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template remove<FirstType>(key, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_remove<KeyType, SecondType, RestTypes...>(type_index - 1, key, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_remove(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template remove<LastType>(key, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove(
+        const KeyType& key) {
+    // STEP 1 - get key
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive remove
+    return this->template type_recursive_remove<KeyType, CascadeTypes...>(subgroup_type_index, key, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::get(
+        const typename SubgroupType::KeyType& key,
+        const persistent::version_t& version,
+        bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_GET_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p get as a subgroup member
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+                // local get
+                auto obj = subgroup_handle.get_ref().get(key, version, stable);
+                auto pending_results = std::make_shared<PendingResults<const typename SubgroupType::ObjectType>>();
+                pending_results->fulfill_map({node_id});
+                pending_results->set_value(node_id, obj);
+                auto query_results = pending_results->get_future();
+                return std::move(*query_results);
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(get)>(node_id, key, version, stable, false);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(get)>(node_id, key, version, stable, false);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(get)>(node_id, key, version, stable, false);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::multi_get(
+        const typename SubgroupType::KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_GET_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p multi_get as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_get)>(node_id, key);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p multi_get as an external caller.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_get)>(node_id, key);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(multi_get)>(node_id, key);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_get(
+        uint32_t type_index,
+        const KeyType& key,
+        const persistent::version_t& version,
+        bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get<FirstType>(key, version, stable, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get<KeyType, SecondType, RestTypes...>(type_index - 1, key, version, stable, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_get(
+        uint32_t type_index,
+        const KeyType& key,
+        const persistent::version_t& version,
+        bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get<LastType>(key, version, stable, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+auto ServiceClient<CascadeTypes...>::get(
+        // const std::decay_t<typename std::result_of_t<decltype(&ObjectType::get_key_ref())>>& key,
+        const KeyType& key,
+        const persistent::version_t& version,
+        bool stable) {
+    // STEP 1 - get key
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_get<KeyType, CascadeTypes...>(subgroup_type_index, key, version, stable, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_multi_get(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template multi_get<FirstType>(key, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_multi_get<KeyType, SecondType, RestTypes...>(type_index - 1, key, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_multi_get(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template multi_get<LastType>(key, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+auto ServiceClient<CascadeTypes...>::multi_get(
+        const KeyType& key) {
+    // STEP 1 - get key
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_multi_get<KeyType, CascadeTypes...>(subgroup_type_index, key, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::get_by_time(
+        const typename SubgroupType::KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p get_by_time
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                // as a shard member.
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(get_by_time)>(node_id, key, ts_us, stable);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p get_by_time as an external caller
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(get_by_time)>(node_id, key, ts_us, stable);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>();
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(get_by_time)>(node_id, key, ts_us, stable);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_get_by_time(
+        uint32_t type_index,
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_by_time<FirstType>(key, ts_us, stable, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get_by_time<KeyType, SecondType, RestTypes...>(type_index - 1, key, ts_us, stable, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_get_by_time(
+        uint32_t type_index,
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_by_time<LastType>(key, ts_us, stable, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+auto ServiceClient<CascadeTypes...>::get_by_time(
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable) {
+    // STEP 1 - get key
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get_by_time
+    return this->template type_recursive_get_by_time<KeyType, CascadeTypes...>(subgroup_type_index, key, ts_us, stable, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size(
+        const typename SubgroupType::KeyType& key,
+        const persistent::version_t& version,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_GET_SIZE_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p get_size as a subgroup_member
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                // as a shard member.
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(get_size)>(node_id, key, version, stable, false);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p get_size as an external caller
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(get_size)>(node_id, key, version, stable, false);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(get_size)>(node_id, key, version, stable, false);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size(
+        uint32_t type_index,
+        const KeyType& key,
+        const persistent::version_t& version,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_size<FirstType>(key, version, stable, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get_size<KeyType, SecondType, RestTypes...>(type_index - 1, key, version, stable, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size(
+        uint32_t type_index,
+        const KeyType& key,
+        const persistent::version_t& version,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_size<LastType>(key, version, stable, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size(
+        const KeyType& key,
+        const persistent::version_t& version,
+        const bool stable) {
+    // STEP 1 - verify the keys
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_get_size<KeyType, CascadeTypes...>(subgroup_type_index, key, version, stable, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::multi_get_size(
+        const typename SubgroupType::KeyType& key,
+        uint32_t subgroup_index, uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_GET_SIZE_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p multi_get_size as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_get_size)>(node_id, key);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p multi_get_size as an external caller.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_get_size)>(node_id, key);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(multi_get_size)>(node_id, key);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_multi_get_size(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template multi_get_size<FirstType>(key, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_multi_get_size<KeyType, SecondType, RestTypes...>(type_index - 1, key, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_multi_get_size(
+        uint32_t type_index,
+        const KeyType& key,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template multi_get_size<LastType>(key, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::multi_get_size(const KeyType& key) {
+    // STEP 1 - verify the keys
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_multi_get_size<KeyType, CascadeTypes...>(subgroup_type_index, key, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size_by_time(
+        const typename SubgroupType::KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        try {
+            // do p2p get_size_by_time as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(get_size_by_time)>(node_id, key, ts_us, stable);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p get_size_by_time as an external caller.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(get_size_by_time)>(node_id, key, ts_us, stable);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, key);
+        return caller.template p2p_send<RPC_NAME(get_size_by_time)>(node_id, key, ts_us, stable);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size_by_time(
+        uint32_t type_index,
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_size_by_time<FirstType>(key, ts_us, stable, subgroup_index, shard_index);
+    } else {
+        return this->template type_recursive_get_size_by_time<KeyType, SecondType, RestTypes...>(type_index - 1, key, ts_us, stable, subgroup_index, shard_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType, typename LastType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size_by_time(
+        uint32_t type_index,
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(type_index == 0) {
+        return this->template get_size_by_time<LastType>(key, ts_us, stable, subgroup_index, shard_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size_by_time(
+        const KeyType& key,
+        const uint64_t& ts_us,
+        const bool stable) {
+    // STEP 1 - verify the keys
+    if constexpr(!std::is_convertible_v<KeyType, std::string>) {
+        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
+    }
+
+    // STEP 2 - get shard
+    uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(key);
+
+    // STEP 3 - call recursive get
+    return this->template type_recursive_get_size_by_time<KeyType, CascadeTypes...>(subgroup_type_index, key, ts_us, stable, subgroup_index, shard_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::list_keys(
+        const persistent::version_t& version,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_LIST_KEYS_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        try {
+            // do p2p list_keys as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id, "", version, stable);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p list_keys as an external client.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id, "", version, stable);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        return caller.template p2p_send<RPC_NAME(list_keys)>(node_id, "", version, stable);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_list_keys(
+        uint32_t type_index,
+        const persistent::version_t& version,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __list_keys<FirstType>(version, stable, object_pool_pathname);
+    } else {
+        return this->template type_recursive_list_keys<SecondType, RestTypes...>(type_index - 1, version, stable, object_pool_pathname);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_list_keys(
+        uint32_t type_index,
+        const persistent::version_t& version,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __list_keys<LastType>(version, stable, object_pool_pathname);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__list_keys(
+        const persistent::version_t& version,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    uint32_t subgroup_index = opm.subgroup_index;
+    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
+    for(uint32_t shard_index = 0; shard_index < shards; shard_index++) {
+        if(!is_external_client()) {
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+            try {
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                    node_id = group_ptr->get_my_id();
+                }
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id, object_pool_pathname, version, stable);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id, object_pool_pathname, version, stable);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            }
+        } else {
+            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+            auto shard_keys = caller.template p2p_send<RPC_NAME(list_keys)>(node_id, object_pool_pathname, version, stable);
+            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+        }
+    }
+    return result;
+}
+
+template <typename... CascadeTypes>
+auto ServiceClient<CascadeTypes...>::list_keys(
+        const persistent::version_t& version,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    volatile uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(object_pool_pathname + "/_");
+    return this->template type_recursive_list_keys<CascadeTypes...>(subgroup_type_index, version, stable, object_pool_pathname);
+}
+
+template <typename ReturnType>
+inline ReturnType wait_for_future(derecho::rpc::QueryResults<ReturnType>& result) {
+    // iterate through ReplyMap
+    for(auto& reply_future : result.get()) {
+        ReturnType reply = static_cast<ReturnType>(reply_future.second.get());
+        return reply;
+    }
+    return ReturnType();
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+std::vector<KeyType> ServiceClient<CascadeTypes...>::wait_list_keys(
+        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<KeyType>>>>& future) {
+    std::vector<KeyType> result;
+    // iterate over each shard's Query result
+    for(auto& query_result : future) {
+        std::vector<KeyType> reply = wait_for_future<std::vector<KeyType>>(*(query_result.get()));
+        std::move(reply.begin(), reply.end(), std::back_inserter(result));
+    }
+    return result;
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::multi_list_keys(
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_LIST_KEYS_START, 0);
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        try {
+            // do p2p multi_list_keys as a subgroup member.
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, "");
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p multi_list_keys as an external client.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+            return subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, "");
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        return caller.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, "");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_multi_list_keys(
+        uint32_t type_index,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __multi_list_keys<FirstType>(object_pool_pathname);
+    } else {
+        return this->template type_recursive_multi_list_keys<SecondType, RestTypes...>(type_index - 1, object_pool_pathname);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_multi_list_keys(
+        uint32_t type_index,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __multi_list_keys<LastType>(object_pool_pathname);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__multi_list_keys(const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    uint32_t subgroup_index = opm.subgroup_index;
+    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
+    for(uint32_t shard_index = 0; shard_index < shards; shard_index++) {
+        if(!is_external_client()) {
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, object_pool_pathname);
+            try {
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                    node_id = group_ptr->get_my_id();
+                }
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, object_pool_pathname);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, object_pool_pathname);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            }
+        } else {
+            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, object_pool_pathname);
+            auto shard_keys = caller.template p2p_send<RPC_NAME(multi_list_keys)>(node_id, object_pool_pathname);
+            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+        }
+    }
+    return result;
+}
+
+template <typename... CascadeTypes>
+auto ServiceClient<CascadeTypes...>::multi_list_keys(const std::string& object_pool_pathname) {
+    volatile uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(object_pool_pathname + "/_");
+    return this->template type_recursive_multi_list_keys<CascadeTypes...>(subgroup_type_index, object_pool_pathname);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::list_keys_by_time(
+        const uint64_t& ts_us,
+        const bool stable,
+        uint32_t subgroup_index,
+        uint32_t shard_index) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        try {
+            // do p2p list_keys_by_time as a subgroup member
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                node_id = group_ptr->get_my_id();
+            }
+            return subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, "", ts_us, stable);
+        } catch(derecho::invalid_subgroup_exception& ex) {
+            // do p2p list_keys_by_time as an external client.
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, "", ts_us, stable);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        // call as an external client (ExternalClientCaller).
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        return caller.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, "", ts_us, stable);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+auto ServiceClient<CascadeTypes...>::type_recursive_list_keys_by_time(
+        uint32_t type_index,
+        const uint64_t& ts_us,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __list_keys_by_time<FirstType>(ts_us, stable, object_pool_pathname);
+    } else {
+        return this->template type_recursive_list_keys_by_time<SecondType, RestTypes...>(type_index - 1, ts_us, stable, object_pool_pathname);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+auto ServiceClient<CascadeTypes...>::type_recursive_list_keys_by_time(
+        uint32_t type_index,
+        const uint64_t& ts_us,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    if(type_index == 0) {
+        return this->template __list_keys_by_time<LastType>(ts_us, stable, object_pool_pathname);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__list_keys_by_time(
+        const uint64_t& ts_us,
+        const bool stable,
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    uint32_t subgroup_index = opm.subgroup_index;
+    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
+    for(uint32_t shard_index = 0; shard_index < shards; shard_index++) {
+        if(!is_external_client()) {
+            std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, object_pool_pathname);
+            try {
+                // do p2p list_keys_by_time as a subgroup member.
+                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+                if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+                    node_id = group_ptr->get_my_id();
+                }
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, object_pool_pathname, ts_us, stable);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            } catch(derecho::invalid_subgroup_exception& ex) {
+                // do p2p list_keys_by_time as an external client.
+                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, object_pool_pathname, ts_us, stable);
+                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+            }
+        } else {
+            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+            // call as an external client (ExternalClientCaller).
+            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, object_pool_pathname);
+            auto shard_keys = caller.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id, object_pool_pathname, ts_us, stable);
+            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
+        }
+    }
+    return result;
+}
+
+template <typename... CascadeTypes>
+auto ServiceClient<CascadeTypes...>::list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname) {
+    volatile uint32_t subgroup_type_index, subgroup_index, shard_index;
+    std::tie(subgroup_type_index, subgroup_index, shard_index) = this->template key_to_shard(object_pool_pathname + "/_");
+    return this->template type_recursive_list_keys_by_time<CascadeTypes...>(subgroup_type_index, ts_us, stable, object_pool_pathname);
+}
+
+template <typename... CascadeTypes>
+void ServiceClient<CascadeTypes...>::refresh_object_pool_metadata_cache() {
+    std::unordered_map<std::string, ObjectPoolMetadataCacheEntry> refreshed_metadata;
+    uint32_t num_shards = this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
+    for(uint32_t shard = 0; shard < num_shards; shard++) {
+        auto results = this->template multi_list_keys<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX, shard);
+        for(auto& reply : results.get()) {         // only once
+            for(auto& key : reply.second.get()) {  // iterate over keys
+                // we only read the stable version.
+                auto opm_result = this->template get<CascadeMetadataService<CascadeTypes...>>(key, CURRENT_VERSION, false, METADATA_SERVICE_SUBGROUP_INDEX, shard);
+                for(auto& opm_reply : opm_result.get()) {  // only once
+                    refreshed_metadata.emplace(key, opm_reply.second.get());
+                    break;
+                }
+            }
+            break;
+        }
+    }
+
+    std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
+    this->object_pool_metadata_cache = std::move(refreshed_metadata);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::create_object_pool(
+        const std::string& pathname, const uint32_t subgroup_index,
+        const sharding_policy_t sharding_policy, const std::unordered_map<std::string, uint32_t>& object_locations,
+        const std::string& affinity_set_regex) {
+    uint32_t subgroup_type_index = ObjectPoolMetadata<CascadeTypes...>::template get_subgroup_type_index<SubgroupType>();
+    if(subgroup_type_index == ObjectPoolMetadata<CascadeTypes...>::invalid_subgroup_type_index) {
+        dbg_default_crit("Create object pool failed because of invalid SubgroupType:{}", typeid(SubgroupType).name());
+        throw derecho::derecho_exception(std::string("Create object pool failed because SubgroupType is invalid:") + typeid(SubgroupType).name());
+    }
+    ObjectPoolMetadata<CascadeTypes...> opm(pathname, subgroup_type_index, subgroup_index, sharding_policy, object_locations, affinity_set_regex, false);
+    // clear local cache entry.
+    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
+    if(object_pool_metadata_cache.find(pathname) == object_pool_metadata_cache.end()) {
+        rlck.unlock();
+    } else {
+        rlck.unlock();
+        std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
+        object_pool_metadata_cache.erase(pathname);
+    }
+    // determine the shard index by hashing
+    uint32_t metadata_service_shard_index = std::hash<std::string>{}(pathname) % this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
+
+    return this->template put<CascadeMetadataService<CascadeTypes...>>(opm, METADATA_SERVICE_SUBGROUP_INDEX, metadata_service_shard_index);
+}
+
+template <typename... CascadeTypes>
+derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove_object_pool(const std::string& pathname) {
+    // determine the shard index by hashing
+    uint32_t metadata_service_shard_index = std::hash<std::string>{}(pathname) % this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
+
+    // check if this object pool exist in metadata service.
+    auto opm = find_object_pool(pathname);
+    // remove it from local cache.
+    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
+    if(object_pool_metadata_cache.find(pathname) == object_pool_metadata_cache.end()) {
+        // no entry in cache
+        rlck.unlock();
+    } else {
+        // remove from cache
+        rlck.unlock();
+        std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
+        object_pool_metadata_cache.erase(pathname);
+        wlck.unlock();
+    }
+    if(opm.is_valid() && !opm.is_null()) {
+        if(opm.deleted) {
+            throw derecho::derecho_exception(std::string("object pool:") + pathname + " has been deleted already.");
+        }
+        opm.deleted = true;
+        opm.set_previous_version(CURRENT_VERSION, opm.version);  // only check previous_version_by_key
+        return this->template put<CascadeMetadataService<CascadeTypes...>>(opm, METADATA_SERVICE_SUBGROUP_INDEX, metadata_service_shard_index);
+    }
+
+    // we didn't find the object pool, but we do the normal 'remove', which has no effect but return a version.
+    dbg_default_warn("deleteing a non-existing objectpool:{}.", pathname);
+    return this->template remove<CascadeMetadataService<CascadeTypes...>>(pathname, METADATA_SERVICE_SUBGROUP_INDEX, metadata_service_shard_index);
+}
+
+template <typename... CascadeTypes>
+ObjectPoolMetadata<CascadeTypes...> ServiceClient<CascadeTypes...>::internal_find_object_pool(
+        const std::string& pathname,
+        std::shared_lock<std::shared_mutex>& rlck) {
+    auto components = str_tokenizer(pathname);
+    std::string prefix;
+    for(const auto& comp : components) {
+        prefix = prefix + PATH_SEPARATOR + comp;
+        if(object_pool_metadata_cache.find(prefix) != object_pool_metadata_cache.end()) {
+            return object_pool_metadata_cache.at(prefix).opm;
+        }
+    }
+    rlck.unlock();
+
+    // refresh and try again.
+    refresh_object_pool_metadata_cache();
+    prefix = "";
+    rlck.lock();
+    for(const auto& comp : components) {
+        prefix = prefix + PATH_SEPARATOR + comp;
+        if(object_pool_metadata_cache.find(prefix) != object_pool_metadata_cache.end()) {
+            return object_pool_metadata_cache.at(prefix).opm;
+        }
+    }
+    return ObjectPoolMetadata<CascadeTypes...>::IV;
+}
+
+template <typename... CascadeTypes>
+ObjectPoolMetadata<CascadeTypes...> ServiceClient<CascadeTypes...>::find_object_pool(const std::string& pathname) {
+    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
+    return this->internal_find_object_pool(pathname, rlck);
+}
+
+template <typename... CascadeTypes>
+template <typename KeyType>
+std::pair<ObjectPoolMetadata<CascadeTypes...>, std::string> ServiceClient<CascadeTypes...>::find_object_pool_and_affinity_set_by_key(
+        const KeyType& key) {
+    std::string object_pool_pathname = get_pathname<KeyType>(key);
+    if(object_pool_pathname.empty()) {
+        throw derecho::derecho_exception(std::string("Key:") + key + " does not belong to any object pool.");
+    }
+
+    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
+    auto opm = this->internal_find_object_pool(object_pool_pathname, rlck);
+
+    std::string affinity_set = "";
+    if(opm.is_valid() && !opm.is_null() && !opm.deleted) {
+        affinity_set = object_pool_metadata_cache.at(opm.pathname).to_affinity_set(key);
+    }
+
+    return {opm, affinity_set};
+}
+
+template <typename... CascadeTypes>
+std::vector<std::string> ServiceClient<CascadeTypes...>::list_object_pools(bool include_deleted, bool refresh) {
+    if(refresh) {
+        this->refresh_object_pool_metadata_cache();
+    }
+
+    std::vector<std::string> ret;
+    std::shared_lock rlck(this->object_pool_metadata_cache_mutex);
+    for(auto& op : this->object_pool_metadata_cache) {
+        if(op.second.opm.deleted) {
+            if(include_deleted) {
+                ret.emplace_back(op.first + "(!)");
+            }
+        } else {
+            ret.emplace_back(op.first);
+        }
+    }
+
+    return ret;
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+bool ServiceClient<CascadeTypes...>::register_notification_handler(
+        const cascade_notification_handler_t& handler,
+        const uint32_t subgroup_index) {
+    return register_notification_handler<SubgroupType>(handler, std::string{}, subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+bool ServiceClient<CascadeTypes...>::register_notification_handler(
+        const cascade_notification_handler_t& handler,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index) {
+    if(!is_external_client()) {
+        throw derecho_exception(std::string(__PRETTY_FUNCTION__) + "Cannot register notification handler because external_group_ptr is null.");
+    }
+
+    std::unique_lock<std::mutex> type_registry_lock(this->notification_handler_registry_mutex);
+    auto& per_type_registry = notification_handler_registry.template get<SubgroupType>();
+    // Register Cascade's root handler:
+    // if subgroup_index exists in the per_type_registry, Cascade's root handler is registered already.
+    if(per_type_registry.find(subgroup_index) == per_type_registry.cend()) {
+        per_type_registry.emplace(subgroup_index, SubgroupNotificationHandler<SubgroupType>{});
+        // register to subgroup_caller
+        auto& subgroup_caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        // to do ... register it.
+        per_type_registry.at(subgroup_index).initialize(subgroup_caller);
+    }
+    auto& subgroup_handlers = per_type_registry.at(subgroup_index);
+
+    // Register the handler
+    std::lock_guard<std::mutex> subgroup_handlers_lock(*subgroup_handlers.object_pool_notification_handlers_mutex);
+    bool ret = (subgroup_handlers.object_pool_notification_handlers.find(object_pool_pathname) != subgroup_handlers.object_pool_notification_handlers.cend());
+
+    if(handler) {
+        subgroup_handlers.object_pool_notification_handlers[object_pool_pathname] = handler;
+    } else {
+        subgroup_handlers.object_pool_notification_handlers[object_pool_pathname].reset();
+    }
+    return ret;
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+bool ServiceClient<CascadeTypes...>::type_recursive_register_notification_handler(
+        uint32_t type_index,
+        const cascade_notification_handler_t& handler,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index) {
+    if(type_index == 0) {
+        return this->template register_notification_handler<FirstType>(handler, object_pool_pathname, subgroup_index);
+    } else {
+        return this->template type_recursive_register_notification_handler<SecondType, RestTypes...>(
+                type_index - 1, handler, object_pool_pathname, subgroup_index);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+bool ServiceClient<CascadeTypes...>::type_recursive_register_notification_handler(
+        uint32_t type_index,
+        const cascade_notification_handler_t& handler,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index) {
+    if(type_index == 0) {
+        return this->template register_notification_handler<LastType>(handler, object_pool_pathname, subgroup_index);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+bool ServiceClient<CascadeTypes...>::register_notification_handler(
+        const cascade_notification_handler_t& handler,
+        const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+
+    return this->template type_recursive_register_notification_handler<CascadeTypes...>(
+            opm.subgroup_type_index, handler, object_pool_pathname, opm.subgroup_index);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::notify(
+        const Blob& msg,
+        const uint32_t subgroup_index,
+        const node_id_t client_id) const {
+    notify<SubgroupType>(msg, "", subgroup_index, client_id);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::notify(
+        const Blob& msg,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index,
+        const node_id_t client_id) const {
+    if(is_external_client()) {
+        throw derecho_exception(std::string(__PRETTY_FUNCTION__) + "Cannot notify an external client from an external client.");
+    }
+
+    auto& client_handle = group_ptr->template get_client_callback<SubgroupType>(subgroup_index);
+
+    // TODO: redesign to avoid memory copies.
+    CascadeNotificationMessage cascade_notification_message(object_pool_pathname, msg);
+    derecho::NotificationMessage derecho_notification_message(CASCADE_NOTIFICATION_MESSAGE_TYPE, mutils::bytes_size(cascade_notification_message));
+    mutils::to_bytes(cascade_notification_message, derecho_notification_message.body);
+
+    client_handle.template p2p_send<RPC_NAME(notify)>(client_id, derecho_notification_message);
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+void ServiceClient<CascadeTypes...>::type_recursive_notify(
+        uint32_t type_index,
+        const Blob& msg,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index,
+        const node_id_t client_id) const {
+    if(type_index == 0) {
+        this->template notify<FirstType>(msg, object_pool_pathname, subgroup_index, client_id);
+    } else {
+        this->template type_recursive_notify<SecondType, RestTypes...>(type_index - 1, msg, object_pool_pathname, subgroup_index, client_id);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+void ServiceClient<CascadeTypes...>::type_recursive_notify(
+        uint32_t type_index,
+        const Blob& msg,
+        const std::string& object_pool_pathname,
+        const uint32_t subgroup_index,
+        const node_id_t client_id) const {
+    if(type_index == 0) {
+        this->template notify<LastType>(msg, object_pool_pathname, subgroup_index, client_id);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+void ServiceClient<CascadeTypes...>::notify(
+        const Blob& msg,
+        const std::string& object_pool_pathname,
+        const node_id_t client_id) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+    this->template type_recursive_notify<CascadeTypes...>(opm.subgroup_type_index, msg, object_pool_pathname, opm.subgroup_index, client_id);
+}
+
+#ifdef ENABLE_EVALUATION
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::dump_timestamp(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template ordered_send<RPC_NAME(ordered_dump_timestamp_log)>(filename);
+        } else {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, filename);
+            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log)>(node_id, filename);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, filename);
+        return caller.template p2p_send<RPC_NAME(dump_timestamp_log)>(node_id, filename);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename FirstType, typename SecondType, typename... RestTypes>
+void ServiceClient<CascadeTypes...>::type_recursive_dump(
+        uint32_t type_index,
+        uint32_t subgroup_index,
+        const std::string& filename) {
+    if(type_index == 0) {
+        this->template dump_timestamp<FirstType>(subgroup_index, filename);
+    } else {
+        this->template type_recursive_dump<SecondType, RestTypes...>(type_index - 1, subgroup_index, filename);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename LastType>
+void ServiceClient<CascadeTypes...>::type_recursive_dump(
+        uint32_t type_index,
+        uint32_t subgroup_index,
+        const std::string& filename) {
+    if(type_index == 0) {
+        this->template dump_timestamp<LastType>(subgroup_index, filename);
+    } else {
+        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
+    }
+}
+
+template <typename... CascadeTypes>
+void ServiceClient<CascadeTypes...>::dump_timestamp(const std::string& filename, const std::string& object_pool_pathname) {
+    auto opm = find_object_pool(object_pool_pathname);
+    if(!opm.is_valid() || opm.is_null() || opm.deleted) {
+        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
+    }
+
+    this->template type_recursive_dump<CascadeTypes...>(opm.subgroup_type_index, opm.subgroup_index, filename);
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+void ServiceClient<CascadeTypes...>::dump_timestamp(const uint32_t subgroup_index, const std::string& filename) {
+    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
+    for(uint32_t shard_index = 0; shard_index < shards; shard_index++) {
+        auto result = this->template dump_timestamp<SubgroupType>(filename, subgroup_index, shard_index);
+        result.get();
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::dump_timestamp_workaround(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index, const node_id_t node_id) {
+    if(!is_external_client()) {
+        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
+        if(static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
+            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id, filename);
+        } else {
+            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
+            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id, filename);
+        }
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        return caller.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id, filename);
+    }
+}
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+derecho::rpc::QueryResults<double> ServiceClient<CascadeTypes...>::perf_put(const uint32_t message_size, const uint64_t duration_sec, const uint32_t subgroup_index, const uint32_t shard_index) {
+    if(!is_external_client()) {
+        // 'perf_put' must be issued from an external client.
+        throw derecho::derecho_exception{"perf_put must be issued from an external client."};
+    } else {
+        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
+        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
+        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index, shard_index, 0);
+        return caller.template p2p_send<RPC_NAME(perf_put)>(node_id, message_size, duration_sec);
+    }
+}
+#endif  // ENABLE_EVALUATION
+
+#ifndef __WITHOUT_SERVICE_SINGLETONS__
+template <typename... CascadeTypes>
+const std::vector<std::type_index> ServiceClient<CascadeTypes...>::subgroup_type_order{typeid(CascadeTypes)...};
+
+template <typename... CascadeTypes>
+const uint32_t ServiceClient<CascadeTypes...>::invalid_subgroup_type_index = 0xffffffff;
+
+template <typename... CascadeTypes>
+template <typename SubgroupType>
+uint32_t ServiceClient<CascadeTypes...>::get_subgroup_type_index() {
+    uint32_t index = 0;
+    while(index < subgroup_type_order.size()) {
+        if(std::type_index(typeid(SubgroupType)) == subgroup_type_order.at(index)) {
+            return index;
+        }
+        index++;
+    }
+    return invalid_subgroup_type_index;
+}
+
+template <typename... CascadeTypes>
+std::unique_ptr<ServiceClient<CascadeTypes...>> ServiceClient<CascadeTypes...>::service_client_singleton_ptr;
+
+template <typename... CascadeTypes>
+std::mutex ServiceClient<CascadeTypes...>::singleton_mutex;
+
+template <typename... CascadeTypes>
+void ServiceClient<CascadeTypes...>::initialize(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr) {
+    std::lock_guard<std::mutex> lock_guard(singleton_mutex);
+    if(!service_client_singleton_ptr) {
+        dbg_default_trace("initializing ServiceClient singleton as cascade member, group pointer={:p}", static_cast<void*>(_group_ptr));
+        service_client_singleton_ptr = std::unique_ptr<ServiceClient<CascadeTypes...>>(new ServiceClient<CascadeTypes...>(_group_ptr));
+    }
+}
+
+template <typename... CascadeTypes>
+ServiceClient<CascadeTypes...>& ServiceClient<CascadeTypes...>::get_service_client() {
+    if(!service_client_singleton_ptr) {
+        std::lock_guard<std::mutex> lock_guard(singleton_mutex);
+        // test again in case another thread has initialized it already.
+        if(!service_client_singleton_ptr) {
+            dbg_default_trace("initializing ServiceClient singleton as external client");
+            service_client_singleton_ptr = std::unique_ptr<ServiceClient<CascadeTypes...>>(new ServiceClient<CascadeTypes...>(nullptr));
+        }
+    }
+    return *service_client_singleton_ptr;
+}
+#endif  //__WITHOUT_SERVICE_SINGLETONS__
+
+}  // namespace cascade
+}  // namespace derecho
diff --git a/include/cascade/detail/service_impl.hpp b/include/cascade/detail/service_impl.hpp
index 7d992e54..ef407301 100644
--- a/include/cascade/detail/service_impl.hpp
+++ b/include/cascade/detail/service_impl.hpp
@@ -1,14 +1,14 @@
-#include <derecho/core/derecho_exception.hpp>
-#include <derecho/core/detail/rpc_utils.hpp>
-#include <derecho/core/notification.hpp>
-#include <vector>
-#include <map>
-#include <typeindex>
-#include <variant>
-#include <derecho/core/derecho.hpp>
 #include <cascade/config.h>
 #include <cascade/data_flow_graph.hpp>
+#include <cascade/service_client.hpp>
+#include <cascade/utils.hpp>
 #include <chrono>
+#include <derecho/core/derecho.hpp>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <variant>
+#include <vector>
 
 using namespace std::chrono_literals;
 
@@ -139,2107 +139,6 @@ void Service<CascadeTypes...>::wait() {
 }
 #endif//__WITHOUT_SERVICE_SINGLETONS__
 
-template <typename CascadeType>
-std::unique_ptr<CascadeType> client_stub_factory() {
-    return std::make_unique<CascadeType>();
-}
-
-
-#ifdef ENABLE_EVALUATION
-#define LOG_SERVICE_CLIENT_TIMESTAMP(tag,msgid) \
-    TimestampLogger::log(tag,this->get_my_id(),msgid);
-#else
-#define LOG_SERVICE_CLIENT_TIMESTAMP(tag,msgid)
-#endif
-
-
-template <typename... CascadeTypes>
-ServiceClient<CascadeTypes...>::ServiceClient(derecho::Group<CascadeMetadataService<CascadeTypes...>,CascadeTypes...>* _group_ptr):
-    external_group_ptr(nullptr),
-    group_ptr(_group_ptr) {
-    if (group_ptr == nullptr) {
-        this->external_group_ptr =
-            std::make_unique<derecho::ExternalGroupClient<CascadeMetadataService<CascadeTypes...>,CascadeTypes...>>(
-                    client_stub_factory<CascadeMetadataService<CascadeTypes...>>,
-                    client_stub_factory<CascadeTypes>...);
-    }
-}
-
-template <typename... CascadeTypes>
-bool ServiceClient<CascadeTypes...>::is_external_client() const {
-    return (group_ptr == nullptr) && (external_group_ptr != nullptr);
-}
-
-template <typename... CascadeTypes>
-node_id_t ServiceClient<CascadeTypes...>::get_my_id() const {
-    if (!is_external_client()) {
-        return group_ptr->get_my_id();
-    } else {
-        return external_group_ptr->get_my_id();
-    }
-}
-
-template <typename... CascadeTypes>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_members() const {
-    if (!is_external_client()) {
-        return group_ptr->get_members();
-    } else {
-        return external_group_ptr->get_members();
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_shard_members(uint32_t subgroup_index, uint32_t shard_index) const {
-    if (!is_external_client()) {
-        std::vector<std::vector<node_id_t>> subgroup_members = group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
-        if (subgroup_members.size() > shard_index) {
-            return subgroup_members[shard_index];
-        } else {
-            return {};
-        }
-    } else {
-        return external_group_ptr->template get_shard_members<SubgroupType>(subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::type_recursive_get_shard_members(uint32_t type_index,
-        uint32_t subgroup_index, uint32_t shard_index) const {
-    if (type_index == 0) {
-        return this->template get_shard_members<FirstType>(subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get_shard_members<SecondType, RestTypes...>(type_index-1,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::type_recursive_get_shard_members(uint32_t type_index,
-        uint32_t subgroup_index, uint32_t shard_index) const {
-    if (type_index == 0) {
-        return this->template get_shard_members<LastType>(subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
-    }
-}
-
-template <typename... CascadeTypes>
-std::vector<node_id_t> ServiceClient<CascadeTypes...>::get_shard_members(
-        const std::string& object_pool_pathname,uint32_t shard_index) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    return this->template type_recursive_get_shard_members<CascadeTypes...>(opm.subgroup_type_index,opm.subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::get_subgroup_members(uint32_t subgroup_index) const {
-    if (!is_external_client()) {
-        return group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
-    } else {
-        return external_group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType,typename...RestTypes>
-std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::type_recursive_get_subgroup_members(
-        uint32_t type_index, uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_subgroup_members<FirstType>(subgroup_index);
-    } else {
-        return this->template type_recursive_get_subgroup_members<SecondType,RestTypes...>(type_index-1,subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::type_recursive_get_subgroup_members(
-        uint32_t type_index, uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_subgroup_members<LastType>(subgroup_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
-    }
-}
-
-template <typename... CascadeTypes>
-std::vector<std::vector<node_id_t>> ServiceClient<CascadeTypes...>::get_subgroup_members(
-        const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    return this->template type_recursive_get_subgroup_members<CascadeTypes...>(opm.subgroup_type_index,opm.subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-uint32_t ServiceClient<CascadeTypes...>::get_number_of_subgroups() const {
-    if (!is_external_client()) {
-        return group_ptr->template get_num_subgroups<SubgroupType>();
-    } else {
-        return external_group_ptr->template get_number_of_subgroups<SubgroupType>();
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards(uint32_t subgroup_index) const {
-    if (!is_external_client()) {
-        return group_ptr->template get_subgroup_members<SubgroupType>(subgroup_index).size();
-    } else {
-        return external_group_ptr->template get_number_of_shards<SubgroupType>(subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-uint32_t ServiceClient<CascadeTypes...>::type_recursive_get_number_of_shards (
-        uint32_t type_index,uint32_t subgroup_index) const {
-    if (type_index == 0 ) {
-        return this->template get_number_of_shards<FirstType>(subgroup_index);
-    } else {
-        return this->template type_recursive_get_number_of_shards<SecondType,RestTypes...>(type_index-1,subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-uint32_t ServiceClient<CascadeTypes...>::type_recursive_get_number_of_shards (
-        uint32_t type_index,uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_number_of_shards<LastType>(subgroup_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
-    }
-}
-
-template <typename... CascadeTypes>
-uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards (
-        uint32_t subgroup_type_index,uint32_t subgroup_index) const {
-    return type_recursive_get_number_of_shards<CascadeTypes...>(subgroup_type_index,subgroup_index);
-}
-
-template <typename... CascadeTypes>
-uint32_t ServiceClient<CascadeTypes...>::get_number_of_shards (
-        const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    return get_number_of_shards(opm.subgroup_type_index,opm.subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-int32_t ServiceClient<CascadeTypes...>::get_my_shard(uint32_t subgroup_index) const {
-    if (!is_external_client()) {
-        return group_ptr->template get_my_shard<SubgroupType>(subgroup_index);
-    } else {
-        return -1;
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-int32_t ServiceClient<CascadeTypes...>::type_recursive_get_my_shard (
-        uint32_t type_index,uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_my_shard<FirstType>(subgroup_index);
-    } else {
-        return this->template type_recursive_get_number_of_shards<SecondType,RestTypes...>(type_index-1,subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-int32_t ServiceClient<CascadeTypes...>::type_recursive_get_my_shard (
-        uint32_t type_index, uint32_t subgroup_index) const {
-    if (type_index == 0) {
-        return this->template get_my_shard<LastType>(subgroup_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + " type index is out of boundary");
-    }
-}
-
-template <typename... CascadeTypes>
-int32_t ServiceClient<CascadeTypes...>::get_my_shard (
-        uint32_t subgroup_type_index, uint32_t subgroup_index) const {
-    return this->template type_recursive_get_my_shard<CascadeTypes...>(subgroup_type_index,subgroup_index);
-}
-
-template <typename... CascadeTypes>
-int32_t ServiceClient<CascadeTypes...>::get_my_shard (
-        const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    return get_my_shard(opm.subgroup_type_index,opm.subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::set_member_selection_policy(uint32_t subgroup_index,uint32_t shard_index,
-        ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id) {
-    // write lock policies
-    std::unique_lock wlck(this->member_selection_policies_mutex);
-    // update map
-    this->member_selection_policies[std::make_tuple(std::type_index(typeid(SubgroupType)),subgroup_index,shard_index)] =
-            std::make_tuple(policy,user_specified_node_id);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::tuple<ShardMemberSelectionPolicy,node_id_t> ServiceClient<CascadeTypes...>::get_member_selection_policy(
-        uint32_t subgroup_index, uint32_t shard_index) const {
-    // read lock policies
-    std::shared_lock rlck(this->member_selection_policies_mutex);
-    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)),subgroup_index,shard_index);
-    // read map
-    if (member_selection_policies.find(key) != member_selection_policies.end()) {
-        return member_selection_policies.at(key);
-    } else {
-        return std::make_tuple(DEFAULT_SHARD_MEMBER_SELECTION_POLICY,INVALID_NODE_ID);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::refresh_member_cache_entry(uint32_t subgroup_index,
-                                                          uint32_t shard_index) {
-    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)),subgroup_index,shard_index);
-    auto members = get_shard_members<SubgroupType>(subgroup_index,shard_index);
-    std::shared_lock rlck(member_cache_mutex);
-    if (member_cache.find(key) == member_cache.end()) {
-        rlck.unlock();
-        std::unique_lock wlck(member_cache_mutex);
-        member_cache[key] = members;
-    } else {
-        member_cache[key].swap(members);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-std::tuple<uint32_t,uint32_t,uint32_t> ServiceClient<CascadeTypes...>::key_to_shard(
-        const KeyType& key,
-        bool check_object_location) {
-
-    auto pair = find_object_pool_and_affinity_set_by_key(key);
-
-    auto& opm = std::get<0>(pair);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to identify the object_pool from key:" + key);
-    }
-    auto& affinity_set = std::get<1>(pair);
-
-    return std::tuple<uint32_t,uint32_t,uint32_t>{opm.subgroup_type_index,opm.subgroup_index,
-        opm.key_to_shard_index(key,affinity_set,get_number_of_shards(opm.subgroup_type_index,opm.subgroup_index),check_object_location)};
-}
-
-template <typename... CascadeTypes>
-ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::ObjectPoolMetadataCacheEntry(
-        const ObjectPoolMetadata<CascadeTypes...>& _opm): opm(_opm), database(nullptr) {
-    if (opm.affinity_set_regex.size() > 0) {
-        hs_compile_error_t* compile_err;
-        if (hs_compile(opm.affinity_set_regex.c_str(), HS_FLAG_DOTALL|HS_FLAG_SOM_LEFTMOST, HS_MODE_BLOCK, NULL, &database,
-                       &compile_err) != HS_SUCCESS) {
-            hs_free_compile_error(compile_err);
-            dbg_default_error("Compilation of affinity set regex:" + opm.affinity_set_regex + " failed with message:" +
-                    compile_err->message);
-            throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) +
-                    ": compilation of affinity_set_regex:" +
-                    opm.affinity_set_regex +
-                    " failed with message:" +
-                    compile_err->message);
-        }
-    }
-}
-
-template <typename... CascadeTypes>
-ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::~ObjectPoolMetadataCacheEntry() {
-    if (this->database != nullptr) {
-        hs_free_database(database);
-        this->database = nullptr;
-    }
-}
-
-template <typename... CascadeTypes>
-inline std::string ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::to_affinity_set(
-        const std::string& key_string) {
-    if (key_string.size() > 0 && this->opm.affinity_set_regex.size() > 0) {
-        if (scratch == nullptr) {
-            if (hs_alloc_scratch(database, &scratch) != HS_SUCCESS) {
-                hs_free_database(database);
-                dbg_default_error("failed to allocate hyperscan scratch space.");
-                throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) +
-                        " failed to allocate hyperscan scratch space.");
-            }
-        }
-
-        struct hs_scan_ctxt {
-            unsigned long long from = 0;
-            unsigned long long to = 0;
-        } ctxt;
-
-        hs_scan(database, key_string.c_str(), key_string.size(), HS_FLAG_SOM_LEFTMOST, scratch,
-                [](unsigned int /*id*/, unsigned long long from,
-                   unsigned long long to, unsigned int /*flags*/,
-                   void* ctxt)->int {
-                    struct hs_scan_ctxt* p_hs_ctxt = static_cast<struct hs_scan_ctxt*>(ctxt);
-                    p_hs_ctxt->from = from;
-                    p_hs_ctxt->to = to;
-                    return 0; // do the longest match
-                },
-                &ctxt);
-        if (ctxt.to > ctxt.from) {
-            return key_string.substr(ctxt.from,(ctxt.to-ctxt.from));
-        }
-    }
-
-    return key_string;
-}
-
-template <typename... CascadeTypes>
-thread_local hs_scratch_t* ServiceClient<CascadeTypes...>::ObjectPoolMetadataCacheEntry::scratch = nullptr;
-
-template <typename... CascadeTypes>
-template <typename SubgroupType,typename KeyTypeForHashing>
-node_id_t ServiceClient<CascadeTypes...>::pick_member_by_policy(uint32_t subgroup_index,
-                                                                uint32_t shard_index,
-                                                                const KeyTypeForHashing& key_for_hashing,
-                                                                bool retry) {
-    ShardMemberSelectionPolicy policy;
-    node_id_t last_specified_node_id_or_index;
-
-    std::tie(policy,last_specified_node_id_or_index) = get_member_selection_policy<SubgroupType>(subgroup_index,shard_index);
-
-    if (policy == ShardMemberSelectionPolicy::UserSpecified) {
-        return last_specified_node_id_or_index;
-    }
-
-    auto key = std::make_tuple(std::type_index(typeid(SubgroupType)),subgroup_index,shard_index);
-
-    if (member_cache.find(key) == member_cache.end() || retry) {
-        refresh_member_cache_entry<SubgroupType>(subgroup_index,shard_index);
-    }
-
-    std::shared_lock rlck(member_cache_mutex);
-
-    node_id_t node_id = last_specified_node_id_or_index;
-
-    switch(policy) {
-    case ShardMemberSelectionPolicy::FirstMember:
-        node_id = member_cache.at(key).front();
-        break;
-    case ShardMemberSelectionPolicy::LastMember:
-        node_id = member_cache.at(key).back();
-        break;
-    case ShardMemberSelectionPolicy::Random:
-        node_id = member_cache.at(key)[get_time()%member_cache.at(key).size()]; // use time as random source.
-        break;
-    case ShardMemberSelectionPolicy::FixedRandom:
-        if (node_id == INVALID_NODE_ID || retry) {
-            node_id = member_cache.at(key)[get_time()%member_cache.at(key).size()]; // use time as random source.
-        }
-        break;
-    case ShardMemberSelectionPolicy::RoundRobin:
-        {
-            std::shared_lock rlck(member_selection_policies_mutex);
-            node_id = static_cast<uint32_t>(node_id+1)%member_cache.at(key).size();
-            auto new_policy = std::make_tuple(ShardMemberSelectionPolicy::RoundRobin,node_id);
-            member_selection_policies[key].swap(new_policy);
-        }
-        node_id = member_cache.at(key)[node_id];
-        break;
-    case ShardMemberSelectionPolicy::KeyHashing:
-        {
-            uint64_t hash = 0;
-            if constexpr (std::is_integral_v<KeyTypeForHashing>) {
-                hash = static_cast<uint64_t>(key_for_hashing);
-            } else if constexpr (std::is_convertible_v<KeyTypeForHashing,std::string>) {
-                hash = std::hash<std::string>{}(std::string(key_for_hashing));
-            } else {
-                dbg_default_warn("Key type {} is neither integral nor string, falling back to FirstMember policy. {}:{}",
-                        typeid(KeyTypeForHashing).name(), __FILE__, __LINE__);
-            }
-            node_id = member_cache.at(key)[hash % member_cache.at(key).size()];
-        }
-        break;
-    default:
-        throw derecho::derecho_exception("Unknown member selection policy:" + std::to_string(static_cast<unsigned int>(policy)) );
-    }
-
-    return node_id;
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::put(
-        const typename SubgroupType::ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_PUT_START,
-            (std::is_base_of<IHasMessageID,typename SubgroupType::ObjectType>::value?value.get_message_id():0));
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            // ordered put as a shard member
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template ordered_send<RPC_NAME(ordered_put)>(value,as_trigger);
-        } else {
-            // p2p put
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-            try {
-                // as a subgroup member
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                return subgroup_handle.template p2p_send<RPC_NAME(put)>(node_id,value,as_trigger);
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                // as an external caller
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                return subgroup_handle.template p2p_send<RPC_NAME(put)>(node_id,value,as_trigger);
-            }
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-        return caller.template p2p_send<RPC_NAME(put)>(node_id,value,as_trigger);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_put(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    if (type_index == 0) {
-        return this->template put<FirstType>(value,subgroup_index,shard_index,as_trigger);
-    } else {
-        return this->template type_recursive_put<ObjectType, SecondType, RestTypes...>(type_index-1,value,subgroup_index,shard_index,as_trigger);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename LastType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_put(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    if (type_index == 0) {
-        return this->template put<LastType>(value,subgroup_index,shard_index,as_trigger);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::put(
-        const ObjectType& value, bool as_trigger) {
-
-    // STEP 1 - get key
-    if constexpr (!std::is_base_of_v<ICascadeObject<std::string,ObjectType>,ObjectType>) {
-        throw derecho::derecho_exception(std::string("ServiceClient<>::put() only support object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(value.get_key_ref());
-
-    // STEP 3 - call recursive put
-    return this->template type_recursive_put<ObjectType,CascadeTypes...>(subgroup_type_index,value,subgroup_index,shard_index,as_trigger);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::put_and_forget(
-        const typename SubgroupType::ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_PUT_AND_FORGET_START,
-            (std::is_base_of<IHasMessageID,typename SubgroupType::ObjectType>::value?value.get_message_id():0));
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            // do ordered put as a shard member (Replicated).
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            subgroup_handle.template ordered_send<RPC_NAME(ordered_put_and_forget)>(value,as_trigger);
-        } else {
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-            // do p2p put
-            try{
-                // as a subgroup member
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                subgroup_handle.template p2p_send<RPC_NAME(put_and_forget)>(node_id,value,as_trigger);
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                // as an external caller
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                subgroup_handle.template p2p_send<RPC_NAME(put_and_forget)>(node_id,value,as_trigger);
-            }
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-        caller.template p2p_send<RPC_NAME(put_and_forget)>(node_id,value,as_trigger);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-void ServiceClient<CascadeTypes...>::type_recursive_put_and_forget(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    if (type_index == 0) {
-        put_and_forget<FirstType>(value,subgroup_index,shard_index,as_trigger);
-    } else {
-        type_recursive_put_and_forget<ObjectType,SecondType,RestTypes...>(type_index-1,value,subgroup_index,shard_index,as_trigger);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename LastType>
-void ServiceClient<CascadeTypes...>::type_recursive_put_and_forget(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index,
-        bool as_trigger) {
-    if (type_index == 0) {
-        put_and_forget<LastType>(value,subgroup_index,shard_index,as_trigger);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType>
-void ServiceClient<CascadeTypes...>::put_and_forget(const ObjectType& value,bool as_trigger) {
-    // STEP 1 - get key
-    if constexpr (!std::is_base_of_v<ICascadeObject<std::string,ObjectType>,ObjectType>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(value.get_key_ref());
-
-    // STEP 3 - call recursive put_and_forget
-    this->template type_recursive_put_and_forget<ObjectType,CascadeTypes...>(subgroup_type_index,value,subgroup_index,shard_index,as_trigger);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::trigger_put(
-        const typename SubgroupType::ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_TRIGGER_PUT_START,
-            (std::is_base_of<IHasMessageID,typename SubgroupType::ObjectType>::value?value.get_message_id():0));
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index){
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-            dbg_default_trace("trigger_put to node {}",node_id);
-            return subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(node_id,value);
-        } else {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-            dbg_default_trace("trigger_put to node {}",node_id);
-            return subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(node_id,value);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,value.get_key_ref());
-        dbg_default_trace("trigger_put to node {}",node_id);
-        return caller.template p2p_send<RPC_NAME(trigger_put)>(node_id,value);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::type_recursive_trigger_put(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return trigger_put<FirstType>(value,subgroup_index,shard_index);
-    } else {
-        return type_recursive_trigger_put<ObjectType,SecondType,RestTypes...>(type_index-1,value,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType, typename LastType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::type_recursive_trigger_put(
-        uint32_t type_index,
-        const ObjectType& value,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return trigger_put<LastType>(value,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename ObjectType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::trigger_put(
-        const ObjectType& value) {
-    // STEP 1 - get key
-    if constexpr (!std::is_base_of_v<ICascadeObject<std::string,ObjectType>,ObjectType>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports object of type ICascadeObject<std::string,ObjectType>,but we get ") + typeid(ObjectType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(value.get_key_ref());
-
-    // STEP 3 - call recursive trigger_put
-    return this->template type_recursive_trigger_put<ObjectType,CascadeTypes...>(subgroup_type_index,value,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::collective_trigger_put(
-        const typename SubgroupType::ObjectType& value,
-        uint32_t subgroup_index,
-        std::unordered_map<node_id_t,std::unique_ptr<derecho::rpc::QueryResults<void>>>& nodes_and_futures) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_COLLECTIVE_TRIGGER_PUT_START,
-            (std::is_base_of<IHasMessageID,typename SubgroupType::ObjectType>::value?value.get_message_id():0));
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (group_ptr->template get_my_shard<SubgroupType>(subgroup_index) != -1) {
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            for (auto& kv: nodes_and_futures) {
-                nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
-                        std::move(subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(kv.first,value)));
-            }
-        } else {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            for (auto& kv: nodes_and_futures) {
-                nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
-                        std::move(subgroup_handle.template p2p_send<RPC_NAME(trigger_put)>(kv.first,value)));
-            }
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        for (auto& kv: nodes_and_futures) {
-            nodes_and_futures[kv.first] = std::make_unique<derecho::rpc::QueryResults<void>>(
-                    std::move(caller.template p2p_send<RPC_NAME(trigger_put)>(kv.first,value)));
-        }
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove(
-        const typename SubgroupType::KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_REMOVE_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            // do ordered remove as a member (Replicated).
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template ordered_send<RPC_NAME(ordered_remove)>(key);
-        } else {
-            // do p2p remove
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-            try {
-                // as a subgroup member
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                return subgroup_handle.template p2p_send<RPC_NAME(remove)>(node_id,key);
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                // as an external caller
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                return subgroup_handle.template p2p_send<RPC_NAME(remove)>(node_id,key);
-            }
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(remove)>(node_id,key);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_remove(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template remove<FirstType>(key,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_remove<KeyType,SecondType,RestTypes...>(type_index-1,key,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::type_recursive_remove(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template remove<LastType>(key,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove(
-        const KeyType& key) {
-    // STEP 1 - get key
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive remove
-    return this->template type_recursive_remove<KeyType,CascadeTypes...>(subgroup_type_index,key,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::get(
-        const typename SubgroupType::KeyType& key,
-        const persistent::version_t& version,
-        bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_GET_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p get as a subgroup member
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-                // local get
-                auto obj = subgroup_handle.get_ref().get(key,version,stable);
-                auto pending_results = std::make_shared<PendingResults<const typename SubgroupType::ObjectType>>();
-                pending_results->fulfill_map({node_id});
-                pending_results->set_value(node_id,obj);
-                auto query_results = pending_results->get_future();
-                return std::move(*query_results);
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(get)>(node_id,key,version,stable,false);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(get)>(node_id,key,version,stable,false);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(get)>(node_id,key,version,stable,false);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::multi_get(
-        const typename SubgroupType::KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_GET_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p multi_get as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_get)>(node_id,key);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p multi_get as an external caller.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_get)>(node_id,key);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(multi_get)>(node_id,key);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_get(
-        uint32_t type_index,
-        const KeyType& key,
-        const persistent::version_t& version,
-        bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get<FirstType>(key,version,stable,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get<KeyType,SecondType,RestTypes...>(type_index-1,key,version,stable,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_get(
-        uint32_t type_index,
-        const KeyType& key,
-        const persistent::version_t& version,
-        bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get<LastType>(key,version,stable,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-auto ServiceClient<CascadeTypes...>::get(
-        // const std::decay_t<typename std::result_of_t<decltype(&ObjectType::get_key_ref())>>& key,
-        const KeyType& key,
-        const persistent::version_t& version,
-        bool stable) {
-    // STEP 1 - get key
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_get<KeyType,CascadeTypes...>(subgroup_type_index,key,version,stable,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_multi_get(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template multi_get<FirstType>(key,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_multi_get<KeyType,SecondType,RestTypes...>(type_index-1,key,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_multi_get(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template multi_get<LastType>(key,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-auto ServiceClient<CascadeTypes...>::multi_get(
-        const KeyType& key) {
-    // STEP 1 - get key
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_multi_get<KeyType,CascadeTypes...>(subgroup_type_index,key,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> ServiceClient<CascadeTypes...>::get_by_time(
-        const typename SubgroupType::KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p get_by_time
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                // as a shard member.
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(get_by_time)>(node_id,key,ts_us,stable);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p get_by_time as an external caller
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(get_by_time)>(node_id,key,ts_us,stable);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>();
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(get_by_time)>(node_id,key,ts_us,stable);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_get_by_time(
-        uint32_t type_index,
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_by_time<FirstType>(key,ts_us,stable,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get_by_time<KeyType,SecondType,RestTypes...>(type_index-1,key,ts_us,stable,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_get_by_time(
-        uint32_t type_index,
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_by_time<LastType>(key,ts_us,stable,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-auto ServiceClient<CascadeTypes...>::get_by_time(
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable) {
-    // STEP 1 - get key
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get_by_time
-    return this->template type_recursive_get_by_time<KeyType,CascadeTypes...>(subgroup_type_index,key,ts_us,stable,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size(
-        const typename SubgroupType::KeyType& key,
-        const persistent::version_t& version,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_GET_SIZE_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p get_size as a subgroup_member
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                // as a shard member.
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(get_size)>(node_id,key,version,stable,false);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p get_size as an external caller
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(get_size)>(node_id,key,version,stable,false);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(get_size)>(node_id,key,version,stable,false);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size(
-        uint32_t type_index,
-        const KeyType& key,
-        const persistent::version_t& version,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_size<FirstType>(key,version,stable,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get_size<KeyType,SecondType,RestTypes...>(type_index-1,key,version,stable,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size(
-        uint32_t type_index,
-        const KeyType& key,
-        const persistent::version_t& version,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_size<LastType>(key,version,stable,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size(
-        const KeyType& key,
-        const persistent::version_t& version,
-        const bool stable) {
-    // STEP 1 - verify the keys
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_get_size<KeyType,CascadeTypes...>(subgroup_type_index,key,version,stable,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::multi_get_size(
-        const typename SubgroupType::KeyType& key,
-        uint32_t subgroup_index, uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_GET_SIZE_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p multi_get_size as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_get_size)>(node_id,key);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p multi_get_size as an external caller.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_get_size)>(node_id,key);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(multi_get_size)>(node_id,key);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_multi_get_size(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template multi_get_size<FirstType>(key,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_multi_get_size<KeyType,SecondType,RestTypes...>(type_index-1,key,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_multi_get_size(
-        uint32_t type_index,
-        const KeyType& key,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template multi_get_size<LastType>(key,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::multi_get_size(const KeyType& key) {
-    // STEP 1 - verify the keys
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_multi_get_size<KeyType,CascadeTypes...>(subgroup_type_index,key,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size_by_time(
-        const typename SubgroupType::KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        try {
-            // do p2p get_size_by_time as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(get_size_by_time)>(node_id,key,ts_us,stable);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p get_size_by_time as an external caller.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(get_size_by_time)>(node_id,key,ts_us,stable);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,key);
-        return caller.template p2p_send<RPC_NAME(get_size_by_time)>(node_id,key,ts_us,stable);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size_by_time(
-        uint32_t type_index,
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_size_by_time<FirstType>(key,ts_us,stable,subgroup_index,shard_index);
-    } else {
-        return this->template type_recursive_get_size_by_time<KeyType,SecondType,RestTypes...>(type_index-1,key,ts_us,stable,subgroup_index,shard_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType, typename LastType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::type_recursive_get_size_by_time(
-        uint32_t type_index,
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (type_index == 0) {
-        return this->template get_size_by_time<LastType>(key,ts_us,stable,subgroup_index,shard_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-derecho::rpc::QueryResults<uint64_t> ServiceClient<CascadeTypes...>::get_size_by_time(
-        const KeyType& key,
-        const uint64_t& ts_us,
-        const bool stable) {
-    // STEP 1 - verify the keys
-    if constexpr (!std::is_convertible_v<KeyType,std::string>) {
-        throw derecho::derecho_exception(__PRETTY_FUNCTION__ + std::string(" only supports string key,but we get ") + typeid(KeyType).name());
-    }
-
-    // STEP 2 - get shard
-    uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(key);
-
-    // STEP 3 - call recursive get
-    return this->template type_recursive_get_size_by_time<KeyType,CascadeTypes...>(subgroup_type_index,key,ts_us,stable,subgroup_index,shard_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::list_keys(
-        const persistent::version_t& version,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_LIST_KEYS_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        try {
-            // do p2p list_keys as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id,"",version,stable);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p list_keys as an external client.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id,"",version,stable);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        return caller.template p2p_send<RPC_NAME(list_keys)>(node_id,"",version,stable);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_list_keys(
-        uint32_t type_index,
-        const persistent::version_t& version,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __list_keys<FirstType>(version,stable,object_pool_pathname);
-    } else {
-        return this->template type_recursive_list_keys<SecondType, RestTypes...>(type_index-1, version, stable, object_pool_pathname);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_list_keys(
-        uint32_t type_index,
-        const persistent::version_t& version,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __list_keys<LastType>(version,stable,object_pool_pathname);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__list_keys(
-        const persistent::version_t& version,
-        const bool stable,
-        const std::string& object_pool_pathname){
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    uint32_t subgroup_index = opm.subgroup_index;
-    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
-    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
-    for (uint32_t shard_index = 0; shard_index < shards; shard_index ++){
-        if (!is_external_client()) {
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-            try {
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                    node_id = group_ptr->get_my_id();
-                }
-                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id,object_pool_pathname,version,stable);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                auto shard_keys= subgroup_handle.template p2p_send<RPC_NAME(list_keys)>(node_id,object_pool_pathname,version,stable);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            }
-        } else {
-            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-            auto shard_keys = caller.template p2p_send<RPC_NAME(list_keys)>(node_id,object_pool_pathname,version,stable);
-            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-        }
-    }
-    return result;
-}
-
-template <typename... CascadeTypes>
-auto ServiceClient<CascadeTypes...>::list_keys(
-        const persistent::version_t& version,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    volatile uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(object_pool_pathname+"/_");
-    return this->template type_recursive_list_keys<CascadeTypes...>(subgroup_type_index,version,stable,object_pool_pathname);
-}
-
-template <typename ReturnType>
-inline ReturnType wait_for_future(derecho::rpc::QueryResults<ReturnType>& result){
-    // iterate through ReplyMap
-    for (auto& reply_future: result.get()) {
-        ReturnType reply = static_cast<ReturnType>(reply_future.second.get());
-        return reply;
-    }
-    return ReturnType();
-}
-
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-std::vector<KeyType> ServiceClient<CascadeTypes...>::wait_list_keys(
-        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<KeyType>>>>& future){
-    std::vector<KeyType> result;
-    // iterate over each shard's Query result
-    for(auto& query_result: future){
-        std::vector<KeyType> reply = wait_for_future<std::vector<KeyType>>(*(query_result.get()));
-        std::move(reply.begin(), reply.end(), std::back_inserter(result));
-    }
-    return result;
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::multi_list_keys(
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    LOG_SERVICE_CLIENT_TIMESTAMP(TLT_SERVICE_CLIENT_MULTI_LIST_KEYS_START,0);
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        try {
-            // do p2p multi_list_keys as a subgroup member.
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,"");
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p multi_list_keys as an external client.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-            return subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,"");
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        return caller.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,"");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_multi_list_keys (
-        uint32_t type_index,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __multi_list_keys<FirstType>(object_pool_pathname);
-    } else {
-        return this->template type_recursive_multi_list_keys<SecondType, RestTypes...>(type_index-1,object_pool_pathname);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_multi_list_keys (
-        uint32_t type_index,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __multi_list_keys<LastType>(object_pool_pathname);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__multi_list_keys(const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    uint32_t subgroup_index = opm.subgroup_index;
-    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
-    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
-    for (uint32_t shard_index = 0; shard_index < shards; shard_index ++){
-        if (!is_external_client()) {
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,object_pool_pathname);
-            try {
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                    node_id = group_ptr->get_my_id();
-                }
-                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,object_pool_pathname);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                auto shard_keys= subgroup_handle.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,object_pool_pathname);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            }
-        } else {
-            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,object_pool_pathname);
-            auto shard_keys = caller.template p2p_send<RPC_NAME(multi_list_keys)>(node_id,object_pool_pathname);
-            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-        }
-    }
-    return result;
-}
-
-template <typename... CascadeTypes>
-auto ServiceClient<CascadeTypes...>::multi_list_keys(const std::string& object_pool_pathname) {
-    volatile uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(object_pool_pathname+"/_");
-    return this->template type_recursive_multi_list_keys<CascadeTypes...>(subgroup_type_index,object_pool_pathname);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> ServiceClient<CascadeTypes...>::list_keys_by_time(
-        const uint64_t& ts_us,
-        const bool stable,
-        uint32_t subgroup_index,
-        uint32_t shard_index) {
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        try {
-            // do p2p list_keys_by_time as a subgroup member
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                node_id = group_ptr->get_my_id();
-            }
-            return subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,"",ts_us,stable);
-        } catch (derecho::invalid_subgroup_exception& ex) {
-            // do p2p list_keys_by_time as an external client.
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,"",ts_us,stable);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        // call as an external client (ExternalClientCaller).
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        return caller.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,"",ts_us,stable);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType, typename SecondType, typename... RestTypes>
-auto ServiceClient<CascadeTypes...>::type_recursive_list_keys_by_time(
-        uint32_t type_index,
-        const uint64_t& ts_us,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __list_keys_by_time<FirstType>(ts_us,stable,object_pool_pathname);
-    } else {
-        return this->template type_recursive_list_keys_by_time<SecondType, RestTypes...>(type_index-1,ts_us,stable,object_pool_pathname);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-auto ServiceClient<CascadeTypes...>::type_recursive_list_keys_by_time(
-        uint32_t type_index,
-        const uint64_t& ts_us,
-        const bool stable,
-        const std::string& object_pool_pathname) {
-    if (type_index == 0) {
-        return this->template __list_keys_by_time<LastType>(ts_us,stable,object_pool_pathname);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> ServiceClient<CascadeTypes...>::__list_keys_by_time(
-        const uint64_t& ts_us,
-        const bool stable,
-        const std::string& object_pool_pathname){
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    uint32_t subgroup_index = opm.subgroup_index;
-    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
-    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>> result;
-    for (uint32_t shard_index = 0; shard_index < shards; shard_index ++){
-        if (!is_external_client()) {
-            std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,object_pool_pathname);
-            try {
-                // do p2p list_keys_by_time as a subgroup member.
-                auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-                if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-                    node_id = group_ptr->get_my_id();
-                }
-                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,object_pool_pathname,ts_us,stable);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            } catch (derecho::invalid_subgroup_exception& ex) {
-                // do p2p list_keys_by_time as an external client.
-                auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-                auto shard_keys = subgroup_handle.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,object_pool_pathname,ts_us,stable);
-                result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-            }
-        } else {
-            std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-            // call as an external client (ExternalClientCaller).
-            auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,object_pool_pathname);
-            auto shard_keys = caller.template p2p_send<RPC_NAME(list_keys_by_time)>(node_id,object_pool_pathname,ts_us,stable);
-            result.emplace_back(std::make_unique<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>(std::move(shard_keys)));
-        }
-    }
-    return result;
-}
-
-template <typename... CascadeTypes>
-auto ServiceClient<CascadeTypes...>::list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname) {
-    volatile uint32_t subgroup_type_index,subgroup_index,shard_index;
-    std::tie(subgroup_type_index,subgroup_index,shard_index) = this->template key_to_shard(object_pool_pathname+"/_");
-    return this->template type_recursive_list_keys_by_time<CascadeTypes...>(subgroup_type_index,ts_us,stable,object_pool_pathname);
-}
-
-template <typename... CascadeTypes>
-void ServiceClient<CascadeTypes...>::refresh_object_pool_metadata_cache() {
-    std::unordered_map<std::string,ObjectPoolMetadataCacheEntry> refreshed_metadata;
-    uint32_t num_shards = this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
-    for(uint32_t shard=0;shard<num_shards;shard++) {
-        auto results = this->template multi_list_keys<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX,shard);
-        for (auto& reply : results.get()) { // only once
-            for(auto& key: reply.second.get()) { // iterate over keys
-                // we only read the stable version.
-                auto opm_result = this->template get<CascadeMetadataService<CascadeTypes...>>(key,CURRENT_VERSION,false,METADATA_SERVICE_SUBGROUP_INDEX,shard);
-                for (auto& opm_reply:opm_result.get()) { // only once
-                    refreshed_metadata.emplace(key,opm_reply.second.get());
-                    break;
-                }
-            }
-            break;
-        }
-    }
-
-    std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
-    this->object_pool_metadata_cache = std::move(refreshed_metadata);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::create_object_pool(
-        const std::string& pathname, const uint32_t subgroup_index,
-        const sharding_policy_t sharding_policy, const std::unordered_map<std::string,uint32_t>& object_locations,
-        const std::string& affinity_set_regex) {
-    uint32_t subgroup_type_index = ObjectPoolMetadata<CascadeTypes...>::template get_subgroup_type_index<SubgroupType>();
-    if (subgroup_type_index == ObjectPoolMetadata<CascadeTypes...>::invalid_subgroup_type_index) {
-        dbg_default_crit("Create object pool failed because of invalid SubgroupType:{}", typeid(SubgroupType).name());
-        throw derecho::derecho_exception(std::string("Create object pool failed because SubgroupType is invalid:")+typeid(SubgroupType).name());
-    }
-    ObjectPoolMetadata<CascadeTypes...> opm(pathname,subgroup_type_index,subgroup_index,sharding_policy,object_locations,affinity_set_regex,false);
-    // clear local cache entry.
-    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
-    if (object_pool_metadata_cache.find(pathname)==object_pool_metadata_cache.end()) {
-        rlck.unlock();
-    } else {
-        rlck.unlock();
-        std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
-        object_pool_metadata_cache.erase(pathname);
-    }
-    // determine the shard index by hashing
-    uint32_t metadata_service_shard_index = std::hash<std::string>{}(pathname) % this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
-
-    return this->template put<CascadeMetadataService<CascadeTypes...>>(opm,METADATA_SERVICE_SUBGROUP_INDEX,metadata_service_shard_index);
-}
-
-template <typename... CascadeTypes>
-derecho::rpc::QueryResults<version_tuple> ServiceClient<CascadeTypes...>::remove_object_pool(const std::string& pathname) {
-    // determine the shard index by hashing
-    uint32_t metadata_service_shard_index = std::hash<std::string>{}(pathname) % this->template get_number_of_shards<CascadeMetadataService<CascadeTypes...>>(METADATA_SERVICE_SUBGROUP_INDEX);
-
-
-    // check if this object pool exist in metadata service.
-    auto opm = find_object_pool(pathname);
-    // remove it from local cache.
-    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
-    if (object_pool_metadata_cache.find(pathname) == object_pool_metadata_cache.end()) {
-        // no entry in cache
-        rlck.unlock();
-    } else {
-        // remove from cache
-        rlck.unlock();
-        std::unique_lock<std::shared_mutex> wlck(object_pool_metadata_cache_mutex);
-        object_pool_metadata_cache.erase(pathname);
-        wlck.unlock();
-    }
-    if (opm.is_valid() && !opm.is_null()) {
-        if (opm.deleted) {
-            throw derecho::derecho_exception(std::string("object pool:")+pathname+" has been deleted already.");
-        }
-        opm.deleted = true;
-        opm.set_previous_version(CURRENT_VERSION,opm.version); // only check previous_version_by_key
-        return this->template put<CascadeMetadataService<CascadeTypes...>>(opm,METADATA_SERVICE_SUBGROUP_INDEX,metadata_service_shard_index);
-    }
-
-    // we didn't find the object pool, but we do the normal 'remove', which has no effect but return a version.
-    dbg_default_warn("deleteing a non-existing objectpool:{}.", pathname);
-    return this->template remove<CascadeMetadataService<CascadeTypes...>>(pathname,METADATA_SERVICE_SUBGROUP_INDEX,metadata_service_shard_index);
-}
-
-template <typename... CascadeTypes>
-ObjectPoolMetadata<CascadeTypes...> ServiceClient<CascadeTypes...>::internal_find_object_pool(
-        const std::string& pathname,
-        std::shared_lock<std::shared_mutex>& rlck) {
-    auto components = str_tokenizer(pathname);
-    std::string prefix;
-    for (const auto& comp: components) {
-        prefix = prefix + PATH_SEPARATOR + comp;
-        if (object_pool_metadata_cache.find(prefix) != object_pool_metadata_cache.end()) {
-            return object_pool_metadata_cache.at(prefix).opm;
-        }
-    }
-    rlck.unlock();
-
-    // refresh and try again.
-    refresh_object_pool_metadata_cache();
-    prefix = "";
-    rlck.lock();
-    for (const auto& comp: components) {
-        prefix = prefix + PATH_SEPARATOR + comp;
-        if (object_pool_metadata_cache.find(prefix) != object_pool_metadata_cache.end()) {
-            return object_pool_metadata_cache.at(prefix).opm;
-        }
-    }
-    return ObjectPoolMetadata<CascadeTypes...>::IV;
-}
-
-template <typename... CascadeTypes>
-ObjectPoolMetadata<CascadeTypes...> ServiceClient<CascadeTypes...>::find_object_pool(const std::string& pathname) {
-    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
-    return this->internal_find_object_pool(pathname,rlck);
-}
-
-template <typename... CascadeTypes>
-template <typename KeyType>
-std::pair<ObjectPoolMetadata<CascadeTypes...>,std::string> ServiceClient<CascadeTypes...>::find_object_pool_and_affinity_set_by_key(
-        const KeyType& key) {
-    std::string object_pool_pathname = get_pathname<KeyType>(key);
-    if (object_pool_pathname.empty()) {
-        throw derecho::derecho_exception(std::string("Key:") + key + " does not belong to any object pool.");
-    }
-
-    std::shared_lock<std::shared_mutex> rlck(object_pool_metadata_cache_mutex);
-    auto opm = this->internal_find_object_pool(object_pool_pathname,rlck);
-
-    std::string affinity_set = "";
-    if (opm.is_valid() && !opm.is_null() && !opm.deleted) {
-        affinity_set = object_pool_metadata_cache.at(opm.pathname).to_affinity_set(key);
-    }
-
-    return {opm,affinity_set};
-}
-
-template <typename... CascadeTypes>
-std::vector<std::string> ServiceClient<CascadeTypes...>::list_object_pools(bool include_deleted, bool refresh) {
-    if (refresh) {
-        this->refresh_object_pool_metadata_cache();
-    }
-
-    std::vector<std::string> ret;
-    std::shared_lock rlck(this->object_pool_metadata_cache_mutex);
-    for (auto& op:this->object_pool_metadata_cache) {
-        if (op.second.opm.deleted) {
-            if (include_deleted) {
-                ret.emplace_back(op.first+"(!)");
-            }
-        } else {
-            ret.emplace_back(op.first);
-        }
-    }
-
-    return ret;
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-bool ServiceClient<CascadeTypes...>::register_notification_handler(
-        const cascade_notification_handler_t& handler,
-        const uint32_t subgroup_index) {
-    return register_notification_handler<SubgroupType>(handler,std::string{},subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-bool ServiceClient<CascadeTypes...>::register_notification_handler(
-        const cascade_notification_handler_t& handler,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index) {
-    if (!is_external_client()) {
-        throw derecho_exception(std::string(__PRETTY_FUNCTION__) +
-            "Cannot register notification handler because external_group_ptr is null.");
-    }
-
-    std::unique_lock<std::mutex> type_registry_lock(this->notification_handler_registry_mutex);
-    auto& per_type_registry = notification_handler_registry.template get<SubgroupType>();
-    // Register Cascade's root handler:
-    // if subgroup_index exists in the per_type_registry, Cascade's root handler is registered already.
-    if (per_type_registry.find(subgroup_index) == per_type_registry.cend()) {
-        per_type_registry.emplace(subgroup_index,SubgroupNotificationHandler<SubgroupType>{});
-        // register to subgroup_caller
-        auto& subgroup_caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        // to do ... register it.
-        per_type_registry.at(subgroup_index).initialize(subgroup_caller);
-    }
-    auto& subgroup_handlers = per_type_registry.at(subgroup_index);
-
-    // Register the handler
-    std::lock_guard<std::mutex> subgroup_handlers_lock(*subgroup_handlers.object_pool_notification_handlers_mutex);
-    bool ret = (subgroup_handlers.object_pool_notification_handlers.find(object_pool_pathname) !=
-                subgroup_handlers.object_pool_notification_handlers.cend());
-
-    if (handler) {
-        subgroup_handlers.object_pool_notification_handlers[object_pool_pathname] = handler;
-    } else {
-        subgroup_handlers.object_pool_notification_handlers[object_pool_pathname].reset();
-    }
-    return ret;
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-bool ServiceClient<CascadeTypes...>::type_recursive_register_notification_handler(
-        uint32_t type_index,
-        const cascade_notification_handler_t& handler,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index) {
-    if (type_index == 0) {
-        return this->template register_notification_handler<FirstType>(handler,object_pool_pathname,subgroup_index);
-    } else {
-        return this->template type_recursive_register_notification_handler<SecondType,RestTypes...>(
-            type_index-1,handler,object_pool_pathname,subgroup_index);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-bool ServiceClient<CascadeTypes...>::type_recursive_register_notification_handler(
-        uint32_t type_index,
-        const cascade_notification_handler_t& handler,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index) {
-    if (type_index == 0) {
-        return this->template register_notification_handler<LastType>(handler,object_pool_pathname,subgroup_index);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-bool ServiceClient<CascadeTypes...>::register_notification_handler(
-        const cascade_notification_handler_t& handler,
-        const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-
-    return this->template type_recursive_register_notification_handler<CascadeTypes...>(
-        opm.subgroup_type_index,handler,object_pool_pathname,opm.subgroup_index);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::notify(
-        const Blob& msg,
-        const uint32_t subgroup_index,
-        const node_id_t client_id) const {
-    notify<SubgroupType>(msg,"",subgroup_index,client_id);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::notify(
-        const Blob& msg,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index,
-        const node_id_t client_id) const {
-    if (is_external_client()) {
-        throw derecho_exception(std::string(__PRETTY_FUNCTION__) +
-                "Cannot notify an external client from an external client.");
-    }
-
-    auto& client_handle = group_ptr->template get_client_callback<SubgroupType>(subgroup_index);
-
-    //TODO: redesign to avoid memory copies.
-    CascadeNotificationMessage cascade_notification_message(object_pool_pathname,msg);
-    derecho::NotificationMessage derecho_notification_message(CASCADE_NOTIFICATION_MESSAGE_TYPE, mutils::bytes_size(cascade_notification_message));
-    mutils::to_bytes(cascade_notification_message,derecho_notification_message.body);
-
-    client_handle.template p2p_send<RPC_NAME(notify)>(client_id,derecho_notification_message);
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType,typename SecondType, typename...RestTypes>
-void ServiceClient<CascadeTypes...>::type_recursive_notify(
-        uint32_t type_index,
-        const Blob& msg,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index,
-        const node_id_t client_id) const {
-    if (type_index == 0) {
-        this->template notify<FirstType>(msg,object_pool_pathname,subgroup_index,client_id);
-    } else {
-        this->template type_recursive_notify<SecondType,RestTypes...>(type_index-1,msg,object_pool_pathname,subgroup_index,client_id);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-void ServiceClient<CascadeTypes...>::type_recursive_notify(
-        uint32_t type_index,
-        const Blob& msg,
-        const std::string& object_pool_pathname,
-        const uint32_t subgroup_index,
-        const node_id_t client_id) const {
-    if (type_index == 0) {
-        this->template notify<LastType>(msg,object_pool_pathname,subgroup_index,client_id);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-void ServiceClient<CascadeTypes...>::notify(
-        const Blob& msg,
-        const std::string& object_pool_pathname,
-        const node_id_t client_id) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-    this->template type_recursive_notify<CascadeTypes...>(opm.subgroup_type_index,msg,object_pool_pathname,opm.subgroup_index,client_id);
-}
-
-#ifdef ENABLE_EVALUATION
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::dump_timestamp(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index) {
-
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template ordered_send<RPC_NAME(ordered_dump_timestamp_log)>(filename);
-        } else {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,filename);
-            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log)>(node_id,filename);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,filename);
-        return caller.template p2p_send<RPC_NAME(dump_timestamp_log)>(node_id,filename);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename FirstType, typename SecondType, typename... RestTypes>
-void ServiceClient<CascadeTypes...>::type_recursive_dump(
-        uint32_t type_index,
-        uint32_t subgroup_index,
-        const std::string& filename) {
-    if (type_index == 0) {
-        this->template dump_timestamp<FirstType>(subgroup_index,filename);
-    } else {
-        this->template type_recursive_dump<SecondType, RestTypes...>(type_index-1,subgroup_index,filename);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename LastType>
-void ServiceClient<CascadeTypes...>::type_recursive_dump(
-        uint32_t type_index,
-        uint32_t subgroup_index,
-        const std::string& filename) {
-    if (type_index == 0) {
-        this->template dump_timestamp<LastType>(subgroup_index,filename);
-    } else {
-        throw derecho::derecho_exception(std::string(__PRETTY_FUNCTION__) + ": type index is out of boundary.");
-    }
-}
-
-template <typename... CascadeTypes>
-void ServiceClient<CascadeTypes...>::dump_timestamp(const std::string& filename, const std::string& object_pool_pathname) {
-    auto opm = find_object_pool(object_pool_pathname);
-    if (!opm.is_valid() || opm.is_null() || opm.deleted) {
-        throw derecho::derecho_exception("Failed to find object_pool:" + object_pool_pathname);
-    }
-
-    this->template type_recursive_dump<CascadeTypes...>(opm.subgroup_type_index,opm.subgroup_index,filename);
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-void ServiceClient<CascadeTypes...>::dump_timestamp(const uint32_t subgroup_index,const std::string& filename){
-    uint32_t shards = get_number_of_shards<SubgroupType>(subgroup_index);
-    for (uint32_t shard_index = 0; shard_index < shards; shard_index ++){
-        auto result = this->template dump_timestamp<SubgroupType>(filename,subgroup_index,shard_index);
-        result.get();
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<void> ServiceClient<CascadeTypes...>::dump_timestamp_workaround(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index, const node_id_t node_id) {
-    if (!is_external_client()) {
-        std::lock_guard<std::mutex> lck(this->group_ptr_mutex);
-        if (static_cast<uint32_t>(group_ptr->template get_my_shard<SubgroupType>(subgroup_index)) == shard_index) {
-            auto& subgroup_handle = group_ptr->template get_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id, filename);
-        } else {
-            auto& subgroup_handle = group_ptr->template get_nonmember_subgroup<SubgroupType>(subgroup_index);
-            return subgroup_handle.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id,filename);
-        }
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        return caller.template p2p_send<RPC_NAME(dump_timestamp_log_workaround)>(node_id,filename);
-    }
-}
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-derecho::rpc::QueryResults<double> ServiceClient<CascadeTypes...>::perf_put(const uint32_t message_size, const uint64_t duration_sec, const uint32_t subgroup_index, const uint32_t shard_index) {
-    if (!is_external_client()) {
-        // 'perf_put' must be issued from an external client.
-        throw derecho::derecho_exception{"perf_put must be issued from an external client."};
-    } else {
-        std::lock_guard<std::mutex> lck(this->external_group_ptr_mutex);
-        auto& caller = external_group_ptr->template get_subgroup_caller<SubgroupType>(subgroup_index);
-        node_id_t node_id = pick_member_by_policy<SubgroupType>(subgroup_index,shard_index,0);
-        return caller.template p2p_send<RPC_NAME(perf_put)>(node_id,message_size,duration_sec);
-    }
-}
-#endif//ENABLE_EVALUATION
-
-#ifndef __WITHOUT_SERVICE_SINGLETONS__
-template <typename... CascadeTypes>
-const std::vector<std::type_index> ServiceClient<CascadeTypes...>::subgroup_type_order{typeid(CascadeTypes)...};
-
-template <typename... CascadeTypes>
-const uint32_t ServiceClient<CascadeTypes...>::invalid_subgroup_type_index = 0xffffffff;
-
-template <typename... CascadeTypes>
-template <typename SubgroupType>
-uint32_t ServiceClient<CascadeTypes...>::get_subgroup_type_index() {
-    uint32_t index = 0;
-    while (index < subgroup_type_order.size()) {
-        if ( std::type_index(typeid(SubgroupType)) == subgroup_type_order.at(index)) {
-            return index;
-        }
-        index ++;
-    }
-    return invalid_subgroup_type_index;
-}
-
-template <typename... CascadeTypes>
-std::unique_ptr<ServiceClient<CascadeTypes...>> ServiceClient<CascadeTypes...>::service_client_singleton_ptr;
-
-template <typename... CascadeTypes>
-std::mutex ServiceClient<CascadeTypes...>::singleton_mutex;
-
-template <typename... CascadeTypes>
-void ServiceClient<CascadeTypes...>::initialize(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr) {
-    std::lock_guard<std::mutex> lock_guard(singleton_mutex);
-    if (!service_client_singleton_ptr) {
-        dbg_default_trace("initializing ServiceClient singleton as cascade member, group pointer={:p}",static_cast<void*>(_group_ptr));
-        service_client_singleton_ptr = std::unique_ptr<ServiceClient<CascadeTypes...>>(new ServiceClient<CascadeTypes...>(_group_ptr));
-    }
-}
-
-template <typename... CascadeTypes>
-ServiceClient<CascadeTypes...>& ServiceClient<CascadeTypes...>::get_service_client() {
-    if (!service_client_singleton_ptr) {
-        std::lock_guard<std::mutex> lock_guard(singleton_mutex);
-        // test again in case another thread has initialized it already.
-        if (!service_client_singleton_ptr) {
-            dbg_default_trace("initializing ServiceClient singleton as external client");
-            service_client_singleton_ptr = std::unique_ptr<ServiceClient<CascadeTypes...>>(new ServiceClient<CascadeTypes...>(nullptr));
-        }
-    }
-    return *service_client_singleton_ptr;
-}
-#endif//__WITHOUT_SERVICE_SINGLETONS__
 
 template <typename... CascadeTypes>
 ExecutionEngine<CascadeTypes...>::ExecutionEngine() {
diff --git a/include/cascade/service.hpp b/include/cascade/service.hpp
index 891d26f0..bd2d8fd1 100644
--- a/include/cascade/service.hpp
+++ b/include/cascade/service.hpp
@@ -6,40 +6,38 @@
  * @brief   This file includes the cascade service templates
  *
  * Type neutral templates components go here. Since the server binary and client library has to be type aware (because
- * they are pre-compiled), we separate the api and implementation of them in type-awared header files as follows:
+ * they are pre-compiled), we separate the api and implementation of them in type-aware header files as follows:
  * - service_types.hpp contains the predefined types for derecho Subgroups, which are specialized from
- *   derecho::cascade::VolatileCascadeStore/PersistentCascadeStore templates.
- * - service_client_api.hpp contains the client API definition.
- * - service_server_api.hpp contains the server API definition. Huh, Server API??? YES! because the application need to
- *   specify their 'onData()' behaviours by implementing the APIs in service_server_api.hpp as a shared library. The
- *   server will load them on restart.
+ *   derecho::cascade::VolatileCascadeStore/PersistentCascadeStore templates, and the server API definition,
+ *   which is a specialization of the Service template.
+ * - service_client_api.hpp contains the client API definition, which is a specialization of ServiceClient.
  */
 
+#include "cascade.hpp"
+#include "data_flow_graph.hpp"
+#include "detail/prefix_registry.hpp"
+#include "object_pool_metadata.hpp"
+#include "user_defined_logic_manager.hpp"
+#include "utils.hpp"
+
+#include <atomic>
+#include <condition_variable>
 #include <cstdint>
+#include <derecho/conf/conf.hpp>
 #include <derecho/core/notification.hpp>
 #include <derecho/mutils-serialization/SerializationSupport.hpp>
 #include <derecho/persistent/PersistentInterface.hpp>
+#include <functional>
+#include <iostream>
+#include <map>
 #include <memory>
 #include <mutex>
-#include <shared_mutex>
-#include <typeinfo>
-#include <tuple>
-#include <derecho/utils/time.h>
-#include <list>
-#include <condition_variable>
 #include <thread>
-#include <functional>
-#include <iostream>
+#include <tuple>
+#include <typeinfo>
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
-#include <derecho/conf/conf.hpp>
-#include "cascade.hpp"
-#include "utils.hpp"
-#include "object_pool_metadata.hpp"
-#include "user_defined_logic_manager.hpp"
-#include "data_flow_graph.hpp"
-#include "detail/prefix_registry.hpp"
 
 namespace derecho {
 namespace cascade {
@@ -81,15 +79,10 @@ namespace cascade {
 #define METADATA_SERVICE_SUBGROUP_INDEX (0)
 
 
-    /* The cascade context to be defined later */
-    template <typename... CascadeTypes>
-    class CascadeContext;
-
+    /* The cascade execution engine to be defined later */
     template <typename... CascadeTypes>
     class ExecutionEngine;
 
-    /* The Action to be defined later */
-    struct Action;
     /**
      * The off-critical data path handler API
      */
@@ -347,1377 +340,9 @@ namespace cascade {
         }
     };
 
-
-    /** The notification handler type */
-    using cascade_notification_handler_t = std::function<void(const Blob&)>;
-
-    /** The CascadeNotificationMessage type */
-#define CASCADE_NOTIFICATION_MESSAGE_TYPE   (0x100000000ull)
-    struct CascadeNotificationMessage: public mutils::ByteRepresentable {
-        /** The object pool pathname, empty string for raw cascade notification message */
-        std::string object_pool_pathname;
-        /** data */
-        Blob blob;
-
-        /** TODO: the default serialization support macro might contain unnecessary copies. Check it!!! */
-        DEFAULT_SERIALIZATION_SUPPORT(CascadeNotificationMessage,object_pool_pathname,blob);
-
-        /** constructors */
-        CascadeNotificationMessage():
-            object_pool_pathname(),
-            blob() {}
-        CascadeNotificationMessage(CascadeNotificationMessage&& other):
-            object_pool_pathname(other.object_pool_pathname),
-            blob(std::move(other.blob)) {}
-        CascadeNotificationMessage(const CascadeNotificationMessage& other):
-            object_pool_pathname(other.object_pool_pathname),
-            blob(other.blob) {}
-        CascadeNotificationMessage(const std::string& _object_pool_pathname,
-                const Blob& _blob) :
-            object_pool_pathname(_object_pool_pathname),
-            blob(_blob) {}
-    };
-
-    /**
-     * This is the structure for the server side notification handlers
-     */
-    template <typename SubgroupType>
-    struct SubgroupNotificationHandler {
-        // key: object_pool_pathname
-        // value: an option for the handler
-        // The handler for "" key is the default handler, which will always be triggered.
-        std::unordered_map<std::string, std::optional<cascade_notification_handler_t>> object_pool_notification_handlers;
-        mutable std::unique_ptr<std::mutex> object_pool_notification_handlers_mutex;
-
-        SubgroupNotificationHandler():
-            object_pool_notification_handlers_mutex(std::make_unique<std::mutex>()) {}
-
-        template <typename T>
-        inline void initialize(derecho::ExternalClientCaller<SubgroupType,T>& subgroup_caller) {
-            dbg_default_trace("SubgroupNotificationHandler(this={:x}) is initialized for SubgroupType:{}",
-                    reinterpret_cast<uint64_t>(this),typeid(SubgroupType).name());
-            subgroup_caller.register_notification_handler(
-                    [this](const derecho::NotificationMessage& msg){
-                        dbg_default_trace("subgroup notification handler is triggered with this={:x}, msg type={}, size={} bytes",
-                                reinterpret_cast<uint64_t>(this), msg.message_type, msg.size);
-                        (*this)(msg);
-                    });
-        }
-
-        inline void operator ()(const derecho::NotificationMessage& msg) {
-            dbg_default_trace("SubgroupNotificationHandler(this={:x}) is triggered with message_type={:x}, size={} bytes",
-                    reinterpret_cast<uint64_t>(this),msg.message_type, msg.size);
-            if (msg.message_type != CASCADE_NOTIFICATION_MESSAGE_TYPE) {
-                return;
-            }
-            // mutils::deserialize_and_run<CascadeNotificationMessage>(nullptr, msg.body,
-            mutils::deserialize_and_run(nullptr, msg.body,
-                    [this](const CascadeNotificationMessage& cascade_message)->void {
-                        dbg_default_trace("Handling cascade_message: {}. size={} bytes",
-                                cascade_message.object_pool_pathname,cascade_message.blob.size);
-                        std::lock_guard<std::mutex> lck(*object_pool_notification_handlers_mutex);
-                        // call default handler
-                        if (object_pool_notification_handlers.find("") !=
-                            object_pool_notification_handlers.cend()) {
-                            if (object_pool_notification_handlers.at("").has_value()) {
-                                (*object_pool_notification_handlers.at(""))(cascade_message.blob);
-                            }
-                        }
-                        // call object pool handler
-                        if (object_pool_notification_handlers.find(cascade_message.object_pool_pathname) !=
-                            object_pool_notification_handlers.cend()) {
-                            if (object_pool_notification_handlers.at(cascade_message.object_pool_pathname).has_value()) {
-                                (*object_pool_notification_handlers.at(cascade_message.object_pool_pathname))(cascade_message.blob);
-                            }
-                        }
-                    });
-        }
-    };
-
-    template <typename SubgroupType>
-    using per_type_notification_handler_registry_t =
-        std::unordered_map<uint32_t,SubgroupNotificationHandler<SubgroupType>>;
-    /**
-     * The ServiceClient template class contains all APIs needed to read/write data. The four core APIs are put, remove,
-     * get, and get_by_time. We also provide a set of helper APIs for the client to get the group topology. The core APIs
-     * target a specific subgroup and shard of the service, or a specific object pool (which maps to a subgroup).
-     * The client uses a ShardMemberSelectionPolicy to determine which member of that subgroup/shard to communicate with.
-     */
+    /* Forward declaration of the ServiceClient, to be defined in service_client.hpp */
     template <typename... CascadeTypes>
-    class ServiceClient {
-        static_assert(have_same_object_type<CascadeTypes...>());
-    private:
-        // default caller as an external client.
-        std::unique_ptr<derecho::ExternalGroupClient<CascadeMetadataService<CascadeTypes...>,CascadeTypes...>> external_group_ptr;
-        mutable std::mutex external_group_ptr_mutex;
-        // caller as a group member.
-        derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* group_ptr;
-        mutable std::mutex group_ptr_mutex;
-        // cascade server side notification handler registry.
-        mutable mutils::KindMap<per_type_notification_handler_registry_t,CascadeTypes...> notification_handler_registry;
-        mutable std::mutex notification_handler_registry_mutex;
-        /**
-         * 'member_selection_policies' is a map from derecho shard to its member selection policy.
-         * We use a 3-tuple consisting of subgroup type index, subgroup index, and shard index to identify a shard. And
-         * the policy is defined by a 2-tuple with the ShardMemberSelectionPolicy enum and a user specified node id, in
-         * case of ShardMemorySelectionPolicy::UserSpecified. The user specified node id is used as member index if the
-         * policy is ShardMemberSelectionPolicy::RoundRobin
-         *
-         * The default member selection policy is defined as DEFAULT_SHARD_MEMBER_SELECTION_POLICY.
-         */
-        std::unordered_map<
-            std::tuple<std::type_index,uint32_t,uint32_t>,
-            std::tuple<ShardMemberSelectionPolicy,node_id_t>,
-            do_hash<std::tuple<std::type_index,uint32_t,uint32_t>>> member_selection_policies;
-        mutable std::shared_mutex member_selection_policies_mutex;
-        /**
-         * 'member_cache' is a map from derecho shard to its member list. This cache is used to accelerate the member
-         * choices process. If the client cannot connect to the cached member (after a couple of retries), it will refresh
-         * the corresponding cache entry.
-         */
-        std::unordered_map<
-            std::tuple<std::type_index,uint32_t,uint32_t>,
-            std::vector<node_id_t>,
-            do_hash<std::tuple<std::type_index,uint32_t,uint32_t>>> member_cache;
-        mutable std::shared_mutex member_cache_mutex;
-        /**
-         * 'object_pool_info_cache' is a local cache for object pool metadata. This cache is used to accelerate the
-         * object access process. If an object pool does not exists, it will be loaded from metadata service.
-         *
-         * Each entry of the object_pool_info_cache is an object of type ObjectPoolMetadataCacheEntry. Such an object
-         * caches an object pool metadata object (opm) along with the affinity set regex processing data structures.
-         */
-        class ObjectPoolMetadataCacheEntry {
-        public:
-            ObjectPoolMetadata<CascadeTypes...> opm;
-            /**
-             * The constructor
-             * @param[in] _opm object pool metadata
-             */
-            ObjectPoolMetadataCacheEntry(const ObjectPoolMetadata<CascadeTypes...>& _opm);
-
-            /**
-             * The destructor
-             */
-            virtual ~ObjectPoolMetadataCacheEntry();
-
-            /**
-             * Convert a key string to corresponding affinity set string.
-             * @param[in] key_string
-             *
-             * @return affinity set string
-             */
-            inline std::string to_affinity_set(const std::string& key_string);
-        private:
-            /* the database storing compiled regex */
-            hs_database_t*                      database;
-            /* the scratch for the regex */
-            thread_local static hs_scratch_t*   scratch;
-        };
-
-        std::unordered_map<
-            std::string,
-            ObjectPoolMetadataCacheEntry> object_pool_metadata_cache;
-        mutable std::shared_mutex object_pool_metadata_cache_mutex;
-
-        /**
-         * Pick a member by a given a policy.
-         * @param[in] subgroup_index
-         * @param[in] shard_index
-         * @param[in] key_for_hashing   - only for KeyHashing policy, ignored otherwise.
-         * @param[in] retry             - if true, refresh the member_cache.
-         */
-        template <typename SubgroupType, typename KeyTypeForHashing>
-        node_id_t pick_member_by_policy(uint32_t subgroup_index,
-                                        uint32_t shard_index,
-                                        const KeyTypeForHashing& key_for_hashing,
-                                        bool retry = false);
-
-        /**
-         * Refresh(or fill) a member cache entry.
-         * @param[in] subgroup_index
-         * @param[in] shard_index
-         */
-        template <typename SubgroupType>
-        void refresh_member_cache_entry(uint32_t subgroup_index, uint32_t shard_index);
-
-        /**
-         * Metadata API Helper: turn a string key to subgroup type index, subgroup index, and shard index.
-         */
-        template <typename KeyType>
-        std::tuple<uint32_t,uint32_t,uint32_t> key_to_shard(
-                const KeyType& key, bool check_object_location = true);
-
-        /**
-         * The Constructor
-         * We prevent calling the constructor explicitly, because the ServiceClient is a singleton.
-         * @param[in] _group_ptr The caller can pass a pointer pointing to a derecho group object. If the pointer is
-         *                   valid, the implementation will reply on the group object instead of creating an external
-         *                   client to communicate with group members.
-         */
-        ServiceClient(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr=nullptr);
-
-    public:
-        /**
-         * ServiceClient can be an external client or a cascade server. is_external_client() test this condition.
-         * The external client implementation is based on ExternalGroupClient<> while the cascade node implementation is
-         * based on Group<>.
-         *
-         * @return true for external client; other wise false.
-         */
-        inline bool is_external_client() const;
-
-        /**
-         * Derecho group helpers: They derive the API in derecho::ExternalClient.
-         * - get_my_id                  return my local node id.
-         * - get_members                returns all members in the top-level Derecho group.
-         * - get_subgroup_members       returns a vector of vectors of node ids: [[node ids in shard 0],[node ids in shard 1],...]
-         * - get_shard_members          returns the members in a shard specified by subgroup id(or subgroup type/index pair) and
-         *                              shard index.
-         * - get_number_of_subgroups    returns the number of subgroups of a given type
-         * - get_number_of_shards       returns the number of shards of a given subgroup
-         * - get_my_shard               returns the shard number that this node is a member of in the specific
-         *                              subgroup (by subgroup type and index), or -1 if this node is not a member
-         *                              of any shard in the specified subgroup.
-         * During view change, the Client might experience failure if the member is gone. In such a case, the client needs
-         * refresh its local member cache by calling get_shard_members.
-         */
-        node_id_t get_my_id() const;
-
-        std::vector<node_id_t> get_members() const;
-
-        template <typename SubgroupType>
-        std::vector<std::vector<node_id_t>> get_subgroup_members(uint32_t subgroup_index) const;
-    protected:
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        std::vector<std::vector<node_id_t>> type_recursive_get_subgroup_members(uint32_t type_index, uint32_t subgroup_index) const;
-        template <typename LastType>
-        std::vector<std::vector<node_id_t>> type_recursive_get_subgroup_members(uint32_t type_index, uint32_t subgroup_index) const;
-    public:
-        std::vector<std::vector<node_id_t>> get_subgroup_members(const std::string& object_pool_pathname);
-
-        template <typename SubgroupType>
-        std::vector<node_id_t> get_shard_members(uint32_t subgroup_index,uint32_t shard_index) const;
-    protected:
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        std::vector<node_id_t> type_recursive_get_shard_members(uint32_t type_index,
-                uint32_t subgroup_index, uint32_t shard_index) const;
-        template <typename LastType>
-        std::vector<node_id_t> type_recursive_get_shard_members(uint32_t type_index,
-                uint32_t subgroup_index, uint32_t shard_index) const;
-    public:
-        std::vector<node_id_t> get_shard_members(const std::string& object_pool_pathname,uint32_t shard_index);
-
-        template <typename SubgroupType>
-        uint32_t get_number_of_subgroups() const;
-
-        template <typename SubgroupType>
-        uint32_t get_number_of_shards(uint32_t subgroup_index) const;
-
-        // type recursive helpers for get_number_of_shards
-    protected:
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        uint32_t type_recursive_get_number_of_shards(uint32_t type_index, uint32_t subgroup_index) const;
-        template <typename LastType>
-        uint32_t type_recursive_get_number_of_shards(uint32_t type_index, uint32_t subgroup_index) const;
-    public:
-        /**
-         * This get_number_of_shards() overload the typed version.
-         * @param[in] subgroup_type_index   - the type index of the subrgoup type.
-         * @param[in] subgroup_index        - the subgroup index in the given type.
-         */
-        uint32_t get_number_of_shards(uint32_t subgroup_type_index, uint32_t subgroup_index) const;
-
-        /**
-         * This get_number_of_shards(), pick subgroup using object pool pathname.
-         * @param[in] object_pool_pathname  - the object pool name
-         */
-        uint32_t get_number_of_shards(const std::string& object_pool_pathname);
-
-        template <typename SubgroupType>
-        int32_t get_my_shard(uint32_t subgroup_index) const;
-    protected:
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        int32_t type_recursive_get_my_shard(uint32_t type_index, uint32_t subgroup_index) const;
-        template <typename LastType>
-        int32_t type_recursive_get_my_shard(uint32_t type_index, uint32_t subgroup_index) const;
-    public:
-        /**
-         * @fn int32_t get_my_shard(uint32_t subgroup_type_index, uint32_t subgroup_index) const
-         * @brief find the shard I belong to, given the subgroup specified by type and index.
-         * @param[in]   subgroup_type_index     - the type index of the subgroup type.
-         * @param[in]   subgroup_index          - the subgroup index in the given type.
-         * @return  The number of the shard, or -1 if current node is not in the specified subgroup.
-         */
-        int32_t get_my_shard(uint32_t subgroup_type_index, uint32_t subgroup_index) const;
-
-        /**
-         * @fn int32_t get_my_shard(const std::string& object_pool_pathname)
-         * @brief find the shard I belong to, given the object pool specified by object pool path name.
-         * @param[in]   object_pool_pathname    - the object pool path name.
-         * @return  The number of the shard, or -1 if current node is not in the specified subgroup.
-         */
-        int32_t get_my_shard(const std::string& object_pool_pathname);
-
-        /**
-         * Updates the member selection policy for a shard.
-         *
-         * @tparam SubgroupType The Cascade subgroup type the shard is in
-         * @param[in] subgroup_index
-         * @param[in] shard_index
-         * @param[in] policy - the new policy
-         * @param[in] user_specified_node_id - optional, the node ID to contact if the policy is UserSpecified
-         */
-        template <typename SubgroupType>
-        void set_member_selection_policy(uint32_t subgroup_index,uint32_t shard_index,
-                ShardMemberSelectionPolicy policy,node_id_t user_specified_node_id=INVALID_NODE_ID);
-
-        /**
-         * Reads the member selection policy for a shard.
-         *
-         * @tparam SubgroupType The Cascade subgroup type the shard is in
-         * @param[in] subgroup_index
-         * @param[in] shard_index
-         * @return a 2-tuple of policy and user_specified_node_id.
-         */
-        template <typename SubgroupType>
-        std::tuple<ShardMemberSelectionPolicy,node_id_t> get_member_selection_policy(
-                uint32_t subgroup_index, uint32_t shard_index) const;
-
-        /**
-         * "put" writes an object to a given subgroup/shard.
-         *
-         * @param[in] object            the object to write.
-         *                          User provided SubgroupType::ObjectType must have the following two members:
-         *                          - SubgroupType::ObjectType::key of SubgroupType::KeyType, which must be set to a
-         *                            valid key.
-         *                          - SubgroupType::ObjectType::ver of std::tuple<persistent::version_t, uint64_t>.
-         *                            Similar to the return object, this member is a two tuple with the first member
-         *                            for a version and the second for a timestamp. A caller of put can specify either
-         *                            of the version and timestamp meaning what is the latest version/timestamp the caller
-         *                            has seen. Cascade will reject the write if the corresponding key has been updated
-         *                            already. TODO: should we make it an optional feature?
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
-         *                              used to update the state.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<version_tuple> put(const typename SubgroupType::ObjectType& object,
-                uint32_t subgroup_index, uint32_t shard_index, bool as_trigger = false);
-        /**
-         * "type_recursive_put" is a helper function for internal use only.
-         * @param[in]   type_index  the index of the subgroup type in the CascadeTypes... list. And the FirstType,
-         *                          SecondType, ..., RestTypes should be in the same order.
-         * @param[in]   object      the object to write
-         * @param[in]   subgroup_index
-         *                          the subgroup index in the subgroup type designated by type_index
-         * @param[in]   shard_index the shard index
-         * @param[in]   as_trigger  If true, the object will NOT apply to the K/V store. The object will only be
-         *                          used to update the state.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-    protected:
-        template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<version_tuple> type_recursive_put(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index,
-                bool as_trigger = false);
-
-        template <typename ObjectType, typename LastType>
-        derecho::rpc::QueryResults<version_tuple> type_recursive_put(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index,
-                bool as_trigger = false);
-    public:
-        /**
-         * object pool version of "put"
-         * @param[in] object            the object to write, the object pool is extracted from the object key.
-         * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
-         *                              used to update the state.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-        template <typename ObjectType>
-        derecho::rpc::QueryResults<version_tuple> put(const ObjectType& object, bool as_trigger = false);
-
-        /**
-         * "put_and_forget" writes an object to a given subgroup/shard, but no return value.
-         *
-         * @param[in] object            the object to write.
-         *                          User provided SubgroupType::ObjectType must have the following two members:
-         *                          - SubgroupType::ObjectType::key of SubgroupType::KeyType, which must be set to a
-         *                            valid key.
-         *                          - SubgroupType::ObjectType::ver of std::tuple<persistent::version_t, uint64_t>.
-         *                            Similar to the return object, this member is a two tuple with the first member
-         *                            for a version and the second for a timestamp. A caller of put can specify either
-         *                            of the version and timestamp meaning what is the latest version/timestamp the caller
-         *                            has seen. Cascade will reject the write if the corresponding key has been updated
-         *                            already. TODO: should we make it an optional feature?
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
-         *                              used to update the state.
-         */
-        template <typename SubgroupType>
-        void put_and_forget(const typename SubgroupType::ObjectType& object,
-                uint32_t subgroup_index, uint32_t shard_index, bool as_trigger = false);
-
-    protected:
-        /**
-         * "type_recursive_put_and_forget" is a helper function for internal use only.
-         * @param[in] type_index    the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] object        the object to write
-         * @param[in] subgroup_index
-         *                          the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index   the shard index
-         * @param[in] as_trigger    If true, the object will NOT apply to the K/V store. The object will only be
-         *                          used to update the state.
-         */
-        template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-        void type_recursive_put_and_forget(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index,
-                bool as_trigger = false);
-
-        template <typename ObjectType, typename LastType>
-        void type_recursive_put_and_forget(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index,
-                bool as_trigger = false);
-    public:
-        /**
-         * object pool version of "put_and_forget"
-         * @param[in] object        the object to write, the object pool is extracted from the object key.
-         * @param[in] as_trigger    If true, the object will NOT apply to the K/V store. The object will only be
-         *                          used to update the state.
-         */
-        template <typename ObjectType>
-        void put_and_forget(const ObjectType& object, bool as_trigger = false);
-
-        /**
-         * "trigger_put" writes an object to a given subgroup/shard.
-         *
-         * @param[in] object            the object to write.
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a void future.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<void> trigger_put(const typename SubgroupType::ObjectType& object,
-                uint32_t subgroup_index, uint32_t shard_index);
-    protected:
-        /**
-         * "type_recursive_trigger_put" is a helper function for internal use only.
-         * @param[in]   type_index  the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in]   object      the object to write
-         * @param[in]   subgroup_index
-         *                          the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index   the shard index
-         *
-         * @return future
-         */
-        template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<void> type_recursive_trigger_put(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename ObjectType, typename LastType>
-        derecho::rpc::QueryResults<void> type_recursive_trigger_put(
-                uint32_t type_index,
-                const ObjectType& object,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-        /**
-         * object pool version of "trigger_put"
-         * @param[in] object    the object to write, the object pool is extracted from the object key.
-         */
-        template <typename ObjectType>
-        derecho::rpc::QueryResults<void> trigger_put(const ObjectType& object);
-        /**
-         * "collective_trigger_put" writes an object to a set of nodes.
-         *
-         * Please notice that returning from QueryResults<void>::get() only means that the message has been sent by the
-         * sender. It does NOT guarantee that the message is/will be successfully processed by the remote side. However,
-         * we agree that QueryResults<void> should reflect exceptions or errors either on local or remote side, which is
-         * not enabled so far. TODO: Track exception in derecho::rpc::QueryResults<void>
-         *
-         * @param[in] object            the object to write.
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] nodes_and_futures map from node ids to futures.
-         */
-        template <typename SubgroupType>
-        void collective_trigger_put(const typename SubgroupType::ObjectType& object,
-                uint32_t subgroup_index,
-                std::unordered_map<node_id_t,std::unique_ptr<derecho::rpc::QueryResults<void>>>& nodes_and_futures);
-
-        /**
-         * "remove" deletes an object with the given key.
-         *
-         * @param[in] key               the object key
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<version_tuple> remove(const typename SubgroupType::KeyType& key,
-                uint32_t subgroup_index, uint32_t shard_index);
-
-    protected:
-        /**
-         * "type_recursive_remove" is a helper function for internal use only.
-         * @param[in]   type_index              the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in]   key                     the key
-         * @param[in]   subgroup_index          the subgroup index in the subgroup type designated by type_index
-         * @param[in]   shard_index             the shard index
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<version_tuple> type_recursive_remove(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        derecho::rpc::QueryResults<version_tuple> type_recursive_remove(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-        /**
-         * object pool version of "remove"
-         * @param[in]   key             the object key
-         *
-         * @return  returns a future
-         */
-        template <typename KeyType>
-        derecho::rpc::QueryResults<version_tuple> remove(const KeyType& key);
-
-        /**
-         * "get" retrieve the object of a given key
-         *
-         * @param[in] key               the object key
-         * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, get will either
-         *                              read the current object from memory of the replica that handles the get request
-         *                              (if stable is false), or read the latest stable version that is persisted (if
-         *                              stable is true). Note that in any case "get" will contact only a single replica;
-         *                              to use an atomic multicast to get the latest version that is present on all replicas,
-         *                              use multi_get.
-         * @param[in] stable            if true, get will wait until the requested version's persistent data is safe, meaning the
-         *                              persistent data is persisted on all replicas.
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved object.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> get(
-                const typename SubgroupType::KeyType& key,
-                const persistent::version_t& version = CURRENT_VERSION,
-                bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-    protected:
-        /**
-         * "type_recursive_get" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] version           the version
-         * @param[in] stable            stable or not?
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the object.
-         */
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_get(
-                uint32_t type_index,
-                const KeyType& key,
-                const persistent::version_t& version,
-                bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        auto type_recursive_get(
-                uint32_t type_index,
-                const KeyType& key,
-                const persistent::version_t& version,
-                bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-        /**
-         * object pool version of "get"
-         *
-         * @param[in] key               the object key; the object pool is extracted from this key
-         * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, get will either
-         *                              read the current object from memory of the replica that handles the get request
-         *                              (if stable is false), or read the latest stable version that is persisted (if
-         *                              stable is true). Note that in any case "get" will contact only a single replica;
-         *                              to use an atomic multicast to get the latest version that is present on all replicas,
-         *                              use multi_get.
-         * @param[in] stable            if true, get will wait until the requested version's persistent data is safe, meaning the
-         *                              persistent data is persisted on all replicas.
-         * @return a future to the retrieved object.
-         */
-        template <typename KeyType>
-        auto get(
-                const KeyType& key,
-                const persistent::version_t& version = CURRENT_VERSION,
-                bool stable = true);
-
-        /**
-         * "multi_get" retrieves the latest version of the object for a given key using an atomic broadcast.
-         * This ensures that the get request is mutually exclusive and linearizable with any concurrent put
-         * requests to the same key.
-         *
-         * @param[in] key               the object key
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved object, including a response from each replica in the shard.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> multi_get(const typename SubgroupType::KeyType& key,
-                uint32_t subgroup_index, uint32_t shard_index);
-
-        /**
-         * "type_recursive_multi_get"   is a helper function for internal use only.
-         * @param[in]   type_index      the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                              SecondType, .../ RestTypes should be in the same order.
-         * @param[in]   key             the key
-         * @param[in]   subgroup_index  the subgroup index in the subgroup type designated by type_index
-         * @param[in]   shard_index     the shard index
-         *
-         * @return a future for the object.
-         */
-    protected:
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_multi_get(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        auto type_recursive_multi_get(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-
-        /**
-         * object pool version of "multi_get"
-         *
-         * @param[in] key               the object key; the object pool is extracted from this key
-         *
-         * @return a future for the retrieved object, including a response from each replica in the object pool
-         */
-        template <typename KeyType>
-        auto multi_get(const KeyType& key);
-
-        /**
-         * "get_by_time" retrieve the object of a given key
-         *
-         * @param[in] key               the object key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved object.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> get_by_time(
-                const typename SubgroupType::KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-        /**
-         * "type_recursive_get_by_time" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the object.
-         */
-    protected:
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_get_by_time(
-                uint32_t type_index,
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        auto type_recursive_get_by_time(
-                uint32_t type_index,
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-
-        /**
-         * object pool version of "get_by_time"
-         *
-         * @param[in] key               the object key; the object pool is extracted from this key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         *
-         * @return a future for the retrieved object.
-         */
-        template <typename KeyType>
-        auto get_by_time(
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable = true);
-
-        /**
-         * "get_size" retrieve size of the object of a given key
-         *
-         * @param[in] key               the object key
-         * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, this will either
-         *                              get the size of the current object from memory of the replica that handles the request
-         *                              (if stable is false), or get the size of the latest stable version that is persisted
-         *                              (if stable is true). Note that in any case "get_size" will contact only a single replica;
-         *                              to use an atomic multicast to check the latest version that is present on all replicas,
-         *                              use multi_get_size.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved size.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<uint64_t> get_size(
-                const typename SubgroupType::KeyType& key,
-                const persistent::version_t& version,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-    protected:
-        /**
-         * "type_recursive_get_size" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] version           version
-         * @param[in] stable            stable get size or not
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the retrieved size.
-         */
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_get_size(
-                uint32_t type_index,
-                const KeyType& key,
-                const persistent::version_t& version,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_get_size(
-                uint32_t type_index,
-                const KeyType& key,
-                const persistent::version_t& version,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-    public:
-        /**
-         * object pool version of "get_size"
-         * @param[in] key               the object key
-         * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, this will either
-         *                              get the size of the current object from memory of the replica that handles the request
-         *                              (if stable is false), or get the size of the latest stable version that is persisted
-         *                              (if stable is true). Note that in any case "get_size" will contact only a single replica;
-         *                              to use an atomic multicast to check the latest version that is present on all replicas,
-         *                              use multi_get_size.
-         * @param[in] stable            stable get or not
-         * @return a future for the retrieved size.
-         */
-        template <typename KeyType>
-        derecho::rpc::QueryResults<uint64_t> get_size(
-                const KeyType& key,
-                const persistent::version_t& version,
-                const bool stable = true);
-
-        /**
-         * "multi_get_size" retrieves the size of the current version of the object with a given key, using an atomic broadcast.
-         *
-         * @param[in] key               the object key
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved size.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<uint64_t> multi_get_size(
-                const typename SubgroupType::KeyType& key,
-                uint32_t subgroup_index, uint32_t shard_index);
-
-        /**
-         * "type_recursive_multi_get_size" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the object.
-         */
-    protected:
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_multi_get_size(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_multi_get_size(
-                uint32_t type_index,
-                const KeyType& key,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-    public:
-        /**
-         * object pool version of "multi_get_size"
-         *
-         * @param[in] key               the object key; the object pool is extracted from this key
-         *
-         * @return a future for the retrieved size.
-         */
-        template <typename KeyType>
-        derecho::rpc::QueryResults<uint64_t> multi_get_size(const KeyType& key);
-
-        /**
-         * "get_size_by_time" retrieve size of the object of a given key
-         *
-         * @param[in] key               the object key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future to the retrieved size.
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<uint64_t> get_size_by_time(
-                const typename SubgroupType::KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-    protected:
-        /**
-         * "type_recursive_get_size" is a helper function for internal use only.
-         * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
-         *                          SecondType, .../ RestTypes should be in the same order.
-         * @param[in] key               the key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
-         * @param[in] shard_index       the shard index
-         *
-         * @return a future for the object.
-         */
-        template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_get_size_by_time(
-                uint32_t type_index,
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-        template <typename KeyType, typename LastType>
-        derecho::rpc::QueryResults<uint64_t> type_recursive_get_size_by_time(
-                uint32_t type_index,
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable,
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-    public:
-
-        /**
-         * object pool version of "get_size_by_time"
-         *
-         * @param[in] key               the object key
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         *
-         * @return a future for the retrieved size.
-         */
-        template <typename KeyType>
-        derecho::rpc::QueryResults<uint64_t> get_size_by_time(
-                const KeyType& key,
-                const uint64_t& ts_us,
-                const bool stable = true);
-
-        /**
-         * "list_keys" retrieve the list of keys in a shard
-         *
-         * @param[in] version           the version at which to list the keys; all keys that existed at or before this version
-         *                              will be included. If equal to CURRENT_VERSION, list_keys will list all keys currently
-         *                              in memory at the replica that handles the request.
-         * @param[in] stable            if true, list_keys will wait until the requested version's persistent data is safe, meaning the
-         *                              persistent data is persisted on all replicas, before listing the keys present at that version.
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future for the vector of keys
-         * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> list_keys(
-                const persistent::version_t& version,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-    protected:
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_list_keys(
-                uint32_t type_index,
-                const persistent::version_t& version,
-                const bool stable,
-                const std::string& object_pool_pathname);
-        template <typename LastType>
-        auto type_recursive_list_keys(
-                uint32_t type_index,
-                const persistent::version_t& version,
-                const bool stable,
-                const std::string& object_pool_pathname);
-        template <typename SubgroupType>
-        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
-            __list_keys(const persistent::version_t& version, const bool stable, const std::string& object_pool_pathname);
-    public:
-        /**
-         * @brief object pool version of "list_keys"; lists all keys in an object pool
-         *
-         * @param[in] version               the version at which to list the keys; all keys that existed at or before this version
-         *                                  will be included. If equal to CURRENT_VERSION, list_keys will list all keys currently
-         *                                  in memory at the replica that handles the request.
-         * @param[in] stable                if true, list_keys will wait until the requested version's persistent data is safe, meaning the
-         *                                  persistent data is persisted on all replicas, before listing the keys present at that version.
-         * @param[in] object_pool_pathname  the object pathname
-         *
-         * @return a vector of futures for key lists, with one key list for each shard in the object pool.
-         * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
-         */
-        auto list_keys(const persistent::version_t& version, const bool stable, const std::string& object_pool_pathname);
-
-        /**
-         * A function that helps unpack the return value of list_keys (object-pool version). Iterates through the
-         * vector of QueryResults and waits for each one to complete, then combines the resulting vectors of keys
-         * into a single vector of keys (across all shards in the object pool).
-         *
-         * @tparam KeyType The type of a key in this object pool
-         * @param future The vector-of-futures returned by list_keys or multi_list_keys
-         * @return a vector of keys in the object pool
-         */
-        template <typename KeyType>
-        std::vector<KeyType> wait_list_keys(
-                                std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<KeyType>>>>& future);
-
-        /**
-         * "multi_list_keys" retrieves the list of keys in a shard at the current version, using an atomic multicast
-         *
-         * @param[in] subgroup_index   the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future for the list of keys.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> multi_list_keys(
-                uint32_t subgroup_index,
-                uint32_t shard_index);
-
-    protected:
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_multi_list_keys(
-                uint32_t type_index,
-                const std::string& object_pool_pathname);
-        template <typename LastType>
-        auto type_recursive_multi_list_keys(
-                uint32_t type_index,
-                const std::string& object_pool_pathname);
-        template <typename SubgroupType>
-        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
-            __multi_list_keys(const std::string& object_pool_pathname);
-    public:
-        /**
-         * object pool version of "multi_list_keys"
-         *
-         * @param[in] object_pool_pathname  the object pathname
-         *
-         * @return a vector of futures for key lists, with one key list for each shard in the object pool.
-         * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
-         */
-        auto multi_list_keys(const std::string& object_pool_pathname);
-
-        /**
-         * "list_keys_by_time" retrieves the list of keys in a shard at a specific time
-         *
-         * @param[in] ts_us             Wall clock time in microseconds.
-         * @param[in] stable            stable get or not
-         * @param[in] subgroup_index    the subgroup index of CascadeType
-         * @param[in] shard_index       the shard index.
-         *
-         * @return a future for the list of keys
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> list_keys_by_time(
-                const uint64_t& ts_us,
-                const bool stable = true,
-                uint32_t subgroup_index = 0,
-                uint32_t shard_index = 0);
-
-    protected:
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        auto type_recursive_list_keys_by_time(
-                uint32_t type_index,
-                const uint64_t& ts_us,
-                const bool stable,
-                const std::string& object_pool_pathname);
-        template <typename LastType>
-        auto type_recursive_list_keys_by_time(
-                uint32_t type_index,
-                const uint64_t& ts_us,
-                const bool stable,
-                const std::string& object_pool_pathname);
-        template <typename SubgroupType>
-        std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
-            __list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname);
-    public:
-        /**
-        * object pool version of "list_keys_by_time"
-        *
-        * @param[in] ts_us                  timestamp
-        * @param[in] stable                 stable flag
-        * @param[in] object_pool_pathname   the object pathname
-        *
-        * @return a vector of futures for key lists, with one key list for each shard in the object pool.
-        * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
-        */
-        auto list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname);
-
-        /**
-         * Object Pool Management API: refresh object pool cache
-         * We load 'unstable' (committed but maybe not persisted) metadata here.
-         */
-        void refresh_object_pool_metadata_cache();
-
-        /**
-         * Object Pool Management API: create object pool
-         *
-         * @tparam SubgroupType     Type of the subgroup for the created object pool
-         * @param[in]  pathname         Object pool's pathname as identifier.
-         * @param[in]  subgroup_index   Index of the subgroup
-         * @param[in]  sharding_policy  The default sharding policy for this object pool
-         * @param[in]  object_locations The set of special object locations.
-         * @param[in]  affinity_set_regex
-         *                          The affinity set regex.
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<version_tuple> create_object_pool(
-                const std::string& pathname, const uint32_t subgroup_index,
-                const sharding_policy_t sharding_policy = HASH,
-                const std::unordered_map<std::string,uint32_t>& object_locations = {},
-                const std::string& affinity_set_regex = "");
-
-        /**
-         * ObjectPoolManagement API: remove object pool
-         *
-         * @param[in]  pathname         Object pool pathname
-         *
-         * @return a future to the version and timestamp of the put operation.
-         */
-        derecho::rpc::QueryResults<version_tuple> remove_object_pool(const std::string& pathname);
-    private:
-        /**
-         * ObjectPoolManagement API: find object pool
-         *
-         * @param[in]  pathname         Object pool pathname
-         * @param[in]  rlck             shared lock, which needs to be hold.
-         *
-         * @return the object pool metadata
-         */
-        ObjectPoolMetadata<CascadeTypes...> internal_find_object_pool(const std::string& pathname,
-                                                                      std::shared_lock<std::shared_mutex>& rlck);
-    public:
-        /**
-         * ObjectPoolManagement API: find object pool
-         *
-         * @param[in]  pathname         Object pool pathname
-         *
-         * @return the object pool metadata
-         */
-        ObjectPoolMetadata<CascadeTypes...> find_object_pool(const std::string& pathname);
-
-        /**
-         * ObjectPoolManagement API: find object pool and affinity_set from key
-         *
-         * @param[in]  key              The key of an object.
-         *
-         * @return the object pool metadata along with the affinity set string
-         */
-        template <typename KeyType>
-        std::pair<ObjectPoolMetadata<CascadeTypes...>,std::string>
-            find_object_pool_and_affinity_set_by_key(const KeyType& key);
-
-        /**
-         * ObjectPoolManagement API: list all the object pools by pathnames
-         *
-         * @param[in] include_deleted   show deleted pools with an exclaimation point(!).
-         * @param[in] refresh           false for cached object ids, true for refreshed ids.
-         *
-         * @return the pool ids.
-         */
-        std::vector<std::string> list_object_pools(bool include_deleted, bool refresh = false);
-
-        /**
-         * Register an notification handler to a subgroup. If such a handler has been registered, it will be replaced
-         * by the new one.
-         *
-         * @tparam SubgroupType     The Subgroup Type
-         * @param[in] handler           The handler to reigster
-         * @param[in] subgroup_index    Index of the subgroup
-         *
-         * @return true if a previous notification handler is replaced.
-         */
-        template <typename SubgroupType>
-        bool register_notification_handler(
-                const cascade_notification_handler_t& handler,
-                const uint32_t subgroup_index = 0);
-
-    protected:
-        template <typename SubgroupType>
-        bool register_notification_handler(
-                const cascade_notification_handler_t& handler,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index);
-        template <typename FirstType,typename SecondType, typename...RestTypes>
-        bool type_recursive_register_notification_handler(
-                uint32_t type_index,
-                const cascade_notification_handler_t& handler,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index);
-        template <typename LastType>
-        bool type_recursive_register_notification_handler(
-                uint32_t type_index,
-                const cascade_notification_handler_t& handler,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index);
-
-    public:
-        /**
-         * Register notification handler(object pool version). If such a handler has been registered, it will be
-         * replaced by the new one.
-         *
-         * @tparam SubgroupType         The Subgroup Type
-         * @param[in] handler               The handler to reigster
-         * @param[in] object_pool_pathname  To with object pool is this handler registered.
-         *
-         * @return true if a previous notification handler is replaced.
-         */
-        bool register_notification_handler(
-                const cascade_notification_handler_t& handler,
-                const std::string& object_pool_pathname);
-
-        /**
-         * Send a notification message to an external client.
-         *
-         * @tparam SubgroupType     The Subgroup Type
-         * @param[in] msg               The message to send
-         * @param[in] subgroup_index    The subgroup index
-         * @param[in] client_id         The node id of the external client to be notified
-         */
-        template <typename SubgroupType>
-        void notify(const Blob& msg,
-                const uint32_t subgroup_index,
-                const node_id_t client_id) const;
-    protected:
-        template <typename SubgroupType>
-        void notify(const Blob& msg,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index,
-                const node_id_t client_id) const;
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        void type_recursive_notify(
-                uint32_t type_index,
-                const Blob& msg,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index,
-                const node_id_t client_id) const;
-        template <typename LastType>
-        void type_recursive_notify(
-                uint32_t type_index,
-                const Blob& msg,
-                const std::string& object_pool_pathname,
-                const uint32_t subgroup_index,
-                const node_id_t client_id) const;
-    public:
-        /**
-         * Send a notification message to an external client.
-         *
-         * @param[in] msg                   The messgae to send
-         * @param[in] object_pool_pathname  In which object_pool the notification is in.
-         * @param[in] client_id             The client id
-         */
-        void notify(const Blob& msg,
-                const std::string& object_pool_pathname,
-                const node_id_t client_id);
-
-#ifdef ENABLE_EVALUATION
-        /**
-         * Dump the timestamp log entries into a file on each of the nodes in a shard.
-         *
-         * @param[in] filename         - the output filename
-         * @param[in] subgroup_index   - the subgroup index
-         * @param[in] shard_index      - the shard index
-         *
-         * @return query results
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<void> dump_timestamp(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index);
-
-        /**
-         * The object store version:
-         *
-         * @param[in] filename             -   the filename
-         * @param[in] object_pool_pathname -   the object pool pathname
-         */
-        void dump_timestamp(const std::string& filename, const std::string& object_pool_pathname);
-
-        /**
-         * Dump the timestamp log entries into a file on each of the nodes in a subgroup.
-         *
-         * @param[in] filename         - the output filename
-         * @param[in] subgroup_index   - the subgroup index
-         */
-        template <typename SubgroupType>
-        void dump_timestamp(const uint32_t subgroup_index, const std::string& filename);
-
-    protected:
-        template <typename FirstType, typename SecondType, typename... RestTypes>
-        void type_recursive_dump(uint32_t type_index, uint32_t subgroup_index, const std::string& filename);
-
-        template <typename LastType>
-        void type_recursive_dump(uint32_t type_index, uint32_t subgroup_index, const std::string& filename);
-
-    public:
-#ifdef DUMP_TIMESTAMP_WORKAROUND
-        /**
-         * Dump the timestamp log entries into a file on a specific node.
-         *
-         * @param[in] filename         - the output filename
-         * @param[in] subgroup_index   - the subgroup index
-         * @param[in] shard_index      - the shard index
-         * @param[in] node_id          - the given node id.
-         *
-         * @return a vector of query results.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<void> dump_timestamp_workaround(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index, const node_id_t node_id);
-#endif
-
-        /**
-         * Evaluate the ordered put performance inside a shard. Please note that those put does not involve the
-         * external client data path.
-         *
-         * @param[in] message_size      - the message size for the shard. TODO: we should be able to retrieve the maximum
-         *                            message size from SubgroupType, subgroup_index and shard_index. How?
-         * @param[in] duration_sec      - the duration of the test in seconds.
-         * @param[in] subgroup_index    - the subgroup index
-         * @param[in] shard_index       - the shard index
-         *
-         * @return the value in ops.
-         */
-        template <typename SubgroupType>
-        derecho::rpc::QueryResults<double> perf_put(const uint32_t message_size, const uint64_t duration_sec, const uint32_t subgroup_index, const uint32_t shard_index);
-#endif//ENABLE_EVALUATION
-
-        const static std::vector<std::type_index> subgroup_type_order;
-        const static uint32_t invalid_subgroup_type_index;
-        /**
-         * Get type index
-         * @return the the subgroup type index
-         */
-        template <typename SubgroupType>
-        static uint32_t get_subgroup_type_index();
-
-        /* singleton */
-    private:
-        static std::unique_ptr<ServiceClient> service_client_singleton_ptr;
-        static std::mutex                     singleton_mutex;
-    public:
-        /**
-         * Initialize the service_client_single_ptr singleton with a cascade service. This can only be called once
-         * before any get_service_client() is called.
-         * @param[in] _group_ptr The caller can pass a pointer pointing to a derecho group object. If the pointer is
-         *                   valid, the implementation will reply on the group object instead of creating an external
-         *                   client to communicate with group members.
-         */
-        static void initialize(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr);
-
-        /**
-         * Get the singleton ServiceClient API. If it does not exists, initialize it as an external client.
-         */
-        static ServiceClient& get_service_client();
-    }; // ServiceClient
-
+    class ServiceClient;
 
     /**
      * configuration keys
diff --git a/include/cascade/service_client.hpp b/include/cascade/service_client.hpp
new file mode 100644
index 00000000..ce6e015a
--- /dev/null
+++ b/include/cascade/service_client.hpp
@@ -0,0 +1,1386 @@
+#pragma once
+
+#include "object.hpp"
+#include "object_pool_metadata.hpp"
+#include "service.hpp"
+#include <cascade/config.h>
+
+#include <cstdint>
+#include <derecho/core/notification.hpp>
+#include <derecho/mutils-serialization/SerializationSupport.hpp>
+#include <derecho/persistent/PersistentInterface.hpp>
+#include <functional>
+#include <hs/hs.h>
+#include <memory>
+#include <mutex>
+#include <mutils-containers/KindMap.hpp>
+#include <optional>
+#include <shared_mutex>
+#include <string>
+#include <tuple>
+#include <unordered_map>
+#include <vector>
+
+namespace derecho {
+namespace cascade {
+/** The notification handler type */
+using cascade_notification_handler_t = std::function<void(const Blob&)>;
+
+/** The CascadeNotificationMessage type */
+#define CASCADE_NOTIFICATION_MESSAGE_TYPE (0x100000000ull)
+struct CascadeNotificationMessage : public mutils::ByteRepresentable {
+    /** The object pool pathname, empty string for raw cascade notification message */
+    std::string object_pool_pathname;
+    /** data */
+    Blob blob;
+
+    /** TODO: the default serialization support macro might contain unnecessary copies. Check it!!! */
+    DEFAULT_SERIALIZATION_SUPPORT(CascadeNotificationMessage, object_pool_pathname, blob);
+
+    /** constructors */
+    CascadeNotificationMessage() : object_pool_pathname(),
+                                   blob() {}
+    CascadeNotificationMessage(CascadeNotificationMessage&& other) : object_pool_pathname(other.object_pool_pathname),
+                                                                     blob(std::move(other.blob)) {}
+    CascadeNotificationMessage(const CascadeNotificationMessage& other) : object_pool_pathname(other.object_pool_pathname),
+                                                                          blob(other.blob) {}
+    CascadeNotificationMessage(const std::string& _object_pool_pathname,
+                               const Blob& _blob) : object_pool_pathname(_object_pool_pathname),
+                                                    blob(_blob) {}
+};
+
+/**
+ * This is the structure for the server side notification handlers
+ */
+template <typename SubgroupType>
+struct SubgroupNotificationHandler {
+    /**
+     * key: object_pool_pathname
+     * value: an optional std::function for the handler
+     * The handler for "" key is the default handler, which will always be triggered.
+     */
+    std::unordered_map<std::string, std::optional<cascade_notification_handler_t>> object_pool_notification_handlers;
+    mutable std::unique_ptr<std::mutex> object_pool_notification_handlers_mutex;
+
+    SubgroupNotificationHandler() : object_pool_notification_handlers_mutex(std::make_unique<std::mutex>()) {}
+
+    template <typename T>
+    void initialize(derecho::ExternalClientCaller<SubgroupType, T>& subgroup_caller);
+
+    void operator()(const derecho::NotificationMessage& msg);
+};
+
+template <typename SubgroupType>
+using per_type_notification_handler_registry_t = std::unordered_map<uint32_t, SubgroupNotificationHandler<SubgroupType>>;
+/**
+ * The ServiceClient template class contains all APIs needed to read/write data. The four core APIs are put, remove,
+ * get, and get_by_time. We also provide a set of helper APIs for the client to get the group topology. The core APIs
+ * target a specific subgroup and shard of the service, or a specific object pool (which maps to a subgroup).
+ * The client uses a ShardMemberSelectionPolicy to determine which member of that subgroup/shard to communicate with.
+ */
+template <typename... CascadeTypes>
+class ServiceClient {
+    static_assert(have_same_object_type<CascadeTypes...>());
+
+private:
+    // default caller as an external client.
+    std::unique_ptr<derecho::ExternalGroupClient<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>> external_group_ptr;
+    mutable std::mutex external_group_ptr_mutex;
+    // caller as a group member.
+    derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* group_ptr;
+    mutable std::mutex group_ptr_mutex;
+    // cascade server side notification handler registry.
+    mutable mutils::KindMap<per_type_notification_handler_registry_t, CascadeTypes...> notification_handler_registry;
+    mutable std::mutex notification_handler_registry_mutex;
+    /**
+     * 'member_selection_policies' is a map from derecho shard to its member selection policy.
+     * We use a 3-tuple consisting of subgroup type index, subgroup index, and shard index to identify a shard. And
+     * the policy is defined by a 2-tuple with the ShardMemberSelectionPolicy enum and a user specified node id, in
+     * case of ShardMemorySelectionPolicy::UserSpecified. The user specified node id is used as member index if the
+     * policy is ShardMemberSelectionPolicy::RoundRobin
+     *
+     * The default member selection policy is defined as DEFAULT_SHARD_MEMBER_SELECTION_POLICY.
+     */
+    std::unordered_map<
+            std::tuple<std::type_index, uint32_t, uint32_t>,
+            std::tuple<ShardMemberSelectionPolicy, node_id_t>,
+            do_hash<std::tuple<std::type_index, uint32_t, uint32_t>>>
+            member_selection_policies;
+    mutable std::shared_mutex member_selection_policies_mutex;
+    /**
+     * 'member_cache' is a map from derecho shard to its member list. This cache is used to accelerate the member
+     * choices process. If the client cannot connect to the cached member (after a couple of retries), it will refresh
+     * the corresponding cache entry.
+     */
+    std::unordered_map<
+            std::tuple<std::type_index, uint32_t, uint32_t>,
+            std::vector<node_id_t>,
+            do_hash<std::tuple<std::type_index, uint32_t, uint32_t>>>
+            member_cache;
+    mutable std::shared_mutex member_cache_mutex;
+    /**
+     * 'object_pool_info_cache' is a local cache for object pool metadata. This cache is used to accelerate the
+     * object access process. If an object pool does not exists, it will be loaded from metadata service.
+     *
+     * Each entry of the object_pool_info_cache is an object of type ObjectPoolMetadataCacheEntry. Such an object
+     * caches an object pool metadata object (opm) along with the affinity set regex processing data structures.
+     */
+    class ObjectPoolMetadataCacheEntry {
+    public:
+        ObjectPoolMetadata<CascadeTypes...> opm;
+        /**
+         * The constructor
+         * @param[in] _opm object pool metadata
+         */
+        ObjectPoolMetadataCacheEntry(const ObjectPoolMetadata<CascadeTypes...>& _opm);
+
+        /**
+         * The destructor
+         */
+        virtual ~ObjectPoolMetadataCacheEntry();
+
+        /**
+         * Convert a key string to corresponding affinity set string.
+         * @param[in] key_string
+         *
+         * @return affinity set string
+         */
+        inline std::string to_affinity_set(const std::string& key_string);
+
+    private:
+        /* the database storing compiled regex */
+        hs_database_t* database;
+        /* the scratch for the regex */
+        thread_local static hs_scratch_t* scratch;
+    };
+
+    std::unordered_map<
+            std::string,
+            ObjectPoolMetadataCacheEntry>
+            object_pool_metadata_cache;
+    mutable std::shared_mutex object_pool_metadata_cache_mutex;
+
+    /**
+     * Pick a member by a given a policy.
+     * @param[in] subgroup_index
+     * @param[in] shard_index
+     * @param[in] key_for_hashing   - only for KeyHashing policy, ignored otherwise.
+     * @param[in] retry             - if true, refresh the member_cache.
+     */
+    template <typename SubgroupType, typename KeyTypeForHashing>
+    node_id_t pick_member_by_policy(uint32_t subgroup_index,
+                                    uint32_t shard_index,
+                                    const KeyTypeForHashing& key_for_hashing,
+                                    bool retry = false);
+
+    /**
+     * Refresh(or fill) a member cache entry.
+     * @param[in] subgroup_index
+     * @param[in] shard_index
+     */
+    template <typename SubgroupType>
+    void refresh_member_cache_entry(uint32_t subgroup_index, uint32_t shard_index);
+
+    /**
+     * Metadata API Helper: turn a string key to subgroup type index, subgroup index, and shard index.
+     */
+    template <typename KeyType>
+    std::tuple<uint32_t, uint32_t, uint32_t> key_to_shard(
+            const KeyType& key, bool check_object_location = true);
+
+    /**
+     * The Constructor
+     * We prevent calling the constructor explicitly, because the ServiceClient is a singleton.
+     * @param[in] _group_ptr The caller can pass a pointer pointing to a derecho group object. If the pointer is
+     *                   valid, the implementation will reply on the group object instead of creating an external
+     *                   client to communicate with group members.
+     */
+    ServiceClient(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr = nullptr);
+
+public:
+    /**
+     * ServiceClient can be an external client or a cascade server. is_external_client() test this condition.
+     * The external client implementation is based on ExternalGroupClient<> while the cascade node implementation is
+     * based on Group<>.
+     *
+     * @return true for external client; other wise false.
+     */
+    inline bool is_external_client() const;
+
+    /**
+     * Derecho group helpers: They derive the API in derecho::ExternalClient.
+     * - get_my_id                  return my local node id.
+     * - get_members                returns all members in the top-level Derecho group.
+     * - get_subgroup_members       returns a vector of vectors of node ids: [[node ids in shard 0],[node ids in shard 1],...]
+     * - get_shard_members          returns the members in a shard specified by subgroup id(or subgroup type/index pair) and
+     *                              shard index.
+     * - get_number_of_subgroups    returns the number of subgroups of a given type
+     * - get_number_of_shards       returns the number of shards of a given subgroup
+     * - get_my_shard               returns the shard number that this node is a member of in the specific
+     *                              subgroup (by subgroup type and index), or -1 if this node is not a member
+     *                              of any shard in the specified subgroup.
+     * During view change, the Client might experience failure if the member is gone. In such a case, the client needs
+     * refresh its local member cache by calling get_shard_members.
+     */
+    node_id_t get_my_id() const;
+
+    std::vector<node_id_t> get_members() const;
+
+    template <typename SubgroupType>
+    std::vector<std::vector<node_id_t>> get_subgroup_members(uint32_t subgroup_index) const;
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    std::vector<std::vector<node_id_t>> type_recursive_get_subgroup_members(uint32_t type_index, uint32_t subgroup_index) const;
+    template <typename LastType>
+    std::vector<std::vector<node_id_t>> type_recursive_get_subgroup_members(uint32_t type_index, uint32_t subgroup_index) const;
+
+public:
+    std::vector<std::vector<node_id_t>> get_subgroup_members(const std::string& object_pool_pathname);
+
+    template <typename SubgroupType>
+    std::vector<node_id_t> get_shard_members(uint32_t subgroup_index, uint32_t shard_index) const;
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    std::vector<node_id_t> type_recursive_get_shard_members(uint32_t type_index,
+                                                            uint32_t subgroup_index, uint32_t shard_index) const;
+    template <typename LastType>
+    std::vector<node_id_t> type_recursive_get_shard_members(uint32_t type_index,
+                                                            uint32_t subgroup_index, uint32_t shard_index) const;
+
+public:
+    std::vector<node_id_t> get_shard_members(const std::string& object_pool_pathname, uint32_t shard_index);
+
+    template <typename SubgroupType>
+    uint32_t get_number_of_subgroups() const;
+
+    template <typename SubgroupType>
+    uint32_t get_number_of_shards(uint32_t subgroup_index) const;
+
+    // type recursive helpers for get_number_of_shards
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    uint32_t type_recursive_get_number_of_shards(uint32_t type_index, uint32_t subgroup_index) const;
+    template <typename LastType>
+    uint32_t type_recursive_get_number_of_shards(uint32_t type_index, uint32_t subgroup_index) const;
+
+public:
+    /**
+     * This get_number_of_shards() overload the typed version.
+     * @param[in] subgroup_type_index   - the type index of the subrgoup type.
+     * @param[in] subgroup_index        - the subgroup index in the given type.
+     */
+    uint32_t get_number_of_shards(uint32_t subgroup_type_index, uint32_t subgroup_index) const;
+
+    /**
+     * This get_number_of_shards(), pick subgroup using object pool pathname.
+     * @param[in] object_pool_pathname  - the object pool name
+     */
+    uint32_t get_number_of_shards(const std::string& object_pool_pathname);
+
+    template <typename SubgroupType>
+    int32_t get_my_shard(uint32_t subgroup_index) const;
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    int32_t type_recursive_get_my_shard(uint32_t type_index, uint32_t subgroup_index) const;
+    template <typename LastType>
+    int32_t type_recursive_get_my_shard(uint32_t type_index, uint32_t subgroup_index) const;
+
+public:
+    /**
+     * @fn int32_t get_my_shard(uint32_t subgroup_type_index, uint32_t subgroup_index) const
+     * @brief find the shard I belong to, given the subgroup specified by type and index.
+     * @param[in]   subgroup_type_index     - the type index of the subgroup type.
+     * @param[in]   subgroup_index          - the subgroup index in the given type.
+     * @return  The number of the shard, or -1 if current node is not in the specified subgroup.
+     */
+    int32_t get_my_shard(uint32_t subgroup_type_index, uint32_t subgroup_index) const;
+
+    /**
+     * @fn int32_t get_my_shard(const std::string& object_pool_pathname)
+     * @brief find the shard I belong to, given the object pool specified by object pool path name.
+     * @param[in]   object_pool_pathname    - the object pool path name.
+     * @return  The number of the shard, or -1 if current node is not in the specified subgroup.
+     */
+    int32_t get_my_shard(const std::string& object_pool_pathname);
+
+    /**
+     * Updates the member selection policy for a shard.
+     *
+     * @tparam SubgroupType The Cascade subgroup type the shard is in
+     * @param[in] subgroup_index
+     * @param[in] shard_index
+     * @param[in] policy - the new policy
+     * @param[in] user_specified_node_id - optional, the node ID to contact if the policy is UserSpecified
+     */
+    template <typename SubgroupType>
+    void set_member_selection_policy(uint32_t subgroup_index, uint32_t shard_index,
+                                     ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id = INVALID_NODE_ID);
+
+    /**
+     * Reads the member selection policy for a shard.
+     *
+     * @tparam SubgroupType The Cascade subgroup type the shard is in
+     * @param[in] subgroup_index
+     * @param[in] shard_index
+     * @return a 2-tuple of policy and user_specified_node_id.
+     */
+    template <typename SubgroupType>
+    std::tuple<ShardMemberSelectionPolicy, node_id_t> get_member_selection_policy(
+            uint32_t subgroup_index, uint32_t shard_index) const;
+
+    /**
+     * "put" writes an object to a given subgroup/shard.
+     *
+     * @param[in] object            the object to write.
+     *                          User provided SubgroupType::ObjectType must have the following two members:
+     *                          - SubgroupType::ObjectType::key of SubgroupType::KeyType, which must be set to a
+     *                            valid key.
+     *                          - SubgroupType::ObjectType::ver of std::tuple<persistent::version_t, uint64_t>.
+     *                            Similar to the return object, this member is a two tuple with the first member
+     *                            for a version and the second for a timestamp. A caller of put can specify either
+     *                            of the version and timestamp meaning what is the latest version/timestamp the caller
+     *                            has seen. Cascade will reject the write if the corresponding key has been updated
+     *                            already. TODO: should we make it an optional feature?
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
+     *                              used to update the state.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<version_tuple> put(const typename SubgroupType::ObjectType& object,
+                                                  uint32_t subgroup_index, uint32_t shard_index, bool as_trigger = false);
+    /**
+     * "type_recursive_put" is a helper function for internal use only.
+     * @param[in]   type_index  the index of the subgroup type in the CascadeTypes... list. And the FirstType,
+     *                          SecondType, ..., RestTypes should be in the same order.
+     * @param[in]   object      the object to write
+     * @param[in]   subgroup_index
+     *                          the subgroup index in the subgroup type designated by type_index
+     * @param[in]   shard_index the shard index
+     * @param[in]   as_trigger  If true, the object will NOT apply to the K/V store. The object will only be
+     *                          used to update the state.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+protected:
+    template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<version_tuple> type_recursive_put(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index,
+            bool as_trigger = false);
+
+    template <typename ObjectType, typename LastType>
+    derecho::rpc::QueryResults<version_tuple> type_recursive_put(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index,
+            bool as_trigger = false);
+
+public:
+    /**
+     * object pool version of "put"
+     * @param[in] object            the object to write, the object pool is extracted from the object key.
+     * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
+     *                              used to update the state.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+    template <typename ObjectType>
+    derecho::rpc::QueryResults<version_tuple> put(const ObjectType& object, bool as_trigger = false);
+
+    /**
+     * "put_and_forget" writes an object to a given subgroup/shard, but no return value.
+     *
+     * @param[in] object            the object to write.
+     *                          User provided SubgroupType::ObjectType must have the following two members:
+     *                          - SubgroupType::ObjectType::key of SubgroupType::KeyType, which must be set to a
+     *                            valid key.
+     *                          - SubgroupType::ObjectType::ver of std::tuple<persistent::version_t, uint64_t>.
+     *                            Similar to the return object, this member is a two tuple with the first member
+     *                            for a version and the second for a timestamp. A caller of put can specify either
+     *                            of the version and timestamp meaning what is the latest version/timestamp the caller
+     *                            has seen. Cascade will reject the write if the corresponding key has been updated
+     *                            already. TODO: should we make it an optional feature?
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     * @param[in] as_trigger        If true, the object will NOT apply to the K/V store. The object will only be
+     *                              used to update the state.
+     */
+    template <typename SubgroupType>
+    void put_and_forget(const typename SubgroupType::ObjectType& object,
+                        uint32_t subgroup_index, uint32_t shard_index, bool as_trigger = false);
+
+protected:
+    /**
+     * "type_recursive_put_and_forget" is a helper function for internal use only.
+     * @param[in] type_index    the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] object        the object to write
+     * @param[in] subgroup_index
+     *                          the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index   the shard index
+     * @param[in] as_trigger    If true, the object will NOT apply to the K/V store. The object will only be
+     *                          used to update the state.
+     */
+    template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+    void type_recursive_put_and_forget(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index,
+            bool as_trigger = false);
+
+    template <typename ObjectType, typename LastType>
+    void type_recursive_put_and_forget(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index,
+            bool as_trigger = false);
+
+public:
+    /**
+     * object pool version of "put_and_forget"
+     * @param[in] object        the object to write, the object pool is extracted from the object key.
+     * @param[in] as_trigger    If true, the object will NOT apply to the K/V store. The object will only be
+     *                          used to update the state.
+     */
+    template <typename ObjectType>
+    void put_and_forget(const ObjectType& object, bool as_trigger = false);
+
+    /**
+     * "trigger_put" writes an object to a given subgroup/shard.
+     *
+     * @param[in] object            the object to write.
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a void future.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<void> trigger_put(const typename SubgroupType::ObjectType& object,
+                                                 uint32_t subgroup_index, uint32_t shard_index);
+
+protected:
+    /**
+     * "type_recursive_trigger_put" is a helper function for internal use only.
+     * @param[in]   type_index  the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in]   object      the object to write
+     * @param[in]   subgroup_index
+     *                          the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index   the shard index
+     *
+     * @return future
+     */
+    template <typename ObjectType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<void> type_recursive_trigger_put(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename ObjectType, typename LastType>
+    derecho::rpc::QueryResults<void> type_recursive_trigger_put(
+            uint32_t type_index,
+            const ObjectType& object,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "trigger_put"
+     * @param[in] object    the object to write, the object pool is extracted from the object key.
+     */
+    template <typename ObjectType>
+    derecho::rpc::QueryResults<void> trigger_put(const ObjectType& object);
+    /**
+     * "collective_trigger_put" writes an object to a set of nodes.
+     *
+     * Please notice that returning from QueryResults<void>::get() only means that the message has been sent by the
+     * sender. It does NOT guarantee that the message is/will be successfully processed by the remote side. However,
+     * we agree that QueryResults<void> should reflect exceptions or errors either on local or remote side, which is
+     * not enabled so far. TODO: Track exception in derecho::rpc::QueryResults<void>
+     *
+     * @param[in] object            the object to write.
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] nodes_and_futures map from node ids to futures.
+     */
+    template <typename SubgroupType>
+    void collective_trigger_put(const typename SubgroupType::ObjectType& object,
+                                uint32_t subgroup_index,
+                                std::unordered_map<node_id_t, std::unique_ptr<derecho::rpc::QueryResults<void>>>& nodes_and_futures);
+
+    /**
+     * "remove" deletes an object with the given key.
+     *
+     * @param[in] key               the object key
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<version_tuple> remove(const typename SubgroupType::KeyType& key,
+                                                     uint32_t subgroup_index, uint32_t shard_index);
+
+protected:
+    /**
+     * "type_recursive_remove" is a helper function for internal use only.
+     * @param[in]   type_index              the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in]   key                     the key
+     * @param[in]   subgroup_index          the subgroup index in the subgroup type designated by type_index
+     * @param[in]   shard_index             the shard index
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<version_tuple> type_recursive_remove(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    derecho::rpc::QueryResults<version_tuple> type_recursive_remove(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "remove"
+     * @param[in]   key             the object key
+     *
+     * @return  returns a future
+     */
+    template <typename KeyType>
+    derecho::rpc::QueryResults<version_tuple> remove(const KeyType& key);
+
+    /**
+     * "get" retrieve the object of a given key
+     *
+     * @param[in] key               the object key
+     * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, get will either
+     *                              read the current object from memory of the replica that handles the get request
+     *                              (if stable is false), or read the latest stable version that is persisted (if
+     *                              stable is true). Note that in any case "get" will contact only a single replica;
+     *                              to use an atomic multicast to get the latest version that is present on all replicas,
+     *                              use multi_get.
+     * @param[in] stable            if true, get will wait until the requested version's persistent data is safe, meaning the
+     *                              persistent data is persisted on all replicas.
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved object.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> get(
+            const typename SubgroupType::KeyType& key,
+            const persistent::version_t& version = CURRENT_VERSION,
+            bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    /**
+     * "type_recursive_get" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] version           the version
+     * @param[in] stable            stable or not?
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the object.
+     */
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_get(
+            uint32_t type_index,
+            const KeyType& key,
+            const persistent::version_t& version,
+            bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    auto type_recursive_get(
+            uint32_t type_index,
+            const KeyType& key,
+            const persistent::version_t& version,
+            bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "get"
+     *
+     * @param[in] key               the object key; the object pool is extracted from this key
+     * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, get will either
+     *                              read the current object from memory of the replica that handles the get request
+     *                              (if stable is false), or read the latest stable version that is persisted (if
+     *                              stable is true). Note that in any case "get" will contact only a single replica;
+     *                              to use an atomic multicast to get the latest version that is present on all replicas,
+     *                              use multi_get.
+     * @param[in] stable            if true, get will wait until the requested version's persistent data is safe, meaning the
+     *                              persistent data is persisted on all replicas.
+     * @return a future to the retrieved object.
+     */
+    template <typename KeyType>
+    auto get(
+            const KeyType& key,
+            const persistent::version_t& version = CURRENT_VERSION,
+            bool stable = true);
+
+    /**
+     * "multi_get" retrieves the latest version of the object for a given key using an atomic broadcast.
+     * This ensures that the get request is mutually exclusive and linearizable with any concurrent put
+     * requests to the same key.
+     *
+     * @param[in] key               the object key
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved object, including a response from each replica in the shard.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> multi_get(const typename SubgroupType::KeyType& key,
+                                                                                  uint32_t subgroup_index, uint32_t shard_index);
+
+    /**
+     * "type_recursive_multi_get"   is a helper function for internal use only.
+     * @param[in]   type_index      the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                              SecondType, .../ RestTypes should be in the same order.
+     * @param[in]   key             the key
+     * @param[in]   subgroup_index  the subgroup index in the subgroup type designated by type_index
+     * @param[in]   shard_index     the shard index
+     *
+     * @return a future for the object.
+     */
+protected:
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_multi_get(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    auto type_recursive_multi_get(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "multi_get"
+     *
+     * @param[in] key               the object key; the object pool is extracted from this key
+     *
+     * @return a future for the retrieved object, including a response from each replica in the object pool
+     */
+    template <typename KeyType>
+    auto multi_get(const KeyType& key);
+
+    /**
+     * "get_by_time" retrieve the object of a given key
+     *
+     * @param[in] key               the object key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved object.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<const typename SubgroupType::ObjectType> get_by_time(
+            const typename SubgroupType::KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+    /**
+     * "type_recursive_get_by_time" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the object.
+     */
+protected:
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_get_by_time(
+            uint32_t type_index,
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    auto type_recursive_get_by_time(
+            uint32_t type_index,
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "get_by_time"
+     *
+     * @param[in] key               the object key; the object pool is extracted from this key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     *
+     * @return a future for the retrieved object.
+     */
+    template <typename KeyType>
+    auto get_by_time(
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable = true);
+
+    /**
+     * "get_size" retrieve size of the object of a given key
+     *
+     * @param[in] key               the object key
+     * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, this will either
+     *                              get the size of the current object from memory of the replica that handles the request
+     *                              (if stable is false), or get the size of the latest stable version that is persisted
+     *                              (if stable is true). Note that in any case "get_size" will contact only a single replica;
+     *                              to use an atomic multicast to check the latest version that is present on all replicas,
+     *                              use multi_get_size.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved size.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<uint64_t> get_size(
+            const typename SubgroupType::KeyType& key,
+            const persistent::version_t& version,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    /**
+     * "type_recursive_get_size" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] version           version
+     * @param[in] stable            stable get size or not
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the retrieved size.
+     */
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_get_size(
+            uint32_t type_index,
+            const KeyType& key,
+            const persistent::version_t& version,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_get_size(
+            uint32_t type_index,
+            const KeyType& key,
+            const persistent::version_t& version,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "get_size"
+     * @param[in] key               the object key
+     * @param[in] version           the version of the object to read. If equal to CURRENT_VERSION, this will either
+     *                              get the size of the current object from memory of the replica that handles the request
+     *                              (if stable is false), or get the size of the latest stable version that is persisted
+     *                              (if stable is true). Note that in any case "get_size" will contact only a single replica;
+     *                              to use an atomic multicast to check the latest version that is present on all replicas,
+     *                              use multi_get_size.
+     * @param[in] stable            stable get or not
+     * @return a future for the retrieved size.
+     */
+    template <typename KeyType>
+    derecho::rpc::QueryResults<uint64_t> get_size(
+            const KeyType& key,
+            const persistent::version_t& version,
+            const bool stable = true);
+
+    /**
+     * "multi_get_size" retrieves the size of the current version of the object with a given key, using an atomic broadcast.
+     *
+     * @param[in] key               the object key
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved size.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<uint64_t> multi_get_size(
+            const typename SubgroupType::KeyType& key,
+            uint32_t subgroup_index, uint32_t shard_index);
+
+    /**
+     * "type_recursive_multi_get_size" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the object.
+     */
+protected:
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_multi_get_size(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_multi_get_size(
+            uint32_t type_index,
+            const KeyType& key,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "multi_get_size"
+     *
+     * @param[in] key               the object key; the object pool is extracted from this key
+     *
+     * @return a future for the retrieved size.
+     */
+    template <typename KeyType>
+    derecho::rpc::QueryResults<uint64_t> multi_get_size(const KeyType& key);
+
+    /**
+     * "get_size_by_time" retrieve size of the object of a given key
+     *
+     * @param[in] key               the object key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future to the retrieved size.
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<uint64_t> get_size_by_time(
+            const typename SubgroupType::KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    /**
+     * "type_recursive_get_size" is a helper function for internal use only.
+     * @param[in] type_index        the index of the subgroup type in the CascadeTypes... list. and the FirstType,
+     *                          SecondType, .../ RestTypes should be in the same order.
+     * @param[in] key               the key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index    the subgroup index in the subgroup type designated by type_index
+     * @param[in] shard_index       the shard index
+     *
+     * @return a future for the object.
+     */
+    template <typename KeyType, typename FirstType, typename SecondType, typename... RestTypes>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_get_size_by_time(
+            uint32_t type_index,
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+    template <typename KeyType, typename LastType>
+    derecho::rpc::QueryResults<uint64_t> type_recursive_get_size_by_time(
+            uint32_t type_index,
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable,
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+public:
+    /**
+     * object pool version of "get_size_by_time"
+     *
+     * @param[in] key               the object key
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     *
+     * @return a future for the retrieved size.
+     */
+    template <typename KeyType>
+    derecho::rpc::QueryResults<uint64_t> get_size_by_time(
+            const KeyType& key,
+            const uint64_t& ts_us,
+            const bool stable = true);
+
+    /**
+     * "list_keys" retrieve the list of keys in a shard
+     *
+     * @param[in] version           the version at which to list the keys; all keys that existed at or before this version
+     *                              will be included. If equal to CURRENT_VERSION, list_keys will list all keys currently
+     *                              in memory at the replica that handles the request.
+     * @param[in] stable            if true, list_keys will wait until the requested version's persistent data is safe, meaning the
+     *                              persistent data is persisted on all replicas, before listing the keys present at that version.
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future for the vector of keys
+     * TODO: check if the user application is responsible for reclaim the future by reading it sometime.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> list_keys(
+            const persistent::version_t& version,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_list_keys(
+            uint32_t type_index,
+            const persistent::version_t& version,
+            const bool stable,
+            const std::string& object_pool_pathname);
+    template <typename LastType>
+    auto type_recursive_list_keys(
+            uint32_t type_index,
+            const persistent::version_t& version,
+            const bool stable,
+            const std::string& object_pool_pathname);
+    template <typename SubgroupType>
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
+    __list_keys(const persistent::version_t& version, const bool stable, const std::string& object_pool_pathname);
+
+public:
+    /**
+     * @brief object pool version of "list_keys"; lists all keys in an object pool
+     *
+     * @param[in] version               the version at which to list the keys; all keys that existed at or before this version
+     *                                  will be included. If equal to CURRENT_VERSION, list_keys will list all keys currently
+     *                                  in memory at the replica that handles the request.
+     * @param[in] stable                if true, list_keys will wait until the requested version's persistent data is safe, meaning the
+     *                                  persistent data is persisted on all replicas, before listing the keys present at that version.
+     * @param[in] object_pool_pathname  the object pathname
+     *
+     * @return a vector of futures for key lists, with one key list for each shard in the object pool.
+     * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
+     */
+    auto list_keys(const persistent::version_t& version, const bool stable, const std::string& object_pool_pathname);
+
+    /**
+     * A function that helps unpack the return value of list_keys (object-pool version). Iterates through the
+     * vector of QueryResults and waits for each one to complete, then combines the resulting vectors of keys
+     * into a single vector of keys (across all shards in the object pool).
+     *
+     * @tparam KeyType The type of a key in this object pool
+     * @param future The vector-of-futures returned by list_keys or multi_list_keys
+     * @return a vector of keys in the object pool
+     */
+    template <typename KeyType>
+    std::vector<KeyType> wait_list_keys(
+            std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<KeyType>>>>& future);
+
+    /**
+     * "multi_list_keys" retrieves the list of keys in a shard at the current version, using an atomic multicast
+     *
+     * @param[in] subgroup_index   the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future for the list of keys.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> multi_list_keys(
+            uint32_t subgroup_index,
+            uint32_t shard_index);
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_multi_list_keys(
+            uint32_t type_index,
+            const std::string& object_pool_pathname);
+    template <typename LastType>
+    auto type_recursive_multi_list_keys(
+            uint32_t type_index,
+            const std::string& object_pool_pathname);
+    template <typename SubgroupType>
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
+    __multi_list_keys(const std::string& object_pool_pathname);
+
+public:
+    /**
+     * object pool version of "multi_list_keys"
+     *
+     * @param[in] object_pool_pathname  the object pathname
+     *
+     * @return a vector of futures for key lists, with one key list for each shard in the object pool.
+     * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
+     */
+    auto multi_list_keys(const std::string& object_pool_pathname);
+
+    /**
+     * "list_keys_by_time" retrieves the list of keys in a shard at a specific time
+     *
+     * @param[in] ts_us             Wall clock time in microseconds.
+     * @param[in] stable            stable get or not
+     * @param[in] subgroup_index    the subgroup index of CascadeType
+     * @param[in] shard_index       the shard index.
+     *
+     * @return a future for the list of keys
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>> list_keys_by_time(
+            const uint64_t& ts_us,
+            const bool stable = true,
+            uint32_t subgroup_index = 0,
+            uint32_t shard_index = 0);
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    auto type_recursive_list_keys_by_time(
+            uint32_t type_index,
+            const uint64_t& ts_us,
+            const bool stable,
+            const std::string& object_pool_pathname);
+    template <typename LastType>
+    auto type_recursive_list_keys_by_time(
+            uint32_t type_index,
+            const uint64_t& ts_us,
+            const bool stable,
+            const std::string& object_pool_pathname);
+    template <typename SubgroupType>
+    std::vector<std::unique_ptr<derecho::rpc::QueryResults<std::vector<typename SubgroupType::KeyType>>>>
+    __list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname);
+
+public:
+    /**
+     * object pool version of "list_keys_by_time"
+     *
+     * @param[in] ts_us                  timestamp
+     * @param[in] stable                 stable flag
+     * @param[in] object_pool_pathname   the object pathname
+     *
+     * @return a vector of futures for key lists, with one key list for each shard in the object pool.
+     * The return value's type will look like vector<unique_ptr<QueryResults<vector<KeyType>>>>, where KeyType is either string or uint64_t
+     */
+    auto list_keys_by_time(const uint64_t& ts_us, const bool stable, const std::string& object_pool_pathname);
+
+    /**
+     * Object Pool Management API: refresh object pool cache
+     * We load 'unstable' (committed but maybe not persisted) metadata here.
+     */
+    void refresh_object_pool_metadata_cache();
+
+    /**
+     * Object Pool Management API: create object pool
+     *
+     * @tparam SubgroupType     Type of the subgroup for the created object pool
+     * @param[in]  pathname         Object pool's pathname as identifier.
+     * @param[in]  subgroup_index   Index of the subgroup
+     * @param[in]  sharding_policy  The default sharding policy for this object pool
+     * @param[in]  object_locations The set of special object locations.
+     * @param[in]  affinity_set_regex
+     *                          The affinity set regex.
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<version_tuple> create_object_pool(
+            const std::string& pathname, const uint32_t subgroup_index,
+            const sharding_policy_t sharding_policy = HASH,
+            const std::unordered_map<std::string, uint32_t>& object_locations = {},
+            const std::string& affinity_set_regex = "");
+
+    /**
+     * ObjectPoolManagement API: remove object pool
+     *
+     * @param[in]  pathname         Object pool pathname
+     *
+     * @return a future to the version and timestamp of the put operation.
+     */
+    derecho::rpc::QueryResults<version_tuple> remove_object_pool(const std::string& pathname);
+
+private:
+    /**
+     * ObjectPoolManagement API: find object pool
+     *
+     * @param[in]  pathname         Object pool pathname
+     * @param[in]  rlck             shared lock, which needs to be hold.
+     *
+     * @return the object pool metadata
+     */
+    ObjectPoolMetadata<CascadeTypes...> internal_find_object_pool(const std::string& pathname,
+                                                                  std::shared_lock<std::shared_mutex>& rlck);
+
+public:
+    /**
+     * ObjectPoolManagement API: find object pool
+     *
+     * @param[in]  pathname         Object pool pathname
+     *
+     * @return the object pool metadata
+     */
+    ObjectPoolMetadata<CascadeTypes...> find_object_pool(const std::string& pathname);
+
+    /**
+     * ObjectPoolManagement API: find object pool and affinity_set from key
+     *
+     * @param[in]  key              The key of an object.
+     *
+     * @return the object pool metadata along with the affinity set string
+     */
+    template <typename KeyType>
+    std::pair<ObjectPoolMetadata<CascadeTypes...>, std::string>
+    find_object_pool_and_affinity_set_by_key(const KeyType& key);
+
+    /**
+     * ObjectPoolManagement API: list all the object pools by pathnames
+     *
+     * @param[in] include_deleted   show deleted pools with an exclaimation point(!).
+     * @param[in] refresh           false for cached object ids, true for refreshed ids.
+     *
+     * @return the pool ids.
+     */
+    std::vector<std::string> list_object_pools(bool include_deleted, bool refresh = false);
+
+    /**
+     * Register an notification handler to a subgroup. If such a handler has been registered, it will be replaced
+     * by the new one.
+     *
+     * @tparam SubgroupType     The Subgroup Type
+     * @param[in] handler           The handler to reigster
+     * @param[in] subgroup_index    Index of the subgroup
+     *
+     * @return true if a previous notification handler is replaced.
+     */
+    template <typename SubgroupType>
+    bool register_notification_handler(
+            const cascade_notification_handler_t& handler,
+            const uint32_t subgroup_index = 0);
+
+protected:
+    template <typename SubgroupType>
+    bool register_notification_handler(
+            const cascade_notification_handler_t& handler,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index);
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    bool type_recursive_register_notification_handler(
+            uint32_t type_index,
+            const cascade_notification_handler_t& handler,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index);
+    template <typename LastType>
+    bool type_recursive_register_notification_handler(
+            uint32_t type_index,
+            const cascade_notification_handler_t& handler,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index);
+
+public:
+    /**
+     * Register notification handler(object pool version). If such a handler has been registered, it will be
+     * replaced by the new one.
+     *
+     * @tparam SubgroupType         The Subgroup Type
+     * @param[in] handler               The handler to reigster
+     * @param[in] object_pool_pathname  To with object pool is this handler registered.
+     *
+     * @return true if a previous notification handler is replaced.
+     */
+    bool register_notification_handler(
+            const cascade_notification_handler_t& handler,
+            const std::string& object_pool_pathname);
+
+    /**
+     * Send a notification message to an external client.
+     *
+     * @tparam SubgroupType     The Subgroup Type
+     * @param[in] msg               The message to send
+     * @param[in] subgroup_index    The subgroup index
+     * @param[in] client_id         The node id of the external client to be notified
+     */
+    template <typename SubgroupType>
+    void notify(const Blob& msg,
+                const uint32_t subgroup_index,
+                const node_id_t client_id) const;
+
+protected:
+    template <typename SubgroupType>
+    void notify(const Blob& msg,
+                const std::string& object_pool_pathname,
+                const uint32_t subgroup_index,
+                const node_id_t client_id) const;
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    void type_recursive_notify(
+            uint32_t type_index,
+            const Blob& msg,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index,
+            const node_id_t client_id) const;
+    template <typename LastType>
+    void type_recursive_notify(
+            uint32_t type_index,
+            const Blob& msg,
+            const std::string& object_pool_pathname,
+            const uint32_t subgroup_index,
+            const node_id_t client_id) const;
+
+public:
+    /**
+     * Send a notification message to an external client.
+     *
+     * @param[in] msg                   The messgae to send
+     * @param[in] object_pool_pathname  In which object_pool the notification is in.
+     * @param[in] client_id             The client id
+     */
+    void notify(const Blob& msg,
+                const std::string& object_pool_pathname,
+                const node_id_t client_id);
+
+#ifdef ENABLE_EVALUATION
+    /**
+     * Dump the timestamp log entries into a file on each of the nodes in a shard.
+     *
+     * @param[in] filename         - the output filename
+     * @param[in] subgroup_index   - the subgroup index
+     * @param[in] shard_index      - the shard index
+     *
+     * @return query results
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<void> dump_timestamp(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index);
+
+    /**
+     * The object store version:
+     *
+     * @param[in] filename             -   the filename
+     * @param[in] object_pool_pathname -   the object pool pathname
+     */
+    void dump_timestamp(const std::string& filename, const std::string& object_pool_pathname);
+
+    /**
+     * Dump the timestamp log entries into a file on each of the nodes in a subgroup.
+     *
+     * @param[in] filename         - the output filename
+     * @param[in] subgroup_index   - the subgroup index
+     */
+    template <typename SubgroupType>
+    void dump_timestamp(const uint32_t subgroup_index, const std::string& filename);
+
+protected:
+    template <typename FirstType, typename SecondType, typename... RestTypes>
+    void type_recursive_dump(uint32_t type_index, uint32_t subgroup_index, const std::string& filename);
+
+    template <typename LastType>
+    void type_recursive_dump(uint32_t type_index, uint32_t subgroup_index, const std::string& filename);
+
+public:
+#ifdef DUMP_TIMESTAMP_WORKAROUND
+    /**
+     * Dump the timestamp log entries into a file on a specific node.
+     *
+     * @param[in] filename         - the output filename
+     * @param[in] subgroup_index   - the subgroup index
+     * @param[in] shard_index      - the shard index
+     * @param[in] node_id          - the given node id.
+     *
+     * @return a vector of query results.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<void> dump_timestamp_workaround(const std::string& filename, const uint32_t subgroup_index, const uint32_t shard_index, const node_id_t node_id);
+#endif
+
+    /**
+     * Evaluate the ordered put performance inside a shard. Please note that those put does not involve the
+     * external client data path.
+     *
+     * @param[in] message_size      - the message size for the shard. TODO: we should be able to retrieve the maximum
+     *                            message size from SubgroupType, subgroup_index and shard_index. How?
+     * @param[in] duration_sec      - the duration of the test in seconds.
+     * @param[in] subgroup_index    - the subgroup index
+     * @param[in] shard_index       - the shard index
+     *
+     * @return the value in ops.
+     */
+    template <typename SubgroupType>
+    derecho::rpc::QueryResults<double> perf_put(const uint32_t message_size, const uint64_t duration_sec, const uint32_t subgroup_index, const uint32_t shard_index);
+#endif  // ENABLE_EVALUATION
+
+    const static std::vector<std::type_index> subgroup_type_order;
+    const static uint32_t invalid_subgroup_type_index;
+    /**
+     * Get type index
+     * @return the the subgroup type index
+     */
+    template <typename SubgroupType>
+    static uint32_t get_subgroup_type_index();
+
+    /* singleton */
+private:
+    static std::unique_ptr<ServiceClient> service_client_singleton_ptr;
+    static std::mutex singleton_mutex;
+
+public:
+    /**
+     * Initialize the service_client_single_ptr singleton with a cascade service. This can only be called once
+     * before any get_service_client() is called.
+     * @param[in] _group_ptr The caller can pass a pointer pointing to a derecho group object. If the pointer is
+     *                   valid, the implementation will reply on the group object instead of creating an external
+     *                   client to communicate with group members.
+     */
+    static void initialize(derecho::Group<CascadeMetadataService<CascadeTypes...>, CascadeTypes...>* _group_ptr);
+
+    /**
+     * Get the singleton ServiceClient API. If it does not exists, initialize it as an external client.
+     */
+    static ServiceClient& get_service_client();
+};  // ServiceClient
+
+}  // namespace cascade
+}  // namespace derecho
+
+#include "detail/service_client_impl.hpp"
diff --git a/include/cascade/service_client_api.hpp b/include/cascade/service_client_api.hpp
index 04d7bc9d..cbaa224d 100644
--- a/include/cascade/service_client_api.hpp
+++ b/include/cascade/service_client_api.hpp
@@ -1,6 +1,7 @@
 #pragma once
 #include "service_types.hpp"
 #include "service.hpp"
+#include "service_client.hpp"
 
 #ifdef HAS_BOOLINQ
 #include <boolinq/boolinq.h>
@@ -60,7 +61,7 @@ class CascadeShardLinq : public boolinq::Linq<CascadeShardLinqStorageType<Cascad
 template <typename CascadeType, typename ServiceClientType>
 CascadeShardLinq<CascadeType,ServiceClientType> from_shard(
         std::vector<typename CascadeType::KeyType>& key_list,
-        ServiceClientType& capi, uint32_t subgroup_index, 
+        ServiceClientType& capi, uint32_t subgroup_index,
         uint32_t shard_index, persistent::version_t version) {
     /* load keys. */
     auto result = capi.template list_keys<CascadeType>(version, true, subgroup_index, shard_index);
@@ -95,7 +96,7 @@ CascadeShardLinq<CascadeType,ServiceClientType> from_shard(
  * @param capi      The cascade client.
  * @param subgroup_index
  * @param shard_index
- * @param ts_us     The unix epoch time in microsecond. 
+ * @param ts_us     The unix epoch time in microsecond.
  * @return a Linq object
  */
 template <typename CascadeType, typename ServiceClientType>
@@ -138,11 +139,11 @@ class CascadeVersionLinq : public boolinq::Linq<persistent::version_t, typename
 
 public:
     CascadeVersionLinq() : boolinq::Linq<persistent::version_t,typename CascadeType::ObjectType>() {};
-    
-    CascadeVersionLinq(ServiceClientType& capi, 
-        uint32_t sgidx, 
-        uint32_t shidx, 
-        const typename CascadeType::KeyType& objkey, 
+
+    CascadeVersionLinq(ServiceClientType& capi,
+        uint32_t sgidx,
+        uint32_t shidx,
+        const typename CascadeType::KeyType& objkey,
         persistent::version_t ver,
                        std::function<typename CascadeType::ObjectType(persistent::version_t&)> nextFunc) :
 
@@ -165,7 +166,7 @@ class CascadeVersionLinq : public boolinq::Linq<persistent::version_t, typename
  */
 template <typename CascadeType, typename ServiceClientType>
 CascadeVersionLinq<CascadeType,ServiceClientType> from_versions(
-    const typename CascadeType::KeyType& key, 
+    const typename CascadeType::KeyType& key,
  ServiceClientType &capi, uint32_t subgroup_index,
     uint32_t shard_index, persistent::version_t version) {
 
@@ -174,12 +175,12 @@ CascadeVersionLinq<CascadeType,ServiceClientType> from_versions(
    if (ver == INVALID_VERSION) {
        throw boolinq::LinqEndException();
    }
-  
+
             do {
                 auto result = capi.template get<CascadeType>(key,ver,true/*always use stable data*/,subgroup_index,shard_index);
                 for (auto& reply_future:result.get()) {
                     auto object = reply_future.second.get();
-                    ver = object.previous_version_by_key; 
+                    ver = object.previous_version_by_key;
                     if (!object.is_null())
                         return object;
                 }
@@ -210,7 +211,7 @@ class CascadeSubgroupLinq : public boolinq::Linq<CascadeSubgroupLinqStorageType<
       persistent::version_t ver,
       std::vector<CascadeShardLinq<CascadeType,ServiceClientType>>& shard_linq_list,
       std::function<typename CascadeType::ObjectType(CascadeSubgroupLinqStorageType<CascadeType,ServiceClientType>&)> nextFunc) :
-  
+
   boolinq::Linq<CascadeSubgroupLinqStorageType<CascadeType,ServiceClientType>, typename CascadeType::ObjectType>(std::make_pair(shard_linq_list.begin(),shard_linq_list.end()), nextFunc),
   client_api(capi),
   subgroup_index(sgidx),
@@ -219,10 +220,10 @@ class CascadeSubgroupLinq : public boolinq::Linq<CascadeSubgroupLinqStorageType<
 
 /**
  * Create a Linq iterating the objects in a subgroup.
- * @param shardidx_to_keys      The map that contains mappings between shard index and the corresponding 
- *                              key lists. The key lists are used to store the generated key_list in [from_shard] 
+ * @param shardidx_to_keys      The map that contains mappings between shard index and the corresponding
+ *                              key lists. The key lists are used to store the generated key_list in [from_shard]
  *                              function call. Please keep it alive throughout the life time of the Link object.
- * @param shard_linq_list       This is an output argument to keep the generated ShardLinq for each shard in the 
+ * @param shard_linq_list       This is an output argument to keep the generated ShardLinq for each shard in the
  *                              in the subgroup. Please keep it alive throughout the life time of the Link object.
  * @param capi                  The cascade client.
  * @param sgidx
@@ -231,10 +232,10 @@ class CascadeSubgroupLinq : public boolinq::Linq<CascadeSubgroupLinqStorageType<
  */
 template <typename CascadeType, typename ServiceClientType>
 CascadeSubgroupLinq<CascadeType,ServiceClientType> from_subgroup(
-    std::unordered_map<uint32_t, std::vector<typename CascadeType::KeyType>>& shardidx_to_keys, 
+    std::unordered_map<uint32_t, std::vector<typename CascadeType::KeyType>>& shardidx_to_keys,
  std::vector<CascadeShardLinq<CascadeType,ServiceClientType>>& shard_linq_list,
  ServiceClientType& capi, uint32_t sgidx, persistent::version_t ver) {
- 
+
     uint32_t num_shards = capi.template get_number_of_shards<CascadeType>(sgidx);
     std::cout << "num_shards=" << num_shards << std::endl;
     for (uint32_t shidx=0;shidx<num_shards;shidx++) {
@@ -242,7 +243,7 @@ CascadeSubgroupLinq<CascadeType,ServiceClientType> from_subgroup(
         shard_linq_list.emplace_back(from_shard<CascadeType, ServiceClientType>(shardidx_to_keys[shidx],capi,sgidx,shidx,ver));
     }
     std::cout << "done prepare shard iterators." << std::endl;
- 
+
  return CascadeSubgroupLinq<CascadeType,ServiceClientType>(capi,sgidx,ver,shard_linq_list,
      [&shard_linq_list](CascadeSubgroupLinqStorageType<CascadeType,ServiceClientType>& _storage) {
             if (_storage.first == _storage.second) {
@@ -253,12 +254,12 @@ CascadeSubgroupLinq<CascadeType,ServiceClientType> from_subgroup(
           try {
            auto obj = _storage.first->next();
      return obj;
-       } 
+       }
     catch(boolinq::LinqEndException &) {
             _storage.first++;
-       } 
+       }
    } while (_storage.first != _storage.second);
-         
+
    throw boolinq::LinqEndException();
      });
 }
@@ -301,7 +302,7 @@ class CascadeObjpoolLinq : public boolinq::Linq<CascadeObjectpoolLinqStorageType
 template <typename CascadeType, typename ServiceClientType>
 CascadeObjpoolLinq<CascadeType,ServiceClientType> from_objectpool(
         ServiceClientType& capi,
-        std::vector<typename CascadeType::KeyType>& key_list, 
+        std::vector<typename CascadeType::KeyType>& key_list,
         persistent::version_t version,
         const std::string objpool_path) {
     /* load keys. */
diff --git a/src/service/server.cpp b/src/service/server.cpp
index c7850d45..0f29d25b 100644
--- a/src/service/server.cpp
+++ b/src/service/server.cpp
@@ -2,6 +2,7 @@
 
 #include "cascade/cascade.hpp"
 #include "cascade/service.hpp"
+#include "cascade/service_client.hpp"
 #include "cascade/service_types.hpp"
 #include "cascade/object.hpp"
 
diff --git a/src/service/server.hpp b/src/service/server.hpp
index ee348cb5..1c1e31ad 100644
--- a/src/service/server.hpp
+++ b/src/service/server.hpp
@@ -3,6 +3,7 @@
 #include <cascade/cascade.hpp>
 #include <cascade/object.hpp>
 #include <cascade/service.hpp>
+#include <cascade/service_client.hpp>
 #include <cascade/service_types.hpp>
 #include <cascade/utils.hpp>
 
@@ -73,13 +74,13 @@ class CascadeServiceCDPO : public CriticalDataPathObserver<CascadeType> {
                         // dfg_ocdpos.second is a set of ocdpo info object of type prefix_ocdpo_info_t.
                         for(auto oiit = dfg_ocdpos.second.begin(); oiit != dfg_ocdpos.second.end();) {
                             if((oiit->hook != DataFlowGraph::VertexHook::BOTH) && (
-                                (oiit->hook == DataFlowGraph::VertexHook::ORDERED_PUT && is_trigger) || 
+                                (oiit->hook == DataFlowGraph::VertexHook::ORDERED_PUT && is_trigger) ||
                                 (oiit->hook == DataFlowGraph::VertexHook::TRIGGER_PUT && !is_trigger))) {
                                 // not my hook, skip it.
                                 oiit = dfg_ocdpos.second.erase(oiit);
                             } else if (is_trigger) {
                                 new_actions = true;
-                                if (oiit->execution_environment != 
+                                if (oiit->execution_environment !=
                                     DataFlowGraph::VertexExecutionEnvironment::PTHREAD) {
                                     has_mproc_udl = true;
                                 }
@@ -133,13 +134,13 @@ class CascadeServiceCDPO : public CriticalDataPathObserver<CascadeType> {
                                 value_ptr,
                                 oi.output_map  // outputs
                         );
-    
+
 #ifdef ENABLE_EVALUATION
                         ActionPostExtraInfo apei;
                         apei.uint64_val = 0;
                         apei.info.is_trigger = is_trigger;
 #endif
-    
+
 #ifdef ENABLE_EVALUATION
                         apei.info.stateful = oi.statefulness;
 #endif