add blazingly fast instance_cache implementation for K2

builtin-functions/kphp-light/stdlib/instance-cache.txt

@@ -1,13 +1,10 @@
 <?php
 
-/** @kphp-extern-func-info cpp_template_call interruptible */
+/** @kphp-extern-func-info cpp_template_call */
 function instance_cache_fetch(string $type, string $key, bool $even_if_expired = false) ::: instance<^1>;
 
-/** @kphp-extern-func-info interruptible */
 function instance_cache_store(string $key, object $value, int $ttl = 0) ::: bool;
 
-/** @kphp-extern-func-info interruptible */
 function instance_cache_update_ttl(string $key, int $ttl = 0) ::: bool;
 
-/** @kphp-extern-func-info interruptible */
 function instance_cache_delete(string $key) ::: bool;

compiler/code-gen/declarations.cpp

@@ -5,6 +5,7 @@
 #include "compiler/code-gen/declarations.h"
 
 #include "common/algorithms/compare.h"
+#include "common/algorithms/hashes.h"
 
 #include "compiler/code-gen/common.h"
 #include "compiler/code-gen/const-globals-batched-mem.h"
@@ -578,6 +579,7 @@ void ClassDeclaration::compile_inner_methods(CodeGenerator &W, ClassPtr klass) {
   compile_has_wakeup_flag(W, klass);
   compile_get_class(W, klass);
   compile_get_hash(W, klass);
+  compile_class_name_hash(W, klass);
   compile_accept_visitor_methods(W, klass);
   compile_msgpack_declarations(W, klass);
   compile_virtual_builtin_functions(W, klass);
@@ -752,6 +754,12 @@ void ClassDeclaration::compile_get_hash(CodeGenerator &W, ClassPtr klass) {
   compile_class_method(FunctionSignatureGenerator(W).set_const_this(), klass, "int get_hash()", klass->get_hash());
 }
 
+void ClassDeclaration::compile_class_name_hash(CodeGenerator &W, ClassPtr klass) {
+  // hash of the class name, computed once at compile time -- same for every instance,
+  // unlike the virtual get_hash() it can be read without an instance at hand.
+  W << "constexpr static uint64_t CLASS_NAME_HASH{" << vk::murmur_hash<uint64_t>(klass->name.data(), klass->name.size()) << "ULL};" << NL << NL;
+}
+
 void ClassDeclaration::compile_accept_visitor(CodeGenerator &W, ClassPtr klass, const char *visitor_type) {
   compile_class_method(FunctionSignatureGenerator(W), klass, fmt_format("void accept({} &visitor)", visitor_type), "generic_accept(visitor)");
 }
@@ -907,7 +915,7 @@ void ClassDeclaration::compile_accept_json_visitor(CodeGenerator &W, ClassPtr kl
 
 void ClassDeclaration::compile_accept_visitor_methods(CodeGenerator &W, ClassPtr klass) {
   bool need_generic_accept =
-    klass->need_to_array_debug_visitor || (klass->need_instance_cache_visitors && !G->is_output_mode_k2()) || (klass->need_instance_memory_estimate_visitor);
+    klass->need_to_array_debug_visitor || klass->need_instance_cache_visitors || klass->need_instance_memory_estimate_visitor;
 
   if (!need_generic_accept && klass->json_encoders.empty()) {
     return;
@@ -939,6 +947,13 @@ void ClassDeclaration::compile_accept_visitor_methods(CodeGenerator &W, ClassPtr
     compile_accept_visitor(W, klass, "InstanceDeepDestroyVisitor");
   }
 
+  if (klass->need_instance_cache_visitors && G->is_output_mode_k2()) {
+    W << NL;
+    compile_accept_visitor(W, klass, "kphp::visitors::instance_deep_copy_visitor");
+    W << NL;
+    compile_accept_visitor(W, klass, "kphp::visitors::instance_deep_estimate_size_visitor");
+  }
+
   compile_accept_json_visitor(W, klass);
 }
 
@@ -960,8 +975,14 @@ void ClassDeclaration::compile_virtual_builtin_functions(CodeGenerator &W, Class
   compile_class_method(FunctionSignatureGenerator(W).set_const_this(), klass,
                        "size_t virtual_builtin_sizeof()", "sizeof(*this)");
 
+  compile_class_method(FunctionSignatureGenerator(W).set_const_this(), klass,
+                       "size_t virtual_builtin_alignof()", "alignof(" + klass->src_name + ")");
+
   compile_class_method(FunctionSignatureGenerator(W).set_const_this(), klass,
                        klass->src_name + "* virtual_builtin_clone()", "new " + klass->src_name + "{*this}");
+
+  compile_class_method(FunctionSignatureGenerator(W).set_const_this(), klass,
+                       klass->src_name + "* virtual_builtin_construct_at(void* ptr)", "new (ptr) " + klass->src_name + "{*this}");
 }
 
 void ClassDeclaration::compile_wakeup(CodeGenerator &W, ClassPtr klass) {
@@ -1059,7 +1080,7 @@ void ClassDeclaration::compile_job_worker_shared_memory_piece_methods(CodeGenera
 
 void ClassMembersDefinition::compile(CodeGenerator &W) const {
   bool need_generic_accept =
-    klass->need_to_array_debug_visitor || (klass->need_instance_cache_visitors && !G->is_output_mode_k2()) || (klass->need_instance_memory_estimate_visitor);
+    klass->need_to_array_debug_visitor || klass->need_instance_cache_visitors || klass->need_instance_memory_estimate_visitor;
 
   if (!need_generic_accept && !klass->is_serializable && klass->json_encoders.empty()) {
     return;
@@ -1100,6 +1121,13 @@ void ClassMembersDefinition::compile(CodeGenerator &W) const {
     compile_generic_accept_instantiations(W, klass, "InstanceDeepDestroyVisitor");
   }
 
+  if (klass->need_instance_cache_visitors && G->is_output_mode_k2()) {
+    W << NL;
+    compile_generic_accept_instantiations(W, klass, "kphp::visitors::instance_deep_copy_visitor");
+    W << NL;
+    compile_generic_accept_instantiations(W, klass, "kphp::visitors::instance_deep_estimate_size_visitor");
+  }
+
   W << NL;
   compile_accept_json_visitor(W, klass);
 

compiler/code-gen/declarations.h

@@ -125,6 +125,7 @@ struct ClassDeclaration : CodeGenRootCmd {
   static void compile_has_wakeup_flag(CodeGenerator &W, ClassPtr klass);
   static void compile_get_class(CodeGenerator &W, ClassPtr klass);
   static void compile_get_hash(CodeGenerator &W, ClassPtr klass);
+  static void compile_class_name_hash(CodeGenerator &W, ClassPtr klass);
   static void compile_accept_visitor_methods(CodeGenerator &W, ClassPtr klass);
   static void compile_msgpack_declarations(CodeGenerator &W, ClassPtr klass);
   static void compile_virtual_builtin_functions(CodeGenerator &W, ClassPtr klass);

compiler/pipes/final-check.cpp

@@ -44,7 +44,6 @@ void check_class_immutableness(ClassPtr klass) {
 std::vector<ClassPtr> find_not_ic_compatibility_derivatives(ClassPtr klass);
 
 void check_fields_ic_compatibility(ClassPtr klass) {
-  // In case of K2 mode, all checks about serializability have already done
   bool flag = false;
   if (!klass->process_fields_ic_compatibility.compare_exchange_strong(flag, true, std::memory_order_acq_rel)) {
     return;
@@ -69,7 +68,6 @@ void check_fields_ic_compatibility(ClassPtr klass) {
 }
 
 void check_derivatives_ic_compatibility(ClassPtr klass) {
-  // In case of K2 mode, all checks about serializability have already done
   std::vector<ClassPtr> descendants = find_not_ic_compatibility_derivatives(klass);
   for (const auto &element : descendants) {
     kphp_error(false, fmt_format("Can not store polymorphic type {} with mutable derived class {}", klass->name, element->name));
@@ -134,13 +132,9 @@ void check_instance_cache_fetch_call(VertexAdaptor<op_func_call> call) {
 
   kphp_error(klass->is_immutable || klass->is_interface(),
              fmt_format("Can not fetch instance of mutable class {} with instance_cache_fetch call", klass->name));
-  kphp_error(klass->is_serializable,
-             fmt_format("Can not fetch instance of non-serializable class {} with instance_cache_fetch call", klass->name));
 
-  if (G->is_output_mode_k2()) {
-    // To be able to store instances in request cache
-    klass->deeply_require_may_be_mixed_base();
-  } else {
+  if (!G->is_output_mode_k2()) {
+    // in K2 mode fetch just reinterprets the shared memory block, so no visitor codegen is needed
     klass->deeply_require_instance_cache_visitor();
   }
 }
@@ -153,12 +147,6 @@ void check_instance_cache_store_call(VertexAdaptor<op_func_call> call) {
 
   kphp_error_return(klass->is_immutable || klass->is_interface(),
                     fmt_format("Can not store instance of mutable class {} with instance_cache_store call", klass->name));
-  kphp_error_return(klass->is_serializable, fmt_format("Can not store instance of non-serializable class {} with instance_cache_store call", klass->name));
-
-  if (G->is_output_mode_k2()) {
-    // To be able to store instances in request cache
-    klass->deeply_require_may_be_mixed_base();
-  }
 
   check_fields_ic_compatibility(klass);
   check_derivatives_ic_compatibility(klass);

runtime-common/core/class-instance/class-instance-decl.inl

@@ -1,6 +1,9 @@
 #pragma once
 
+#include <cstddef>
+
 #include "common/smart_ptrs/intrusive_ptr.h"
+#include "common/wrappers/span.h"
 
 #ifndef INCLUDED_FROM_KPHP_CORE
 #error "this file must be included only from runtime-core.h"
@@ -71,8 +74,17 @@ public:
 
   inline class_instance& operator=(const Optional<bool>& null) noexcept;
   inline class_instance clone() const;
+  // copies the instance into externally provided memory (no allocation/ownership)
+  // memory must be aligned to alignof(T) and >= estimate_memory_usage() bytes
+  // caller must pin it with a special ExtraRefCnt (e.g. for_instance_cache), since the instance never frees it.
+  // Returns a null instance if memory is unfit.
+  inline class_instance clone_in(vk::span<std::byte> memory) const noexcept;
   template<class... Args>
   inline class_instance<T> alloc(Args&&... args) __attribute__((always_inline));
+  // constructs an instance in externally provided memory (no allocation/ownership)
+  // leaves it null if memory is smaller than sizeof(T) or misaligned
+  template<class... Args>
+  inline class_instance<T> alloc(vk::span<std::byte> memory, Args&&... args) noexcept __attribute__((always_inline));
   inline class_instance<T> empty_alloc() __attribute__((always_inline));
   inline void destroy() {
     o.reset();
@@ -97,11 +109,26 @@ public:
     return o->virtual_builtin_sizeof();
   }
 
+  template<class S = T>
+  std::enable_if_t<!std::is_polymorphic<S>{}, size_t> alignment() const noexcept {
+    return alignof(T);
+  }
+
+  template<class S = T>
+  std::enable_if_t<std::is_polymorphic<S>{}, size_t> alignment() const noexcept {
+    return o->virtual_builtin_alignof();
+  }
+
   template<class S = T>
   std::enable_if_t<!std::is_polymorphic<S>{}, class_instance> virtual_builtin_clone() const noexcept {
     return clone();
   }
 
+  template<class S = T>
+  std::enable_if_t<!std::is_polymorphic<S>{}, class_instance> virtual_builtin_clone_in(vk::span<std::byte> memory) const noexcept {
+    return clone_in(memory);
+  }
+
   template<class S = T>
   std::enable_if_t<std::is_polymorphic<S>{}, class_instance> virtual_builtin_clone() const noexcept {
     // TODO this is used only for job workers. Should we use this logic for other?
@@ -113,6 +140,19 @@ public:
     return res;
   }
 
+  template<class S = T>
+  std::enable_if_t<std::is_polymorphic<S>{}, class_instance> virtual_builtin_clone_in(vk::span<std::byte> memory) const noexcept {
+    class_instance res;
+    if (o) {
+      if (unlikely(memory.size() < o->virtual_builtin_sizeof() || reinterpret_cast<std::uintptr_t>(memory.data()) % o->virtual_builtin_alignof() != 0)) {
+        return res;
+      }
+      res.o = vk::intrusive_ptr<T>{o->virtual_builtin_construct_at(memory.data())};
+      res.o->set_refcnt(1);
+    }
+    return res;
+  }
+
   template<class S = T>
   std::enable_if_t<!std::is_polymorphic<S>{}, void*> get_base_raw_ptr() const noexcept {
     return get();
@@ -198,6 +238,8 @@ public:
 private:
   class_instance<T> clone_impl(std::true_type /*is empty*/) const;
   class_instance<T> clone_impl(std::false_type /*is empty*/) const;
+  class_instance<T> clone_in_impl(vk::span<std::byte> memory, std::true_type /*is empty*/) const noexcept;
+  class_instance<T> clone_in_impl(vk::span<std::byte> memory, std::false_type /*is empty*/) const noexcept;
 };
 
 template<class T, class... Args>

runtime-common/core/class-instance/class-instance.inl

@@ -26,10 +26,31 @@ class_instance<T> class_instance<T>::clone_impl(std::false_type /*is empty*/) co
   return res;
 }
 
+template<class T>
+class_instance<T> class_instance<T>::clone_in_impl(vk::span<std::byte> /* memory */, std::true_type /*is empty*/) const noexcept {
+  return class_instance<T>{}.empty_alloc();
+}
+
+template<class T>
+class_instance<T> class_instance<T>::clone_in_impl(vk::span<std::byte> memory, std::false_type /*is empty*/) const noexcept {
+  class_instance<T> res;
+  if (o) {
+    res.alloc(memory, *o); // res stays null if the memory is insufficient or misaligned
+    if (likely(!res.is_null())) {
+      res.o->set_refcnt(1);
+    }
+  }
+  return res;
+}
+
 template<class T>
 class_instance<T> class_instance<T>::clone() const {
   return clone_impl(std::is_empty<T>{});
 }
+template<class T>
+class_instance<T> class_instance<T>::clone_in(vk::span<std::byte> memory) const noexcept {
+  return clone_in_impl(memory, std::is_empty<T>{});
+}
 
 template<class T>
 template<class... Args>
@@ -40,6 +61,19 @@ class_instance<T> class_instance<T>::alloc(Args&&... args) {
   return *this;
 }
 
+template<class T>
+template<class... Args>
+class_instance<T> class_instance<T>::alloc(vk::span<std::byte> memory, Args&&... args) noexcept {
+  static_assert(!std::is_empty<T>{}, "class T may not be empty");
+  php_assert(!o);
+  if (unlikely(memory.size() < sizeof(T) || reinterpret_cast<std::uintptr_t>(memory.data()) % alignof(T) != 0)) {
+    return *this;
+  }
+  T* ptr = new (memory.data()) T{std::forward<Args>(args)...};
+  new (&o) vk::intrusive_ptr<T>(ptr);
+  return *this;
+}
+
 template<class T>
 inline class_instance<T> class_instance<T>::empty_alloc() {
   static_assert(std::is_empty<T>{}, "class T must be empty");

runtime-common/core/core-types/decl/array_decl.inl

@@ -4,6 +4,10 @@
 
 #pragma once
 
+#include <cstddef>
+#include <optional>
+
+#include "common/wrappers/span.h"
 #include "runtime-common/core/core-types/decl/array_iterator.h"
 #include "runtime-common/core/include.h"
 
@@ -132,7 +136,6 @@ private:
     inline static size_t sizeof_vector(uint32_t int_size) noexcept __attribute__((always_inline));
     inline static size_t sizeof_map(uint32_t int_size) noexcept __attribute__((always_inline));
     inline static size_t estimate_size(int64_t& new_int_size, bool is_vector);
-    inline static array_inner* create(int64_t new_int_size, bool is_vector);
 
     inline static array_inner* empty_array() __attribute__((always_inline));
 
@@ -186,6 +189,36 @@ private:
     inline array_inner& operator=(const array_inner& other) = delete;
   };
 
+  class allocation {
+    vk::span<std::byte> mem_;
+    int64_t int_size_{0};
+    bool is_vector_{false};
+
+    inline allocation(vk::span<std::byte> memory, int64_t int_size, bool is_vector) noexcept
+        : mem_{memory},
+          int_size_{int_size},
+          is_vector_{is_vector} {}
+
+  public:
+    // allocates script memory for an array of the given size
+    inline static allocation allocate(int64_t new_int_size, bool is_vector) noexcept;
+    // takes ownership of the beginning of externally provided memory (no allocation):
+    // returns std::nullopt if the memory is smaller than estimate_size(new_int_size, is_vector) or misaligned
+    inline static std::optional<allocation> from_external(vk::span<std::byte> memory, int64_t new_int_size, bool is_vector) noexcept;
+
+    vk::span<std::byte> memory() const noexcept {
+      return mem_;
+    }
+    int64_t int_size() const noexcept {
+      return int_size_;
+    }
+    bool is_vector() const noexcept {
+      return is_vector_;
+    }
+  };
+
+  inline static array_inner* create_from_allocation(allocation alloc) noexcept;
+
   inline bool mutate_if_vector_shared(uint32_t mul = 1);
   inline bool mutate_to_size_if_vector_shared(int64_t int_size);
   inline void mutate_to_size(int64_t int_size);
@@ -197,7 +230,13 @@ private:
   inline void convert_to_map();
 
   template<class T1>
-  inline void copy_from(const array<T1>& other);
+  inline void copy_from(const array<T1>& other) noexcept;
+
+  template<class T1>
+  inline bool copy_from(vk::span<std::byte> memory, const array<T1>& other) noexcept;
+
+  template<class T1>
+  inline void copy_from_impl(array_inner* new_array, const array<T1>& other) noexcept;
 
   template<class T1>
   inline void move_from(array<T1>&& other) noexcept;
@@ -228,6 +267,12 @@ public:
   template<class T1, class = enable_if_constructible_or_unknown<T, T1>>
   inline array(array<T1>&& other) noexcept __attribute__((always_inline));
 
+  // copies other into externally provided memory (no allocation/ownership).
+  // Memory must be aligned to alignof(array_inner) and have at least other.calculate_memory_for_copying() bytes.
+  // The array never frees this memory, so the caller must protect it with a special ExtraRefCnt (e.g. for_instance_cache).
+  // Returns std::nullopt if the memory is unfit.
+  inline static std::optional<array> copy_in(vk::span<std::byte> memory, const array& other) noexcept;
+
   template<class... Args>
   inline static array create(Args&&... args) __attribute__((always_inline));
 
@@ -440,6 +485,10 @@ public:
   size_t estimate_memory_usage() const noexcept;
   size_t calculate_memory_for_copying() const noexcept;
 
+  static constexpr size_t alignment() noexcept {
+    return alignof(array_inner);
+  }
+
   template<typename U>
   static array<T> convert_from(const array<U>&);