Copycat L3 indexer and GraphQL

rebar.config

@@ -176,7 +176,7 @@
 ]}.
 
 {deps, [
-    {elmdb, {git, "https://github.com/permaweb/elmdb-rs.git", {ref, "06ccf937abc250cb22c782d568efcaa39f5452ff"}}},
+    {elmdb, {git, "https://github.com/permaweb/elmdb-rs.git", {ref, "68316484a5bd45bcc4b180a60883db7272c5dbde"}}},
     {b64rs, {git, "https://github.com/permaweb/b64rs.git", {ref, "94b7d8e51d9a44f3bd12b7d138dd0d2cb74c169f"}}},
     {base32, "1.0.0"},
     {cowlib, "2.16.0"},

rebar.lock

@@ -11,7 +11,7 @@
  {<<"ddskerl">>,{pkg,<<"ddskerl">>,<<"0.4.2">>},1},
  {<<"elmdb">>,
   {git,"https://github.com/permaweb/elmdb-rs.git",
-       {ref,"06ccf937abc250cb22c782d568efcaa39f5452ff"}},
+       {ref,"68316484a5bd45bcc4b180a60883db7272c5dbde"}},
   0},
  {<<"eqwalizer_support">>,
   {git_subdir,"https://github.com/whatsapp/eqwalizer.git",

src/core/include/hb_store_arweave.hrl

@@ -0,0 +1,3 @@
+-define(SCOPE_PARENT, <<"parent">>).
+-define(SCOPE_OFFSET, <<"offset">>).
+

src/core/monitor/hb_event.erl

@@ -405,18 +405,8 @@ check_overload(Last, N) ->
             case erlang:process_info(self(), message_queue_len) of
                 {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH ->
                     {memory, MemorySize} = erlang:process_info(self(), memory),
-                    case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of
-                        1 ->
-                            ?debug_print(
-                                {warning,
-                                    prometheus_event_queue_overloading,
-                                    {queue, Len},
-                                    {last_event, Last},
-                                    {memory_bytes, MemorySize}
-                                }
-                            );
-                        _ -> ignored
-                    end,
+                    % If the size of this process is too large, exit such that
+                    % we can be restarted by the next caller.
                     case MemorySize of
                         MemorySize when MemorySize > ?MAX_MEMORY ->
                             ?debug_print(
@@ -683,4 +673,4 @@ wait_drain_loop(Pid, Deadline) ->
         undefined ->
             error(event_server_dead)
     end.
--endif.
+-endif.

src/core/monitor/hb_prometheus.erl

@@ -1,7 +1,7 @@
 %%% @doc HyperBEAM wrapper for Prometheus metrics.
 -module(hb_prometheus).
 -export([ensure_started/0, declare/2, measure_and_report/2, measure_and_report/3]).
--export([observe/2, observe/3, inc/2, inc/3, inc/4, dec/2, dec/3, dec/4]).
+-export([observe/2, observe/3, inc/2, inc/3, inc/4, dec/2, dec/3, dec/4, set/4]).
 -define(STARTED_CACHE_KEY, {?MODULE, started}).
 
 %% @doc Ensure the Prometheus application has been started. Caches startup
@@ -118,4 +118,13 @@ dec(Type, Metrics, Labels, Value) ->
     end.
 
 do_dec(gauge, Name, Labels, Value) ->
-    prometheus_gauge:dec(Name, Labels, Value).
+    prometheus_gauge:dec(Name, Labels, Value).
+
+set(gauge, Name, Labels, Value) ->
+    case ensure_started() of
+        ok ->
+            try prometheus_gauge:set(Name, Labels, Value)
+            catch error:mfa_already_exists -> ok
+            end;
+        _ -> ok
+    end.

src/core/resolver/hb_opts.erl

@@ -20,6 +20,7 @@
 -include("include/hb.hrl").
 -include("include/hb_opts.hrl").
 -include("include/hb_arweave_nodes.hrl").
+-include("include/hb_store_arweave.hrl").
 -include("../../_build/hb_preloaded_index.hrl").
 
 -ifndef(PRELOADED_DEVICES_INDEX_MESSAGE_ID).
@@ -283,6 +284,12 @@ raw_default_message() ->
         <<"relay-http-client">> => httpc,
         % The default codec to use for commitment signatures.
         <<"commitment-device">> => <<"httpsig@1.0">>,
+        % Copycat-specific options.
+        <<"copycat-memory-cap">> => 6 * 1024 * 1024 * 1024,
+        <<"copycat-memory-budget">> => 6 * 1024 * 1024 * 1024,
+        <<"copycat-depth-recursion-cap">> => 6, % 2x the deepest we've seen to date
+        <<"arweave-block-workers">> => 3,
+        <<"copycat-scope">> => [?SCOPE_OFFSET, ?SCOPE_PARENT],
         % Dev options
         <<"mode">> => debug,
         <<"profiling">> => true,
@@ -1258,4 +1265,4 @@ ensure_node_history_test() ->
                 ]
         },
     ?assertEqual({error, invalid_values}, ensure_node_history(InvalidItems, RequiredOpts)).
--endif.
+-endif.

src/core/store/hb_store.erl

@@ -561,8 +561,6 @@ start_one(Store = #{ <<"store-module">> := Mod }, Req, Opts) ->
     end.
 
 call_store_start(Mod, Store, Req, Opts) ->
-    %% function_exported doesn't load the module. We need to call ensure_loaded
-    %% here since is the first time we call a function to load the module.
     code:ensure_loaded(Mod),
     case erlang:function_exported(Mod, start, 3) of
         true -> Mod:start(Store, Req, Opts);