Separate the mechanisms and APIs for dependent memory and custom blocks

otherlibs/runtime_events/runtime_events.ml

@@ -33,6 +33,7 @@ type runtime_counter =
 | EV_C_MAJOR_HEAP_POOL_FRAG_WORDS
 | EV_C_MAJOR_HEAP_POOL_LIVE_BLOCKS
 | EV_C_MAJOR_HEAP_LARGE_BLOCKS
+| EV_C_REQUEST_MINOR_REALLOC_DEPENDENT_TABLE
 
 type runtime_phase =
 | EV_EXPLICIT_GC_SET
@@ -83,6 +84,7 @@ type runtime_phase =
 | EV_COMPACT_EVACUATE
 | EV_COMPACT_FORWARD
 | EV_COMPACT_RELEASE
+| EV_MINOR_DEPENDENT
 
 type lifecycle =
   EV_RING_START
@@ -121,6 +123,8 @@ let runtime_counter_name counter =
       "major_heap_pool_live_blocks"
   | EV_C_MAJOR_HEAP_LARGE_BLOCKS ->
       "major_heap_large_blocks"
+  | EV_C_REQUEST_MINOR_REALLOC_DEPENDENT_TABLE ->
+      "request_minor_realloc_dependent_table"
 
 let runtime_phase_name phase =
   match phase with
@@ -172,6 +176,7 @@ let runtime_phase_name phase =
   | EV_COMPACT_EVACUATE -> "compaction_evacuate"
   | EV_COMPACT_FORWARD -> "compaction_forward"
   | EV_COMPACT_RELEASE -> "compaction_release"
+  | EV_MINOR_DEPENDENT -> "minor_dependent"
 
 let lifecycle_name lifecycle =
   match lifecycle with

otherlibs/runtime_events/runtime_events.mli

@@ -87,6 +87,9 @@ Live blocks of a Domain's major heap pools.
 (**
 Live blocks of a Domain's major heap large allocations.
 @since 5.1 *)
+| EV_C_REQUEST_MINOR_REALLOC_DEPENDENT_TABLE
+(** Reallocation of the table of dependent memory from minor heap
+    @since 5.3 *)
 
 (** The type for span events emitted by the runtime. *)
 type runtime_phase =
@@ -138,6 +141,7 @@ type runtime_phase =
 | EV_COMPACT_EVACUATE
 | EV_COMPACT_FORWARD
 | EV_COMPACT_RELEASE
+| EV_MINOR_DEPENDENT
 
 (** Lifecycle events for the ring itself. *)
 type lifecycle =

runtime/caml/bigarray.h

@@ -81,7 +81,7 @@ enum caml_ba_subarray {
 struct caml_ba_proxy {
   atomic_uintnat refcount;      /* Reference count */
   void * data;                  /* Pointer to base of actual data */
-  uintnat size;                 /* Size of data in bytes (if mapped file) */
+  uintnat size;                 /* Size of data in bytes */
 };
 
 struct caml_ba_array {

runtime/caml/custom.h

@@ -53,6 +53,7 @@ extern "C" {
 
 
 CAMLextern uintnat caml_custom_major_ratio;
+CAMLextern uintnat caml_custom_minor_ratio;
 
 CAMLextern value caml_alloc_custom(const struct custom_operations * ops,
                                    uintnat size, /*size in bytes*/

runtime/caml/domain_state.tbl

@@ -90,6 +90,10 @@ DOMAIN_STATE(uintnat, allocated_words_direct)
 /* Number of words allocated directly to the major heap since the latest
    slice. (subset of allocated_words) */
 
+DOMAIN_STATE(uintnat, allocated_dependent_bytes)
+/* Number of external bytes whose pointers were promoted or allocated in
+   the major heap since latest slice. */
+
 DOMAIN_STATE(uintnat, swept_words)
 
 DOMAIN_STATE(uintnat, major_slice_epoch)
@@ -138,11 +142,8 @@ DOMAIN_STATE(int, unique_id)
 DOMAIN_STATE(value, dls_root)
 /* Domain-local state */
 
-DOMAIN_STATE(double, extra_heap_resources)
-DOMAIN_STATE(double, extra_heap_resources_minor)
-
-DOMAIN_STATE(uintnat, dependent_size)
-DOMAIN_STATE(uintnat, dependent_allocated)
+/* How much external memory is currenty held by the minor and major heap. */
+DOMAIN_STATE(uintnat, minor_dependent_bsz)
 
 /* How much work needs to be done (by all domains) before we stop this slice. */
 DOMAIN_STATE(intnat, slice_target)
@@ -165,6 +166,9 @@ DOMAIN_STATE(struct caml_intern_state*, intern_state)
 DOMAIN_STATE(uintnat, stat_minor_words)
 DOMAIN_STATE(uintnat, stat_promoted_words)
 DOMAIN_STATE(uintnat, stat_major_words)
+DOMAIN_STATE(uintnat, stat_minor_dependent_bytes)
+DOMAIN_STATE(uintnat, stat_promoted_dependent_bytes)
+DOMAIN_STATE(uintnat, stat_major_dependent_bytes)
 DOMAIN_STATE(intnat, stat_forced_major_collections)
 DOMAIN_STATE(uintnat, stat_blocks_marked)
 

runtime/caml/gc_stats.h

@@ -41,6 +41,7 @@ struct heap_stats {
   intnat large_words;
   intnat large_max_words;
   intnat large_blocks;
+  intnat dependent_bytes;
 };
 
 /* Note: accumulating stats then removing them is not a no-op, as
@@ -56,6 +57,9 @@ struct alloc_stats {
   uint64_t minor_words;
   uint64_t promoted_words;
   uint64_t major_words;
+  uint64_t minor_dependent_bytes;
+  uint64_t promoted_dependent_bytes;
+  uint64_t major_dependent_bytes;
   uint64_t forced_major_collections;
 };
 void caml_accum_alloc_stats(

runtime/caml/minor_gc.h

@@ -57,15 +57,20 @@ struct caml_ephe_ref_table CAML_TABLE_STRUCT(struct caml_ephe_ref_elt);
 
 struct caml_custom_elt {
   value block;     /* The finalized block in the minor heap. */
-  mlsize_t mem;    /* The parameters for adjusting GC speed. */
-  mlsize_t max;
 };
 struct caml_custom_table CAML_TABLE_STRUCT(struct caml_custom_elt);
 
+struct caml_dependent_elt {
+  value block;     /* The finalized block in the minor heap. */
+  mlsize_t mem;    /* The size in bytes of the dependent memory. */
+};
+struct caml_dependent_table CAML_TABLE_STRUCT(struct caml_dependent_elt);
+
 struct caml_minor_tables {
   struct caml_ref_table major_ref;
   struct caml_ephe_ref_table ephe_ref;
   struct caml_custom_table custom;
+  struct caml_dependent_table dependent;
 };
 
 CAMLextern void caml_minor_collection (void);
@@ -85,6 +90,7 @@ void caml_alloc_table (struct caml_ref_table *tbl, asize_t sz, asize_t rsv);
 extern void caml_realloc_ref_table (struct caml_ref_table *);
 extern void caml_realloc_ephe_ref_table (struct caml_ephe_ref_table *);
 extern void caml_realloc_custom_table (struct caml_custom_table *);
+extern void caml_realloc_dependent_table (struct caml_dependent_table *);
 struct caml_minor_tables* caml_alloc_minor_tables(void);
 void caml_free_minor_tables(struct caml_minor_tables*);
 void caml_empty_minor_heap_setup(caml_domain_state* domain, void*);
@@ -117,8 +123,7 @@ Caml_inline void add_to_ephe_ref_table (struct caml_ephe_ref_table *tbl,
   CAMLassert(ephe_ref->offset < Wosize_val(ephe_ref->ephe));
 }
 
-Caml_inline void add_to_custom_table (struct caml_custom_table *tbl, value v,
-                                        mlsize_t mem, mlsize_t max)
+Caml_inline void add_to_custom_table (struct caml_custom_table *tbl, value v)
 {
   struct caml_custom_elt *elt;
   if (tbl->ptr >= tbl->limit){
@@ -127,8 +132,20 @@ Caml_inline void add_to_custom_table (struct caml_custom_table *tbl, value v,
   }
   elt = tbl->ptr++;
   elt->block = v;
+}
+
+Caml_inline void add_to_dependent_table (struct caml_dependent_table *tbl,
+                                         value v,
+                                         mlsize_t mem)
+{
+  struct caml_dependent_elt *elt;
+  if (tbl->ptr >= tbl->limit){
+    CAMLassert (tbl->ptr == tbl->limit);
+    caml_realloc_dependent_table (tbl);
+  }
+  elt = tbl->ptr++;
+  elt->block = v;
   elt->mem = mem;
-  elt->max = max;
 }
 
 #endif /* CAML_INTERNALS */

runtime/caml/runtime_events.h

@@ -118,7 +118,8 @@ typedef enum {
     EV_COMPACT,
     EV_COMPACT_EVACUATE,
     EV_COMPACT_FORWARD,
-    EV_COMPACT_RELEASE
+    EV_COMPACT_RELEASE,
+    EV_MINOR_DEPENDENT,
 } ev_runtime_phase;
 
 typedef enum {
@@ -139,6 +140,7 @@ typedef enum {
     EV_C_MAJOR_HEAP_POOL_FRAG_WORDS,
     EV_C_MAJOR_HEAP_POOL_LIVE_BLOCKS,
     EV_C_MAJOR_HEAP_LARGE_BLOCKS,
+    EV_C_REQUEST_MINOR_REALLOC_DEPENDENT_TABLE,
 } ev_runtime_counter;
 
 typedef enum {

runtime/caml/shared_heap.h

@@ -39,6 +39,9 @@ value* caml_shared_try_alloc(struct caml_heap_state*,
 /* If we were to grow the shared heap, how much would we grow it? */
 uintnat caml_shared_heap_grow_bsize(void);
 
+/* Update the dependent_bytes field of the heap stats. */
+void caml_add_dependent_bytes (struct caml_heap_state *local, intnat n);
+
 /* Copy the domain-local heap stats into a heap stats sample. */
 void caml_collect_heap_stats_sample(
   struct caml_heap_state* local,

runtime/custom.c

@@ -51,22 +51,8 @@ mlsize_t caml_custom_get_max_major (void)
          * caml_custom_major_ratio;
 }
 
-/* [mem] is an amount of out-of-heap resources, in the same units as
-   [max_major] and [max_minor]. When the cumulated amount of such
-   resources reaches [max_minor] (for resources held by the minor
-   heap) we do a minor collection; when it reaches [max_major] (for
-   resources held by the major heap), we guarantee that a major cycle
-   is done.
-
-   If [max_major] is 0, then [mem] is a number of bytes and the actual
-   limit is [caml_custom_get_max_major ()] computed at the
-   time when the custom block is promoted to the major heap.
-*/
 static value alloc_custom_gen (const struct custom_operations * ops,
                                uintnat bsz,
-                               mlsize_t mem,
-                               mlsize_t max_major,
-                               mlsize_t max_minor,
                                int minor_ok,
                                int local)
 {
@@ -83,21 +69,13 @@ static value alloc_custom_gen (const struct custom_operations * ops,
   else if (wosize <= Max_young_wosize && minor_ok) {
     result = caml_alloc_small(wosize, Custom_tag);
     Custom_ops_val(result) = ops;
-    if (ops->finalize != NULL || mem != 0) {
+    if (ops->finalize != NULL) {
       /* Record the extra resources in case the block gets promoted. */
-      add_to_custom_table (&Caml_state->minor_tables->custom, result,
-                           mem, max_major);
-      /* Keep track of extra resources held by custom block in
-         minor heap. */
-      if (mem != 0) {
-        caml_adjust_minor_gc_speed (mem, max_minor);
-      }
+      add_to_custom_table (&Caml_state->minor_tables->custom, result);
     }
   } else {
     result = caml_alloc_shr(wosize, Custom_tag);
     Custom_ops_val(result) = ops;
-    caml_adjust_gc_speed(mem, max_major);
-    result = caml_check_urgent_gc(result);
   }
   CAMLreturn(result);
 }
@@ -108,23 +86,12 @@ Caml_inline mlsize_t get_max_minor (void)
     Bsize_wsize (Caml_state->minor_heap_wsz) / 100 * caml_custom_minor_ratio;
 }
 
-static value caml_alloc_custom0(const struct custom_operations * ops,
-                                uintnat bsz,
-                                mlsize_t mem,
-                                mlsize_t max,
-                                int local)
-{
-  mlsize_t max_major = max;
-  mlsize_t max_minor = max == 0 ? get_max_minor() : max;
-  return alloc_custom_gen (ops, bsz, mem, max_major, max_minor, 1, local);
-}
-
 CAMLexport value caml_alloc_custom(const struct custom_operations * ops,
                                    uintnat bsz,
                                    mlsize_t mem,
                                    mlsize_t max)
 {
-  return caml_alloc_custom0(ops, bsz, mem, max, 0);
+  return alloc_custom_gen(ops, bsz, /* minor_ok: */ 1, /* local: */ 0);
 }
 
 CAMLexport value caml_alloc_custom_local(const struct custom_operations * ops,
@@ -136,7 +103,7 @@ CAMLexport value caml_alloc_custom_local(const struct custom_operations * ops,
     caml_invalid_argument(
       "caml_alloc_custom_local: finalizers not supported");
 
-  return caml_alloc_custom0(ops, bsz, mem, max, 1);
+  return alloc_custom_gen(ops, bsz, /* minor_ok: */ 1, /* local: */ 1);
 }
 
 CAMLexport value caml_alloc_custom_mem(const struct custom_operations * ops,
@@ -150,19 +117,30 @@ CAMLexport value caml_alloc_custom_mem(const struct custom_operations * ops,
   } else {
     max_minor_single = max_minor * caml_custom_minor_max_bsz / 100;
   }
-  value v = alloc_custom_gen (ops, bsz, mem, 0,
-                              max_minor, (mem < max_minor_single), 0);
+
+  value v = alloc_custom_gen (ops, bsz,
+                              /* minor_ok: */ (mem <= max_minor_single),
+                              /* local: */ 0);
   size_t mem_words = (mem + sizeof(value) - 1) / sizeof(value);
   caml_memprof_sample_block(v, mem_words, mem_words, CAML_MEMPROF_SRC_CUSTOM);
   return v;
 }
 
-CAMLexport value caml_alloc_custom_dep(const struct custom_operations * ops,
-                                       uintnat size, mlsize_t mem)
+/* For each block allocated with [caml_alloc_custom_dep],
+   the finalizer must call [caml_free_dependent_memory].
+   [bsz] is the size in bytes of the payload inside the heap-allocated
+   block, and [mem] is the size in bytes of the external memory
+   held by this block.
+*/
+CAMLexport value caml_alloc_custom_dep (const struct custom_operations * ops,
+                                        uintnat bsz,
+                                        mlsize_t mem)
 {
-  /* For now, alias caml_alloc_custom_mem, but this implementation
-     is to be replaced */
-  return caml_alloc_custom_mem(ops, size, mem);
+  CAMLparam0();
+  CAMLlocal1(result);
+  result = caml_alloc_custom_mem(ops, bsz, mem);
+  caml_alloc_dependent_memory (result, mem);
+  CAMLreturn(result);
 }
 
 struct custom_operations_list {

runtime/domain.c

@@ -658,11 +658,8 @@ static void domain_create(uintnat initial_minor_heap_wsize,
 
   CAMLassert(!interruptor_has_pending(s));
 
-  domain_state->extra_heap_resources = 0.0;
-  domain_state->extra_heap_resources_minor = 0.0;
-
-  domain_state->dependent_size = 0;
-  domain_state->dependent_allocated = 0;
+  domain_state->allocated_dependent_bytes = 0;
+  domain_state->minor_dependent_bsz = 0;
 
   domain_state->major_work_done_between_slices = 0;
 

runtime/gc_stats.c

@@ -38,6 +38,7 @@ void caml_accum_heap_stats(struct heap_stats* acc, const struct heap_stats* h)
   acc->large_max_words = intnat_max(acc->large_max_words, acc->large_words);
   acc->large_max_words = intnat_max(acc->large_max_words, h->large_max_words);
   acc->large_blocks += h->large_blocks;
+  acc->dependent_bytes += h->dependent_bytes;
 }
 
 void caml_remove_heap_stats(struct heap_stats* acc, const struct heap_stats* h)
@@ -48,6 +49,7 @@ void caml_remove_heap_stats(struct heap_stats* acc, const struct heap_stats* h)
   acc->pool_frag_words -= h->pool_frag_words;
   acc->large_words -= h->large_words;
   acc->large_blocks -= h->large_blocks;
+  acc->dependent_bytes -= h->dependent_bytes;
 }
 
 void caml_accum_alloc_stats(
@@ -57,6 +59,9 @@ void caml_accum_alloc_stats(
   acc->minor_words += s->minor_words;
   acc->promoted_words += s->promoted_words;
   acc->major_words += s->major_words;
+  acc->minor_dependent_bytes += s->minor_dependent_bytes;
+  acc->promoted_dependent_bytes += s->promoted_dependent_bytes;
+  acc->major_dependent_bytes += s->major_dependent_bytes;
   acc->forced_major_collections += s->forced_major_collections;
 }
 
@@ -67,6 +72,9 @@ void caml_collect_alloc_stats_sample(
   sample->minor_words = local->stat_minor_words;
   sample->promoted_words = local->stat_promoted_words;
   sample->major_words = local->stat_major_words;
+  sample->minor_dependent_bytes = local->stat_minor_dependent_bytes;
+  sample->promoted_dependent_bytes = local->stat_promoted_dependent_bytes;
+  sample->major_dependent_bytes = local->stat_major_dependent_bytes;
   sample->forced_major_collections = local->stat_forced_major_collections;
 }
 
@@ -75,6 +83,9 @@ void caml_reset_domain_alloc_stats(caml_domain_state *local)
   local->stat_minor_words = 0;
   local->stat_promoted_words = 0;
   local->stat_major_words = 0;
+  local->stat_minor_dependent_bytes = 0;
+  local->stat_promoted_dependent_bytes = 0;
+  local->stat_major_dependent_bytes = 0;
   local->stat_forced_major_collections = 0;
 }
 

runtime/intern.c

@@ -715,7 +715,7 @@ static void intern_rec(struct caml_intern_state* s,
           s->intern_obj_table[s->obj_counter++] = v;
         if (ops->finalize != NULL && Is_young(v)) {
           /* Remember that the block has a finalizer. */
-          add_to_custom_table (&d->minor_tables->custom, v, 0, 1);
+          add_to_custom_table (&d->minor_tables->custom, v);
         }
         break;
       }