From 4de86ec4f989f49be9f9869e02e1c580087149fd Mon Sep 17 00:00:00 2001 From: Erik Boasson Date: Mon, 12 Aug 2024 16:00:37 +0200 Subject: [PATCH] Add setting for controlling thread affinity This adds a Thread/Scheduling/Affinity option, which is a list of unsigned 32-bit integers indicating in some manner what CPUs to bind the thread to. Silently ignored on platforms that do not support it (or where Cyclone doesn't provide support for it yet). Currently only on Linux, where it is interpreted as a list of CPU ids. Signed-off-by: Erik Boasson --- docs/manual/config/config_file_reference.rst | 24 +++++-- docs/manual/options.md | 20 ++++-- etc/cyclonedds.rnc | 16 +++-- etc/cyclonedds.xsd | 18 ++++-- src/core/ddsi/defconfig.c | 10 +-- src/core/ddsi/include/dds/ddsi/ddsi_config.h | 6 ++ src/core/ddsi/src/ddsi__cfgelems.h | 10 +++ src/core/ddsi/src/ddsi_config.c | 68 ++++++++++++++++++++ src/core/ddsi/src/ddsi_thread.c | 2 + src/ddsrt/include/dds/ddsrt/threads.h | 3 + src/ddsrt/src/threads.c | 3 + src/ddsrt/src/threads/posix/threads.c | 22 +++++++ src/tools/_confgen/_confgen.h | 1 + src/tools/_confgen/generate_defconfig.c | 8 +++ 14 files changed, 184 insertions(+), 27 deletions(-) diff --git a/docs/manual/config/config_file_reference.rst b/docs/manual/config/config_file_reference.rst index ffe85c57b5..a526e4da16 100644 --- a/docs/manual/config/config_file_reference.rst +++ b/docs/manual/config/config_file_reference.rst @@ -2520,11 +2520,23 @@ The default value is: ```` //CycloneDDS/Domain/Threads/Thread/Scheduling ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Children: :ref:`Class`, :ref:`Priority` +Children: :ref:`Affinity`, :ref:`Class`, :ref:`Priority` This element configures the scheduling properties of the thread. +.. _`//CycloneDDS/Domain/Threads/Thread/Scheduling/Affinity`: + +//CycloneDDS/Domain/Threads/Thread/Scheduling/Affinity +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Text + +This element specifies the thread affinity using a string of comma-separated unsigned 32-bit integers. The notional meaning of the string is that it lists the IDs of the CPU cores to use, but some platforms may use a different mapping. Ignored if unsupported by the platform. + +The default value is: ```` + + .. _`//CycloneDDS/Domain/Threads/Thread/Scheduling/Class`: //CycloneDDS/Domain/Threads/Thread/Scheduling/Class @@ -2704,14 +2716,14 @@ The categorisation of tracing output is incomplete and hence most of the verbosi The default value is: ``none`` .. - generated from ddsi_config.h[83ad19f1a665710b0c82b3ac6b861e6c8e83913f] + generated from ddsi_config.h[e6e75c7c07b3b91a92715063cfd8abdd0fbd8b08] generated from ddsi__cfgunits.h[bd22f0c0ed210501d0ecd3b07c992eca549ef5aa] - generated from ddsi__cfgelems.h[194217161977869610495a7889bbc1e6bc976ce1] - generated from ddsi_config.c[a439a20e32fe327db26f2f10028d0056e46c1a0b] - generated from _confgen.h[e32eabfc35e9f3a7dcb63b19ed148c0d17c6e5fc] + generated from ddsi__cfgelems.h[69679834d0a592a339803ed27e3966adc900d592] + generated from ddsi_config.c[8d7ef0ae962a47cb2138de27ac0f6751e3393c66] + generated from _confgen.h[9554f1d72645c0b8bb66ffbfbc3c0fb664fc1a43] generated from _confgen.c[237308acd53897a34e8c643e16e05a61d73ffd65] generated from generate_rnc.c[b50e4b7ab1d04b2bc1d361a0811247c337b74934] generated from generate_md.c[789b92e422631684352909cfb8bf43f6ceb16a01] generated from generate_rst.c[3c4b523fbb57c8e4a7e247379d06a8021ccc21c4] generated from generate_xsd.c[6b6818d7f17a35d56c376c04ec1410427f34c0f0] - generated from generate_defconfig.c[63ca9d8ae2f1ce2e761c9d4c0510a45eb062d830] + generated from generate_defconfig.c[631cafee70a6f9480e0267db8ffe883d806f5f70] diff --git a/docs/manual/options.md b/docs/manual/options.md index 217bb09790..bd01a1bb4c 100644 --- a/docs/manual/options.md +++ b/docs/manual/options.md @@ -1754,11 +1754,19 @@ The default value is: `` ##### //CycloneDDS/Domain/Threads/Thread/Scheduling -Children: [Class](#cycloneddsdomainthreadsthreadschedulingclass), [Priority](#cycloneddsdomainthreadsthreadschedulingpriority) +Children: [Affinity](#cycloneddsdomainthreadsthreadschedulingaffinity), [Class](#cycloneddsdomainthreadsthreadschedulingclass), [Priority](#cycloneddsdomainthreadsthreadschedulingpriority) This element configures the scheduling properties of the thread. +###### //CycloneDDS/Domain/Threads/Thread/Scheduling/Affinity +Text + +This element specifies the thread affinity using a string of comma-separated unsigned 32-bit integers. The notional meaning of the string is that it lists the IDs of the CPU cores to use, but some platforms may use a different mapping. Ignored if unsupported by the platform. + +The default value is: `` + + ###### //CycloneDDS/Domain/Threads/Thread/Scheduling/Class One of: realtime, timeshare, default @@ -1898,14 +1906,14 @@ While none prevents any message from being written to a DDSI2 log file. The categorisation of tracing output is incomplete and hence most of the verbosity levels and categories are not of much use in the current release. This is an ongoing process and here we describe the target situation rather than the current situation. Currently, the most useful verbosity levels are config, fine and finest. The default value is: `none` - + - - - + + + - + diff --git a/etc/cyclonedds.rnc b/etc/cyclonedds.rnc index f1d4c36265..0370114513 100644 --- a/etc/cyclonedds.rnc +++ b/etc/cyclonedds.rnc @@ -1215,6 +1215,12 @@ MIIEpAIBAAKCAQEA3HIh...AOBaaqSV37XBUJg==

This element configures the scheduling properties of the thread.

""" ] ] element Scheduling { [ a:documentation [ xml:lang="en" """ +

This element specifies the thread affinity using a string of comma-separated unsigned 32-bit integers. The notional meaning of the string is that it lists the IDs of the CPU cores to use, but some platforms may use a different mapping. Ignored if unsupported by the platform.

+

The default value is: <empty>

""" ] ] + element Affinity { + text + }? + & [ a:documentation [ xml:lang="en" """

This element specifies the thread scheduling class (realtime, timeshare or default). The user may need special privileges from the underlying operating system to be able to assign some of the privileged scheduling classes.

The default value is: default

""" ] ] element Class { @@ -1313,14 +1319,14 @@ MIIEpAIBAAKCAQEA3HIh...AOBaaqSV37XBUJg==
duration_inf = xsd:token { pattern = "inf|0|(\d+(\.\d*)?([Ee][\-+]?\d+)?|\.\d+([Ee][\-+]?\d+)?) *([num]?s|min|hr|day)" } memsize = xsd:token { pattern = "0|(\d+(\.\d*)?([Ee][\-+]?\d+)?|\.\d+([Ee][\-+]?\d+)?) *([kMG]i?)?B" } } -# generated from ddsi_config.h[83ad19f1a665710b0c82b3ac6b861e6c8e83913f] +# generated from ddsi_config.h[e6e75c7c07b3b91a92715063cfd8abdd0fbd8b08] # generated from ddsi__cfgunits.h[bd22f0c0ed210501d0ecd3b07c992eca549ef5aa] -# generated from ddsi__cfgelems.h[194217161977869610495a7889bbc1e6bc976ce1] -# generated from ddsi_config.c[a439a20e32fe327db26f2f10028d0056e46c1a0b] -# generated from _confgen.h[e32eabfc35e9f3a7dcb63b19ed148c0d17c6e5fc] +# generated from ddsi__cfgelems.h[69679834d0a592a339803ed27e3966adc900d592] +# generated from ddsi_config.c[8d7ef0ae962a47cb2138de27ac0f6751e3393c66] +# generated from _confgen.h[9554f1d72645c0b8bb66ffbfbc3c0fb664fc1a43] # generated from _confgen.c[237308acd53897a34e8c643e16e05a61d73ffd65] # generated from generate_rnc.c[b50e4b7ab1d04b2bc1d361a0811247c337b74934] # generated from generate_md.c[789b92e422631684352909cfb8bf43f6ceb16a01] # generated from generate_rst.c[3c4b523fbb57c8e4a7e247379d06a8021ccc21c4] # generated from generate_xsd.c[6b6818d7f17a35d56c376c04ec1410427f34c0f0] -# generated from generate_defconfig.c[63ca9d8ae2f1ce2e761c9d4c0510a45eb062d830] +# generated from generate_defconfig.c[631cafee70a6f9480e0267db8ffe883d806f5f70] diff --git a/etc/cyclonedds.xsd b/etc/cyclonedds.xsd index c314d2fdb2..15e0a9c9c5 100644 --- a/etc/cyclonedds.xsd +++ b/etc/cyclonedds.xsd @@ -1813,11 +1813,19 @@ MIIEpAIBAAKCAQEA3HIh...AOBaaqSV37XBUJg==<br> + + + + +<p>This element specifies the thread affinity using a string of comma-separated unsigned 32-bit integers. The notional meaning of the string is that it lists the IDs of the CPU cores to use, but some platforms may use a different mapping. Ignored if unsupported by the platform.</p> +<p>The default value is: <code>&lt;empty&gt;</code></p> + + @@ -1971,14 +1979,14 @@ MIIEpAIBAAKCAQEA3HIh...AOBaaqSV37XBUJg==<br> - + - - - + + + - + diff --git a/src/core/ddsi/defconfig.c b/src/core/ddsi/defconfig.c index b84a164347..53614ab3ab 100644 --- a/src/core/ddsi/defconfig.c +++ b/src/core/ddsi/defconfig.c @@ -99,14 +99,14 @@ void ddsi_config_init_default (struct ddsi_config *cfg) cfg->ssl_min_version.minor = 3; #endif /* DDS_HAS_TCP_TLS */ } -/* generated from ddsi_config.h[83ad19f1a665710b0c82b3ac6b861e6c8e83913f] */ +/* generated from ddsi_config.h[e6e75c7c07b3b91a92715063cfd8abdd0fbd8b08] */ /* generated from ddsi__cfgunits.h[bd22f0c0ed210501d0ecd3b07c992eca549ef5aa] */ -/* generated from ddsi__cfgelems.h[194217161977869610495a7889bbc1e6bc976ce1] */ -/* generated from ddsi_config.c[a439a20e32fe327db26f2f10028d0056e46c1a0b] */ -/* generated from _confgen.h[e32eabfc35e9f3a7dcb63b19ed148c0d17c6e5fc] */ +/* generated from ddsi__cfgelems.h[69679834d0a592a339803ed27e3966adc900d592] */ +/* generated from ddsi_config.c[8d7ef0ae962a47cb2138de27ac0f6751e3393c66] */ +/* generated from _confgen.h[9554f1d72645c0b8bb66ffbfbc3c0fb664fc1a43] */ /* generated from _confgen.c[237308acd53897a34e8c643e16e05a61d73ffd65] */ /* generated from generate_rnc.c[b50e4b7ab1d04b2bc1d361a0811247c337b74934] */ /* generated from generate_md.c[789b92e422631684352909cfb8bf43f6ceb16a01] */ /* generated from generate_rst.c[3c4b523fbb57c8e4a7e247379d06a8021ccc21c4] */ /* generated from generate_xsd.c[6b6818d7f17a35d56c376c04ec1410427f34c0f0] */ -/* generated from generate_defconfig.c[63ca9d8ae2f1ce2e761c9d4c0510a45eb062d830] */ +/* generated from generate_defconfig.c[631cafee70a6f9480e0267db8ffe883d806f5f70] */ diff --git a/src/core/ddsi/include/dds/ddsi/ddsi_config.h b/src/core/ddsi/include/dds/ddsi/ddsi_config.h index 0abc6fe808..e240e85a72 100644 --- a/src/core/ddsi/include/dds/ddsi/ddsi_config.h +++ b/src/core/ddsi/include/dds/ddsi/ddsi_config.h @@ -120,12 +120,18 @@ struct ddsi_config_maybe_duration { dds_duration_t value; }; +struct ddsi_config_uint32_array { + uint32_t n; + uint32_t *xs; +}; + struct ddsi_config_thread_properties_listelem { struct ddsi_config_thread_properties_listelem *next; char *name; ddsrt_sched_t sched_class; struct ddsi_config_maybe_int32 schedule_priority; struct ddsi_config_maybe_uint32 stack_size; + struct ddsi_config_uint32_array affinity; }; struct ddsi_config_peer_listelem diff --git a/src/core/ddsi/src/ddsi__cfgelems.h b/src/core/ddsi/src/ddsi__cfgelems.h index 2b17969bb6..9f67e84f85 100644 --- a/src/core/ddsi/src/ddsi__cfgelems.h +++ b/src/core/ddsi/src/ddsi__cfgelems.h @@ -846,6 +846,16 @@ static struct cfgelem thread_properties_sched_cfgelems[] = { "special privileges from the underlying operating system to be able to " "assign some of the privileged priorities.

" )), + STRING("Affinity", NULL, 1, "", + MEMBEROF(ddsi_config_thread_properties_listelem, affinity), + FUNCTIONS(0, uf_uint32_array, ff_uint32_array, pf_uint32_array), + DESCRIPTION( + "

This element specifies the thread affinity using a string of " + "comma-separated unsigned 32-bit integers. The notional meaning " + "of the string is that it lists the IDs of the CPU cores to use, " + "but some platforms may use a different mapping. Ignored if " + "unsupported by the platform.

" + )), END_MARKER }; diff --git a/src/core/ddsi/src/ddsi_config.c b/src/core/ddsi/src/ddsi_config.c index 4db60cf602..40c97a178b 100644 --- a/src/core/ddsi/src/ddsi_config.c +++ b/src/core/ddsi/src/ddsi_config.c @@ -192,6 +192,7 @@ DU(deaf_mute); DUPF(min_tls_version); #endif DUPF(shm_loglevel); +DUPF(uint32_array); #undef DUPF #undef DU #undef PF @@ -199,6 +200,7 @@ DUPF(shm_loglevel); #define DF(fname) static void fname (struct ddsi_cfgst *cfgst, void *parent, struct cfgelem const * const cfgelem) DF(ff_free); DF(ff_networkAddresses); +DF(ff_uint32_array); #undef DF #define DI(fname) static int fname (struct ddsi_cfgst *cfgst, void *parent, struct cfgelem const * const cfgelem) @@ -1595,6 +1597,72 @@ static enum update_result uf_deaf_mute (struct ddsi_cfgst *cfgst, void *parent, return uf_boolean (cfgst, parent, cfgelem, first, value); } +static enum update_result uf_uint32_array (struct ddsi_cfgst *cfgst, void *parent, struct cfgelem const * const cfgelem, UNUSED_ARG (int first), const char *value) +{ + struct ddsi_config_uint32_array * const elem = cfg_address (cfgst, parent, cfgelem); + if (*value == 0) // Short-circuit the common, trivial case of an empty string + { + elem->n = 0; + elem->xs = NULL; + return URES_SUCCESS; + } + else + { + const char *v = strchr (value, ','); + uint32_t maxn = 1; + while (v) + { + maxn++; + v = strchr (v + 1, ','); + } + uint32_t *xs = ddsrt_malloc (maxn * sizeof (*xs)); + uint32_t idx = 0; + char *valuecopy = ddsrt_strdup (value), *cursor = valuecopy, *tok; + while ((tok = ddsrt_strsep (&cursor, ",")) != NULL) + { + assert (idx < maxn); + int64_t i64; + enum update_result res = uf_int64_unit (cfgst, &i64, tok, NULL, 1, 0, UINT32_MAX); + if (res != URES_SUCCESS) + { + ddsrt_free (valuecopy); + ddsrt_free (xs); + return res; + } + assert (i64 >= 0 && i64 <= UINT32_MAX); + xs[idx++] = (uint32_t) i64; + } + elem->n = idx; + elem->xs = xs; + ddsrt_free (valuecopy); + return URES_SUCCESS; + } +} + +static void ff_uint32_array (struct ddsi_cfgst *cfgst, void *parent, struct cfgelem const * const cfgelem) +{ + struct ddsi_config_uint32_array * const elem = cfg_address (cfgst, parent, cfgelem); + ddsrt_free (elem->xs); +} + +static void pf_uint32_array (struct ddsi_cfgst *cfgst, void *parent, struct cfgelem const * const cfgelem, uint32_t sources) +{ + struct ddsi_config_uint32_array const * const elem = cfg_address (cfgst, parent, cfgelem); + size_t bufsize = 11 * elem->n + 1; + char *buf = ddsrt_malloc (bufsize); + size_t pos = 0; + buf[0] = 0; + // pos < bufsize by construction (uint32_t: 10 digits + 1 for a comma, the extra byte is there for the n = 0 case) + for (uint32_t i = 0; i < elem->n; i++) + { + int n = snprintf (buf + pos, bufsize - pos, "%s%"PRIu32, (i > 0 ? "," : ""), elem->xs[i]); + if (n > 0) + pos += (size_t) n; + } + cfg_logelem (cfgst, sources, "%s", buf); + ddsrt_free (buf); +} + static struct ddsi_cfgst_node *lookup_or_create_elem_record (struct ddsi_cfgst *cfgst, struct cfgelem const * const cfgelem, void *parent, uint32_t source) { struct ddsi_cfgst_node *n; diff --git a/src/core/ddsi/src/ddsi_thread.c b/src/core/ddsi/src/ddsi_thread.c index 5f9b0e3bf2..b2d30b28d8 100644 --- a/src/core/ddsi/src/ddsi_thread.c +++ b/src/core/ddsi/src/ddsi_thread.c @@ -334,6 +334,8 @@ static dds_return_t create_thread_int (struct ddsi_thread_state **ts1_out, const tattr.schedClass = tprops->sched_class; /* explicit default value in the enum */ if (!tprops->stack_size.isdefault) tattr.stackSize = tprops->stack_size.value; + tattr.schedAffinityN = tprops->affinity.n; + tattr.schedAffinitySet = tprops->affinity.xs; } if (gv) { diff --git a/src/ddsrt/include/dds/ddsrt/threads.h b/src/ddsrt/include/dds/ddsrt/threads.h index 0b3bb94428..d6b937f021 100644 --- a/src/ddsrt/include/dds/ddsrt/threads.h +++ b/src/ddsrt/include/dds/ddsrt/threads.h @@ -68,6 +68,9 @@ typedef struct { ddsrt_sched_t schedClass; /** Specifies the thread priority */ int32_t schedPriority; + /** Specifies thread affinity, N = 0, Set = NULL or N > 0 and Set[0..N-1] contains N CPU ids */ + uint32_t schedAffinityN; + uint32_t *schedAffinitySet; /** Specifies the thread stack size */ uint32_t stackSize; } ddsrt_threadattr_t; diff --git a/src/ddsrt/src/threads.c b/src/ddsrt/src/threads.c index 902ba8676f..6631467d70 100644 --- a/src/ddsrt/src/threads.c +++ b/src/ddsrt/src/threads.c @@ -11,6 +11,7 @@ #include #include "dds/ddsrt/threads.h" +#include "dds/ddsrt/heap.h" void ddsrt_threadattr_init ( @@ -19,5 +20,7 @@ ddsrt_threadattr_init ( assert(attr != NULL); attr->schedClass = DDSRT_SCHED_DEFAULT; attr->schedPriority = 0; + attr->schedAffinityN = 0; + attr->schedAffinitySet = NULL; attr->stackSize = 0; } diff --git a/src/ddsrt/src/threads/posix/threads.c b/src/ddsrt/src/threads/posix/threads.c index 466a0a646e..b3d7398abc 100644 --- a/src/ddsrt/src/threads/posix/threads.c +++ b/src/ddsrt/src/threads/posix/threads.c @@ -410,6 +410,28 @@ ddsrt_thread_create ( } #endif } + +#if defined __linux + if (tattr.schedAffinityN > 0) + { + cpu_set_t cpuset; + CPU_ZERO (&cpuset); + for (uint32_t i = 0; i < tattr.schedAffinityN; i++) + { + if (tattr.schedAffinitySet[i] >= CPU_SETSIZE) + { + DDS_ERROR ("os_threadCreate(%s): CPU id %"PRIu32" out of range when setting affinity\n", name, tattr.schedAffinitySet[i]); + goto err; + } + CPU_SET(tattr.schedAffinitySet[i], &cpuset); + } + if ((result = pthread_attr_setaffinity_np (&pattr, sizeof (cpuset), &cpuset)) != 0) + { + DDS_ERROR("ddsrt_thread_create(%s): pthread_attr_setinheritsched(EXPLICIT) failed with error %d\n", name, result); + goto err; + } + } +#endif /* Construct context structure & start thread */ ctx = ddsrt_malloc (sizeof (thread_context_t)); diff --git a/src/tools/_confgen/_confgen.h b/src/tools/_confgen/_confgen.h index cd64e8ae90..aa25fc9077 100644 --- a/src/tools/_confgen/_confgen.h +++ b/src/tools/_confgen/_confgen.h @@ -53,6 +53,7 @@ void gendef_pf_transport_selector (FILE *fp, void *parent, struct cfgelem const void gendef_pf_many_sockets_mode (FILE *fp, void *parent, struct cfgelem const * const cfgelem); void gendef_pf_standards_conformance (FILE *fp, void *parent, struct cfgelem const * const cfgelem); void gendef_pf_shm_loglevel (FILE *fp, void *parent, struct cfgelem const * const cfgelem); +void gendef_pf_uint32_array (FILE *out, void *parent, struct cfgelem const * const cfgelem); struct cfgunit { const char *name; diff --git a/src/tools/_confgen/generate_defconfig.c b/src/tools/_confgen/generate_defconfig.c index 1122998307..b0924a43b8 100644 --- a/src/tools/_confgen/generate_defconfig.c +++ b/src/tools/_confgen/generate_defconfig.c @@ -193,6 +193,14 @@ void gendef_pf_standards_conformance (FILE *out, void *parent, struct cfgelem co void gendef_pf_shm_loglevel (FILE *out, void *parent, struct cfgelem const * const cfgelem) { gendef_pf_int (out, parent, cfgelem); } +void gendef_pf_uint32_array (FILE *out, void *parent, struct cfgelem const * const cfgelem) { + (void) out; + struct ddsi_config_uint32_array const * const p = cfg_address (parent, cfgelem); + if (p->n != 0) { + fprintf (stderr, "generate_defconfig internal error: non-empty uint32_array not handled\n"); + abort (); + } +} static void gen_defaults (FILE *out, void *parent, struct cfgelem const * const cfgelem) {