cpus_allowed: use __NRPROCESSORS_CONF instead of __SC_NPROCESSORS_ONL…

…N for non-sequential CPU ids

When disabling SMT on some systems, the ID of some available CPU may
be larger than the value of sysconf(_SC_NPROCESSORS_ONLN). Without
this patch, fio complains that the expected CPU ID is invalid. Here's
an example from my server:

    $ ./fio --cpus_allowed=14 --ioengine=libaio --direct=1 --name=test --numjobs=1 --blocksize=128k --iodepth=1 --rw=read --filename=/dev/nvme0n1
    fio: CPU 14 too large (max=11)
    fio: failed parsing cpus_allowed=14

System information:

    $ lscpu
    Architecture:         x86_64
    CPU op-mode(s):       32-bit, 64-bit
    Byte Order:           Little Endian
    CPU(s):               20
    On-line CPU(s) list:  0,2,4,6,8,10,12,14,16-19
    Off-line CPU(s) list: 1,3,5,7,9,11,13,15
    ...
    Model name:           12th Gen Intel(R) Core(TM) i7-12700
    BIOS Model name:      12th Gen Intel(R) Core(TM) i7-12700
    ...

    $ uname -a
    Linux localhost.localdomain 4.18.0-348.el8.x86_64 axboe#1 SMP Tue Oct 19 15:14:17 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux

    $ cat /etc/redhat-release
    CentOS Linux release 8.5.2111

    $ cat /proc/cmdline
    BOOT_IMAGE=(hd0,gpt2)/vmlinuz-4.18.0-348.el8.x86_64 root=/dev/mapper/cl-root ro nosmt isolcpus=0,2,4,6,8,10,12,14 crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet

    $ cat test.c
    #include <stdio.h>
    #include <unistd.h>
    int main(int argc, char *argv[])
    {
        printf("_SC_NPROCESSORS_ONLN=%ld _SC_NPROCESSORS_CONF=%ld\n",
               sysconf(_SC_NPROCESSORS_ONLN), sysconf(_SC_NPROCESSORS_CONF));
    }

    $ gcc test.c

    $ ./a.out
    _SC_NPROCESSORS_ONLN=12 _SC_NPROCESSORS_CONF=20

Signed-off-by: mayuanpeng <[email protected]>

gettime.c

@@ -671,7 +671,7 @@ static int clock_cmp(const void *p1, const void *p2)
 int fio_monotonic_clocktest(int debug)
 {
 	struct clock_thread *cthreads;
-	unsigned int seen_cpus, nr_cpus = cpus_online();
+	unsigned int seen_cpus, nr_cpus = cpus_configured();
 	struct clock_entry *entries;
 	unsigned long nr_entries, tentries, failed = 0;
 	struct clock_entry *prev, *this;

idletime.c

@@ -189,7 +189,7 @@ void fio_idle_prof_init(void)
 	pthread_condattr_t cattr;
 	struct idle_prof_thread *ipt;
 
-	ipc.nr_cpus = cpus_online();
+	ipc.nr_cpus = cpus_configured();
 	ipc.status = IDLE_PROF_STATUS_OK;
 
 	if (ipc.opt == IDLE_PROF_OPT_NONE)

options.c

@@ -627,7 +627,7 @@ static int str_exitall_cb(void)
 int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu_index)
 {
 	unsigned int i, index, cpus_in_mask;
-	const long max_cpu = cpus_online();
+	const long max_cpu = cpus_configured();
 
 	cpus_in_mask = fio_cpu_count(mask);
 	if (!cpus_in_mask)
@@ -666,7 +666,7 @@ static int str_cpumask_cb(void *data, unsigned long long *val)
 		return 1;
 	}
 
-	max_cpu = cpus_online();
+	max_cpu = cpus_configured();
 
 	for (i = 0; i < sizeof(int) * 8; i++) {
 		if ((1 << i) & *val) {
@@ -702,7 +702,7 @@ static int set_cpus_allowed(struct thread_data *td, os_cpu_mask_t *mask,
 	strip_blank_front(&str);
 	strip_blank_end(str);
 
-	max_cpu = cpus_online();
+	max_cpu = cpus_configured();
 
 	while ((cpu = strsep(&str, ",")) != NULL) {
 		char *str2, *cpu2;
@@ -5305,7 +5305,7 @@ void fio_keywords_init(void)
 	sprintf(buf, "%llu", mb_memory);
 	fio_keywords[1].replace = strdup(buf);
 
-	l = cpus_online();
+	l = cpus_configured();
 	sprintf(buf, "%lu", l);
 	fio_keywords[2].replace = strdup(buf);
 }

os/os-hpux.h

@@ -88,9 +88,9 @@ static inline unsigned long long os_phys_mem(void)
 	return ret;
 }
 
-#define FIO_HAVE_CPU_ONLINE_SYSCONF
+#define FIO_HAVE_CPU_CONF_SYSCONF
 
-static inline unsigned int cpus_online(void)
+static inline unsigned int cpus_configured(void)
 {
 	return mpctl(MPC_GETNUMSPUS, 0, NULL);
 }

os/os-linux.h

@@ -251,14 +251,6 @@ static inline int arch_cache_line_size(void)
 		return atoi(size);
 }
 
-#ifdef __powerpc64__
-#define FIO_HAVE_CPU_ONLINE_SYSCONF
-static inline unsigned int cpus_online(void)
-{
-        return sysconf(_SC_NPROCESSORS_CONF);
-}
-#endif
-
 static inline unsigned long long get_fs_free_size(const char *path)
 {
 	unsigned long long ret;

os/os-solaris.h

@@ -119,7 +119,7 @@ static inline int fio_set_odirect(struct fio_file *f)
 
 static inline bool fio_cpu_isset(os_cpu_mask_t *mask, int cpu)
 {
-	const unsigned int max_cpus = sysconf(_SC_NPROCESSORS_ONLN);
+	const unsigned int max_cpus = sysconf(_SC_NPROCESSORS_CONF);
 	unsigned int num_cpus;
 	processorid_t *cpus;
 	bool ret;

os/os-windows.h

@@ -44,7 +44,7 @@
 #define fio_swap64(x)	_byteswap_uint64(x)
 
 #define _SC_PAGESIZE			0x1
-#define _SC_NPROCESSORS_ONLN	0x2
+#define _SC_NPROCESSORS_CONF	0x2
 #define _SC_PHYS_PAGES			0x4
 
 #define SA_RESTART	0
@@ -219,9 +219,6 @@ static inline int fio_mkdir(const char *path, mode_t mode) {
 	return 0;
 }
 
-#define FIO_HAVE_CPU_ONLINE_SYSCONF
-unsigned int cpus_online(void);
-
 int first_set_cpu(os_cpu_mask_t *cpumask);
 int fio_setaffinity(int pid, os_cpu_mask_t cpumask);
 int fio_cpuset_init(os_cpu_mask_t *mask);

os/os.h

@@ -352,18 +352,18 @@ static inline unsigned long long get_fs_free_size(const char *path)
 }
 #endif
 
-#ifndef FIO_HAVE_CPU_ONLINE_SYSCONF
-static inline unsigned int cpus_online(void)
+#ifndef FIO_HAVE_CPU_CONF_SYSCONF
+static inline unsigned int cpus_configured(void)
 {
-	return sysconf(_SC_NPROCESSORS_ONLN);
+	return sysconf(_SC_NPROCESSORS_CONF);
 }
 #endif
 
 #ifndef CPU_COUNT
 #ifdef FIO_HAVE_CPU_AFFINITY
 static inline int CPU_COUNT(os_cpu_mask_t *mask)
 {
-	int max_cpus = cpus_online();
+	int max_cpus = cpus_configured();
 	int nr_cpus, i;
 
 	for (i = 0, nr_cpus = 0; i < max_cpus; i++)

os/windows/cpu-affinity.c

@@ -2,12 +2,6 @@
 
 #include <windows.h>
 
-/* Return all processors regardless of processor group */
-unsigned int cpus_online(void)
-{
-	return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
-}
-
 static void print_mask(os_cpu_mask_t *cpumask)
 {
 	for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)

os/windows/posix.c

@@ -216,10 +216,18 @@ long sysconf(int name)
 	MEMORYSTATUSEX status;
 
 	switch (name) {
-	case _SC_NPROCESSORS_ONLN:
-		val = GetNumLogicalProcessors();
+	case _SC_NPROCESSORS_CONF:
+		/*
+		 * Using GetMaximumProcessorCount introduces a problem in
+		 * gettime.c because Windows does not have
+		 * fio_get_thread_affinity. Log sample (see #1479):
+		 *
+		 *   CPU mask contains processor beyond last active processor index (2)
+		 *   clock setaffinity failed: No error
+		 */
+		val = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
 		if (val == -1)
-			log_err("sysconf(_SC_NPROCESSORS_ONLN) failed\n");
+			log_err("sysconf(_SC_NPROCESSORS_CONF) failed\n");
 
 		break;
 
@@ -1201,4 +1209,4 @@ HANDLE windows_handle_connection(HANDLE hjob, int sk)
 	DisconnectNamedPipe(hpipe);
 	CloseHandle(hpipe);
 	return ret;
-}
+}

server.c

@@ -999,7 +999,7 @@ static int handle_probe_cmd(struct fio_net_cmd *cmd)
 		.os		= FIO_OS,
 		.arch		= FIO_ARCH,
 		.bpp		= sizeof(void *),
-		.cpus		= __cpu_to_le32(cpus_online()),
+		.cpus		= __cpu_to_le32(cpus_configured()),
 	};
 
 	dprint(FD_NET, "server: sending probe reply\n");

t/dedupe.c

@@ -688,7 +688,7 @@ int main(int argc, char *argv[])
 		use_bloom = 0;
 
 	if (!num_threads)
-		num_threads = cpus_online();
+		num_threads = cpus_configured();
 
 	if (argc == optind)
 		return usage(argv);