From 9b5d1b7f8076821467918e5ceff821f90570f864 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 2 Sep 2023 13:33:34 -0400 Subject: [PATCH 001/111] Attempting to improve the DHT abstraction. --- DHT/depend | 47 +++-------- DHT/dht.c | 78 ++++++++++-------- DHT/dht.h | 12 +-- DHT/dhtbcmem.c | 109 +++++++++++++++++-------- DHT/dhtcmem.c | 92 ++++++++++++++++------ DHT/dhtcmem.h | 4 +- DHT/dhtexam1.c | 38 +++++---- DHT/dhtexam2.c | 8 +- DHT/dhtmanex.c | 20 +++-- DHT/dhtmem.c | 73 +++++++++++------ DHT/dhtmem.h | 6 +- DHT/dhtsimpl.c | 52 ++++++------ DHT/dhtstrng.c | 53 ++++++++----- DHT/dhtvalue.h | 60 ++++++++++++-- optimisations/hash.c | 184 +++++++++++++++++++------------------------ optimisations/hash.h | 1 + 16 files changed, 500 insertions(+), 337 deletions(-) diff --git a/DHT/depend b/DHT/depend index ce88a9dce0..8518b65096 100644 --- a/DHT/depend +++ b/DHT/depend @@ -1,70 +1,47 @@ DHT/dht$(OBJ_SUFFIX): DHT/dht.c debugging/assert.h utilities/boolean.h \ DHT/dhtvalue.h DHT/fxf.h DHT/dht.h debugging/trace.h - debugging/assert.h: - utilities/boolean.h: - DHT/dhtvalue.h: - DHT/fxf.h: - DHT/dht.h: - debugging/trace.h: -DHT/dhtcmem$(OBJ_SUFFIX): DHT/dhtcmem.c DHT/dhtvalue.h DHT/fxf.h DHT/dhtcmem.h \ - DHT/dht.h - +DHT/dhtcmem$(OBJ_SUFFIX): DHT/dhtcmem.c debugging/assert.h DHT/dhtvalue.h DHT/fxf.h \ + DHT/dhtcmem.h DHT/dht.h +debugging/assert.h: DHT/dhtvalue.h: - DHT/fxf.h: - DHT/dhtcmem.h: - DHT/dht.h: -DHT/dhtmem$(OBJ_SUFFIX): DHT/dhtmem.c DHT/dhtvalue.h DHT/fxf.h DHT/dhtmem.h \ - DHT/dht.h - +DHT/dhtmem$(OBJ_SUFFIX): DHT/dhtmem.c debugging/assert.h DHT/dhtvalue.h DHT/fxf.h \ + DHT/dhtmem.h DHT/dht.h +debugging/assert.h: DHT/dhtvalue.h: - DHT/fxf.h: - DHT/dhtmem.h: - DHT/dht.h: -DHT/dhtsimpl$(OBJ_SUFFIX): DHT/dhtsimpl.c DHT/dhtvalue.h DHT/fxf.h - +DHT/dhtsimpl$(OBJ_SUFFIX): DHT/dhtsimpl.c debugging/assert.h DHT/dhtvalue.h \ + DHT/fxf.h +debugging/assert.h: DHT/dhtvalue.h: - DHT/fxf.h: -DHT/dhtstrng$(OBJ_SUFFIX): DHT/dhtstrng.c DHT/dhtvalue.h DHT/fxf.h DHT/dht.h - +DHT/dhtstrng$(OBJ_SUFFIX): DHT/dhtstrng.c debugging/assert.h DHT/dhtvalue.h \ + DHT/fxf.h DHT/dht.h +debugging/assert.h: DHT/dhtvalue.h: - DHT/fxf.h: - DHT/dht.h: DHT/dhtvalue$(OBJ_SUFFIX): DHT/dhtvalue.c DHT/dhtvalue.h DHT/fxf.h DHT/dht.h - DHT/dhtvalue.h: - DHT/fxf.h: - DHT/dht.h: DHT/dhtbcmem$(OBJ_SUFFIX): DHT/dhtbcmem.c debugging/assert.h DHT/dhtvalue.h \ DHT/fxf.h DHT/dhtbcmem.h DHT/dht.h - debugging/assert.h: - DHT/dhtvalue.h: - DHT/fxf.h: - DHT/dhtbcmem.h: - DHT/dht.h: DHT/fxf$(OBJ_SUFFIX): DHT/fxf.c debugging/assert.h DHT/fxf.h - debugging/assert.h: - DHT/fxf.h: diff --git a/DHT/dht.c b/DHT/dht.c index 90103df1a2..f68555f902 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -396,14 +396,14 @@ static InternHsElement *stepDirTable(dirEnumerate *enumeration) typedef struct { - dhtHashValue (*Hash)(dhtConstValue); - int (*Equal)(dhtConstValue, dhtConstValue); - dhtConstValue (*DupKey)(dhtConstValue); - dhtConstValue (*DupData)(dhtConstValue); - void (*FreeKey)(dhtValue); - void (*FreeData)(dhtValue); - void (*DumpData)(dhtConstValue,FILE *); - void (*DumpKey)(dhtConstValue,FILE *); + dhtHashValue (*Hash)(dhtKey); + int (*Equal)(dhtKey, dhtKey); + int (*DupKey)(dhtKeyOrValue, dhtKeyOrValue *); + int (*DupData)(dhtKeyOrValue, dhtKeyOrValue *); + void (*FreeKey)(dhtKeyOrValue); + void (*FreeData)(dhtKeyOrValue); + void (*DumpData)(dhtKeyOrValue,FILE *); + void (*DumpKey)(dhtKeyOrValue,FILE *); } Procedures; typedef struct dht { @@ -559,8 +559,8 @@ void dhtDestroy(HashTable *ht) while (b) { InternHsElement *tmp= b; - (ht->procs.FreeKey)((dhtValue)b->HsEl.Key); - (ht->procs.FreeData)((dhtValue)b->HsEl.Data); + (ht->procs.FreeKey)(b->HsEl.Key.key_data); + (ht->procs.FreeData)(b->HsEl.Data.value_data); b= b->Next; FreeInternHsElement(tmp); } @@ -599,9 +599,9 @@ void dhtDumpIndented(int ind, HashTable *ht, FILE *f) while (b) { fprintf(f, "%*s ", ind, ""); - (ht->procs.DumpKey)(b->HsEl.Key, f); + (ht->procs.DumpKey)(b->HsEl.Key.key_data, f); fputs("->", f); - (ht->procs.DumpData)(b->HsEl.Data, f); + (ht->procs.DumpData)(b->HsEl.Data.value_data, f); b= b->Next; fputc('\n', f); hcnt++; @@ -796,14 +796,18 @@ LOCAL void ShrinkHashTable(HashTable *ht) shrinkDirTable(&ht->DirTab); } -LOCAL InternHsElement **LookupInternHsElement(HashTable *ht, dhtConstValue key) +LOCAL InternHsElement **LookupInternHsElement(HashTable *ht, dhtKey key) { uLong h; InternHsElement **phe; TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); - TraceFunctionParam("%p",(void const *)key); +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) + TraceFunctionParam("%jx",(unsigned long long)key.key_data.unsigned_integer); +#else + TraceFunctionParam("%lx",(unsigned long)key.key_data.unsigned_integer); +#endif TraceFunctionParamListEnd(); h = DynamicHash(ht->p, ht->maxp, (ht->procs.Hash)(key)); @@ -827,14 +831,18 @@ LOCAL InternHsElement **LookupInternHsElement(HashTable *ht, dhtConstValue key) return phe; } -void dhtRemoveElement(HashTable *ht, dhtConstValue key) +void dhtRemoveElement(HashTable *ht, dhtKey key) { MYNAME(dhtRemoveElement) InternHsElement **phe, *he; TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); - TraceFunctionParam("%p",(void const *)key); +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) + TraceFunctionParam("%jx",(unsigned long long)key.key_data.unsigned_integer); +#else + TraceFunctionParam("%lx",(unsigned long)key.key_data.unsigned_integer); +#endif TraceFunctionParamListEnd(); phe= LookupInternHsElement(ht, key); @@ -849,8 +857,8 @@ void dhtRemoveElement(HashTable *ht, dhtConstValue key) ht->NextStep= ht->NextStep->Next; *phe= he->Next; - (ht->procs.FreeData)((dhtValue)he->HsEl.Data); - (ht->procs.FreeKey)((dhtValue)he->HsEl.Key); + (ht->procs.FreeData)(he->HsEl.Data.value_data); + (ht->procs.FreeKey)(he->HsEl.Key.key_data); FreeInternHsElement(he); ht->KeyCount--; if (ActualLoadFactor(ht) < ht->MinLoadFactor) @@ -877,21 +885,24 @@ void dhtRemoveElement(HashTable *ht, dhtConstValue key) TraceFunctionResultEnd(); } -dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data) +dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) { InternHsElement **phe, *he; - dhtConstValue KeyV; - dhtConstValue DataV; + dhtKey KeyV; + dhtValue DataV; TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); - TraceFunctionParam("%p",(void const *)key); - TraceFunctionParam("%p",(void const *)data); +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) + TraceFunctionParam("%jx",(unsigned long long)key.key_data.unsigned_integer); + TraceFunctionParam("%jx",(unsigned long long)data.value_data.unsigned_integer); +#else + TraceFunctionParam("%lx",(unsigned long)key.key_data.unsigned_integer); + TraceFunctionParam("%lx",(unsigned long)data.value_data.unsigned_integer); +#endif TraceFunctionParamListEnd(); - assert(key!=0); - KeyV = (ht->procs.DupKey)(key); - if (KeyV==0) + if ((ht->procs.DupKey)(key.key_data, &KeyV.key_data)) { TraceText("key duplication failed\n"); TraceFunctionExit(__func__); @@ -900,10 +911,9 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data return dhtNilElement; } - DataV = data==0 ? 0 : (ht->procs.DupData)(data); - if (data!=0 && DataV==0) + if ((ht->procs.DupData)(data.value_data, &DataV.value_data)) { - (ht->procs.FreeKey)((dhtValue)KeyV); + (ht->procs.FreeKey)(KeyV.key_data); TraceText("data duplication failed\n"); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void *)dhtNilElement); @@ -923,8 +933,8 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data TraceEOL(); if (he==0) { - (ht->procs.FreeKey)((dhtValue)KeyV); - (ht->procs.FreeData)((dhtValue)DataV); + (ht->procs.FreeKey)(KeyV.key_data); + (ht->procs.FreeData)(DataV.value_data); TraceText("allocation of new intern Hs element failed\n"); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void *)dhtNilElement); @@ -941,9 +951,9 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data else { if (ht->DtaPolicy == dhtCopy) - (ht->procs.FreeData)((dhtValue)he->HsEl.Data); + (ht->procs.FreeData)(he->HsEl.Data.value_data); if (ht->KeyPolicy == dhtCopy) - (ht->procs.FreeKey)((dhtValue)he->HsEl.Key); + (ht->procs.FreeKey)(he->HsEl.Key.key_data); } he->HsEl.Key = KeyV; @@ -975,7 +985,7 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data return &he->HsEl; } -dhtElement *dhtLookupElement(HashTable *ht, dhtConstValue key) +dhtElement *dhtLookupElement(HashTable *ht, dhtKey key) { InternHsElement **phe; dhtElement *result; diff --git a/DHT/dht.h b/DHT/dht.h index a2a50c07bd..18b279def7 100644 --- a/DHT/dht.h +++ b/DHT/dht.h @@ -15,10 +15,10 @@ typedef enum { /* Now finally this is the HashElement */ typedef struct { - dhtConstValue Key; - dhtConstValue Data; + dhtKey Key; + dhtValue Data; } dhtElement; -#define dhtNilElement ((dhtElement *)0) +#define dhtNilElement ((dhtElement *)0) struct dht; #define dhtNilHashTable ((struct dht *)0) @@ -26,13 +26,13 @@ struct dht; /* procedures */ struct dht *dhtCreate(dhtValueType KeyType, dhtValuePolicy KeyPolicy, dhtValueType DtaType, dhtValuePolicy DataPolicy); -dhtElement *dhtEnterElement(struct dht *, dhtConstValue key, dhtConstValue data); +dhtElement *dhtEnterElement(struct dht *, dhtKey key, dhtValue data); unsigned int dhtBucketStat (struct dht *, unsigned int *counter, unsigned int n); void dhtDestroy (struct dht *); void dhtDump (struct dht *, FILE *); void dhtDumpIndented (int i, struct dht *, FILE *); -void dhtRemoveElement (struct dht *, dhtConstValue key); -dhtElement *dhtLookupElement (struct dht *, dhtConstValue key); +void dhtRemoveElement (struct dht *, dhtKey key); +dhtElement *dhtLookupElement (struct dht *, dhtKey key); dhtElement *dhtGetFirstElement(struct dht *); dhtElement *dhtGetNextElement (struct dht *); unsigned long dhtKeyCount (struct dht *); diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index 54c474c91a..b1a7c652a9 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -8,25 +8,29 @@ * and in place. */ -#include "debugging/assert.h" #include #include #include #include +#include #if defined(__BORLANDC__) #include #endif /*__BORLANDC__*/ +#include "debugging/assert.h" #include "dhtvalue.h" #include "dhtbcmem.h" #include "dht.h" -static dhtHashValue ConvertBCMemValue(dhtConstValue m) +static dhtHashValue ConvertBCMemValue(dhtKey m) { - BCMemValue const * const toBeConverted = (BCMemValue const *)m; - unsigned int leng = toBeConverted->Leng; - unsigned char const *s = toBeConverted->Data; + BCMemValue const * const toBeConverted = (BCMemValue const *)m.key_data.object_pointer; + unsigned int leng; + unsigned char const *s; + assert(!!toBeConverted); + leng = toBeConverted->Leng; + s = toBeConverted->Data; dhtHashValue hash = 0; unsigned int i; @@ -44,45 +48,88 @@ static dhtHashValue ConvertBCMemValue(dhtConstValue m) return hash; } -static int EqualBCMemValue(dhtConstValue v1, dhtConstValue v2) +static int EqualBCMemValue(dhtKey v1, dhtKey v2) { - BCMemValue const * const value1 = (BCMemValue const *)v1; - BCMemValue const * const value2 = (BCMemValue const *)v2; - size_t const size = sizeof *value1 - sizeof value1->Data + value1->Leng; - - return memcmp(value1,value2,size)==0; + BCMemValue const * const value1 = (BCMemValue const *)v1.key_data.object_pointer; + BCMemValue const * const value2 = (BCMemValue const *)v2.key_data.object_pointer; + unsigned int length; + assert(value1 && value2) + + length = value1->Leng; + if (length != value2->Leng) + return 0; + + return !memcmp(value1->Data, value2->Data, length*sizeof value1->Data[0]); } -static dhtConstValue DupBCMemValue(dhtConstValue v) +static int DupBCMemValue(dhtKeyOrValue kv, dhtKeyOrValue *output) { - BCMemValue const * const original = (BCMemValue const *)v; - size_t const size = (sizeof *original - - sizeof original->Data - + original->Leng); + BCMemValue const *original = (BCMemValue const *)kv.object_pointer; + size_t const num_bytes_in_Data = ((sizeof *original) - offsetof(BCMemValue, Data)); + size_t const size_of_element = sizeof original->Data[0]; + size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); + size_t const remainder = (num_bytes_in_Data % size_of_element); + size_t size = sizeof *original; + BCMemValue *result; + unsigned char length; + + assert(!!original); + + length = original->Leng; + if (length > num_elements_in_Data) + { + size_t const num_new_elements = (length - num_elements_in_Data); + if (num_new_elements > (((size_t)-1) - size + remainder) / size_of_element) + return 1; + size += ((num_new_elements * size_of_element) - remainder); + } - BCMemValue * const result = fxfAlloc(size); - if (result!=0) - memcpy(result,original,size); + result = (BCMemValue *)fxfAlloc(size); + if (result) + { + result->Leng = length; + memcpy(result->Data,original->Data,(length*size_of_element)); + output->object_pointer = result; + return 0; + } - return (dhtConstValue)result; + return 1; } -static void FreeBCMemVal(dhtValue v) +static void FreeBCMemValue(dhtKeyOrValue kv) { - BCMemValue * const freed = (BCMemValue *)v; - size_t const size = sizeof *freed - sizeof freed->Data + freed->Leng; - fxfFree(freed,size); + BCMemValue *freed = (BCMemValue *)kv.object_pointer; + size_t const num_bytes_in_Data = ((sizeof *freed) - offsetof(BCMemValue, Data)); + size_t const size_of_element = sizeof freed->Data[0]; + size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); + size_t const remainder = (num_bytes_in_Data % size_of_element); + size_t size = sizeof *freed; + unsigned char length; + if (freed) + { + length = freed->Leng; + if (length > num_elements_in_Data) + { + size_t const num_new_elements = (length - num_elements_in_Data); + assert(num_new_elements <= ((((size_t)-1) - size + remainder) / size_of_element)); + size += ((num_new_elements * size_of_element) - remainder); + } + fxfFree(freed,size); + } } -static void DumpBCMemValue(dhtConstValue v, FILE *f) +static void DumpBCMemValue(dhtKeyOrValue kv, FILE *f) { - BCMemValue const * const toBeDumped = (BCMemValue const *)v; - unsigned int const length = toBeDumped->Leng; - unsigned int i; + BCMemValue const *toBeDumped = (BCMemValue const *)kv.object_pointer; + unsigned int length; + int i; + + assert(toBeDumped && f); - fprintf(f, "(%d)", toBeDumped->Leng); + length = toBeDumped->Leng; + fprintf(f, "(%d)", (int)toBeDumped->Leng); for (i=0; iData[i] & 0xff); + fprintf(f, "%02x", (toBeDumped->Data[i] & 0xffU)); } dhtValueProcedures dhtBCMemoryProcs = @@ -90,6 +137,6 @@ dhtValueProcedures dhtBCMemoryProcs = ConvertBCMemValue, EqualBCMemValue, DupBCMemValue, - FreeBCMemVal, + FreeBCMemValue, DumpBCMemValue }; diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 9353418cac..03ebd76777 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -11,6 +11,7 @@ #include #include +#include "debugging/assert.h" #include "dhtvalue.h" #include "dhtcmem.h" #include "dht.h" @@ -22,10 +23,14 @@ typedef unsigned long uLong; typedef unsigned char uChar; -static dhtHashValue ConvertCompactMemoryValue(dhtConstValue m) +static dhtHashValue ConvertCompactMemoryValue(dhtKey m) { - uLong leng= ((CompactMemVal const *)m)->Leng; - uChar const *s= ((CompactMemVal const *)m)->Data; + CompactMemVal const * const toBeConverted = (CompactMemVal const *)m.key_data.object_pointer; + uLong leng; + uChar const *s; + assert(!!toBeConverted); + leng= toBeConverted->Leng; + s= toBeConverted->Data; dhtHashValue hash= 0; uLong i; for (i=0; iLeng != ((CompactMemVal const *)v2)->Leng) + CompactMemVal const * const value1 = (CompactMemVal const *)v1.key_data.object_pointer; + CompactMemVal const * const value2 = (CompactMemVal const *)v2.key_data.object_pointer; + assert(value1 && value2); + if (value1->Leng != value2->Leng) return 0; - if (memcmp(((CompactMemVal const *)v1)->Data, - ((CompactMemVal const *)v2)->Data, ((CompactMemVal const *)v1)->Leng)) - return 0; - else - return 1; + return !memcmp(value1->Data, value2->Data, value1->Leng*sizeof value1->Data[0]); } -static dhtConstValue DupCompactMemoryValue(dhtConstValue v) +static int DupCompactMemoryValue(dhtKeyOrValue kv, dhtKeyOrValue *output) { - CompactMemVal *cm= NewCompactMemVal(((CompactMemVal const *)v)->Leng); - if (cm) { - cm->Leng= ((CompactMemVal const *)v)->Leng; - memcpy(cm->Data, ((CompactMemVal const *)v)->Data, cm->Leng); - return (dhtValue)cm; + CompactMemVal const *v = (CompactMemVal const *)kv.object_pointer; + size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); + size_t const size_of_element = sizeof v->Data[0]; + size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); + size_t const remainder = (num_bytes_in_Data % size_of_element); + size_t size = sizeof *v; + CompactMemVal *result; + uLong length; + + assert(!!v); + + length = v->Leng; + if (length > num_elements_in_Data) + { + size_t const num_new_elements = (length - num_elements_in_Data); + if (num_new_elements > (((size_t)-1) - size + remainder) / size_of_element) + return 1; + size += ((num_new_elements * size_of_element) - remainder); + } + + result = (CompactMemVal *)fxfAlloc(size); + if (result) + { + result->Leng = length; + memcpy(result->Data,v->Data,(length*size_of_element)); + output->object_pointer = result; + return 0; } - return (dhtValue)cm; + + return 1; } -static void FreeCompactMemoryValue(dhtValue v) +static void FreeCompactMemoryValue(dhtKeyOrValue kv) { - FreeCompactMemVal(v); + CompactMemVal *v = (CompactMemVal *)kv.object_pointer; + size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); + size_t const size_of_element = sizeof v->Data[0]; + size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); + size_t const remainder = (num_bytes_in_Data % size_of_element); + size_t size = sizeof *v; + uLong length; + if (v) + { + length = v->Leng; + if (length > num_elements_in_Data) + { + size_t const num_new_elements = (length - num_elements_in_Data); + assert(num_new_elements <= ((((size_t)-1) - size + remainder) / size_of_element)); + size += ((num_new_elements * size_of_element) - remainder); + } + fxfFree(v,size); + } } -static void DumpCompactMemoryValue(dhtConstValue v, FILE *f) +static void DumpCompactMemoryValue(dhtKeyOrValue kv, FILE *f) { + CompactMemVal const *v = (CompactMemVal const *)kv.object_pointer; uLong i; - fprintf(f, "(%lu)", ((CompactMemVal const *)v)->Leng); - for (i=0; i<((CompactMemVal const *)v)->Leng; i++) - fprintf(f, "%02x", ((CompactMemVal const *)v)->Data[i] & 0xff); + assert(v && f); + fprintf(f, "(%lu)", v->Leng); + for (i=0; iLeng; i++) + fprintf(f, "%02x", (v->Data[i] & 0xffU)); } dhtValueProcedures dhtCompactMemoryProcs = { diff --git a/DHT/dhtcmem.h b/DHT/dhtcmem.h index 7e1a6c85d8..92fff63df9 100644 --- a/DHT/dhtcmem.h +++ b/DHT/dhtcmem.h @@ -17,7 +17,7 @@ typedef struct CompactMemVal { unsigned long Leng; unsigned char Data[1]; } CompactMemVal; -#define NilCompactMemVal (CompactMemVal *)0 -#define NewCompactMemVal(n) (CompactMemVal *)fxfAlloc(sizeof(CompactMemVal)+(n)*sizeof(uChar)) +#define NilCompactMemVal ((CompactMemVal *)0) +#define NewCompactMemVal(n) ((CompactMemVal *)fxfAlloc(sizeof(CompactMemVal)+(n)*sizeof(uChar))) #define FreeCompactMemVal(v) fxfFree(v, sizeof(CompactMemVal)+((CompactMemVal const *)(v))->Leng*sizeof(uChar)) #endif /*DHTCMEM_INCLUDED*/ diff --git a/DHT/dhtexam1.c b/DHT/dhtexam1.c index 7fb3e9adf2..73bc56128f 100644 --- a/DHT/dhtexam1.c +++ b/DHT/dhtexam1.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ @@ -115,6 +115,8 @@ int main(int argc, char *argv[]) { struct hostent *host; int i; char *hostsfile; + dhtKey k; + dhtValue v; if (argc > 1) { @@ -194,14 +196,18 @@ int main(int argc, char *argv[]) { */ InetAddr= *(uLong *)h; - if (dhtLookupElement(NameToInet, (dhtValue)HostName)) { + k.key_data.object_pointer = HostName; + if (dhtLookupElement(NameToInet, k)) { fprintf(stderr, "Hostname %s already entered\n", host->h_name); } - dhtEnterElement(NameToInet, (dhtValue)HostName, (dhtValue)InetAddr); - if (dhtLookupElement(InetToName, (dhtValue)InetAddr)) { + v.value_data.unsigned_integer = InetAddr; + dhtEnterElement(NameToInet, k, v); + k.key_data.unsigned_integer = InetAddr; + if (dhtLookupElement(InetToName, k)) { fprintf(stderr, "InetAddr 0x%08lx already entered\n", InetAddr); } - dhtEnterElement(InetToName, (dhtValue)InetAddr, (dhtValue)HostName); + v.value_data.object_pointer = HostName; + dhtEnterElement(InetToName, k, v); } #if defined(FXF) fputs("fxf-Info after filling the hash tables\n",stderr); @@ -219,7 +225,7 @@ int main(int argc, char *argv[]) { fDumpMallinfo(stderr); */ - fputs("Testing if we get alle entries from NameToInet via GetFirst and GetNext...\n",stderr); + fputs("Testing if we get all entries from NameToInet via GetFirst and GetNext...\n",stderr); he= dhtGetFirstElement(NameToInet); i= 0; while (he) { @@ -239,33 +245,35 @@ int main(int argc, char *argv[]) { he= dhtGetFirstElement(NameToInet); while (he) { - uLong adr= (uLong)he->Data; + uLong adr= (uLong)he->Data.value_data.unsigned_integer; char *Name; #if defined(USE_MEMVAL) strncpy(str, - (char *)((MemVal *)he->Key)->Data, - ((MemVal *)he->Key)->Leng); - str[((MemVal *)he->Key)->Leng]='\0'; + (char *)((MemVal *)he->Key.object_pointer)->Data, + ((MemVal *)he->Key.object_pointer)->Leng); + str[((MemVal *)he->Key.object_pointer)->Leng]='\0'; Name= str; #else - Name= (char *)he->Key; + Name= (char *)he->Key.key_data.object_pointer; #endif /*USE_MEMVAL*/ printf("%3u.%3u.%3u.%3u = %s ", BYT(adr), BYT(adr>>8), BYT(adr>>16), BYT(adr>>24), Name); dhtRemoveElement(NameToInet, he->Key); - dhtRemoveElement(InetToName, he->Data); + k.key_data = he->Data.value_data; + dhtRemoveElement(InetToName, k); printf(" Deleting and checking consistency (Load=%lu... ", dhtKeyCount(NameToInet)); i=0; hhe= dhtGetFirstElement(NameToInet); fputs(" ", stdout); fflush(stdout); while (hhe) { - dhtElement *he1= dhtLookupElement(InetToName, hhe->Data); - if (strcmp((char *)he1->Data, (char *)hhe->Key) != 0) { + k.key_data = hhe->Data.value_data; + dhtElement *he1= dhtLookupElement(InetToName, k); + if (strcmp((char *)he1->Data.value_data.object_pointer, (char *)hhe->Key.key_data.object_pointer) != 0) { puts("\nSorry, Mismatch"); exit(1); } - if (he1->Key != hhe->Data) { + if (he1->Key.key_data.unsigned_integer != hhe->Data.value_data.unsigned_integer) { puts("\nSorry, Mismatch"); exit(2); } diff --git a/DHT/dhtexam2.c b/DHT/dhtexam2.c index 58a9229e8e..121f82b25d 100644 --- a/DHT/dhtexam2.c +++ b/DHT/dhtexam2.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ @@ -23,6 +23,8 @@ int main(int argc, char *argv[]) { int cnt, EntryCnt= 0; long int tmp; + dhtKey k; + dhtValue v; struct dummy { long l1; @@ -82,7 +84,9 @@ int main(int argc, char *argv[]) { for (cnt=0; cntage= Age; info_to_enter->room= Room; /* the string will be duplicated the info not */ - dhtEnterElement(OurTable, - (dhtValue)string_to_enter, (dhtValue)info_to_enter); + k.key_data.object_pointer = string_to_enter; + v.value_data.object_pointer = info_to_enter; + dhtEnterElement(OurTable, k, v); } /* access table */ while (scanf("%127s", name_to_find) != EOF) { - he= dhtLookupElement(OurTable, (dhtValue)name_to_find); + k.key_data.object_pointer = name_to_find; + he= dhtLookupElement(OurTable, k); if (he != dhtNilElement) { /* if item is in the table */ (void)printf("found %s, age = %d, room = %d\n", - (char *)he->Key, - ((struct info *)he->Data)->age, - ((struct info *)he->Data)->room); + (char *)he->Key.key_data.object_pointer, + ((struct info *)he->Data.value_data.object_pointer)->age, + ((struct info *)he->Data.value_data.object_pointer)->room); } else { (void)printf("no such employee %s\n", name_to_find); } @@ -79,7 +83,7 @@ int main( ) /* now delete all struct info in the table */ he= dhtGetFirstElement(OurTable); while (he) { - free((dhtValue)he->Data); + free(he->Data.value_data.object_pointer); he= dhtGetNextElement(OurTable); } /* now destroy the whole table */ diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 7f573748ff..48c143e959 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -16,6 +16,7 @@ # include #endif /*__BORLANDC__*/ +#include "debugging/assert.h" #include "dhtvalue.h" #include "dhtmem.h" #include "dht.h" @@ -23,10 +24,13 @@ typedef unsigned long uLong; typedef unsigned char uChar; -static dhtHashValue HashMemoryValue(dhtConstValue v) +static dhtHashValue HashMemoryValue(dhtKey k) { - uLong leng= ((MemVal const *)v)->Leng; - uChar const *s= ((MemVal const *)v)->Data; + MemVal const * toBeHashed = (MemVal const *)k.key_data.object_pointer; + assert(!!toBeHashed); + assert(toBeHashed->Data || !toBeHashed->Leng); + uLong leng= toBeHashed->Leng; + uChar const *s= toBeHashed->Data; dhtHashValue hash= 0; uLong i; for (i=0; iLeng != ((MemVal const *)v2)->Leng) - return 0; - if (memcmp(((MemVal const *)v1)->Data, ((MemVal const *)v2)->Data, ((MemVal const *)v1)->Leng)) + MemVal const * value1 = (MemVal const *)v1.key_data.object_pointer; + MemVal const * value2 = (MemVal const *)v2.key_data.object_pointer; + + assert(value1 && value2); + assert(value1->Data || !value1->Leng); + assert(value2->Data || !value2->Leng); + if (value1->Leng != value2->Leng) return 0; - else + if (!value1->Leng) // avoid the potential undefined behavior of using memcmp on NULL pointers return 1; + return !memcmp(value1->Data, value2->Data, value1->Leng*sizeof value1->Data[0]); } - -static dhtConstValue DupMemoryValue(dhtConstValue v) +static int DupMemoryValue(dhtKeyOrValue kv, dhtKeyOrValue *output) { + MemVal const *v= (MemVal const *)kv.object_pointer; MemVal *mv; - + assert(!!v); mv= NewMemVal; if (mv) { - mv->Data= (unsigned char *)fxfAlloc(((MemVal const *)v)->Leng); - if (mv->Data) { - mv->Leng= ((MemVal const *)v)->Leng; - memcpy(mv->Data, ((MemVal const *)v)->Data, mv->Leng); - return (dhtValue)mv; + mv->Leng= v->Leng; + if (mv->Leng) { + mv->Data= (uChar *)fxfAlloc(mv->Leng); + if (mv->Data) { + memcpy(mv->Data, v->Data, mv->Leng*sizeof mv->Data[0]); + output->object_pointer = mv; + return 0; + } else { + FreeMemVal(mv); + return 1; + } } else { - FreeMemVal(mv); - mv = NilMemVal; + mv->Data= NULL; // NULL is a valid pointer if Leng == 0 + output->object_pointer= mv; + return 0; } } - return (dhtValue)mv; + return 1; } -static void FreeMemoryValue(dhtValue v) +static void FreeMemoryValue(dhtKeyOrValue kv) { - DeleteMemVal(v); + MemVal *v= (MemVal *)kv.object_pointer; + if (v) { + fxfFree(v->Data, v->Leng*sizeof v->Data[0]); + fxfFree(v, sizeof *v); + } } -static void DumpMemoryValue(dhtConstValue v, FILE *f) { +static void DumpMemoryValue(dhtKeyOrValue kv, FILE *f) { + MemVal const * v= (MemVal *)kv.object_pointer; uLong i; - fprintf(f, "(%lu)", ((MemVal const *)v)->Leng); - for (i=0; i<((MemVal const *)v)->Leng; i++) - fprintf(f, "%02x", ((MemVal const *)v)->Data[i] & 0xff); + assert(v && f); + assert(v->Data || !v->Leng); + fprintf(f, "(%lu)", v->Leng); + for (i=0; iLeng; i++) + fprintf(f, "%02x", (v->Data[i] & 0xffU)); } dhtValueProcedures dhtMemoryProcs = { diff --git a/DHT/dhtmem.h b/DHT/dhtmem.h index 57774b3538..08be5de399 100644 --- a/DHT/dhtmem.h +++ b/DHT/dhtmem.h @@ -19,9 +19,9 @@ typedef struct MemVal { unsigned long Leng; unsigned char *Data; } MemVal; -#define NilMemVal (MemVal *)0 -#define NewMemVal (MemVal *)fxfAlloc(sizeof(MemVal)) +#define NilMemVal ((MemVal *)0) +#define NewMemVal ((MemVal *)fxfAlloc(sizeof(MemVal))) #define FreeMemVal(v) fxfFree(v, sizeof(MemVal)) -#define DeleteMemVal(v) if (((MemVal const *)(v))!=NilMemVal) fxfFree(((MemVal const *)(v))->Data, ((MemVal const *)(v))->Leng), FreeMemVal(v) +#define DeleteMemVal(v) do {if (((MemVal const *)(v))!=NilMemVal) fxfFree(((MemVal const *)(v))->Data, ((MemVal const *)(v))->Leng), FreeMemVal(v);} while (0) #endif /*DHTMEM_INCLUDED*/ diff --git a/DHT/dhtsimpl.c b/DHT/dhtsimpl.c index 7ee2c4aba5..4e6eeafb1c 100644 --- a/DHT/dhtsimpl.c +++ b/DHT/dhtsimpl.c @@ -7,6 +7,7 @@ * comment with the above copyright notice is keept intact * and in place. */ +#include "debugging/assert.h" #include "dhtvalue.h" #if defined(ARCH64) @@ -30,54 +31,59 @@ b -= c; b -= a; b ^= (a<<18); \ c -= a; c -= b; c ^= (b>>22); \ } -static unsigned long ConvertSimpleValue(dhtConstValue v) +static unsigned long ConvertSimpleValue(dhtKey v) { unsigned long a, b, c; c = 0x9e3779b97f4a7c13LL; - a = v<<1; - b = v; + a = v.unsigned_integer<<1; + b = v.unsigned_integer; mix(a,b,c); return c; } #else -static unsigned long ConvertSimpleValue(dhtConstValue v) +static unsigned long ConvertSimpleValue(dhtKey k) { - size_t c = (size_t)v; - size_t a = 0; - size_t b = 0x9e3779b9; + unsigned long c = k.key_data.unsigned_integer; + unsigned long a = 0; + unsigned long b = 0x9e3779b9U; a -= c; a ^= c >> 13; - b -= c; b -= a; b ^= a << 8; - c -= a; c -= b; c ^= b >> 13; - a -= b; a -= c; a ^= c >> 12; - b -= c; b -= a; b ^= a << 16; - c -= a; c -= b; c ^= b >> 5; - a -= b; a -= c; a ^= c >> 3; - b -= c; b -= a; b ^= a << 10; - c -= a; c -= b; c ^= b >> 15; - return (unsigned long)c; + b -= c; b -= a; b ^= (a << 8); + c -= a; c -= b; c ^= (b >> 13); + a -= b; a -= c; a ^= (c >> 12); + b -= c; b -= a; b ^= (a << 16); + c -= a; c -= b; c ^= (b >> 5); + a -= b; a -= c; a ^= (c >> 3); + b -= c; b -= a; b ^= (a << 10); + c -= a; c -= b; c ^= (b >> 15); + return c; } #endif /*ARCH64*/ -static int EqualSimpleValue(dhtConstValue v1, dhtConstValue v2) +static int EqualSimpleValue(dhtKey k1, dhtKey k2) { - return v1 == v2; + return (k1.key_data.unsigned_integer == k2.key_data.unsigned_integer); } -static dhtConstValue DupSimpleValue(dhtConstValue v) +static int DupSimpleValue(dhtKeyOrValue kv, dhtKeyOrValue *output) { - return v; + assert(!!output); + output->unsigned_integer = kv.unsigned_integer; + return 0; } -static void FreeSimpleValue(dhtValue v) +static void FreeSimpleValue(dhtKeyOrValue kv) { + (void)kv; } -static void DumpSimpleValue(dhtConstValue v, FILE *f) +static void DumpSimpleValue(dhtKeyOrValue kv, FILE *f) { - fprintf(f, "%08lx", (unsigned long)(size_t)v); + assert(!!f); + fprintf(f, "%08lx", (unsigned long)kv.unsigned_integer); } + dhtValueProcedures dhtSimpleProcs = { ConvertSimpleValue, diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 24d6431056..bb9e30634c 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -11,20 +11,23 @@ #include #include +#include "debugging/assert.h" #include "dhtvalue.h" #include "dht.h" -static unsigned long ConvertString(dhtConstValue v) +static dhtHashValue ConvertString(dhtKey k) { /* I found this hash function on * http://ourworld.compuserve.com/homepages/bob_jenkins/doobs.htm * There are other functions, but this one is has some advantages: - * - its small - * - independant from word sizes + * - it's small + * - independent from word sizes * - needs no initialisation */ - unsigned char const *s= (unsigned char const *)v; - unsigned long hash= 0; + unsigned char const *s= (unsigned char const *)k.key_data.object_pointer; + unsigned long hash; + assert(!!s); + hash= 0; while (*s) { hash+= *s++; hash+= hash << 10; @@ -33,32 +36,42 @@ static unsigned long ConvertString(dhtConstValue v) hash+= hash << 3; hash^= hash >> 11; hash+= hash << 15; - return hash; + return (dhtHashValue)hash; } -static int EqualString(dhtConstValue v1, dhtConstValue v2) +static int EqualString(dhtKey v1, dhtKey v2) { - if (strcmp((char const *)v1, (char const *)v2)) - return 0; - else - return 1; + char const *s1= (char const *)v1.key_data.object_pointer; + char const *s2= (char const *)v2.key_data.object_pointer; + assert(s1 && s2); + return !strcmp(s1, s2); } -static dhtConstValue DupString(dhtConstValue v) +static int DupString(dhtKeyOrValue v, dhtKeyOrValue *output) { char *nv; - nv= (char *)fxfAlloc(strlen((char const *)v)+1); - if (nv!=0) - strcpy(nv, (char const *)v); - return (dhtConstValue)nv; + assert(!!output); + char const *original= (char const *)v.object_pointer; + assert(!!original); + nv= (char *)fxfAlloc(strlen(original)+1); + if (nv) { + strcpy(nv, original); + output->object_pointer = nv; + return 0; + } + return 1; } -static void FreeString(dhtValue v) +static void FreeString(dhtKeyOrValue v) { - fxfFree(v, strlen((char const *)v)+1); + char *s= (char *)v.object_pointer; + if (s) + fxfFree(s, strlen(s)+1); } -static void DumpString(dhtConstValue v, FILE *f) +static void DumpString(dhtKeyOrValue v, FILE *f) { - fputs((char const *)v,f); + char const *s= (char const *)v.object_pointer; + assert(s && f); + fputs(s,f); } dhtValueProcedures dhtStringProcs = { diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index befd63b4df..80e67529f9 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -11,6 +11,17 @@ * and in place. */ #include +#ifdef __cplusplus +# if __cplusplus >= 201103L +# include +# endif +# include +#elif defined(__STDC_VERSION__) +# if __STDC_VERSION__ >= 199901L +# include +# endif +# include +#endif #if defined(FXF) #include "fxf.h" @@ -52,16 +63,51 @@ typedef enum { extern char const *dhtValueTypeToString[dhtValueTypeCnt]; -typedef void *dhtValue; -typedef void const *dhtConstValue; +typedef union { +#ifdef __cplusplus +# if __cplusplus >= 201103L + ::std::uintmax_t unsigned_integer; + ::std::intmax_t signed_integer; +# else + unsigned long int unsigned_integer; + long int signed_integer; +# endif + bool boolean; + ::std::sig_atomic_t atomic_integer; +#else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + uintmax_t unsigned_integer; + intmax_t signed_integer; + _Bool boolean; +# else + unsigned long int unsigned_integer; + long int signed_integer; + int boolean; // What else? +# endif + sig_atomic_t atomic_integer; +#endif + const volatile void * object_pointer; + char character; + void (*function_pointer)(void); + long double floating_point; +} dhtKeyOrValue; + +typedef struct { + dhtKeyOrValue key_data; +} dhtKey; + +typedef struct { + dhtKeyOrValue value_data; +} dhtValue; + typedef unsigned long dhtHashValue; typedef struct { - dhtHashValue (*Hash)(dhtConstValue); - int (*Equal)(dhtConstValue, dhtConstValue); - dhtConstValue (*Dup)(dhtConstValue); - void (*Free)(dhtValue); - void (*Dump)(dhtConstValue, FILE *); + dhtHashValue (*Hash)(dhtKey); + int (*Equal)(dhtKey, dhtKey); + int (*Dup)(dhtKeyOrValue, dhtKeyOrValue *); // should return 0 on success (and store the copied value at the second argument) and nonzero on error + void (*Free)(dhtKeyOrValue); + void (*Dump)(dhtKeyOrValue, FILE *); } dhtValueProcedures; #if defined(REGISTER_SIMPLE) diff --git a/optimisations/hash.c b/optimisations/hash.c index bc2f1f519e..9e0153fef4 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -246,26 +246,6 @@ static void (*encode)(stip_length_type min_length, typedef unsigned int data_type; -/* hash table element type defining the data member as we use it in - * this module - */ -typedef struct -{ - dhtValue Key; - data_type data; -} element_t; - -/* Grand union of "element" type and the generic one used by the hash - * table implementation. - * Using this union type rather than casting frm dhtElement * to - * element_t * avoids aliasing issues. - */ -typedef union -{ - dhtElement d; - element_t e; -} hashElement_union_t; - /* Hashing properties of stipulation slices */ typedef struct @@ -651,17 +631,17 @@ static void init_slice_properties(slice_index si) /* Pseudo hash table element - template for fast initialization of * newly created actual table elements */ -static hashElement_union_t template_element; +static dhtElement template_element = {0}; -static void set_value_attack_nosuccess(hashElement_union_t *hue, +static void set_value_attack_nosuccess(dhtElement *e, slice_index si, hash_value_type val) { unsigned int const offset = slice_properties[si].u.d.offsetNoSucc; unsigned int const bits = ((offset < (CHAR_BIT * (sizeof val))) ? (val << offset) : 0); unsigned int const mask = slice_properties[si].u.d.maskNoSucc; - element_t * const e = &hue->e; + data_type tmp; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceFunctionParam("%u",val); @@ -669,27 +649,29 @@ static void set_value_attack_nosuccess(hashElement_union_t *hue, TraceValue("%u",slice_properties[si].size); TraceValue("%u",offset); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->data); - TraceValue("pre:%08x ",e->data); + TraceValue("%p",(void *)&e->Data.value_data.unsigned_integer); + TraceValue("pre:%08x ",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); TraceValue("%08x",bits); TraceEOL(); assert((bits&mask)==bits); - e->data &= ~mask; - e->data |= bits; - TraceValue("post:%08x",e->data); + tmp = (data_type)e->Data.value_data.unsigned_integer; + tmp &= ~mask; + tmp |= bits; + e->Data.value_data.unsigned_integer = tmp; + TraceValue("post:%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); TraceFunctionResultEnd(); } -static void set_value_attack_success(hashElement_union_t *hue, +static void set_value_attack_success(dhtElement *e, slice_index si, hash_value_type val) { unsigned int const offset = slice_properties[si].u.d.offsetSucc; unsigned int const bits = ((offset < (CHAR_BIT * (sizeof val))) ? (val << offset) : 0); unsigned int const mask = slice_properties[si].u.d.maskSucc; - element_t * const e = &hue->e; + data_type tmp; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); @@ -699,28 +681,30 @@ static void set_value_attack_success(hashElement_union_t *hue, TraceValue("%u",slice_properties[si].size); TraceValue("%u",offset); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->data); - TraceValue("pre:%08x ",e->data); + TraceValue("%p",(void *)&e->Data.value_data.unsigned_integer); + TraceValue("pre:%08x ",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); TraceValue("%08x",bits); TraceEOL(); assert((bits&mask)==bits); - e->data &= ~mask; - e->data |= bits; - TraceValue("post:%08x",e->data); + tmp = (data_type)e->Data.value_data.unsigned_integer; + tmp &= ~mask; + tmp |= bits; + e->Data.value_data.unsigned_integer = tmp; + TraceValue("post:%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); TraceFunctionResultEnd(); } -static void set_value_help(hashElement_union_t *hue, +static void set_value_help(dhtElement *e, slice_index si, hash_value_type val) { unsigned int const offset = slice_properties[si].u.h.offsetNoSucc; unsigned int const bits = val << offset; unsigned int const mask = slice_properties[si].u.h.maskNoSucc; - element_t * const e = &hue->e; + data_type tmp; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceFunctionParam("%u",val); @@ -728,31 +712,32 @@ static void set_value_help(hashElement_union_t *hue, TraceValue("%u",slice_properties[si].size); TraceValue("%u",offset); TraceValue("0x%08x ",mask); - TraceValue("%p ",(void *)&e->data); - TraceValue("pre:0x%08x ",e->data); + TraceValue("%p ",(void *)&e->Data.value_data.unsigned_integer); + TraceValue("pre:0x%08x ",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); TraceValue("0x%08x",bits); TraceEOL(); assert((bits&mask)==bits); - e->data &= ~mask; - e->data |= bits; - TraceValue("post:0x%08x",e->data); + tmp = (data_type)e->Data.value_data.unsigned_integer; + tmp &= ~mask; + tmp |= bits; + e->Data.value_data.unsigned_integer = tmp; + TraceValue("post:0x%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); TraceFunctionResultEnd(); } -static hash_value_type get_value_attack_success(hashElement_union_t const *hue, +static hash_value_type get_value_attack_success(dhtElement const *e, slice_index si) { unsigned int const offset = slice_properties[si].u.d.offsetSucc; unsigned int const mask = slice_properties[si].u.d.maskSucc; - element_t const * const e = &hue->e; - data_type const result = (e->data & mask) >> offset; + data_type const result = (((data_type)e->Data.value_data.unsigned_integer) & mask) >> offset; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->data); - TraceValue("%08x",e->data); + TraceValue("%p",(void const *)&e->Data.value_data.unsigned_integer); + TraceValue("%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -761,18 +746,17 @@ static hash_value_type get_value_attack_success(hashElement_union_t const *hue, return result; } -static hash_value_type get_value_attack_nosuccess(hashElement_union_t const *hue, +static hash_value_type get_value_attack_nosuccess(dhtElement const *e, slice_index si) { unsigned int const offset = slice_properties[si].u.d.offsetNoSucc; unsigned int const mask = slice_properties[si].u.d.maskNoSucc; - element_t const * const e = &hue->e; - data_type const result = (e->data & mask) >> offset; + data_type const result = (((data_type)e->Data.value_data.unsigned_integer) & mask) >> offset; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->data); - TraceValue("%08x",e->data); + TraceValue("%p",(void const *)&e->Data.value_data.unsigned_integer); + TraceValue("%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -781,19 +765,18 @@ static hash_value_type get_value_attack_nosuccess(hashElement_union_t const *hue return result; } -static hash_value_type get_value_help(hashElement_union_t const *hue, +static hash_value_type get_value_help(dhtElement const *e, slice_index si) { unsigned int const offset = slice_properties[si].u.h.offsetNoSucc; unsigned int const mask = slice_properties[si].u.h.maskNoSucc; - element_t const * const e = &hue->e; - data_type const result = (e->data & mask) >> offset; + data_type const result = (((data_type)e->Data.value_data.unsigned_integer) & mask) >> offset; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceValue("%u",offset); TraceValue("0x%08x ",mask); - TraceValue("%p ",(void *)&e->data); - TraceValue("0x%08x",e->data); + TraceValue("%p ",(void const *)&e->Data.value_data.unsigned_integer); + TraceValue("0x%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -808,7 +791,7 @@ static hash_value_type get_value_help(hashElement_union_t const *hue, * @param si slice index of slice * @return value of contribution of slice si to *he's value */ -static hash_value_type own_value_of_data_solve(hashElement_union_t const *hue, +static hash_value_type own_value_of_data_solve(dhtElement const *hue, slice_index si) { stip_length_type const length = SLICE_U(si).branch.length; @@ -818,7 +801,7 @@ static hash_value_type own_value_of_data_solve(hashElement_union_t const *hue, hash_value_type success_neg; TraceFunctionEntry(__func__); - TraceFunctionParam("%p",hue); + TraceFunctionParam("%p",(void const *)hue); TraceFunctionParam("%u",si); TraceFunctionParamListEnd(); @@ -842,14 +825,14 @@ static hash_value_type own_value_of_data_solve(hashElement_union_t const *hue, * @param si slice index of help slice * @return value of contribution of slice si to *he's value */ -static hash_value_type own_value_of_data_help(hashElement_union_t const *hue, +static hash_value_type own_value_of_data_help(dhtElement const *hue, slice_index si) { unsigned int const parity = slice_properties[si].u.h.parity; hash_value_type result; TraceFunctionEntry(__func__); - TraceFunctionParam("%p",hue); + TraceFunctionParam("%p",(void const *)hue); TraceFunctionParam("%u",si); TraceFunctionParamListEnd(); @@ -869,14 +852,14 @@ static hash_value_type own_value_of_data_help(hashElement_union_t const *hue, * @param si slice index * @return value of contribuation of the slice to *he's value */ -static hash_value_type value_of_data_from_slice(hashElement_union_t const *hue, +static hash_value_type value_of_data_from_slice(dhtElement const *hue, slice_index si) { hash_value_type result; unsigned int const offset = slice_properties[si].valueOffset; TraceFunctionEntry(__func__); - TraceFunctionParam("%p",hue); + TraceFunctionParam("%p",(void const *)hue); TraceFunctionParam("%u",si); TraceFunctionParamListEnd(); @@ -912,13 +895,13 @@ static hash_value_type value_of_data_from_slice(hashElement_union_t const *hue, * @param he address of hash table element to determine value of * @return value of *he */ -static hash_value_type value_of_data(hashElement_union_t const *hue) +static hash_value_type value_of_data(dhtElement const *hue) { hash_value_type result = 0; unsigned int i; TraceFunctionEntry(__func__); - TraceFunctionParam("%p",hue); + TraceFunctionParam("%p",(void const *)hue); TraceFunctionParamListEnd(); for (i = 0; iKey); #if defined(TESTHASH) if (nrElementsRemovedInCompression + dhtKeyCount(pyhash) != nrElementsAtStartOfCompression) @@ -1636,7 +1615,7 @@ static void SmallEncode(stip_length_type min_length, * element's data field with null values * @param he address of hash table element */ -static void init_elements(hashElement_union_t *hue) +static void init_elements(dhtElement *hue) { unsigned int i; @@ -1677,9 +1656,9 @@ static void init_elements(hashElement_union_t *hue) * @param hb has value (basis for calculation of key) * @return address of element */ -static dhtElement *allocDHTelement(dhtConstValue hb) +static dhtElement *allocDHTelement(dhtKey hb) { - dhtElement *result = dhtEnterElement(pyhash,hb,template_element.d.Data); + dhtElement *result = dhtEnterElement(pyhash,hb,template_element.Data); while (result==dhtNilElement) { unsigned long const nrKeysBeforeCompression = dhtKeyCount(pyhash); @@ -1696,11 +1675,11 @@ static dhtElement *allocDHTelement(dhtConstValue hb) fprintf(stderr, "\nOUT OF SPACE: Unable to create hash table in %s in %s -- aborting.\n", __func__, __FILE__); exit(2); /* TODO: Do we have to exit here? */ } - result = dhtEnterElement(pyhash,hb,template_element.d.Data); + result = dhtEnterElement(pyhash,hb,template_element.Data); break; } else - result = dhtEnterElement(pyhash,hb,template_element.d.Data); + result = dhtEnterElement(pyhash,hb,template_element.Data); } if (result==dhtNilElement) @@ -1814,7 +1793,7 @@ static void inithash(slice_index si) init_slice_properties(si); - template_element.d.Data = 0; + template_element.Data.value_data.unsigned_integer = 0; init_elements(&template_element); is_table_uncompressed = true; /* V3.60 TLi */ @@ -2202,7 +2181,7 @@ static void addtohash_battle_nosuccess(slice_index si, stip_length_type n, stip_length_type min_length_adjusted) { - HashBuffer const * const hb = &hashBuffers[nbply]; + dhtKey hb; hash_value_type const val = (n+1-min_length_adjusted)/2; dhtElement *he; @@ -2211,17 +2190,17 @@ static void addtohash_battle_nosuccess(slice_index si, TraceFunctionParam("%u",min_length_adjusted); TraceFunctionParamListEnd(); + hb.key_data.object_pointer = &hashBuffers[nbply]; he = dhtLookupElement(pyhash,hb); if (he==dhtNilElement) { - hashElement_union_t * const hue = (hashElement_union_t *)allocDHTelement(hb); - set_value_attack_nosuccess(hue,si,val); + he = allocDHTelement(hb); + set_value_attack_nosuccess(he,si,val); } else { - hashElement_union_t * const hue = (hashElement_union_t *)he; - if (get_value_attack_nosuccess(hue,si)val) - set_value_attack_success(hue,si,val); + if (get_value_attack_success(he,si)>val) + set_value_attack_success(he,si,val); } TraceFunctionExit(__func__); @@ -2317,6 +2295,7 @@ stip_length_type delegate_can_attack_in_n(slice_index si, void attack_hashed_tester_solve(slice_index si) { dhtElement const *he; + dhtKey k; slice_index const base = SLICE_U(si).derived_pipe.base; stip_length_type const min_length = SLICE_U(base).branch.min_length; stip_length_type const played = SLICE_U(base).branch.length-solve_nr_remaining; @@ -2334,23 +2313,23 @@ void attack_hashed_tester_solve(slice_index si) (*encode)(min_length,validity_value); - he = dhtLookupElement(pyhash,&hashBuffers[nbply]); + k.key_data.object_pointer = &hashBuffers[nbply]; + he = dhtLookupElement(pyhash,k); if (he==dhtNilElement) solve_result = delegate_can_attack_in_n(si,min_length_adjusted); else { - hashElement_union_t const * const hue = (hashElement_union_t const *)he; stip_length_type const parity = (solve_nr_remaining-min_length_adjusted)%2; /* It is more likely that a position has no solution. */ /* Therefore let's check for "no solution" first. TLi */ - hash_value_type const val_nosuccess = get_value_attack_nosuccess(hue,base); + hash_value_type const val_nosuccess = get_value_attack_nosuccess(he,base); stip_length_type const n_nosuccess = 2*val_nosuccess + min_length_adjusted-parity; if (n_nosuccess>=MOVE_HAS_SOLVED_LENGTH()) solve_result = MOVE_HAS_NOT_SOLVED_LENGTH(); else { - hash_value_type const val_success = get_value_attack_success(hue,base); + hash_value_type const val_success = get_value_attack_success(he,base); stip_length_type const n_success = 2*val_success + min_length_adjusted+2-parity; if (n_success<=MOVE_HAS_SOLVED_LENGTH()) solve_result = n_success; @@ -2385,7 +2364,7 @@ void attack_hashed_tester_solve(slice_index si) static boolean inhash_help(slice_index si) { boolean result; - HashBuffer *hb = &hashBuffers[nbply]; + dhtKey hb; dhtElement const *he; stip_length_type const validity_value = (solve_nr_remaining-1)/2+1; @@ -2404,10 +2383,11 @@ static boolean inhash_help(slice_index si) ifHASHRATE(use_all++); + hb.key_data.object_pointer = &hashBuffers[nbply]; he = dhtLookupElement(pyhash,hb); if (he==dhtNilElement) result = false; - else if (get_value_help((hashElement_union_t const *)he,si)==1) + else if (get_value_help(he,si)==1) { ifHASHRATE(use_pos++); result = true; @@ -2427,21 +2407,19 @@ static boolean inhash_help(slice_index si) */ static void addtohash_help(slice_index si) { - HashBuffer const * const hb = &hashBuffers[nbply]; + dhtKey hb; dhtElement *he; - hashElement_union_t * hue; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceFunctionParamListEnd(); + hb.key_data.object_pointer = &hashBuffers[nbply]; he = dhtLookupElement(pyhash,hb); if (he==dhtNilElement) - hue = (hashElement_union_t *)allocDHTelement(hb); - else - hue = (hashElement_union_t *)he; + he = allocDHTelement(hb); - set_value_help(hue,si,1); + set_value_help(he,si,1); TraceFunctionExit(__func__); TraceFunctionResultEnd(); diff --git a/optimisations/hash.h b/optimisations/hash.h index 2e977a5469..9f454dbb4d 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -13,6 +13,7 @@ #include "pieces/pieces.h" #include "solving/machinery/solve.h" #include "solving/ply.h" +#include #if defined(TESTHASH) # if !defined(HASHRATE) From 25a9e54ca102968471fa0b82d46913ad54bcf500 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 2 Sep 2023 16:12:06 -0400 Subject: [PATCH 002/111] Adding some important documentation. --- DHT/dhtvalue.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 80e67529f9..9cc08d5efd 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -111,6 +111,8 @@ typedef struct { } dhtValueProcedures; #if defined(REGISTER_SIMPLE) +/* dhtSimple verifies equality by comparing unsigned_integer members; + keys should be stored to unsigned_integer members, cast if needed */ extern dhtValueProcedures dhtSimpleProcs; #endif /*REGISTER_SIMPLE*/ #if defined(REGISTER_STRING) From 0f1f1aff6b178166132c4b9149bb33b7d053cf47 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 2 Sep 2023 21:20:31 -0400 Subject: [PATCH 003/111] Simplifying things -- dhtKey IS a dhtValue, specifically one that also defines a hash value and equality. --- DHT/dht.c | 54 +++++++++++++++++++-------------------- DHT/dhtbcmem.c | 16 ++++++------ DHT/dhtcmem.c | 12 ++++----- DHT/dhtexam1.c | 26 +++++++++---------- DHT/dhtexam2.c | 4 +-- DHT/dhtmanex.c | 14 +++++------ DHT/dhtmem.c | 12 ++++----- DHT/dhtsimpl.c | 16 ++++++------ DHT/dhtstrng.c | 12 ++++----- DHT/dhtvalue.h | 14 ++++------- DHT/fxf.c | 2 +- optimisations/hash.c | 60 ++++++++++++++++++++++---------------------- 12 files changed, 119 insertions(+), 123 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index f68555f902..87ecfd525e 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -398,12 +398,12 @@ typedef struct { dhtHashValue (*Hash)(dhtKey); int (*Equal)(dhtKey, dhtKey); - int (*DupKey)(dhtKeyOrValue, dhtKeyOrValue *); - int (*DupData)(dhtKeyOrValue, dhtKeyOrValue *); - void (*FreeKey)(dhtKeyOrValue); - void (*FreeData)(dhtKeyOrValue); - void (*DumpData)(dhtKeyOrValue,FILE *); - void (*DumpKey)(dhtKeyOrValue,FILE *); + int (*DupKey)(dhtValue, dhtValue *); + int (*DupData)(dhtValue, dhtValue *); + void (*FreeKey)(dhtValue); + void (*FreeData)(dhtValue); + void (*DumpData)(dhtValue,FILE *); + void (*DumpKey)(dhtValue,FILE *); } Procedures; typedef struct dht { @@ -559,8 +559,8 @@ void dhtDestroy(HashTable *ht) while (b) { InternHsElement *tmp= b; - (ht->procs.FreeKey)(b->HsEl.Key.key_data); - (ht->procs.FreeData)(b->HsEl.Data.value_data); + (ht->procs.FreeKey)(b->HsEl.Key.value); + (ht->procs.FreeData)(b->HsEl.Data); b= b->Next; FreeInternHsElement(tmp); } @@ -599,9 +599,9 @@ void dhtDumpIndented(int ind, HashTable *ht, FILE *f) while (b) { fprintf(f, "%*s ", ind, ""); - (ht->procs.DumpKey)(b->HsEl.Key.key_data, f); + (ht->procs.DumpKey)(b->HsEl.Key.value, f); fputs("->", f); - (ht->procs.DumpData)(b->HsEl.Data.value_data, f); + (ht->procs.DumpData)(b->HsEl.Data, f); b= b->Next; fputc('\n', f); hcnt++; @@ -804,9 +804,9 @@ LOCAL InternHsElement **LookupInternHsElement(HashTable *ht, dhtKey key) TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) - TraceFunctionParam("%jx",(unsigned long long)key.key_data.unsigned_integer); + TraceFunctionParam("%jx",(unsigned long long)key.value.unsigned_integer); #else - TraceFunctionParam("%lx",(unsigned long)key.key_data.unsigned_integer); + TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); #endif TraceFunctionParamListEnd(); @@ -839,9 +839,9 @@ void dhtRemoveElement(HashTable *ht, dhtKey key) TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) - TraceFunctionParam("%jx",(unsigned long long)key.key_data.unsigned_integer); + TraceFunctionParam("%jx",(unsigned long long)key.value.unsigned_integer); #else - TraceFunctionParam("%lx",(unsigned long)key.key_data.unsigned_integer); + TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); #endif TraceFunctionParamListEnd(); @@ -857,8 +857,8 @@ void dhtRemoveElement(HashTable *ht, dhtKey key) ht->NextStep= ht->NextStep->Next; *phe= he->Next; - (ht->procs.FreeData)(he->HsEl.Data.value_data); - (ht->procs.FreeKey)(he->HsEl.Key.key_data); + (ht->procs.FreeData)(he->HsEl.Data); + (ht->procs.FreeKey)(he->HsEl.Key.value); FreeInternHsElement(he); ht->KeyCount--; if (ActualLoadFactor(ht) < ht->MinLoadFactor) @@ -894,15 +894,15 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) - TraceFunctionParam("%jx",(unsigned long long)key.key_data.unsigned_integer); - TraceFunctionParam("%jx",(unsigned long long)data.value_data.unsigned_integer); + TraceFunctionParam("%jx",(unsigned long long)key.value.unsigned_integer); + TraceFunctionParam("%jx",(unsigned long long)data.unsigned_integer); #else - TraceFunctionParam("%lx",(unsigned long)key.key_data.unsigned_integer); - TraceFunctionParam("%lx",(unsigned long)data.value_data.unsigned_integer); + TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); + TraceFunctionParam("%lx",(unsigned long)data.unsigned_integer); #endif TraceFunctionParamListEnd(); - if ((ht->procs.DupKey)(key.key_data, &KeyV.key_data)) + if ((ht->procs.DupKey)(key.value, &KeyV.value)) { TraceText("key duplication failed\n"); TraceFunctionExit(__func__); @@ -911,9 +911,9 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) return dhtNilElement; } - if ((ht->procs.DupData)(data.value_data, &DataV.value_data)) + if ((ht->procs.DupData)(data, &DataV)) { - (ht->procs.FreeKey)(KeyV.key_data); + (ht->procs.FreeKey)(KeyV.value); TraceText("data duplication failed\n"); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void *)dhtNilElement); @@ -933,8 +933,8 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) TraceEOL(); if (he==0) { - (ht->procs.FreeKey)(KeyV.key_data); - (ht->procs.FreeData)(DataV.value_data); + (ht->procs.FreeKey)(KeyV.value); + (ht->procs.FreeData)(DataV); TraceText("allocation of new intern Hs element failed\n"); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void *)dhtNilElement); @@ -951,9 +951,9 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) else { if (ht->DtaPolicy == dhtCopy) - (ht->procs.FreeData)(he->HsEl.Data.value_data); + (ht->procs.FreeData)(he->HsEl.Data); if (ht->KeyPolicy == dhtCopy) - (ht->procs.FreeKey)(he->HsEl.Key.key_data); + (ht->procs.FreeKey)(he->HsEl.Key.value); } he->HsEl.Key = KeyV; diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index b1a7c652a9..d4f63e69ce 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -25,7 +25,7 @@ static dhtHashValue ConvertBCMemValue(dhtKey m) { - BCMemValue const * const toBeConverted = (BCMemValue const *)m.key_data.object_pointer; + BCMemValue const * const toBeConverted = (BCMemValue const *)m.value.object_pointer; unsigned int leng; unsigned char const *s; assert(!!toBeConverted); @@ -50,8 +50,8 @@ static dhtHashValue ConvertBCMemValue(dhtKey m) static int EqualBCMemValue(dhtKey v1, dhtKey v2) { - BCMemValue const * const value1 = (BCMemValue const *)v1.key_data.object_pointer; - BCMemValue const * const value2 = (BCMemValue const *)v2.key_data.object_pointer; + BCMemValue const * const value1 = (BCMemValue const *)v1.value.object_pointer; + BCMemValue const * const value2 = (BCMemValue const *)v2.value.object_pointer; unsigned int length; assert(value1 && value2) @@ -62,7 +62,7 @@ static int EqualBCMemValue(dhtKey v1, dhtKey v2) return !memcmp(value1->Data, value2->Data, length*sizeof value1->Data[0]); } -static int DupBCMemValue(dhtKeyOrValue kv, dhtKeyOrValue *output) +static int DupBCMemValue(dhtValue kv, dhtValue *output) { BCMemValue const *original = (BCMemValue const *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *original) - offsetof(BCMemValue, Data)); @@ -96,7 +96,7 @@ static int DupBCMemValue(dhtKeyOrValue kv, dhtKeyOrValue *output) return 1; } -static void FreeBCMemValue(dhtKeyOrValue kv) +static void FreeBCMemValue(dhtValue kv) { BCMemValue *freed = (BCMemValue *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *freed) - offsetof(BCMemValue, Data)); @@ -118,16 +118,16 @@ static void FreeBCMemValue(dhtKeyOrValue kv) } } -static void DumpBCMemValue(dhtKeyOrValue kv, FILE *f) +static void DumpBCMemValue(dhtValue kv, FILE *f) { BCMemValue const *toBeDumped = (BCMemValue const *)kv.object_pointer; - unsigned int length; + int length; int i; assert(toBeDumped && f); length = toBeDumped->Leng; - fprintf(f, "(%d)", (int)toBeDumped->Leng); + fprintf(f, "(%d)", length); for (i=0; iData[i] & 0xffU)); } diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 03ebd76777..c53ca6bc3e 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -25,7 +25,7 @@ typedef unsigned char uChar; static dhtHashValue ConvertCompactMemoryValue(dhtKey m) { - CompactMemVal const * const toBeConverted = (CompactMemVal const *)m.key_data.object_pointer; + CompactMemVal const * const toBeConverted = (CompactMemVal const *)m.value.object_pointer; uLong leng; uChar const *s; assert(!!toBeConverted); @@ -46,15 +46,15 @@ static dhtHashValue ConvertCompactMemoryValue(dhtKey m) static int EqualCompactMemoryValue(dhtKey v1, dhtKey v2) { - CompactMemVal const * const value1 = (CompactMemVal const *)v1.key_data.object_pointer; - CompactMemVal const * const value2 = (CompactMemVal const *)v2.key_data.object_pointer; + CompactMemVal const * const value1 = (CompactMemVal const *)v1.value.object_pointer; + CompactMemVal const * const value2 = (CompactMemVal const *)v2.value.object_pointer; assert(value1 && value2); if (value1->Leng != value2->Leng) return 0; return !memcmp(value1->Data, value2->Data, value1->Leng*sizeof value1->Data[0]); } -static int DupCompactMemoryValue(dhtKeyOrValue kv, dhtKeyOrValue *output) +static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) { CompactMemVal const *v = (CompactMemVal const *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); @@ -88,7 +88,7 @@ static int DupCompactMemoryValue(dhtKeyOrValue kv, dhtKeyOrValue *output) return 1; } -static void FreeCompactMemoryValue(dhtKeyOrValue kv) +static void FreeCompactMemoryValue(dhtValue kv) { CompactMemVal *v = (CompactMemVal *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); @@ -110,7 +110,7 @@ static void FreeCompactMemoryValue(dhtKeyOrValue kv) } } -static void DumpCompactMemoryValue(dhtKeyOrValue kv, FILE *f) +static void DumpCompactMemoryValue(dhtValue kv, FILE *f) { CompactMemVal const *v = (CompactMemVal const *)kv.object_pointer; uLong i; diff --git a/DHT/dhtexam1.c b/DHT/dhtexam1.c index 73bc56128f..ef61a25e9d 100644 --- a/DHT/dhtexam1.c +++ b/DHT/dhtexam1.c @@ -196,17 +196,17 @@ int main(int argc, char *argv[]) { */ InetAddr= *(uLong *)h; - k.key_data.object_pointer = HostName; + k.value.object_pointer = HostName; if (dhtLookupElement(NameToInet, k)) { fprintf(stderr, "Hostname %s already entered\n", host->h_name); } - v.value_data.unsigned_integer = InetAddr; + v.unsigned_integer = InetAddr; dhtEnterElement(NameToInet, k, v); - k.key_data.unsigned_integer = InetAddr; + k.value.unsigned_integer = InetAddr; if (dhtLookupElement(InetToName, k)) { fprintf(stderr, "InetAddr 0x%08lx already entered\n", InetAddr); } - v.value_data.object_pointer = HostName; + v.object_pointer = HostName; dhtEnterElement(InetToName, k, v); } #if defined(FXF) @@ -245,35 +245,35 @@ int main(int argc, char *argv[]) { he= dhtGetFirstElement(NameToInet); while (he) { - uLong adr= (uLong)he->Data.value_data.unsigned_integer; + uLong adr= (uLong)he->Data.unsigned_integer; char *Name; #if defined(USE_MEMVAL) strncpy(str, - (char *)((MemVal *)he->Key.object_pointer)->Data, - ((MemVal *)he->Key.object_pointer)->Leng); - str[((MemVal *)he->Key.object_pointer)->Leng]='\0'; + (char *)((MemVal *)he->Key.value.object_pointer)->Data, + ((MemVal *)he->Key.value.object_pointer)->Leng); + str[((MemVal *)he->Key.value.object_pointer)->Leng]='\0'; Name= str; #else - Name= (char *)he->Key.key_data.object_pointer; + Name= (char *)he->Key.value.object_pointer; #endif /*USE_MEMVAL*/ printf("%3u.%3u.%3u.%3u = %s ", BYT(adr), BYT(adr>>8), BYT(adr>>16), BYT(adr>>24), Name); dhtRemoveElement(NameToInet, he->Key); - k.key_data = he->Data.value_data; + k.value = he->Data; dhtRemoveElement(InetToName, k); printf(" Deleting and checking consistency (Load=%lu... ", dhtKeyCount(NameToInet)); i=0; hhe= dhtGetFirstElement(NameToInet); fputs(" ", stdout); fflush(stdout); while (hhe) { - k.key_data = hhe->Data.value_data; + k.value = hhe->Data; dhtElement *he1= dhtLookupElement(InetToName, k); - if (strcmp((char *)he1->Data.value_data.object_pointer, (char *)hhe->Key.key_data.object_pointer) != 0) { + if (strcmp((char const *)he1->Data.object_pointer, (char const *)hhe->Key.value.object_pointer) != 0) { puts("\nSorry, Mismatch"); exit(1); } - if (he1->Key.key_data.unsigned_integer != hhe->Data.value_data.unsigned_integer) { + if (he1->Key.value.unsigned_integer != hhe->Data.unsigned_integer) { puts("\nSorry, Mismatch"); exit(2); } diff --git a/DHT/dhtexam2.c b/DHT/dhtexam2.c index 121f82b25d..fa1941b9ef 100644 --- a/DHT/dhtexam2.c +++ b/DHT/dhtexam2.c @@ -84,8 +84,8 @@ int main(int argc, char *argv[]) { for (cnt=0; cntage= Age; info_to_enter->room= Room; /* the string will be duplicated the info not */ - k.key_data.object_pointer = string_to_enter; - v.value_data.object_pointer = info_to_enter; + k.value.object_pointer = string_to_enter; + v.object_pointer = info_to_enter; dhtEnterElement(OurTable, k, v); } /* access table */ while (scanf("%127s", name_to_find) != EOF) { - k.key_data.object_pointer = name_to_find; + k.value.object_pointer = name_to_find; he= dhtLookupElement(OurTable, k); if (he != dhtNilElement) { /* if item is in the table */ (void)printf("found %s, age = %d, room = %d\n", - (char *)he->Key.key_data.object_pointer, - ((struct info *)he->Data.value_data.object_pointer)->age, - ((struct info *)he->Data.value_data.object_pointer)->room); + (char *)he->Key.value.object_pointer, + ((struct info *)he->Data.object_pointer)->age, + ((struct info *)he->Data.object_pointer)->room); } else { (void)printf("no such employee %s\n", name_to_find); } @@ -83,7 +83,7 @@ int main( ) /* now delete all struct info in the table */ he= dhtGetFirstElement(OurTable); while (he) { - free(he->Data.value_data.object_pointer); + free(he->Data.object_pointer); he= dhtGetNextElement(OurTable); } /* now destroy the whole table */ diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 48c143e959..7f2f65ddbe 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -26,7 +26,7 @@ typedef unsigned char uChar; static dhtHashValue HashMemoryValue(dhtKey k) { - MemVal const * toBeHashed = (MemVal const *)k.key_data.object_pointer; + MemVal const * toBeHashed = (MemVal const *)k.value.object_pointer; assert(!!toBeHashed); assert(toBeHashed->Data || !toBeHashed->Leng); uLong leng= toBeHashed->Leng; @@ -45,8 +45,8 @@ static dhtHashValue HashMemoryValue(dhtKey k) } static int EqualMemoryValue(dhtKey v1, dhtKey v2) { - MemVal const * value1 = (MemVal const *)v1.key_data.object_pointer; - MemVal const * value2 = (MemVal const *)v2.key_data.object_pointer; + MemVal const * value1 = (MemVal const *)v1.value.object_pointer; + MemVal const * value2 = (MemVal const *)v2.value.object_pointer; assert(value1 && value2); assert(value1->Data || !value1->Leng); @@ -57,7 +57,7 @@ static int EqualMemoryValue(dhtKey v1, dhtKey v2) return 1; return !memcmp(value1->Data, value2->Data, value1->Leng*sizeof value1->Data[0]); } -static int DupMemoryValue(dhtKeyOrValue kv, dhtKeyOrValue *output) +static int DupMemoryValue(dhtValue kv, dhtValue *output) { MemVal const *v= (MemVal const *)kv.object_pointer; MemVal *mv; @@ -83,7 +83,7 @@ static int DupMemoryValue(dhtKeyOrValue kv, dhtKeyOrValue *output) } return 1; } -static void FreeMemoryValue(dhtKeyOrValue kv) +static void FreeMemoryValue(dhtValue kv) { MemVal *v= (MemVal *)kv.object_pointer; if (v) { @@ -91,7 +91,7 @@ static void FreeMemoryValue(dhtKeyOrValue kv) fxfFree(v, sizeof *v); } } -static void DumpMemoryValue(dhtKeyOrValue kv, FILE *f) { +static void DumpMemoryValue(dhtValue kv, FILE *f) { MemVal const * v= (MemVal *)kv.object_pointer; uLong i; assert(v && f); diff --git a/DHT/dhtsimpl.c b/DHT/dhtsimpl.c index 4e6eeafb1c..09b00c5298 100644 --- a/DHT/dhtsimpl.c +++ b/DHT/dhtsimpl.c @@ -31,19 +31,19 @@ b -= c; b -= a; b ^= (a<<18); \ c -= a; c -= b; c ^= (b>>22); \ } -static unsigned long ConvertSimpleValue(dhtKey v) +static unsigned long ConvertSimpleValue(dhtKey k) { unsigned long a, b, c; c = 0x9e3779b97f4a7c13LL; - a = v.unsigned_integer<<1; - b = v.unsigned_integer; + a = k.value.unsigned_integer<<1; + b = k.value.unsigned_integer; mix(a,b,c); return c; } #else static unsigned long ConvertSimpleValue(dhtKey k) { - unsigned long c = k.key_data.unsigned_integer; + unsigned long c = k.value.unsigned_integer; unsigned long a = 0; unsigned long b = 0x9e3779b9U; a -= c; @@ -62,22 +62,22 @@ static unsigned long ConvertSimpleValue(dhtKey k) static int EqualSimpleValue(dhtKey k1, dhtKey k2) { - return (k1.key_data.unsigned_integer == k2.key_data.unsigned_integer); + return (k1.value.unsigned_integer == k2.value.unsigned_integer); } -static int DupSimpleValue(dhtKeyOrValue kv, dhtKeyOrValue *output) +static int DupSimpleValue(dhtValue kv, dhtValue *output) { assert(!!output); output->unsigned_integer = kv.unsigned_integer; return 0; } -static void FreeSimpleValue(dhtKeyOrValue kv) +static void FreeSimpleValue(dhtValue kv) { (void)kv; } -static void DumpSimpleValue(dhtKeyOrValue kv, FILE *f) +static void DumpSimpleValue(dhtValue kv, FILE *f) { assert(!!f); fprintf(f, "%08lx", (unsigned long)kv.unsigned_integer); diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index bb9e30634c..35027e55b0 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -24,7 +24,7 @@ static dhtHashValue ConvertString(dhtKey k) * - independent from word sizes * - needs no initialisation */ - unsigned char const *s= (unsigned char const *)k.key_data.object_pointer; + unsigned char const *s= (unsigned char const *)k.value.object_pointer; unsigned long hash; assert(!!s); hash= 0; @@ -41,13 +41,13 @@ static dhtHashValue ConvertString(dhtKey k) static int EqualString(dhtKey v1, dhtKey v2) { - char const *s1= (char const *)v1.key_data.object_pointer; - char const *s2= (char const *)v2.key_data.object_pointer; + char const *s1= (char const *)v1.value.object_pointer; + char const *s2= (char const *)v2.value.object_pointer; assert(s1 && s2); return !strcmp(s1, s2); } -static int DupString(dhtKeyOrValue v, dhtKeyOrValue *output) +static int DupString(dhtValue v, dhtValue *output) { char *nv; assert(!!output); @@ -61,13 +61,13 @@ static int DupString(dhtKeyOrValue v, dhtKeyOrValue *output) } return 1; } -static void FreeString(dhtKeyOrValue v) +static void FreeString(dhtValue v) { char *s= (char *)v.object_pointer; if (s) fxfFree(s, strlen(s)+1); } -static void DumpString(dhtKeyOrValue v, FILE *f) +static void DumpString(dhtValue v, FILE *f) { char const *s= (char const *)v.object_pointer; assert(s && f); diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 9cc08d5efd..69807600b6 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -90,24 +90,20 @@ typedef union { char character; void (*function_pointer)(void); long double floating_point; -} dhtKeyOrValue; +} dhtValue; typedef struct { - dhtKeyOrValue key_data; + dhtValue value; } dhtKey; -typedef struct { - dhtKeyOrValue value_data; -} dhtValue; - typedef unsigned long dhtHashValue; typedef struct { dhtHashValue (*Hash)(dhtKey); int (*Equal)(dhtKey, dhtKey); - int (*Dup)(dhtKeyOrValue, dhtKeyOrValue *); // should return 0 on success (and store the copied value at the second argument) and nonzero on error - void (*Free)(dhtKeyOrValue); - void (*Dump)(dhtKeyOrValue, FILE *); + int (*Dup)(dhtValue, dhtValue *); // should return 0 on success (and store the copied value at the second argument) and nonzero on error + void (*Free)(dhtValue); + void (*Dump)(dhtValue, FILE *); } dhtValueProcedures; #if defined(REGISTER_SIMPLE) diff --git a/DHT/fxf.c b/DHT/fxf.c index 7c7e285c34..b8edfd95e7 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -374,7 +374,7 @@ void *fxfAlloc(size_t size) { static char const * const myname= "fxfAlloc"; #endif SizeHead *sh; - char *ptr; + char *ptr= Nil(char); TMDBG(printf("fxfAlloc - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); DBG((stderr, "%s(%" SIZE_T_PRINTF_SPECIFIER ") =", myname, (size_t_printf_type)size)); diff --git a/optimisations/hash.c b/optimisations/hash.c index 9e0153fef4..c8d5ccdc21 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -649,16 +649,16 @@ static void set_value_attack_nosuccess(dhtElement *e, TraceValue("%u",slice_properties[si].size); TraceValue("%u",offset); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->Data.value_data.unsigned_integer); - TraceValue("pre:%08x ",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); + TraceValue("%p",(void *)&e->Data.unsigned_integer); + TraceValue("pre:%08x ",(unsigned int)(data_type)e->Data.unsigned_integer); TraceValue("%08x",bits); TraceEOL(); assert((bits&mask)==bits); - tmp = (data_type)e->Data.value_data.unsigned_integer; + tmp = (data_type)e->Data.unsigned_integer; tmp &= ~mask; tmp |= bits; - e->Data.value_data.unsigned_integer = tmp; - TraceValue("post:%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); + e->Data.unsigned_integer = tmp; + TraceValue("post:%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); TraceFunctionResultEnd(); @@ -681,16 +681,16 @@ static void set_value_attack_success(dhtElement *e, TraceValue("%u",slice_properties[si].size); TraceValue("%u",offset); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->Data.value_data.unsigned_integer); - TraceValue("pre:%08x ",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); + TraceValue("%p",(void *)&e->Data.unsigned_integer); + TraceValue("pre:%08x ",(unsigned int)(data_type)e->Data.unsigned_integer); TraceValue("%08x",bits); TraceEOL(); assert((bits&mask)==bits); - tmp = (data_type)e->Data.value_data.unsigned_integer; + tmp = (data_type)e->Data.unsigned_integer; tmp &= ~mask; tmp |= bits; - e->Data.value_data.unsigned_integer = tmp; - TraceValue("post:%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); + e->Data.unsigned_integer = tmp; + TraceValue("post:%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -712,16 +712,16 @@ static void set_value_help(dhtElement *e, TraceValue("%u",slice_properties[si].size); TraceValue("%u",offset); TraceValue("0x%08x ",mask); - TraceValue("%p ",(void *)&e->Data.value_data.unsigned_integer); - TraceValue("pre:0x%08x ",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); + TraceValue("%p ",(void *)&e->Data.unsigned_integer); + TraceValue("pre:0x%08x ",(unsigned int)(data_type)e->Data.unsigned_integer); TraceValue("0x%08x",bits); TraceEOL(); assert((bits&mask)==bits); - tmp = (data_type)e->Data.value_data.unsigned_integer; + tmp = (data_type)e->Data.unsigned_integer; tmp &= ~mask; tmp |= bits; - e->Data.value_data.unsigned_integer = tmp; - TraceValue("post:0x%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); + e->Data.unsigned_integer = tmp; + TraceValue("post:0x%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); TraceFunctionResultEnd(); @@ -732,12 +732,12 @@ static hash_value_type get_value_attack_success(dhtElement const *e, { unsigned int const offset = slice_properties[si].u.d.offsetSucc; unsigned int const mask = slice_properties[si].u.d.maskSucc; - data_type const result = (((data_type)e->Data.value_data.unsigned_integer) & mask) >> offset; + data_type const result = (((data_type)e->Data.unsigned_integer) & mask) >> offset; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceValue("%08x ",mask); - TraceValue("%p",(void const *)&e->Data.value_data.unsigned_integer); - TraceValue("%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); + TraceValue("%p",(void const *)&e->Data.unsigned_integer); + TraceValue("%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -751,12 +751,12 @@ static hash_value_type get_value_attack_nosuccess(dhtElement const *e, { unsigned int const offset = slice_properties[si].u.d.offsetNoSucc; unsigned int const mask = slice_properties[si].u.d.maskNoSucc; - data_type const result = (((data_type)e->Data.value_data.unsigned_integer) & mask) >> offset; + data_type const result = (((data_type)e->Data.unsigned_integer) & mask) >> offset; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceValue("%08x ",mask); - TraceValue("%p",(void const *)&e->Data.value_data.unsigned_integer); - TraceValue("%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); + TraceValue("%p",(void const *)&e->Data.unsigned_integer); + TraceValue("%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -770,13 +770,13 @@ static hash_value_type get_value_help(dhtElement const *e, { unsigned int const offset = slice_properties[si].u.h.offsetNoSucc; unsigned int const mask = slice_properties[si].u.h.maskNoSucc; - data_type const result = (((data_type)e->Data.value_data.unsigned_integer) & mask) >> offset; + data_type const result = (((data_type)e->Data.unsigned_integer) & mask) >> offset; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceValue("%u",offset); TraceValue("0x%08x ",mask); - TraceValue("%p ",(void const *)&e->Data.value_data.unsigned_integer); - TraceValue("0x%08x",(unsigned int)(data_type)e->Data.value_data.unsigned_integer); + TraceValue("%p ",(void const *)&e->Data.unsigned_integer); + TraceValue("0x%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -1793,7 +1793,7 @@ static void inithash(slice_index si) init_slice_properties(si); - template_element.Data.value_data.unsigned_integer = 0; + template_element.Data.unsigned_integer = 0; init_elements(&template_element); is_table_uncompressed = true; /* V3.60 TLi */ @@ -2190,7 +2190,7 @@ static void addtohash_battle_nosuccess(slice_index si, TraceFunctionParam("%u",min_length_adjusted); TraceFunctionParamListEnd(); - hb.key_data.object_pointer = &hashBuffers[nbply]; + hb.value.object_pointer = &hashBuffers[nbply]; he = dhtLookupElement(pyhash,hb); if (he==dhtNilElement) { @@ -2230,7 +2230,7 @@ static void addtohash_battle_success(slice_index si, TraceFunctionParam("%u",min_length_adjusted); TraceFunctionParamListEnd(); - hb.key_data.object_pointer = &hashBuffers[nbply]; + hb.value.object_pointer = &hashBuffers[nbply]; he = dhtLookupElement(pyhash,hb); if (he==dhtNilElement) { @@ -2313,7 +2313,7 @@ void attack_hashed_tester_solve(slice_index si) (*encode)(min_length,validity_value); - k.key_data.object_pointer = &hashBuffers[nbply]; + k.value.object_pointer = &hashBuffers[nbply]; he = dhtLookupElement(pyhash,k); if (he==dhtNilElement) solve_result = delegate_can_attack_in_n(si,min_length_adjusted); @@ -2383,7 +2383,7 @@ static boolean inhash_help(slice_index si) ifHASHRATE(use_all++); - hb.key_data.object_pointer = &hashBuffers[nbply]; + hb.value.object_pointer = &hashBuffers[nbply]; he = dhtLookupElement(pyhash,hb); if (he==dhtNilElement) result = false; @@ -2414,7 +2414,7 @@ static void addtohash_help(slice_index si) TraceFunctionParam("%u",si); TraceFunctionParamListEnd(); - hb.key_data.object_pointer = &hashBuffers[nbply]; + hb.value.object_pointer = &hashBuffers[nbply]; he = dhtLookupElement(pyhash,hb); if (he==dhtNilElement) he = allocDHTelement(hb); From ef64367ed0c743f0be66ad509fd206a6d0562749 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 2 Sep 2023 21:30:04 -0400 Subject: [PATCH 004/111] Might as well eliminate a compiler warning while we're at it. --- optimisations/hash.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index c8d5ccdc21..d95403b57b 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1769,7 +1769,8 @@ static boolean is_proofgame(slice_index si) */ boolean is_hashtable_allocated(void) { - return fxfInitialised(); + return !!fxfInitialised(); /* !! just in case fxfInitialised returns a nonzero value other than 1 + and boolean is some type that won't automatically convert it to 1. */ } /* Initialise the hashing machinery for the current stipulation From 2b4c82196f4cc9b8ec66414dd54eafb052151906 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 2 Sep 2023 21:43:37 -0400 Subject: [PATCH 005/111] Making cppcheck happier. --- DHT/dhtsimpl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DHT/dhtsimpl.c b/DHT/dhtsimpl.c index 09b00c5298..8472dc39e7 100644 --- a/DHT/dhtsimpl.c +++ b/DHT/dhtsimpl.c @@ -33,12 +33,12 @@ } static unsigned long ConvertSimpleValue(dhtKey k) { - unsigned long a, b, c; - c = 0x9e3779b97f4a7c13LL; + unsigned long long a, b, c; + c = 0x9e3779b97f4a7c13LLU; a = k.value.unsigned_integer<<1; b = k.value.unsigned_integer; mix(a,b,c); - return c; + return (unsigned long)c; } #else static unsigned long ConvertSimpleValue(dhtKey k) From 258f0555f92062fe9094fdecda89e75d2b14744d Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 2 Sep 2023 21:54:08 -0400 Subject: [PATCH 006/111] I guess this is more generic. --- DHT/dhtsimpl.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DHT/dhtsimpl.c b/DHT/dhtsimpl.c index 8472dc39e7..5e2f408bfa 100644 --- a/DHT/dhtsimpl.c +++ b/DHT/dhtsimpl.c @@ -33,8 +33,13 @@ } static unsigned long ConvertSimpleValue(dhtKey k) { +# if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) unsigned long long a, b, c; c = 0x9e3779b97f4a7c13LLU; +# else + unsigned long a, b, c; + c = 0x9e3779b97f4a7c13LU; +# endif a = k.value.unsigned_integer<<1; b = k.value.unsigned_integer; mix(a,b,c); From 1958bb22196d3e89e759b307d3d65ee72ed76d70 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 07:37:07 -0400 Subject: [PATCH 007/111] Since we want everything to be 0, we don't need the explicit initialiser. --- optimisations/hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index d95403b57b..95a074d6d4 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -631,7 +631,7 @@ static void init_slice_properties(slice_index si) /* Pseudo hash table element - template for fast initialization of * newly created actual table elements */ -static dhtElement template_element = {0}; +static dhtElement template_element; static void set_value_attack_nosuccess(dhtElement *e, From 3fe00c7da0848e3b60dbb2cdb1d8f3afec93cea1 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 08:16:24 -0400 Subject: [PATCH 008/111] Prefer unsigned integers. --- DHT/dhtbcmem.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index d4f63e69ce..dfb3058db0 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -121,13 +121,13 @@ static void FreeBCMemValue(dhtValue kv) static void DumpBCMemValue(dhtValue kv, FILE *f) { BCMemValue const *toBeDumped = (BCMemValue const *)kv.object_pointer; - int length; - int i; + unsigned int length; + unsigned int i; assert(toBeDumped && f); length = toBeDumped->Leng; - fprintf(f, "(%d)", length); + fprintf(f, "(%u)", length); for (i=0; iData[i] & 0xffU)); } From f6d1f70d60a1cfd52b6fc1557f69774ff1c83d7a Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 08:32:22 -0400 Subject: [PATCH 009/111] Fixing some cppcheck style warnings. --- DHT/dht.c | 2 +- DHT/dht.h | 4 ++-- DHT/dhtbcmem.c | 5 ++--- DHT/dhtcmem.c | 5 ++--- DHT/fxf.h | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index 87ecfd525e..3ea6a417c4 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -436,7 +436,7 @@ typedef struct dht { #define ActualLoadFactor(h) (((h)->KeyCount*100)/(h)->DirTab.count) #endif /*OVERFLOW_SAVE*/ -unsigned long dhtKeyCount(dht *h) +unsigned long dhtKeyCount(dht const *h) { return h->KeyCount; } diff --git a/DHT/dht.h b/DHT/dht.h index 18b279def7..9b3b1fa88a 100644 --- a/DHT/dht.h +++ b/DHT/dht.h @@ -30,12 +30,12 @@ dhtElement *dhtEnterElement(struct dht *, dhtKey key, dhtValue data); unsigned int dhtBucketStat (struct dht *, unsigned int *counter, unsigned int n); void dhtDestroy (struct dht *); void dhtDump (struct dht *, FILE *); -void dhtDumpIndented (int i, struct dht *, FILE *); +void dhtDumpIndented (int ind, struct dht *, FILE *); void dhtRemoveElement (struct dht *, dhtKey key); dhtElement *dhtLookupElement (struct dht *, dhtKey key); dhtElement *dhtGetFirstElement(struct dht *); dhtElement *dhtGetNextElement (struct dht *); -unsigned long dhtKeyCount (struct dht *); +unsigned long dhtKeyCount (struct dht const *); char const *dhtErrorMsg (void); extern char dhtError[]; diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index dfb3058db0..5b4295d1c7 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -103,11 +103,10 @@ static void FreeBCMemValue(dhtValue kv) size_t const size_of_element = sizeof freed->Data[0]; size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); size_t const remainder = (num_bytes_in_Data % size_of_element); - size_t size = sizeof *freed; - unsigned char length; if (freed) { - length = freed->Leng; + size_t size = sizeof *freed; + unsigned char length = freed->Leng; if (length > num_elements_in_Data) { size_t const num_new_elements = (length - num_elements_in_Data); diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index c53ca6bc3e..fea9f02b9c 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -95,11 +95,10 @@ static void FreeCompactMemoryValue(dhtValue kv) size_t const size_of_element = sizeof v->Data[0]; size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); size_t const remainder = (num_bytes_in_Data % size_of_element); - size_t size = sizeof *v; - uLong length; if (v) { - length = v->Leng; + size_t size = sizeof *v; + uLong length = v->Leng; if (length > num_elements_in_Data) { size_t const num_new_elements = (length - num_elements_in_Data); diff --git a/DHT/fxf.h b/DHT/fxf.h index 401dfc5dfb..a0fc92f19b 100644 --- a/DHT/fxf.h +++ b/DHT/fxf.h @@ -3,7 +3,7 @@ #include -size_t fxfInit(size_t GlobalSize); /* returns the number of bytes actually allocated */ +size_t fxfInit(size_t Size); /* returns the number of bytes actually allocated */ int fxfInitialised(void); void *fxfAlloc(size_t size); void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize); From a5f410fced87bfc827c83179b3f3aed95534de0a Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 10:31:21 -0400 Subject: [PATCH 010/111] Fixing some typos. --- DHT/dht.h | 2 +- DHT/dhtbcmem.h | 2 +- DHT/dhtcmem.h | 2 +- DHT/dhtmem.h | 2 +- DHT/dhtvalue.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DHT/dht.h b/DHT/dht.h index 9b3b1fa88a..e4c6bf4a4b 100644 --- a/DHT/dht.h +++ b/DHT/dht.h @@ -6,7 +6,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ typedef enum { diff --git a/DHT/dhtbcmem.h b/DHT/dhtbcmem.h index 30f015648a..e044d76b86 100644 --- a/DHT/dhtbcmem.h +++ b/DHT/dhtbcmem.h @@ -6,7 +6,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept + * comment with the above copyright notice is kept * intact and in place. */ diff --git a/DHT/dhtcmem.h b/DHT/dhtcmem.h index 92fff63df9..4f048f5492 100644 --- a/DHT/dhtcmem.h +++ b/DHT/dhtcmem.h @@ -6,7 +6,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept + * comment with the above copyright notice is kept * intact and in place. */ diff --git a/DHT/dhtmem.h b/DHT/dhtmem.h index 08be5de399..d055c18843 100644 --- a/DHT/dhtmem.h +++ b/DHT/dhtmem.h @@ -6,7 +6,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 69807600b6..fd5b7cdd4c 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -7,7 +7,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ #include From de4eeaf367693b1c0ad9561b16ef7558b2a3bdc0 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 10:33:40 -0400 Subject: [PATCH 011/111] Fixing some more typos. --- DHT/dhtbcmem.c | 2 +- DHT/dhtcmem.c | 2 +- DHT/dhtmem.c | 2 +- DHT/dhtsimpl.c | 2 +- DHT/dhtstrng.c | 2 +- DHT/dhtvalue.c | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index 5b4295d1c7..870402bcc5 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index fea9f02b9c..e7db8d6377 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ #include diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 7f2f65ddbe..7262594d8c 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ diff --git a/DHT/dhtsimpl.c b/DHT/dhtsimpl.c index 5e2f408bfa..9bbc71c7ca 100644 --- a/DHT/dhtsimpl.c +++ b/DHT/dhtsimpl.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ #include "debugging/assert.h" diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 35027e55b0..18bc253db7 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ #include diff --git a/DHT/dhtvalue.c b/DHT/dhtvalue.c index ef5761d598..321a08f8fc 100644 --- a/DHT/dhtvalue.c +++ b/DHT/dhtvalue.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ #include "dhtvalue.h" From 3fb8b38a0621f27a781dab9a2cc490574a00c28e Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 15:05:09 -0400 Subject: [PATCH 012/111] Beginning to track dimensions so we can ensure that we don't write outside of bounds. --- optimisations/hash.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index 95a074d6d4..5d6c0dd9be 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -148,6 +148,10 @@ static hash_value_type minimalElementValueAfterCompression; static unsigned int nr_hash_slices; static slice_index hash_slices[max_nr_slices]; +enum { + NUM_ELEMENTS_IN_HASHBUFFER = ((sizeof(HashBuffer) - offsetof(BCMemValue, Data))/sizeof(byte)), + ENSURE_HASHBUFFER_DATA_HAS_AT_LEAST_NR_ROWS_ON_BOARD_ENTRIES = 1/(NUM_ELEMENTS_IN_HASHBUFFER >= nr_rows_on_board) +}; HashBuffer hashBuffers[maxply+1]; @@ -1192,7 +1196,7 @@ static void ProofEncode(stip_length_type min_length, stip_length_type validity_v byte *bp = position+nr_rows_on_board; /* clear the bits for storing the position of pieces */ - memset(position, 0, nr_rows_on_board); + memset(position, 0, nr_rows_on_board * sizeof *position); { boolean even = false; @@ -1512,7 +1516,7 @@ static void LargeEncode(stip_length_type min_length, TraceFunctionParamListEnd(); /* clear the bits for storing the position of pieces */ - memset(position,0,nr_rows_on_board); + memset(position,0,nr_rows_on_board * sizeof *position); for (row=0; row Date: Sun, 3 Sep 2023 15:12:20 -0400 Subject: [PATCH 013/111] Matching style. --- optimisations/hash.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index 5d6c0dd9be..e2440ec4b3 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -148,7 +148,8 @@ static hash_value_type minimalElementValueAfterCompression; static unsigned int nr_hash_slices; static slice_index hash_slices[max_nr_slices]; -enum { +enum +{ NUM_ELEMENTS_IN_HASHBUFFER = ((sizeof(HashBuffer) - offsetof(BCMemValue, Data))/sizeof(byte)), ENSURE_HASHBUFFER_DATA_HAS_AT_LEAST_NR_ROWS_ON_BOARD_ENTRIES = 1/(NUM_ELEMENTS_IN_HASHBUFFER >= nr_rows_on_board) }; From 427e8f7b5adacb3c90209fddf67b5b53b954e397 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 17:17:10 -0400 Subject: [PATCH 014/111] Lots of simplifications, plus other changes. --- DHT/dhtbcmem.c | 24 +++++++++--------------- DHT/dhtbcmem.h | 2 +- DHT/dhtcmem.c | 18 ++++++------------ optimisations/hash.c | 19 +++++++++++++------ optimisations/hash.h | 11 ++++++++--- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index 870402bcc5..f1f28d4a8c 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -67,21 +67,18 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) BCMemValue const *original = (BCMemValue const *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *original) - offsetof(BCMemValue, Data)); size_t const size_of_element = sizeof original->Data[0]; - size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); - size_t const remainder = (num_bytes_in_Data % size_of_element); - size_t size = sizeof *original; BCMemValue *result; - unsigned char length; + unsigned short length; + size_t size = sizeof *original; assert(!!original); length = original->Leng; - if (length > num_elements_in_Data) + if (length > (num_bytes_in_Data / size_of_element)) { - size_t const num_new_elements = (length - num_elements_in_Data); - if (num_new_elements > (((size_t)-1) - size + remainder) / size_of_element) + if (length > ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)) return 1; - size += ((num_new_elements * size_of_element) - remainder); + size += ((length * size_of_element) - num_bytes_in_Data); } result = (BCMemValue *)fxfAlloc(size); @@ -101,17 +98,14 @@ static void FreeBCMemValue(dhtValue kv) BCMemValue *freed = (BCMemValue *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *freed) - offsetof(BCMemValue, Data)); size_t const size_of_element = sizeof freed->Data[0]; - size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); - size_t const remainder = (num_bytes_in_Data % size_of_element); if (freed) { size_t size = sizeof *freed; - unsigned char length = freed->Leng; - if (length > num_elements_in_Data) + unsigned short length = freed->Leng; + if (length > (num_bytes_in_Data / size_of_element)) { - size_t const num_new_elements = (length - num_elements_in_Data); - assert(num_new_elements <= ((((size_t)-1) - size + remainder) / size_of_element)); - size += ((num_new_elements * size_of_element) - remainder); + assert(length <= ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)); + size += ((length * size_of_element) - num_bytes_in_Data); } fxfFree(freed,size); } diff --git a/DHT/dhtbcmem.h b/DHT/dhtbcmem.h index e044d76b86..35b518b0f3 100644 --- a/DHT/dhtbcmem.h +++ b/DHT/dhtbcmem.h @@ -16,7 +16,7 @@ */ typedef struct BCMemValue { - unsigned char Leng; + unsigned short Leng; unsigned char Data[1]; } BCMemValue; diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index e7db8d6377..3a360bc1ab 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -59,8 +59,6 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) CompactMemVal const *v = (CompactMemVal const *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); size_t const size_of_element = sizeof v->Data[0]; - size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); - size_t const remainder = (num_bytes_in_Data % size_of_element); size_t size = sizeof *v; CompactMemVal *result; uLong length; @@ -68,12 +66,11 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) assert(!!v); length = v->Leng; - if (length > num_elements_in_Data) + if (length > (num_bytes_in_Data / size_of_element)) { - size_t const num_new_elements = (length - num_elements_in_Data); - if (num_new_elements > (((size_t)-1) - size + remainder) / size_of_element) + if (length > ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)) return 1; - size += ((num_new_elements * size_of_element) - remainder); + size += ((length * size_of_element) - num_bytes_in_Data); } result = (CompactMemVal *)fxfAlloc(size); @@ -93,17 +90,14 @@ static void FreeCompactMemoryValue(dhtValue kv) CompactMemVal *v = (CompactMemVal *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); size_t const size_of_element = sizeof v->Data[0]; - size_t const num_elements_in_Data = (num_bytes_in_Data / size_of_element); - size_t const remainder = (num_bytes_in_Data % size_of_element); if (v) { size_t size = sizeof *v; uLong length = v->Leng; - if (length > num_elements_in_Data) + if (length > (num_bytes_in_Data / size_of_element)) { - size_t const num_new_elements = (length - num_elements_in_Data); - assert(num_new_elements <= ((((size_t)-1) - size + remainder) / size_of_element)); - size += ((num_new_elements * size_of_element) - remainder); + assert(length <= ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)); + size += ((length * size_of_element) - num_bytes_in_Data); } fxfFree(v,size); } diff --git a/optimisations/hash.c b/optimisations/hash.c index e2440ec4b3..230a4f93fb 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1190,6 +1190,13 @@ static void ProofLargeEncodePiece(byte **bp, ++*bp; } +#define SIGNAL_OUT_OF_ROOM_IN_HASHBUFFER do { \ + fprintf(stderr, "Internal Popeye limit reached in function %s in file %s.", __func__, __FILE__); \ + fputs("Please share with the developers so this limit can be increased or potentially eliminated.\n", stderr) \ + fputs("Computation is now aborting.\n", stderr); \ + exit(1); \ + } while (0) + static void ProofEncode(stip_length_type min_length, stip_length_type validity_value) { HashBuffer *hb = &hashBuffers[nbply]; @@ -1244,8 +1251,8 @@ static void ProofEncode(stip_length_type min_length, stip_length_type validity_v /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=UCHAR_MAX); - hb->cmv.Leng = (unsigned char)(bp-hb->cmv.Data); + assert(bp-hb->cmv.Data<=USHRT_MAX); + hb->cmv.Leng = (bp-hb->cmv.Data); } static unsigned int TellCommonEncodePosLeng(unsigned int len, @@ -1545,8 +1552,8 @@ static void LargeEncode(stip_length_type min_length, /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=UCHAR_MAX); - hb->cmv.Leng = (unsigned char)(bp-hb->cmv.Data); + assert(bp-hb->cmv.Data<=USHRT_MAX); + hb->cmv.Leng = (bp-hb->cmv.Data); TraceFunctionExit(__func__); TraceFunctionResultEnd(); @@ -1609,8 +1616,8 @@ static void SmallEncode(stip_length_type min_length, /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=UCHAR_MAX); - hb->cmv.Leng = (unsigned char)(bp-hb->cmv.Data); + assert(bp-hb->cmv.Data<=USHRT_MAX); + hb->cmv.Leng = (bp-hb->cmv.Data); TraceFunctionExit(__func__); TraceFunctionResultEnd(); diff --git a/optimisations/hash.h b/optimisations/hash.h index 9f454dbb4d..7c473ab9d5 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -12,6 +12,8 @@ #include "DHT/dhtbcmem.h" #include "pieces/pieces.h" #include "solving/machinery/solve.h" +#include "position/underworld.h" +#include "position/position.h" #include "solving/ply.h" #include @@ -26,13 +28,16 @@ typedef unsigned char byte; enum { - hashbuf_length = 256 + MAX_LENGTH_OF_ENCODING = (((nr_rows_on_board * nr_files_on_board) + underworld_capacity) * 6) + + 19 + maxinum + maxply, + hashbuf_padding = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) - + (sizeof(BCMemValue) - offsetof(BCMemValue, Data)) }; -typedef union +typedef struct { BCMemValue cmv; - byte buffer[hashbuf_length]; + byte cmv_Data_padding[hashbuf_padding]; } HashBuffer; extern HashBuffer hashBuffers[maxply+1]; From d5dcfb6b10a4fba577ef392740809187769ba585 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 21:24:35 -0400 Subject: [PATCH 015/111] Restoring the union as there was no strong reason to remove it. We can achieve the same by adjusting hashbuf_length accordingly. --- optimisations/hash.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/optimisations/hash.h b/optimisations/hash.h index 7c473ab9d5..afb7f24628 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -29,15 +29,16 @@ typedef unsigned char byte; enum { MAX_LENGTH_OF_ENCODING = (((nr_rows_on_board * nr_files_on_board) + underworld_capacity) * 6) + - 19 + maxinum + maxply, - hashbuf_padding = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) - - (sizeof(BCMemValue) - offsetof(BCMemValue, Data)) + 18 + maxinum + (1 + maxply), +/* MAX_LENGTH_OF_ENCODING = (((nr_rows_on_board * nr_files_on_board) + underworld_capacity) * 6) + + 18 + maxinum + (en_passant_top[nbply] - en_passant_top[nbply-1]), */ + hashbuf_length = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) + offsetof(BCMemValue, Data) }; -typedef struct +typedef union { BCMemValue cmv; - byte cmv_Data_padding[hashbuf_padding]; + byte buffer[hashbuf_length]; } HashBuffer; extern HashBuffer hashBuffers[maxply+1]; From 488b13bb3cf72432c60b0d3a6e0343f96a8361c5 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 21:36:48 -0400 Subject: [PATCH 016/111] Comparing the right upper bound, the one that indicates how much space we have. --- optimisations/hash.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index 230a4f93fb..53b421ca60 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1251,7 +1251,7 @@ static void ProofEncode(stip_length_type min_length, stip_length_type validity_v /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=USHRT_MAX); + assert(bp-hb->cmv.Data<=MAX_LENGTH_OF_ENCODING); hb->cmv.Leng = (bp-hb->cmv.Data); } @@ -1552,7 +1552,7 @@ static void LargeEncode(stip_length_type min_length, /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=USHRT_MAX); + assert(bp-hb->cmv.Data<=MAX_LENGTH_OF_ENCODING); hb->cmv.Leng = (bp-hb->cmv.Data); TraceFunctionExit(__func__); @@ -1616,7 +1616,7 @@ static void SmallEncode(stip_length_type min_length, /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=USHRT_MAX); + assert(bp-hb->cmv.Data<=MAX_LENGTH_OF_ENCODING); hb->cmv.Leng = (bp-hb->cmv.Data); TraceFunctionExit(__func__); From 196796611c5a7541a8b641c2d0edee80d3791152 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 22:38:45 -0400 Subject: [PATCH 017/111] The Leng member is an unsigned (short) int, so cast to ensure it matches the format specifier. --- debugging/trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debugging/trace.c b/debugging/trace.c index e24e42e4c7..c0657839d1 100644 --- a/debugging/trace.c +++ b/debugging/trace.c @@ -473,7 +473,7 @@ void TraceCurrentHashBuffer(void) HashBuffer const *hb = &hashBuffers[nbply]; unsigned int i; - printf(" #%lu nbply:%u Leng:%u ",level,nbply,hb->cmv.Leng); + printf(" #%lu nbply:%u Leng:%u ",level,nbply,(unsigned int)hb->cmv.Leng); for (i = 0; icmv.Leng; ++i) printf("%02x ",(unsigned int)hb->cmv.Data[i]); putchar('\n'); From f95c1cdb907fc8fd2fdbecb090985ad5a9c5eb58 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 22:42:02 -0400 Subject: [PATCH 018/111] Let's just use the right format specifier. --- debugging/trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debugging/trace.c b/debugging/trace.c index c0657839d1..21c5ebf128 100644 --- a/debugging/trace.c +++ b/debugging/trace.c @@ -473,7 +473,7 @@ void TraceCurrentHashBuffer(void) HashBuffer const *hb = &hashBuffers[nbply]; unsigned int i; - printf(" #%lu nbply:%u Leng:%u ",level,nbply,(unsigned int)hb->cmv.Leng); + printf(" #%lu nbply:%u Leng:%hu ",level,nbply,hb->cmv.Leng); for (i = 0; icmv.Leng; ++i) printf("%02x ",(unsigned int)hb->cmv.Data[i]); putchar('\n'); From 7786ef593b34fcfd65535ec2300c926648992c97 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 3 Sep 2023 22:53:01 -0400 Subject: [PATCH 019/111] Fixing some issues found by VSCode. --- DHT/dhtexam2.c | 2 +- DHT/fxf.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/DHT/dhtexam2.c b/DHT/dhtexam2.c index fa1941b9ef..4f3f6dbf23 100644 --- a/DHT/dhtexam2.c +++ b/DHT/dhtexam2.c @@ -84,7 +84,7 @@ int main(int argc, char *argv[]) { for (cnt=0; cnt +#include size_t fxfInit(size_t Size); /* returns the number of bytes actually allocated */ int fxfInitialised(void); From 591df8917f30b6f308efdb2cd7c27f61d83b35c2 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 4 Sep 2023 06:53:28 -0400 Subject: [PATCH 020/111] Oops, forgot a term. --- optimisations/hash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimisations/hash.h b/optimisations/hash.h index afb7f24628..015248a068 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -29,7 +29,7 @@ typedef unsigned char byte; enum { MAX_LENGTH_OF_ENCODING = (((nr_rows_on_board * nr_files_on_board) + underworld_capacity) * 6) + - 18 + maxinum + (1 + maxply), + 18 + maxinum + nr_rows_on_board + (1 + maxply), /* MAX_LENGTH_OF_ENCODING = (((nr_rows_on_board * nr_files_on_board) + underworld_capacity) * 6) + 18 + maxinum + (en_passant_top[nbply] - en_passant_top[nbply-1]), */ hashbuf_length = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) + offsetof(BCMemValue, Data) From 87c28c9225ec523f58176afb035faac19c96d51d Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 4 Sep 2023 07:15:58 -0400 Subject: [PATCH 021/111] Those functions can be static. --- optimisations/hash.c | 278 +++++++++++++++++++++---------------------- optimisations/hash.h | 8 -- 2 files changed, 139 insertions(+), 147 deletions(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index 53b421ca60..8595956e77 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1197,6 +1197,144 @@ static void ProofLargeEncodePiece(byte **bp, exit(1); \ } while (0) +static byte *SmallEncodePiece(byte *bp, + int row, int col, + piece_walk_type pienam, Flags pspec) +{ + *bp++= (byte)((row<<(CHAR_BIT/2))+col); + if (one_byte_hash) + *bp++ = (byte)pspec + (piece_nbr[pienam] << (CHAR_BIT/2)); + else + { + unsigned int i; + *bp++ = pienam; + for (i = 0; i>(CHAR_BIT*i)) & ByteMask); + } + + return bp; +} + +static byte *CommonEncode(byte *bp, + stip_length_type min_length, + stip_length_type validity_value) +{ + if (CondFlag[messigny]) + { + move_effect_journal_index_type const base = move_effect_journal_base[nbply]; + move_effect_journal_index_type const movement = base+move_effect_journal_index_offset_movement; + if (move_effect_journal[movement].type==move_effect_piece_exchange + && move_effect_journal[movement].reason==move_effect_reason_messigny_exchange) + { + *bp++ = (byte)(move_effect_journal[movement].u.piece_exchange.to - square_a1); + *bp++ = (byte)(move_effect_journal[movement].u.piece_exchange.from - square_a1); + } + else + *bp++ = (byte)UCHAR_MAX; + } + + if (CondFlag[duellist]) + { + *bp++ = (byte)(duellists[White] - square_a1); + *bp++ = (byte)(duellists[Black] - square_a1); + } + + if (CondFlag[blfollow] || CondFlag[whfollow] || CondFlag[champursue]) + { + square const sq_departure = move_effect_journal_get_departure_square(nbply); + if (sq_departure==initsquare) + *bp++ = (byte)UCHAR_MAX; + else + *bp++ = (byte)(sq_departure-square_a1); + } + + if (CondFlag[blacksynchron] || CondFlag[whitesynchron] + || CondFlag[blackantisynchron] || CondFlag[whiteantisynchron]) + { + square const sq_departure = move_effect_journal_get_departure_square(nbply); + if (sq_departure==initsquare) + *bp++ = (byte)UCHAR_MAX; + else + { + move_effect_journal_index_type const base = move_effect_journal_base[nbply]; + move_effect_journal_index_type const movement = base+move_effect_journal_index_offset_movement; + square const sq_arrival = move_effect_journal[movement].u.piece_movement.to; + enum { nr_squares = nr_rows_on_board*nr_files_on_board }; + *bp++= (byte)(sq_num(sq_departure)-sq_num(sq_arrival)+nr_squares); + } + } + + if (CondFlag[imitators]) + { + unsigned int imi_idx; + + /* The number of imitators has to be coded too to avoid + * ambiguities. + */ + *bp++ = (byte)being_solved.number_of_imitators; + for (imi_idx = 0; imi_idx>CHAR_BIT); + *bp++ = (byte)(removedspec&ByteMask); + } + } + else + *bp++ = (byte)0; + } + + assert(validity_value<=(1<slack_length+1) + *bp++ = (byte)(validity_value); + + { + unsigned int i; + + for (i = en_passant_top[nbply-1]+1; i<=en_passant_top[nbply]; ++i) + *bp++ = (byte)(en_passant_multistep_over[i] - square_a1); + } + + *bp++ = being_solved.castling_rights; /* Castling_Flag */ + + if (CondFlag[BGL]) { + memcpy(bp, &BGL_values[White], sizeof BGL_values[White]); + bp += sizeof BGL_values[White]; + memcpy(bp, &BGL_values[Black], sizeof BGL_values[Black]); + bp += sizeof BGL_values[Black]; + } + + if (CondFlag[disparate]) + { + move_effect_journal_index_type const top = move_effect_journal_base[nbply]; + move_effect_journal_index_type const movement = top+move_effect_journal_index_offset_movement; + *bp++ = (byte)move_effect_journal[movement].u.piece_movement.moving; + *bp++ = trait[nbply]; + } + + return bp; +} /* CommonEncode */ + static void ProofEncode(stip_length_type min_length, stip_length_type validity_value) { HashBuffer *hb = &hashBuffers[nbply]; @@ -1371,126 +1509,6 @@ static unsigned int TellSmallEncodePosLeng(void) return result; } /* TellSmallEncodePosLeng */ -byte *CommonEncode(byte *bp, - stip_length_type min_length, - stip_length_type validity_value) -{ - if (CondFlag[messigny]) - { - move_effect_journal_index_type const base = move_effect_journal_base[nbply]; - move_effect_journal_index_type const movement = base+move_effect_journal_index_offset_movement; - if (move_effect_journal[movement].type==move_effect_piece_exchange - && move_effect_journal[movement].reason==move_effect_reason_messigny_exchange) - { - *bp++ = (byte)(move_effect_journal[movement].u.piece_exchange.to - square_a1); - *bp++ = (byte)(move_effect_journal[movement].u.piece_exchange.from - square_a1); - } - else - *bp++ = (byte)UCHAR_MAX; - } - - if (CondFlag[duellist]) - { - *bp++ = (byte)(duellists[White] - square_a1); - *bp++ = (byte)(duellists[Black] - square_a1); - } - - if (CondFlag[blfollow] || CondFlag[whfollow] || CondFlag[champursue]) - { - square const sq_departure = move_effect_journal_get_departure_square(nbply); - if (sq_departure==initsquare) - *bp++ = (byte)UCHAR_MAX; - else - *bp++ = (byte)(sq_departure-square_a1); - } - - if (CondFlag[blacksynchron] || CondFlag[whitesynchron] - || CondFlag[blackantisynchron] || CondFlag[whiteantisynchron]) - { - square const sq_departure = move_effect_journal_get_departure_square(nbply); - if (sq_departure==initsquare) - *bp++ = (byte)UCHAR_MAX; - else - { - move_effect_journal_index_type const base = move_effect_journal_base[nbply]; - move_effect_journal_index_type const movement = base+move_effect_journal_index_offset_movement; - square const sq_arrival = move_effect_journal[movement].u.piece_movement.to; - enum { nr_squares = nr_rows_on_board*nr_files_on_board }; - *bp++= (byte)(sq_num(sq_departure)-sq_num(sq_arrival)+nr_squares); - } - } - - if (CondFlag[imitators]) - { - unsigned int imi_idx; - - /* The number of imitators has to be coded too to avoid - * ambiguities. - */ - *bp++ = (byte)being_solved.number_of_imitators; - for (imi_idx = 0; imi_idx>CHAR_BIT); - *bp++ = (byte)(removedspec&ByteMask); - } - } - else - *bp++ = (byte)0; - } - - assert(validity_value<=(1<slack_length+1) - *bp++ = (byte)(validity_value); - - { - unsigned int i; - - for (i = en_passant_top[nbply-1]+1; i<=en_passant_top[nbply]; ++i) - *bp++ = (byte)(en_passant_multistep_over[i] - square_a1); - } - - *bp++ = being_solved.castling_rights; /* Castling_Flag */ - - if (CondFlag[BGL]) { - memcpy(bp, &BGL_values[White], sizeof BGL_values[White]); - bp += sizeof BGL_values[White]; - memcpy(bp, &BGL_values[Black], sizeof BGL_values[Black]); - bp += sizeof BGL_values[Black]; - } - - if (CondFlag[disparate]) - { - move_effect_journal_index_type const top = move_effect_journal_base[nbply]; - move_effect_journal_index_type const movement = top+move_effect_journal_index_offset_movement; - *bp++ = (byte)move_effect_journal[movement].u.piece_movement.moving; - *bp++ = trait[nbply]; - } - - return bp; -} /* CommonEncode */ - static byte *LargeEncodePiece(byte *bp, byte *position, int row, int col, piece_walk_type pienam, Flags pspec) @@ -1553,30 +1571,12 @@ static void LargeEncode(stip_length_type min_length, bp = CommonEncode(bp,min_length,validity_value); assert(bp-hb->cmv.Data<=MAX_LENGTH_OF_ENCODING); - hb->cmv.Leng = (bp-hb->cmv.Data); + hb->cmv.Leng = bp-hb->cmv.Data; TraceFunctionExit(__func__); TraceFunctionResultEnd(); } /* LargeEncode */ -byte *SmallEncodePiece(byte *bp, - int row, int col, - piece_walk_type pienam, Flags pspec) -{ - *bp++= (byte)((row<<(CHAR_BIT/2))+col); - if (one_byte_hash) - *bp++ = (byte)pspec + (piece_nbr[pienam] << (CHAR_BIT/2)); - else - { - unsigned int i; - *bp++ = pienam; - for (i = 0; i>(CHAR_BIT*i)) & ByteMask); - } - - return bp; -} - static void SmallEncode(stip_length_type min_length, stip_length_type validity_value) { diff --git a/optimisations/hash.h b/optimisations/hash.h index 015248a068..79d72d51d2 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -66,14 +66,6 @@ void HashStats(unsigned int level, char const *trailer); void IncHashRateLevel(void); void DecHashRateLevel(void); -byte *CommonEncode(byte *bp, - stip_length_type min_length, - stip_length_type validity_value); - -byte *SmallEncodePiece(byte *bp, - int row, int col, - piece_walk_type p, Flags pspec); - /* Try to solve in solve_nr_remaining half-moves. * @param si slice index * @note assigns solve_result the length of solution found and written, i.e.: From 1159ebe35aaf7f41ee0e105ce1f0537a54a48e22 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 4 Sep 2023 16:25:37 -0400 Subject: [PATCH 022/111] Trying to compute the needed size at compile-time. --- optimisations/hash.c | 7 --- optimisations/hash.h | 110 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 11 deletions(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index 8595956e77..5f132b56be 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1190,13 +1190,6 @@ static void ProofLargeEncodePiece(byte **bp, ++*bp; } -#define SIGNAL_OUT_OF_ROOM_IN_HASHBUFFER do { \ - fprintf(stderr, "Internal Popeye limit reached in function %s in file %s.", __func__, __FILE__); \ - fputs("Please share with the developers so this limit can be increased or potentially eliminated.\n", stderr) \ - fputs("Computation is now aborting.\n", stderr); \ - exit(1); \ - } while (0) - static byte *SmallEncodePiece(byte *bp, int row, int col, piece_walk_type pienam, Flags pspec) diff --git a/optimisations/hash.h b/optimisations/hash.h index 79d72d51d2..8b5edf0189 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -15,6 +15,7 @@ #include "position/underworld.h" #include "position/position.h" #include "solving/ply.h" +#include "conditions/bgl.h" #include #if defined(TESTHASH) @@ -26,12 +27,113 @@ /* typedefs */ typedef unsigned char byte; +/* Time for some math -- let's figure out the maximum number of bytes a position encoding may take up. + If it's a proofgame then ProofEncode will be used. Otherwise, SmallEncode or LargeEncode will be chosen, + the former if TellSmallEncodePosLeng <= TellLargeEncodePosLeng, the latter otherwise. + - ProofEncode <= nr_rows_on_board + + "num nonempty squares (found by looping over nr_rows_on_board and nr_files_on_board)" * max(ProofLargeEncodePiece, ProofSmallEncodePiece) + + 1 + + nr_ghosts * SmallEncodePiece + + CommonEncode + - LargeEncode = nr_rows_on_board + + "num nonempty squares (found by looping over nr_rows_on_board and nr_files_on_board)" * LargeEncodePiece + + nr_ghosts * LargeEncodePiece + + CommonEncode + - SmallEncode = "num nonempty squares (found by looping over nr_rows_on_board and nr_files_on_board)" * SmallEncodePiece + + nr_ghosts * SmallEncodePiece + + CommonEncode + TellLargeEncodePosLeng = 8 + + "num nonempty squares (found by looping boardnum until 0 is found)" * bytes_per_piece + + (sizeof BGL_values[White] + sizeof BGL_values[Black]) * !!CondFlag[BGL] + + nr_ghosts*bytes_per_piece + + TellCommonEncodePosLeng + TellSmallEncodePosLeng = "num nonempty squares (found by looping boardnum until 0 is found)" * (1 + bytes_per_piece) + + nr_ghosts*bytes_per_piece + + TellCommonEncodePosLeng + SmallEncode will therefore be chosen iff "num nonempty squares" <= (8 + sizeof BGL_values[White] + sizeof BGL_values[Black]) * !!CondFlag[BGL], + hence: + - If SmallEncode is chosen then "num nonempty squares" <= min((nr_rows_on_board * nr_files_on_board), (8 + sizeof BGL_values[White] + sizeof BGL_values[Black])). + - If LargeEncode is chosen then 8 < "num nonempty squares" <= (nr_rows_on_board * nr_files_on_board). + Substituting we find that + - SmallEncode <= min((nr_rows_on_board * nr_files_on_board), (8 + sizeof BGL_values[White] + sizeof BGL_values[Black])) * SmallEncodePiece + + nr_ghosts*bytes_per_piece + + CommonEncode + - LargeEncode <= nr_rows_on_board + + (nr_rows_on_board * nr_files_on_board) * LargeEncodePiece + + nr_ghosts * LargeEncodePiece + + CommonEncode + - ProofEncode <= nr_rows_on_board + + (nr_rows_on_board * nr_files_on_board) * max(ProofLargeEncodePiece, ProofSmallEncodePiece) + + 1 + + nr_ghosts * SmallEncodePiece + + CommonEncode + Meanwhile, inspection reveals that: + - ProofLargeEncodePiece = 2 + - ProofSmallEncodePiece <= 1 + - SmallEncodePiece <= 1 + 1 + bytes_per_spec + - LargeEncodePiece = 1 + bytes_per_spec + - CommonEncode <= 2 + 2 + 1 + 1 + 1 + + being_solved.number_of_imitators + + 1 + 1 + 3 + 1 + + en_passant_top[nbply] - en_passant_top[nbply-1] + + 1 + sizeof BGL_values[White] + sizeof BGL_values[Black] + + 2 + = 16 + sizeof BGL_values[White] + sizeof BGL_values[Black] + + being_solved.number_of_imitators + + en_passant_top[nbply] - en_passant_top[nbply-1] + Substituting yields: + - SmallEncode <= min((nr_rows_on_board * nr_files_on_board), (8 + sizeof BGL_values[White] + sizeof BGL_values[Black])) * (2 + bytes_per_spec) + + nr_ghosts*(2 + bytes_per_spec) + + CommonEncode + - LargeEncode <= nr_rows_on_board + + (nr_rows_on_board * nr_files_on_board) * (1 + bytes_per_spec) + + nr_ghosts * (1 + bytes_per_spec) + + CommonEncode + - ProofEncode <= nr_rows_on_board + + (nr_rows_on_board * nr_files_on_board) * 2 + + 1 + + nr_ghosts * (2 + bytes_per_spec) + + CommonEncode + We need to bound the remaining non-constants. We have + - nr_ghosts <= underworld_capacity + - bytes_per_spec <= 4 + - being_solved.number_of_imitators <= maxinum + The most annoying quantity to bound is en_passant_top[nbply] - en_passant_top[nbply-1]. We must have + - en_passant_top[nbply] - en_passant_top[nbply-1] <= maxply + 1 + but it's likely that tighter bounds are possible. + With all of the above, we can determine the maximum bytes that an encoding should take up. We'll + let the compiler perform the arithmetic. +*/ + +enum { + MAX_EN_PASSANT_TOP_DIFFERENCE = maxply + 1, // TODO: Improve this! + COMMONENCODE_MAX = 16 + + sizeof BGL_values[White] + sizeof BGL_values[Black] + + maxinum + + MAX_EN_PASSANT_TOP_DIFFERENCE, + NUM_NONEMPTY_SMALL_FIRST = (nr_rows_on_board * nr_files_on_board), + NUM_NONEMPTY_SMALL_SECOND = ((8 + sizeof BGL_values[White] + sizeof BGL_values[Black]) * (2 + 4)), + NUM_NONEMPTY_SMALL = ((NUM_NONEMPTY_SMALL_FIRST < NUM_NONEMPTY_SMALL_SECOND) ? NUM_NONEMPTY_SMALL_FIRST : NUM_NONEMPTY_SMALL_SECOND), + SMALLENCODE_MAX = (NUM_NONEMPTY_SMALL * (2 + 4)) + + (underworld_capacity * (2 + 4)) + + COMMONENCODE_MAX, + LARGEENCODE_MAX = nr_rows_on_board + + ((nr_rows_on_board * nr_files_on_board) * (1 + 4)) + + (underworld_capacity * (1 + 4)) + + COMMONENCODE_MAX, + PROOFENCODE_MAX = nr_rows_on_board + + ((nr_rows_on_board * nr_files_on_board) * 2) + + 1 + + (underworld_capacity * (2 + 4)) + + COMMONENCODE_MAX +}; + enum { - MAX_LENGTH_OF_ENCODING = (((nr_rows_on_board * nr_files_on_board) + underworld_capacity) * 6) + - 18 + maxinum + nr_rows_on_board + (1 + maxply), -/* MAX_LENGTH_OF_ENCODING = (((nr_rows_on_board * nr_files_on_board) + underworld_capacity) * 6) + - 18 + maxinum + (en_passant_top[nbply] - en_passant_top[nbply-1]), */ + + MAX_LENGTH_OF_ENCODING = ((SMALLENCODE_MAX > LARGEENCODE_MAX) ? ((SMALLENCODE_MAX > PROOFENCODE_MAX) ? SMALLENCODE_MAX : PROOFENCODE_MAX) + : ((LARGEENCODE_MAX > PROOFENCODE_MAX) ? LARGEENCODE_MAX : PROOFENCODE_MAX)), + ENSURE_MAX_LENGTH_FITS_IN_UNSIGNED_SHORT = 1/(MAX_LENGTH_OF_ENCODING <= USHRT_MAX), hashbuf_length = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) + offsetof(BCMemValue, Data) }; From df217392ff02dd115d8fa320f3a37b1409d01db6 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 5 Sep 2023 06:46:40 -0400 Subject: [PATCH 023/111] Fixing some typos, allowing NULL pointers to be duplicated. --- DHT/dht.c | 8 ++++---- DHT/dhtbcmem.c | 9 +++++++-- DHT/dhtcmem.c | 7 ++++++- DHT/dhtmem.c | 7 ++++++- DHT/dhtsimpl.c | 5 ++--- DHT/dhtstrng.c | 22 ++++++++++++++++------ optimisations/hash.h | 1 - 7 files changed, 41 insertions(+), 18 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index 3ea6a417c4..b64e6de4ca 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -804,7 +804,7 @@ LOCAL InternHsElement **LookupInternHsElement(HashTable *ht, dhtKey key) TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) - TraceFunctionParam("%jx",(unsigned long long)key.value.unsigned_integer); + TraceFunctionParam("%jx",key.value.unsigned_integer); #else TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); #endif @@ -839,7 +839,7 @@ void dhtRemoveElement(HashTable *ht, dhtKey key) TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) - TraceFunctionParam("%jx",(unsigned long long)key.value.unsigned_integer); + TraceFunctionParam("%jx",key.value.unsigned_integer); #else TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); #endif @@ -894,8 +894,8 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) - TraceFunctionParam("%jx",(unsigned long long)key.value.unsigned_integer); - TraceFunctionParam("%jx",(unsigned long long)data.unsigned_integer); + TraceFunctionParam("%jx",key.value.unsigned_integer); + TraceFunctionParam("%jx",data.unsigned_integer); #else TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); TraceFunctionParam("%lx",(unsigned long)data.unsigned_integer); diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index f1f28d4a8c..0af00335e1 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -53,7 +53,7 @@ static int EqualBCMemValue(dhtKey v1, dhtKey v2) BCMemValue const * const value1 = (BCMemValue const *)v1.value.object_pointer; BCMemValue const * const value2 = (BCMemValue const *)v2.value.object_pointer; unsigned int length; - assert(value1 && value2) + assert(value1 && value2); length = value1->Leng; if (length != value2->Leng) @@ -71,7 +71,12 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) unsigned short length; size_t size = sizeof *original; - assert(!!original); + assert(!!output); + if (!original) + { + output->object_pointer = NULL; + return 0; + } length = original->Leng; if (length > (num_bytes_in_Data / size_of_element)) diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 3a360bc1ab..d4cf4a5d64 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -63,7 +63,12 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) CompactMemVal *result; uLong length; - assert(!!v); + assert(!!output); + if (!v) + { + output->object_pointer = NilCompactMemVal; + return 0; + } length = v->Leng; if (length > (num_bytes_in_Data / size_of_element)) diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 7262594d8c..a9fb572945 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -55,13 +55,18 @@ static int EqualMemoryValue(dhtKey v1, dhtKey v2) return 0; if (!value1->Leng) // avoid the potential undefined behavior of using memcmp on NULL pointers return 1; + assert(value1->Leng <= (((size_t)-1)/sizeof value1->Data[0])); return !memcmp(value1->Data, value2->Data, value1->Leng*sizeof value1->Data[0]); } static int DupMemoryValue(dhtValue kv, dhtValue *output) { MemVal const *v= (MemVal const *)kv.object_pointer; MemVal *mv; - assert(!!v); + assert(!!output); + if (!v) { + output->object_pointer = NilMemVal; + return 0; + } mv= NewMemVal; if (mv) { mv->Leng= v->Leng; diff --git a/DHT/dhtsimpl.c b/DHT/dhtsimpl.c index 9bbc71c7ca..e9f5403fb9 100644 --- a/DHT/dhtsimpl.c +++ b/DHT/dhtsimpl.c @@ -73,7 +73,7 @@ static int EqualSimpleValue(dhtKey k1, dhtKey k2) static int DupSimpleValue(dhtValue kv, dhtValue *output) { assert(!!output); - output->unsigned_integer = kv.unsigned_integer; + *output = kv; return 0; } @@ -85,10 +85,9 @@ static void FreeSimpleValue(dhtValue kv) static void DumpSimpleValue(dhtValue kv, FILE *f) { assert(!!f); - fprintf(f, "%08lx", (unsigned long)kv.unsigned_integer); + fprintf(f, "%08jx", kv.unsigned_integer); } - dhtValueProcedures dhtSimpleProcs = { ConvertSimpleValue, diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 18bc253db7..5ce9bff250 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -19,7 +19,7 @@ static dhtHashValue ConvertString(dhtKey k) { /* I found this hash function on * http://ourworld.compuserve.com/homepages/bob_jenkins/doobs.htm - * There are other functions, but this one is has some advantages: + * There are other functions, but this one has some advantages: * - it's small * - independent from word sizes * - needs no initialisation @@ -50,12 +50,18 @@ static int EqualString(dhtKey v1, dhtKey v2) static int DupString(dhtValue v, dhtValue *output) { char *nv; - assert(!!output); + size_t len; char const *original= (char const *)v.object_pointer; - assert(!!original); - nv= (char *)fxfAlloc(strlen(original)+1); + assert(!!output); + if (!original) { + output->object_pointer = NULL; + return 0; + } + len = strlen(original); + assert(len < ((size_t)-1)); + nv= (char *)fxfAlloc(len+1); if (nv) { - strcpy(nv, original); + memcpy(nv, original, len+1); output->object_pointer = nv; return 0; } @@ -65,7 +71,11 @@ static void FreeString(dhtValue v) { char *s= (char *)v.object_pointer; if (s) - fxfFree(s, strlen(s)+1); + { + size_t const len = strlen(s); + assert(len < ((size_t)-1)); + fxfFree(s, len+1); + } } static void DumpString(dhtValue v, FILE *f) { diff --git a/optimisations/hash.h b/optimisations/hash.h index 8b5edf0189..a572c3fd8e 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -130,7 +130,6 @@ enum { enum { - MAX_LENGTH_OF_ENCODING = ((SMALLENCODE_MAX > LARGEENCODE_MAX) ? ((SMALLENCODE_MAX > PROOFENCODE_MAX) ? SMALLENCODE_MAX : PROOFENCODE_MAX) : ((LARGEENCODE_MAX > PROOFENCODE_MAX) ? LARGEENCODE_MAX : PROOFENCODE_MAX)), ENSURE_MAX_LENGTH_FITS_IN_UNSIGNED_SHORT = 1/(MAX_LENGTH_OF_ENCODING <= USHRT_MAX), From c4ef24da3b85e2c46c08e7a81b17f5b75d820005 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 5 Sep 2023 18:56:42 -0400 Subject: [PATCH 024/111] Holding back on expanding the buffers until I figure out why this is SEGFAULTing on the first EXAMPLES problem. --- DHT/dhtstrng.c | 14 ++++++++------ optimisations/hash.c | 17 ++++++++++++++--- optimisations/hash.h | 9 +++++++++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 5ce9bff250..7a20415e8a 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -58,12 +58,14 @@ static int DupString(dhtValue v, dhtValue *output) return 0; } len = strlen(original); - assert(len < ((size_t)-1)); - nv= (char *)fxfAlloc(len+1); - if (nv) { - memcpy(nv, original, len+1); - output->object_pointer = nv; - return 0; + if (len < ((size_t)-1)) { + nv= (char *)fxfAlloc(len+1); + if (nv) { + memcpy(nv, original, len); + nv[len] = '\0'; + output->object_pointer = nv; + return 0; + } } return 1; } diff --git a/optimisations/hash.c b/optimisations/hash.c index 5f132b56be..dc22dacd27 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1328,6 +1328,14 @@ static byte *CommonEncode(byte *bp, return bp; } /* CommonEncode */ +#define SIGNAL_HASHBUFFER_OVERFLOW do { \ + fprintf(stderr, "ERROR: HashBuffer overflow detected in function %s in %s\n" \ + "This is an internal limit of Popeye.\n" \ + "Please notify the developers so this limit can be raised or potentially eliminated.\n" \ + "Popeye will now abort.\n", __func__, __FILE__); \ + abort(); \ + } while (0) + static void ProofEncode(stip_length_type min_length, stip_length_type validity_value) { HashBuffer *hb = &hashBuffers[nbply]; @@ -1382,7 +1390,8 @@ static void ProofEncode(stip_length_type min_length, stip_length_type validity_v /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=MAX_LENGTH_OF_ENCODING); + if ((bp - hb->cmv.Data) > MAX_LENGTH_OF_ENCODING) + SIGNAL_HASHBUFFER_OVERFLOW; hb->cmv.Leng = (bp-hb->cmv.Data); } @@ -1563,7 +1572,8 @@ static void LargeEncode(stip_length_type min_length, /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=MAX_LENGTH_OF_ENCODING); + if ((bp - hb->cmv.Data) > MAX_LENGTH_OF_ENCODING) + SIGNAL_HASHBUFFER_OVERFLOW; hb->cmv.Leng = bp-hb->cmv.Data; TraceFunctionExit(__func__); @@ -1609,7 +1619,8 @@ static void SmallEncode(stip_length_type min_length, /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=MAX_LENGTH_OF_ENCODING); + if ((bp - hb->cmv.Data) > MAX_LENGTH_OF_ENCODING) + SIGNAL_HASHBUFFER_OVERFLOW; hb->cmv.Leng = (bp-hb->cmv.Data); TraceFunctionExit(__func__); diff --git a/optimisations/hash.h b/optimisations/hash.h index a572c3fd8e..7c1582d61c 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -105,6 +105,7 @@ typedef unsigned char byte; let the compiler perform the arithmetic. */ +#if 0 enum { MAX_EN_PASSANT_TOP_DIFFERENCE = maxply + 1, // TODO: Improve this! COMMONENCODE_MAX = 16 @@ -135,6 +136,14 @@ enum ENSURE_MAX_LENGTH_FITS_IN_UNSIGNED_SHORT = 1/(MAX_LENGTH_OF_ENCODING <= USHRT_MAX), hashbuf_length = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) + offsetof(BCMemValue, Data) }; +#else +enum +{ + hashbuf_length = 256, + MAX_LENGTH_OF_ENCODING = (hashbuf_length - offsetof(BCMemValue, Data))/sizeof(byte), + ENSURE_MAX_LENGTH_FITS_IN_UNSIGNED_SHORT = 1/(MAX_LENGTH_OF_ENCODING <= USHRT_MAX), +}; +#endif typedef union { From 69c6c375f24c958bc295040c3c18b6a72086fed8 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 6 Sep 2023 11:18:45 -0400 Subject: [PATCH 025/111] Removing long double floating_point; since fxfAlloc (apparently) can't handle such allocations. --- DHT/dhtvalue.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index fd5b7cdd4c..3161922259 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -89,7 +89,10 @@ typedef union { const volatile void * object_pointer; char character; void (*function_pointer)(void); - long double floating_point; + +/* fxfAlloc can't currently handle the alignment requirements of long double. + TODO: Update fxfAlloc so this isn't a concern. */ +// long double floating_point; } dhtValue; typedef struct { From f8ab1c3aa58de09e68b64d0b38c22e15823e4666 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 6 Sep 2023 11:44:25 -0400 Subject: [PATCH 026/111] Trying to make uses of tabs consistent. --- DHT/dht.h | 32 ++++++++++++++++---------------- DHT/dhtbcmem.h | 4 ++-- DHT/dhtcmem.h | 4 ++-- DHT/dhtmem.c | 10 +++++----- DHT/dhtmem.h | 4 ++-- DHT/dhtstrng.c | 50 +++++++++++++++++++++++++------------------------- DHT/dhtvalue.h | 24 ++++++++++++------------ 7 files changed, 64 insertions(+), 64 deletions(-) diff --git a/DHT/dht.h b/DHT/dht.h index e4c6bf4a4b..da48e683da 100644 --- a/DHT/dht.h +++ b/DHT/dht.h @@ -15,28 +15,28 @@ typedef enum { /* Now finally this is the HashElement */ typedef struct { - dhtKey Key; - dhtValue Data; + dhtKey Key; + dhtValue Data; } dhtElement; #define dhtNilElement ((dhtElement *)0) struct dht; -#define dhtNilHashTable ((struct dht *)0) +#define dhtNilHashTable ((struct dht *)0) /* procedures */ -struct dht *dhtCreate(dhtValueType KeyType, dhtValuePolicy KeyPolicy, - dhtValueType DtaType, dhtValuePolicy DataPolicy); -dhtElement *dhtEnterElement(struct dht *, dhtKey key, dhtValue data); -unsigned int dhtBucketStat (struct dht *, unsigned int *counter, unsigned int n); -void dhtDestroy (struct dht *); -void dhtDump (struct dht *, FILE *); -void dhtDumpIndented (int ind, struct dht *, FILE *); -void dhtRemoveElement (struct dht *, dhtKey key); -dhtElement *dhtLookupElement (struct dht *, dhtKey key); -dhtElement *dhtGetFirstElement(struct dht *); -dhtElement *dhtGetNextElement (struct dht *); -unsigned long dhtKeyCount (struct dht const *); -char const *dhtErrorMsg (void); +struct dht *dhtCreate(dhtValueType KeyType, dhtValuePolicy KeyPolicy, + dhtValueType DtaType, dhtValuePolicy DataPolicy); +dhtElement *dhtEnterElement(struct dht *, dhtKey key, dhtValue data); +unsigned int dhtBucketStat(struct dht *, unsigned int *counter, unsigned int n); +void dhtDestroy(struct dht *); +void dhtDump(struct dht *, FILE *); +void dhtDumpIndented(int ind, struct dht *, FILE *); +void dhtRemoveElement(struct dht *, dhtKey key); +dhtElement *dhtLookupElement(struct dht *, dhtKey key); +dhtElement *dhtGetFirstElement(struct dht *); +dhtElement *dhtGetNextElement(struct dht *); +unsigned long dhtKeyCount(struct dht const *); +char const *dhtErrorMsg(void); extern char dhtError[]; diff --git a/DHT/dhtbcmem.h b/DHT/dhtbcmem.h index 35b518b0f3..55bd635f36 100644 --- a/DHT/dhtbcmem.h +++ b/DHT/dhtbcmem.h @@ -16,8 +16,8 @@ */ typedef struct BCMemValue { - unsigned short Leng; - unsigned char Data[1]; + unsigned short Leng; + unsigned char Data[1]; } BCMemValue; #endif /*DHTBCMEM_INCLUDED*/ diff --git a/DHT/dhtcmem.h b/DHT/dhtcmem.h index 4f048f5492..85b24a1c4a 100644 --- a/DHT/dhtcmem.h +++ b/DHT/dhtcmem.h @@ -14,8 +14,8 @@ * It is allocated via one malloc call. */ typedef struct CompactMemVal { - unsigned long Leng; - unsigned char Data[1]; + unsigned long Leng; + unsigned char Data[1]; } CompactMemVal; #define NilCompactMemVal ((CompactMemVal *)0) #define NewCompactMemVal(n) ((CompactMemVal *)fxfAlloc(sizeof(CompactMemVal)+(n)*sizeof(uChar))) diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index a9fb572945..399d444095 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -34,9 +34,9 @@ static dhtHashValue HashMemoryValue(dhtKey k) dhtHashValue hash= 0; uLong i; for (i=0; i> 6; + hash+= s[i]; + hash+= hash << 10; + hash^= hash >> 6; } hash+= hash << 3; hash^= hash >> 11; @@ -88,7 +88,7 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) } return 1; } -static void FreeMemoryValue(dhtValue kv) +static void FreeMemoryValue(dhtValue kv) { MemVal *v= (MemVal *)kv.object_pointer; if (v) { @@ -96,7 +96,7 @@ static void FreeMemoryValue(dhtValue kv) fxfFree(v, sizeof *v); } } -static void DumpMemoryValue(dhtValue kv, FILE *f) { +static void DumpMemoryValue(dhtValue kv, FILE *f) { MemVal const * v= (MemVal *)kv.object_pointer; uLong i; assert(v && f); diff --git a/DHT/dhtmem.h b/DHT/dhtmem.h index d055c18843..c7640bfa77 100644 --- a/DHT/dhtmem.h +++ b/DHT/dhtmem.h @@ -16,8 +16,8 @@ * hash-table is needed. */ typedef struct MemVal { - unsigned long Leng; - unsigned char *Data; + unsigned long Leng; + unsigned char *Data; } MemVal; #define NilMemVal ((MemVal *)0) #define NewMemVal ((MemVal *)fxfAlloc(sizeof(MemVal))) diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 7a20415e8a..a376ad8012 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -17,26 +17,26 @@ static dhtHashValue ConvertString(dhtKey k) { - /* I found this hash function on - * http://ourworld.compuserve.com/homepages/bob_jenkins/doobs.htm - * There are other functions, but this one has some advantages: - * - it's small - * - independent from word sizes - * - needs no initialisation - */ - unsigned char const *s= (unsigned char const *)k.value.object_pointer; - unsigned long hash; - assert(!!s); - hash= 0; - while (*s) { - hash+= *s++; - hash+= hash << 10; - hash^= hash >> 6; - } - hash+= hash << 3; - hash^= hash >> 11; - hash+= hash << 15; - return (dhtHashValue)hash; + /* I found this hash function on + * http://ourworld.compuserve.com/homepages/bob_jenkins/doobs.htm + * There are other functions, but this one has some advantages: + * - it's small + * - independent from word sizes + * - needs no initialisation + */ + unsigned char const *s= (unsigned char const *)k.value.object_pointer; + unsigned long hash; + assert(!!s); + hash= 0; + while (*s) { + hash+= *s++; + hash+= hash << 10; + hash^= hash >> 6; + } + hash+= hash << 3; + hash^= hash >> 11; + hash+= hash << 15; + return (dhtHashValue)hash; } static int EqualString(dhtKey v1, dhtKey v2) @@ -59,12 +59,12 @@ static int DupString(dhtValue v, dhtValue *output) } len = strlen(original); if (len < ((size_t)-1)) { - nv= (char *)fxfAlloc(len+1); - if (nv) { - memcpy(nv, original, len); + nv= (char *)fxfAlloc(len+1); + if (nv) { + memcpy(nv, original, len); nv[len] = '\0'; - output->object_pointer = nv; - return 0; + output->object_pointer = nv; + return 0; } } return 1; diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 3161922259..9d8232a5d3 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -66,33 +66,33 @@ extern char const *dhtValueTypeToString[dhtValueTypeCnt]; typedef union { #ifdef __cplusplus # if __cplusplus >= 201103L - ::std::uintmax_t unsigned_integer; - ::std::intmax_t signed_integer; + ::std::uintmax_t unsigned_integer; + ::std::intmax_t signed_integer; # else unsigned long int unsigned_integer; - long int signed_integer; + long int signed_integer; # endif bool boolean; ::std::sig_atomic_t atomic_integer; #else # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) - uintmax_t unsigned_integer; - intmax_t signed_integer; - _Bool boolean; + uintmax_t unsigned_integer; + intmax_t signed_integer; + _Bool boolean; # else unsigned long int unsigned_integer; - long int signed_integer; - int boolean; // What else? + long int signed_integer; + int boolean; // What else? # endif sig_atomic_t atomic_integer; #endif const volatile void * object_pointer; - char character; - void (*function_pointer)(void); + char character; + void (*function_pointer)(void); /* fxfAlloc can't currently handle the alignment requirements of long double. TODO: Update fxfAlloc so this isn't a concern. */ -// long double floating_point; +// long double floating_point; } dhtValue; typedef struct { @@ -102,7 +102,7 @@ typedef struct { typedef unsigned long dhtHashValue; typedef struct { - dhtHashValue (*Hash)(dhtKey); + dhtHashValue (*Hash)(dhtKey); int (*Equal)(dhtKey, dhtKey); int (*Dup)(dhtValue, dhtValue *); // should return 0 on success (and store the copied value at the second argument) and nonzero on error void (*Free)(dhtValue); From 14daa9ee3318d351c6f8e0bddf5fa08fa7408527 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 6 Sep 2023 14:00:27 -0400 Subject: [PATCH 027/111] Bumping the required alignment; this allows us to store long doubles (or anything else that may come along, presumably). --- DHT/dhtvalue.h | 5 +---- DHT/fxf.c | 41 ++++++++++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 9d8232a5d3..bef88094ef 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -89,10 +89,7 @@ typedef union { const volatile void * object_pointer; char character; void (*function_pointer)(void); - -/* fxfAlloc can't currently handle the alignment requirements of long double. - TODO: Update fxfAlloc so this isn't a concern. */ -// long double floating_point; + long double floating_point; } dhtValue; typedef struct { diff --git a/DHT/fxf.c b/DHT/fxf.c index b8edfd95e7..23b64f4681 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -35,6 +35,33 @@ # define SIZE_T_PRINTF_SPECIFIER "lu" #endif +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) +# define MAX_ALIGNMENT _Alignof(max_align_t) +#elif (defined(__cplusplus) && (__cplusplus >= 201103L)) +# define MAX_ALIGNMENT alignof(max_align_t) +#else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# include +# endif +union MAX_ALIGNED_TYPE { +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + uintmax_t unsigned_integer; +# elif defined(LLONG_MAX) /* We have long long integer types. */ + unsigned long long int unsigned_integer; +# else + unsigned long int unsigned_integer; +# endif + const volatile void * object_pointer; + void (*function_pointer)(void); + long double floating_point; +}; +# define MAX_ALIGNMENT sizeof(union MAX_ALIGNED_TYPE) +#endif + +enum { + ENSURE_MAX_ALIGNMENT_IS_POWER_OF_TWO = 1/(!(MAX_ALIGNMENT & (MAX_ALIGNMENT - 1U))) +}; + #if !defined(Nil) && !defined(New) && !defined(nNew) /* TODO: Is this the correct check for all of the below lines? */ # define Nil(type) ((type *)0) # define New(type) ((type *)malloc(sizeof(type))) @@ -68,7 +95,7 @@ static inline void * nNewImpl(size_t const nmemb, size_t const size) { /* FiXed and Fast malloc, free * As the name tells: this code implements on top of traditional * malloc/realloc/free a fast version, that relies on a lot of - * allocation/delallocation of fixed sized blocks of memory. For + * allocation/deallocation of fixed sized blocks of memory. For * each size of memory we keep a head pointer and all freed chunks * of memory is threaded on this list. If memory of this size * is requested, we drag it from the list, otherwise we carve it @@ -80,7 +107,7 @@ static inline void * nNewImpl(size_t const nmemb, size_t const size) { typedef struct { unsigned long MallocCount; unsigned long FreeCount; - char * FreeHead; + void * FreeHead; } SizeHead; #if defined(DOS) @@ -360,8 +387,8 @@ void fxfReset(void) * SPARC, HPPA, MIPS. We wouldn't need this when running on an * Intel *86 type of CPU, but also there, aligned access is faster. */ -#define PTRMASK (sizeof(char *)-1) -#define ALIGNED_MINSIZE (sizeof(char *)+PTRMASK) +#define PTRMASK (MAX_ALIGNMENT-1) +#define ALIGNED_MINSIZE (MAX_ALIGNMENT+PTRMASK) #define ALIGN(ptr) (((size_t)ptr+PTRMASK) & (~PTRMASK)) #define GetNextPtr(ptr) (*(char **)ALIGN(ptr)) @@ -395,7 +422,7 @@ void *fxfAlloc(size_t size) { sh= &SizeData[size]; if (sh->FreeHead) { - ptr= sh->FreeHead; + ptr= (char *)sh->FreeHead; sh->FreeHead= GetNextPtr(ptr); sh->FreeCount--; sh->MallocCount++; @@ -474,7 +501,7 @@ void fxfFree(void *ptr, size_t size) } else { SetRange((char *)ptr-Arena,size); - *(char **)ALIGN(ptr)= sh->FreeHead; + *(char **)ALIGN(ptr)= (char *)sh->FreeHead; sh->FreeHead= ptr; ++sh->FreeCount; --sh->MallocCount; @@ -491,7 +518,7 @@ void fxfFree(void *ptr, size_t size) } else { SetRange((char *)ptr-Arena,size); - *(char **)ptr= sh->FreeHead; + *(char **)ptr= (char *)sh->FreeHead; sh->FreeHead= ptr; ++sh->FreeCount; --sh->MallocCount; From 300ef7fc9e85a821923208dc8ba0e681997f99fd Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 6 Sep 2023 14:57:07 -0400 Subject: [PATCH 028/111] If the board is filled then we need 64 piece IDs. --- position/pieceid.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/position/pieceid.h b/position/pieceid.h index e402f0ddd5..f90153555a 100644 --- a/position/pieceid.h +++ b/position/pieceid.h @@ -8,7 +8,7 @@ enum { NullPieceId = 0, MinPieceId = 1, - MaxPieceId = 63 + MaxPieceId = 64 }; typedef unsigned long PieceIdType; From 19c22848bfe0aef9ada07b23330df2024007b5d4 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 6 Sep 2023 17:24:36 -0400 Subject: [PATCH 029/111] Going back to assertions. --- optimisations/hash.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index dc22dacd27..5dd94d0eb1 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1328,14 +1328,6 @@ static byte *CommonEncode(byte *bp, return bp; } /* CommonEncode */ -#define SIGNAL_HASHBUFFER_OVERFLOW do { \ - fprintf(stderr, "ERROR: HashBuffer overflow detected in function %s in %s\n" \ - "This is an internal limit of Popeye.\n" \ - "Please notify the developers so this limit can be raised or potentially eliminated.\n" \ - "Popeye will now abort.\n", __func__, __FILE__); \ - abort(); \ - } while (0) - static void ProofEncode(stip_length_type min_length, stip_length_type validity_value) { HashBuffer *hb = &hashBuffers[nbply]; @@ -1390,9 +1382,8 @@ static void ProofEncode(stip_length_type min_length, stip_length_type validity_v /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - if ((bp - hb->cmv.Data) > MAX_LENGTH_OF_ENCODING) - SIGNAL_HASHBUFFER_OVERFLOW; - hb->cmv.Leng = (bp-hb->cmv.Data); + assert((bp - hb->cmv.Data) <= MAX_LENGTH_OF_ENCODING); + hb->cmv.Leng = (bp - hb->cmv.Data); } static unsigned int TellCommonEncodePosLeng(unsigned int len, @@ -1572,9 +1563,8 @@ static void LargeEncode(stip_length_type min_length, /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - if ((bp - hb->cmv.Data) > MAX_LENGTH_OF_ENCODING) - SIGNAL_HASHBUFFER_OVERFLOW; - hb->cmv.Leng = bp-hb->cmv.Data; + assert((bp - hb->cmv.Data) <= MAX_LENGTH_OF_ENCODING); + hb->cmv.Leng = (bp - hb->cmv.Data); TraceFunctionExit(__func__); TraceFunctionResultEnd(); @@ -1619,9 +1609,8 @@ static void SmallEncode(stip_length_type min_length, /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - if ((bp - hb->cmv.Data) > MAX_LENGTH_OF_ENCODING) - SIGNAL_HASHBUFFER_OVERFLOW; - hb->cmv.Leng = (bp-hb->cmv.Data); + assert((bp - hb->cmv.Data) <= MAX_LENGTH_OF_ENCODING); + hb->cmv.Leng = (bp - hb->cmv.Data); TraceFunctionExit(__func__); TraceFunctionResultEnd(); From cedb9af237e921073f08883f8a86394fa3b40ffe Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 6 Sep 2023 18:13:29 -0400 Subject: [PATCH 030/111] Some reformatting. --- optimisations/hash.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/optimisations/hash.h b/optimisations/hash.h index 7c1582d61c..95dc21369e 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -33,15 +33,15 @@ typedef unsigned char byte; - ProofEncode <= nr_rows_on_board + "num nonempty squares (found by looping over nr_rows_on_board and nr_files_on_board)" * max(ProofLargeEncodePiece, ProofSmallEncodePiece) + 1 - + nr_ghosts * SmallEncodePiece - + CommonEncode + + nr_ghosts * SmallEncodePiece + + CommonEncode - LargeEncode = nr_rows_on_board + "num nonempty squares (found by looping over nr_rows_on_board and nr_files_on_board)" * LargeEncodePiece + nr_ghosts * LargeEncodePiece - + CommonEncode + + CommonEncode - SmallEncode = "num nonempty squares (found by looping over nr_rows_on_board and nr_files_on_board)" * SmallEncodePiece + nr_ghosts * SmallEncodePiece - + CommonEncode + + CommonEncode TellLargeEncodePosLeng = 8 + "num nonempty squares (found by looping boardnum until 0 is found)" * bytes_per_piece + (sizeof BGL_values[White] + sizeof BGL_values[Black]) * !!CondFlag[BGL] @@ -61,12 +61,12 @@ typedef unsigned char byte; - LargeEncode <= nr_rows_on_board + (nr_rows_on_board * nr_files_on_board) * LargeEncodePiece + nr_ghosts * LargeEncodePiece - + CommonEncode + + CommonEncode - ProofEncode <= nr_rows_on_board + (nr_rows_on_board * nr_files_on_board) * max(ProofLargeEncodePiece, ProofSmallEncodePiece) + 1 - + nr_ghosts * SmallEncodePiece - + CommonEncode + + nr_ghosts * SmallEncodePiece + + CommonEncode Meanwhile, inspection reveals that: - ProofLargeEncodePiece = 2 - ProofSmallEncodePiece <= 1 @@ -88,12 +88,12 @@ typedef unsigned char byte; - LargeEncode <= nr_rows_on_board + (nr_rows_on_board * nr_files_on_board) * (1 + bytes_per_spec) + nr_ghosts * (1 + bytes_per_spec) - + CommonEncode + + CommonEncode - ProofEncode <= nr_rows_on_board + (nr_rows_on_board * nr_files_on_board) * 2 + 1 - + nr_ghosts * (2 + bytes_per_spec) - + CommonEncode + + nr_ghosts * (2 + bytes_per_spec) + + CommonEncode We need to bound the remaining non-constants. We have - nr_ghosts <= underworld_capacity - bytes_per_spec <= 4 @@ -105,7 +105,6 @@ typedef unsigned char byte; let the compiler perform the arithmetic. */ -#if 0 enum { MAX_EN_PASSANT_TOP_DIFFERENCE = maxply + 1, // TODO: Improve this! COMMONENCODE_MAX = 16 @@ -136,7 +135,10 @@ enum ENSURE_MAX_LENGTH_FITS_IN_UNSIGNED_SHORT = 1/(MAX_LENGTH_OF_ENCODING <= USHRT_MAX), hashbuf_length = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) + offsetof(BCMemValue, Data) }; -#else + +#if 0 +/* Previously hashbuf_length was the below constant. + MAX_LENGTH_OF_ENCODING wasn't computed or used but should have been. */ enum { hashbuf_length = 256, From 8858f797b450ce07f6191a0991c47a8932a9e1a6 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 6 Sep 2023 20:45:04 -0400 Subject: [PATCH 031/111] Ensuring that fxfAlloc returns properly-aligned pointers. --- DHT/fxf.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 23b64f4681..13871d2cd3 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -389,7 +389,7 @@ void fxfReset(void) */ #define PTRMASK (MAX_ALIGNMENT-1) #define ALIGNED_MINSIZE (MAX_ALIGNMENT+PTRMASK) -#define ALIGN(ptr) (((size_t)ptr+PTRMASK) & (~PTRMASK)) +#define ALIGN(ptr) ((((size_t)(ptr))+PTRMASK) & (~PTRMASK)) #define GetNextPtr(ptr) (*(char **)ALIGN(ptr)) #define PutNextPtr(dst, ptr) *(char **)ALIGN(dst)= ptr @@ -433,13 +433,17 @@ void *fxfAlloc(size_t size) { } else { /* we have to allocate a new piece */ - size_t const sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); + size_t sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); TMDBG(printf(" sizeCurrentSeg:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)sizeCurrentSeg)); if (sizeCurrentSeg>=size) { if (size&PTRMASK) { /* not aligned */ - ptr= BotFreePtr; - BotFreePtr+= size; + char * const BotFreePtrAligned= (char *)ALIGN(BotFreePtr); + sizeCurrentSeg= (size_t)(TopFreePtr-BotFreePtrAligned); + if (sizeCurrentSeg Date: Wed, 6 Sep 2023 20:46:48 -0400 Subject: [PATCH 032/111] Might as well parenthesize the argument. --- DHT/fxf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 13871d2cd3..5d37cf9980 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -392,7 +392,7 @@ void fxfReset(void) #define ALIGN(ptr) ((((size_t)(ptr))+PTRMASK) & (~PTRMASK)) #define GetNextPtr(ptr) (*(char **)ALIGN(ptr)) -#define PutNextPtr(dst, ptr) *(char **)ALIGN(dst)= ptr +#define PutNextPtr(dst, ptr) *(char **)ALIGN(dst)= (ptr) #define TMDBG(x) if (0) x From d63e53d45cfa7fbd4235e58fa87a1b7e25f289dc Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 6 Sep 2023 21:35:09 -0400 Subject: [PATCH 033/111] Trying to make freeing match the alignment that we imposed in fxfAlloc. It would be better to put unaligned allocations at the upper end, but that's for a later change. --- DHT/fxf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 5d37cf9980..d8826c5edb 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -499,8 +499,8 @@ void fxfFree(void *ptr, size_t size) if (size&PTRMASK) { /* unaligned size */ TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(BotFreePtr-(char const*)ptr))); - if ((char *)ptr+size == BotFreePtr) { - BotFreePtr-= size; + if ((char *)ALIGN((char *)ptr+size) == BotFreePtr) { + BotFreePtr= (char *)ptr; TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(TopFreePtr-BotFreePtr))); --sh->MallocCount; } From bd82898a1c6cb631a367e0d7e389a2fc4b1a994b Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 8 Sep 2023 07:15:40 -0400 Subject: [PATCH 034/111] Going with the simplest methods for ensuring that returned pointers are fully aligned, with just a single associated optimization (changing how SizeData indices are handled). Further optimizations are certainly possible. --- DHT/fxf.c | 51 ++++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index d8826c5edb..a308a42972 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -113,7 +113,7 @@ typedef struct { #if defined(DOS) /* MSDOS 16 Bit support (maxmemory <= 1 MB) */ #define SEGMENTED -#define ARENA_SEG_SIZE 32000 +#define ARENA_SEG_SIZE (32000 & ~MAX_ALIGNMENT) #define ARENA_SEG_COUNT ((1024*1024)/ARENA_SEG_SIZE) #define OSNAME "MSDOS" #define OSMAXMEM "1 MB" @@ -121,7 +121,7 @@ typedef struct { /* Win95/Win98/WinME can only allocate chunks up to 255 MB */ /* maxmemory <= 768 MB */ #define SEGMENTED -#define ARENA_SEG_SIZE 1000000 +#define ARENA_SEG_SIZE (1000000 & ~MAX_ALIGNMENT) #define ARENA_SEG_COUNT ((768*1024*1024)/ARENA_SEG_SIZE) #define OSNAME "Win95/Win98/WinME" #define OSMAXMEM "768 MB" @@ -129,9 +129,9 @@ typedef struct { /* The maximum size an fxfAlloc can handle */ #if defined(SEGMENTED) || defined(__TURBOC__) -#define fxfMAXSIZE ((size_t)1024) +#define fxfMAXSIZE (((size_t)1024) & ~MAX_ALIGNMENT) #else -#define fxfMAXSIZE ((size_t)2048) /* this is needed only when sizeof(void*)==8 */ +#define fxfMAXSIZE (((size_t)2048) & ~MAX_ALIGNMENT) /* this is needed only when sizeof(void*)==8 */ #endif /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ @@ -140,7 +140,10 @@ enum fxfMINSIZE = sizeof(size_t) }; -static SizeHead SizeData[fxfMAXSIZE+1]; +/* Below we'll ensure that allocation sizes are all multiples of MAX_ALIGNMENT in the range + [MAX_ALIGNMENT, fxfMAXSIZE]. + If we divide by MAX_ALIGNMENT and subtract 1, we get the range 0, 1, 2, ..., fxfMAXSIZE/MAX_ALIGNMENT - 1. */ +static SizeHead SizeData[fxfMAXSIZE/MAX_ALIGNMENT]; #if defined(SEGMENTED) /* #define ARENA_SEG_SIZE 32000 */ @@ -281,6 +284,10 @@ size_t fxfInit(size_t Size) { #endif if (Arena) free(Arena); + if (Size < MAX_ALIGNMENT) + Size= MAX_ALIGNMENT; + else + Size&= ~MAX_ALIGNMENT; if ((Arena=nNew(Size, char)) == Nil(char)) { ERROR_LOG2("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " bytes\n", myname, (size_t_printf_type) Size); @@ -372,8 +379,8 @@ void fxfReset(void) #if !defined(NDEBUG) { - unsigned int i; - for (i = 1; i<=50; ++i) + size_t i; + for (i = 0; i<(sizeof(SizeData)/sizeof(*SizeData)); ++i) assert(SizeData[i].MallocCount==0); } #endif @@ -389,7 +396,8 @@ void fxfReset(void) */ #define PTRMASK (MAX_ALIGNMENT-1) #define ALIGNED_MINSIZE (MAX_ALIGNMENT+PTRMASK) -#define ALIGN(ptr) ((((size_t)(ptr))+PTRMASK) & (~PTRMASK)) +#define ALIGN_SIZE_T(s) (((s)+PTRMASK) & (~PTRMASK)) +#define ALIGN(ptr) ((void *)ALIGN_SIZE_T((size_t)ptr)) #define GetNextPtr(ptr) (*(char **)ALIGN(ptr)) #define PutNextPtr(dst, ptr) *(char **)ALIGN(dst)= (ptr) @@ -407,7 +415,7 @@ void *fxfAlloc(size_t size) { DBG((stderr, "%s(%" SIZE_T_PRINTF_SPECIFIER ") =", myname, (size_t_printf_type)size)); if (sizefxfMAXSIZE) { @@ -417,10 +425,9 @@ void *fxfAlloc(size_t size) { (size_t_printf_type) fxfMAXSIZE); return Nil(char); } - if ( (size&PTRMASK) && sizeFreeHead) { ptr= (char *)sh->FreeHead; sh->FreeHead= GetNextPtr(ptr); @@ -433,17 +440,13 @@ void *fxfAlloc(size_t size) { } else { /* we have to allocate a new piece */ - size_t sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); + size_t const sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); TMDBG(printf(" sizeCurrentSeg:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)sizeCurrentSeg)); if (sizeCurrentSeg>=size) { if (size&PTRMASK) { /* not aligned */ - char * const BotFreePtrAligned= (char *)ALIGN(BotFreePtr); - sizeCurrentSeg= (size_t)(TopFreePtr-BotFreePtrAligned); - if (sizeCurrentSeg fxfMAXSIZE) { fprintf(stderr, "%s: size=%" SIZE_T_PRINTF_SPECIFIER " >= %" SIZE_T_PRINTF_SPECIFIER "\n", myname, (size_t_printf_type) size, (size_t_printf_type) fxfMAXSIZE); exit(-5); } - if (size < fxfMINSIZE) - size= fxfMINSIZE; - if ((size&PTRMASK) && size Date: Fri, 8 Sep 2023 07:31:55 -0400 Subject: [PATCH 035/111] Need to align before checking agains the max size (in general). --- DHT/fxf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index a308a42972..03d77b6140 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -417,6 +417,7 @@ void *fxfAlloc(size_t size) { if (sizefxfMAXSIZE) { ERROR_LOG3("%s: size=%" SIZE_T_PRINTF_SPECIFIER " > %" SIZE_T_PRINTF_SPECIFIER "\n", @@ -425,7 +426,6 @@ void *fxfAlloc(size_t size) { (size_t_printf_type) fxfMAXSIZE); return Nil(char); } - size= ALIGN_SIZE_T(size); sh= &SizeData[(size/MAX_ALIGNMENT) - 1]; if (sh->FreeHead) { @@ -490,12 +490,12 @@ void fxfFree(void *ptr, size_t size) DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *) ptr, (size_t_printf_type) size)); if (size < fxfMINSIZE) size= fxfMINSIZE; + size= ALIGN_SIZE_T(size); if (size > fxfMAXSIZE) { fprintf(stderr, "%s: size=%" SIZE_T_PRINTF_SPECIFIER " >= %" SIZE_T_PRINTF_SPECIFIER "\n", myname, (size_t_printf_type) size, (size_t_printf_type) fxfMAXSIZE); exit(-5); } - size= ALIGN_SIZE_T(size); sh= &SizeData[(size/MAX_ALIGNMENT)-1]; if (size&PTRMASK) { /* unaligned size */ From c5347f40c459d17e06ad5f192c152b1eca904502 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 8 Sep 2023 08:03:11 -0400 Subject: [PATCH 036/111] Fixing a few more things. --- DHT/fxf.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 03d77b6140..47254f44b2 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -380,7 +380,7 @@ void fxfReset(void) #if !defined(NDEBUG) { size_t i; - for (i = 0; i<(sizeof(SizeData)/sizeof(*SizeData)); ++i) + for (i = 0; i<((sizeof SizeData)/(sizeof *SizeData)); ++i) assert(SizeData[i].MallocCount==0); } #endif @@ -417,7 +417,6 @@ void *fxfAlloc(size_t size) { if (sizefxfMAXSIZE) { ERROR_LOG3("%s: size=%" SIZE_T_PRINTF_SPECIFIER " > %" SIZE_T_PRINTF_SPECIFIER "\n", @@ -427,6 +426,8 @@ void *fxfAlloc(size_t size) { return Nil(char); } + size= ALIGN_SIZE_T(size); + sh= &SizeData[(size/MAX_ALIGNMENT) - 1]; if (sh->FreeHead) { ptr= (char *)sh->FreeHead; @@ -490,12 +491,12 @@ void fxfFree(void *ptr, size_t size) DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *) ptr, (size_t_printf_type) size)); if (size < fxfMINSIZE) size= fxfMINSIZE; - size= ALIGN_SIZE_T(size); if (size > fxfMAXSIZE) { fprintf(stderr, "%s: size=%" SIZE_T_PRINTF_SPECIFIER " >= %" SIZE_T_PRINTF_SPECIFIER "\n", myname, (size_t_printf_type) size, (size_t_printf_type) fxfMAXSIZE); exit(-5); } + size= ALIGN_SIZE_T(size); sh= &SizeData[(size/MAX_ALIGNMENT)-1]; if (size&PTRMASK) { /* unaligned size */ @@ -555,11 +556,11 @@ size_t fxfTotal(void) { size_t UsedBytes = 0; size_t FreeBytes = 0; - unsigned int i; - for (i=0; i<=fxfMAXSIZE; i++,hd++) { + size_t i; + for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - UsedBytes+= hd->MallocCount*i; - FreeBytes+= hd->FreeCount*i; + UsedBytes+= hd->MallocCount*(i+1)*MAX_ALIGNMENT; + FreeBytes+= hd->FreeCount*(i+1)*MAX_ALIGNMENT; } } @@ -590,15 +591,15 @@ void fxfInfo(FILE *f) { size_t UsedBytes = 0; size_t FreeBytes = 0; - unsigned int i; + size_t i; fprintf(f, "%12s %10s%10s\n", "Size", "MallocCnt", "FreeCnt"); - for (i=0; i<=fxfMAXSIZE; i++,hd++) { + for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - fprintf(f, "%12u %10lu%10lu\n", i, hd->MallocCount, hd->FreeCount); + fprintf(f, "%12zu %10lu%10lu\n", (i+1)*MAX_ALIGNMENT, hd->MallocCount, hd->FreeCount); nrUsed+= hd->MallocCount; - UsedBytes+= hd->MallocCount*i; + UsedBytes+= hd->MallocCount*(i+1)*MAX_ALIGNMENT; nrFree+= hd->FreeCount; - FreeBytes+= hd->FreeCount*i; + FreeBytes+= hd->FreeCount*(i+1)*MAX_ALIGNMENT; } } fprintf(f, "%12s %10lu%10lu\n", "Total:", nrUsed, nrFree); From 1c43b0a87ef5cae00c2fb7f5273082a628c0677d Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 8 Sep 2023 19:35:41 -0400 Subject: [PATCH 037/111] Catching more errors, and fixing some rather silly masking bugs. --- DHT/dhtstrng.c | 16 ++++++++-------- DHT/fxf.c | 22 +++++++++++----------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index a376ad8012..401ba6f24c 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -54,16 +54,16 @@ static int DupString(dhtValue v, dhtValue *output) char const *original= (char const *)v.object_pointer; assert(!!output); if (!original) { - output->object_pointer = NULL; + output->object_pointer= NULL; return 0; } - len = strlen(original); - if (len < ((size_t)-1)) { - nv= (char *)fxfAlloc(len+1); + len= strlen(original); + if ((len < ((size_t)-1)) && !original[len]) { + ++len; + nv= (char *)fxfAlloc(len); if (nv) { memcpy(nv, original, len); - nv[len] = '\0'; - output->object_pointer = nv; + output->object_pointer= nv; return 0; } } @@ -74,7 +74,7 @@ static void FreeString(dhtValue v) char *s= (char *)v.object_pointer; if (s) { - size_t const len = strlen(s); + size_t const len= strlen(s); assert(len < ((size_t)-1)); fxfFree(s, len+1); } @@ -86,7 +86,7 @@ static void DumpString(dhtValue v, FILE *f) fputs(s,f); } -dhtValueProcedures dhtStringProcs = { +dhtValueProcedures dhtStringProcs= { ConvertString, EqualString, DupString, diff --git a/DHT/fxf.c b/DHT/fxf.c index 47254f44b2..d7c63ebc1a 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -113,7 +113,7 @@ typedef struct { #if defined(DOS) /* MSDOS 16 Bit support (maxmemory <= 1 MB) */ #define SEGMENTED -#define ARENA_SEG_SIZE (32000 & ~MAX_ALIGNMENT) +#define ARENA_SEG_SIZE (32000 & ~(MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((1024*1024)/ARENA_SEG_SIZE) #define OSNAME "MSDOS" #define OSMAXMEM "1 MB" @@ -121,7 +121,7 @@ typedef struct { /* Win95/Win98/WinME can only allocate chunks up to 255 MB */ /* maxmemory <= 768 MB */ #define SEGMENTED -#define ARENA_SEG_SIZE (1000000 & ~MAX_ALIGNMENT) +#define ARENA_SEG_SIZE (1000000 & ~(MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((768*1024*1024)/ARENA_SEG_SIZE) #define OSNAME "Win95/Win98/WinME" #define OSMAXMEM "768 MB" @@ -129,9 +129,9 @@ typedef struct { /* The maximum size an fxfAlloc can handle */ #if defined(SEGMENTED) || defined(__TURBOC__) -#define fxfMAXSIZE (((size_t)1024) & ~MAX_ALIGNMENT) +#define fxfMAXSIZE (((size_t)1024) & ~(MAX_ALIGNMENT - 1U)) #else -#define fxfMAXSIZE (((size_t)2048) & ~MAX_ALIGNMENT) /* this is needed only when sizeof(void*)==8 */ +#define fxfMAXSIZE (((size_t)2048) & ~(MAX_ALIGNMENT - 1U)) /* this is needed only when sizeof(void*)==8 */ #endif /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ @@ -287,7 +287,7 @@ size_t fxfInit(size_t Size) { if (Size < MAX_ALIGNMENT) Size= MAX_ALIGNMENT; else - Size&= ~MAX_ALIGNMENT; + Size&= ~(MAX_ALIGNMENT - 1U); if ((Arena=nNew(Size, char)) == Nil(char)) { ERROR_LOG2("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " bytes\n", myname, (size_t_printf_type) Size); @@ -394,7 +394,7 @@ void fxfReset(void) * SPARC, HPPA, MIPS. We wouldn't need this when running on an * Intel *86 type of CPU, but also there, aligned access is faster. */ -#define PTRMASK (MAX_ALIGNMENT-1) +#define PTRMASK (MAX_ALIGNMENT-1U) #define ALIGNED_MINSIZE (MAX_ALIGNMENT+PTRMASK) #define ALIGN_SIZE_T(s) (((s)+PTRMASK) & (~PTRMASK)) #define ALIGN(ptr) ((void *)ALIGN_SIZE_T((size_t)ptr)) @@ -559,8 +559,8 @@ size_t fxfTotal(void) { size_t i; for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - UsedBytes+= hd->MallocCount*(i+1)*MAX_ALIGNMENT; - FreeBytes+= hd->FreeCount*(i+1)*MAX_ALIGNMENT; + UsedBytes+= hd->MallocCount*(i+1U)*MAX_ALIGNMENT; + FreeBytes+= hd->FreeCount*(i+1U)*MAX_ALIGNMENT; } } @@ -595,11 +595,11 @@ void fxfInfo(FILE *f) { fprintf(f, "%12s %10s%10s\n", "Size", "MallocCnt", "FreeCnt"); for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - fprintf(f, "%12zu %10lu%10lu\n", (i+1)*MAX_ALIGNMENT, hd->MallocCount, hd->FreeCount); + fprintf(f, "%12zu %10lu%10lu\n", (i+1U)*MAX_ALIGNMENT, hd->MallocCount, hd->FreeCount); nrUsed+= hd->MallocCount; - UsedBytes+= hd->MallocCount*(i+1)*MAX_ALIGNMENT; + UsedBytes+= hd->MallocCount*(i+1U)*MAX_ALIGNMENT; nrFree+= hd->FreeCount; - FreeBytes+= hd->FreeCount*(i+1)*MAX_ALIGNMENT; + FreeBytes+= hd->FreeCount*(i+1U)*MAX_ALIGNMENT; } } fprintf(f, "%12s %10lu%10lu\n", "Total:", nrUsed, nrFree); From 06d16d97623abc3ab75568ae16ac0c15b0d505df Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 8 Sep 2023 20:36:58 -0400 Subject: [PATCH 038/111] Choosing the appropriate format specifier. --- DHT/dhtsimpl.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DHT/dhtsimpl.c b/DHT/dhtsimpl.c index e9f5403fb9..253fee9264 100644 --- a/DHT/dhtsimpl.c +++ b/DHT/dhtsimpl.c @@ -85,7 +85,12 @@ static void FreeSimpleValue(dhtValue kv) static void DumpSimpleValue(dhtValue kv, FILE *f) { assert(!!f); +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) fprintf(f, "%08jx", kv.unsigned_integer); +#else + fprintf(f, "%08lx", kv.unsigned_integer); +#endif } dhtValueProcedures dhtSimpleProcs = From 00039b445170668e9ea78d5237655708301b3b7e Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 9 Sep 2023 13:21:43 -0400 Subject: [PATCH 039/111] An additional sanity check, and no reason to include floating point if we don't need it (and it increases the overall structure size). --- DHT/dhtstrng.c | 2 +- DHT/dhtvalue.h | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 401ba6f24c..8921abc525 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -75,7 +75,7 @@ static void FreeString(dhtValue v) if (s) { size_t const len= strlen(s); - assert(len < ((size_t)-1)); + assert((len < ((size_t)-1)) && !s[len]); fxfFree(s, len+1); } } diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index bef88094ef..6a7104bf4f 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -89,7 +89,9 @@ typedef union { const volatile void * object_pointer; char character; void (*function_pointer)(void); - long double floating_point; +#ifdef DHTVALUE_NEEDS_FLOATING_POINT + long double floating_point; +#endif } dhtValue; typedef struct { From 743f649f139bf1124ab1f4cbf3753a8626995a39 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 9 Sep 2023 17:23:38 -0400 Subject: [PATCH 040/111] Might as well go with the most generic code, even if it's redundant here. The compiler should optimize it away anyway. --- DHT/dhtmem.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 399d444095..57be87117a 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -71,14 +71,16 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) if (mv) { mv->Leng= v->Leng; if (mv->Leng) { - mv->Data= (uChar *)fxfAlloc(mv->Leng); - if (mv->Data) { - memcpy(mv->Data, v->Data, mv->Leng*sizeof mv->Data[0]); - output->object_pointer = mv; - return 0; - } else { - FreeMemVal(mv); - return 1; + if (mv->Leng <= (((size_t)-1)/(sizeof mv->Data[0]))) { + mv->Data= (uChar *)fxfAlloc(mv->Leng*sizeof mv->Data[0]); + if (mv->Data) { + memcpy(mv->Data, v->Data, mv->Leng*sizeof mv->Data[0]); + output->object_pointer = mv; + return 0; + } else { + FreeMemVal(mv); + return 1; + } } } else { mv->Data= NULL; // NULL is a valid pointer if Leng == 0 @@ -92,6 +94,7 @@ static void FreeMemoryValue(dhtValue kv) { MemVal *v= (MemVal *)kv.object_pointer; if (v) { + assert(v->Leng <= (((size_t)-1)/(sizeof v->Data[0]))); fxfFree(v->Data, v->Leng*sizeof v->Data[0]); fxfFree(v, sizeof *v); } From 12a354c107349fdf6aea73522d00b3f55156f03e Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 11 Sep 2023 21:19:00 -0400 Subject: [PATCH 041/111] Simplifying things a bit, IMO. --- DHT/dhtbcmem.c | 11 +++++++---- DHT/dhtcmem.c | 12 +++++++++--- DHT/dhtmem.c | 17 +++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index 0af00335e1..6252a90b4d 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -53,13 +53,16 @@ static int EqualBCMemValue(dhtKey v1, dhtKey v2) BCMemValue const * const value1 = (BCMemValue const *)v1.value.object_pointer; BCMemValue const * const value2 = (BCMemValue const *)v2.value.object_pointer; unsigned int length; + unsigned char const *data1; + unsigned char const *data2; assert(value1 && value2); - length = value1->Leng; - if (length != value2->Leng) - return 0; + data1 = value1->Data; + assert(data1 || !length); + data2 = value2->Data; + assert(data2 || !value2->Leng); - return !memcmp(value1->Data, value2->Data, length*sizeof value1->Data[0]); + return ((length == value2->Leng) && !(length && memcmp(data1, data2, length))); } static int DupBCMemValue(dhtValue kv, dhtValue *output) diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index d4cf4a5d64..675efe720b 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -48,10 +48,16 @@ static int EqualCompactMemoryValue(dhtKey v1, dhtKey v2) { CompactMemVal const * const value1 = (CompactMemVal const *)v1.value.object_pointer; CompactMemVal const * const value2 = (CompactMemVal const *)v2.value.object_pointer; + unsigned long length; + unsigned char const *data1; + unsigned char const *data2; assert(value1 && value2); - if (value1->Leng != value2->Leng) - return 0; - return !memcmp(value1->Data, value2->Data, value1->Leng*sizeof value1->Data[0]); + length = value1->Leng; + data1 = value1->Data; + assert(data1 || !length); + data2 = value2->Data; + assert(data2 || !value2->Leng); + return ((length == value2->Leng) && !(length && memcmp(data1, data2, length))); } static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 57be87117a..dcef99f162 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -47,16 +47,17 @@ static int EqualMemoryValue(dhtKey v1, dhtKey v2) { MemVal const * value1 = (MemVal const *)v1.value.object_pointer; MemVal const * value2 = (MemVal const *)v2.value.object_pointer; + unsigned long length; + unsigned char const *data1; + unsigned char const *data2; assert(value1 && value2); - assert(value1->Data || !value1->Leng); - assert(value2->Data || !value2->Leng); - if (value1->Leng != value2->Leng) - return 0; - if (!value1->Leng) // avoid the potential undefined behavior of using memcmp on NULL pointers - return 1; - assert(value1->Leng <= (((size_t)-1)/sizeof value1->Data[0])); - return !memcmp(value1->Data, value2->Data, value1->Leng*sizeof value1->Data[0]); + length = value1->Leng; + data1 = value1->Data; + assert(data1 || !length); + data2 = value2->Data; + assert(data2 || !value2->Leng); + return ((length == value2->Leng) && !(length && memcmp(data1, data2, length))); } static int DupMemoryValue(dhtValue kv, dhtValue *output) { From cbaf0765df6248164948df11b7d170a37ee2707a Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 11 Sep 2023 21:30:55 -0400 Subject: [PATCH 042/111] Some more assertions. --- DHT/dhtbcmem.c | 7 +++++-- DHT/dhtcmem.c | 7 +++++-- DHT/dhtmem.c | 17 +++++++++++------ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index 6252a90b4d..ab39dfc310 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -62,7 +62,7 @@ static int EqualBCMemValue(dhtKey v1, dhtKey v2) data2 = value2->Data; assert(data2 || !value2->Leng); - return ((length == value2->Leng) && !(length && memcmp(data1, data2, length))); + return ((length == value2->Leng) && !(length && memcmp(data1, data2, length*sizeof *data1))); } static int DupBCMemValue(dhtValue kv, dhtValue *output) @@ -73,6 +73,7 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) BCMemValue *result; unsigned short length; size_t size = sizeof *original; + unsigned char const *data; assert(!!output); if (!original) @@ -82,6 +83,8 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) } length = original->Leng; + data = original->Data; + assert(data || !length); if (length > (num_bytes_in_Data / size_of_element)) { if (length > ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)) @@ -93,7 +96,7 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) if (result) { result->Leng = length; - memcpy(result->Data,original->Data,(length*size_of_element)); + memcpy(result->Data,data,(length*size_of_element)); output->object_pointer = result; return 0; } diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 675efe720b..6a233ef43b 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -57,7 +57,7 @@ static int EqualCompactMemoryValue(dhtKey v1, dhtKey v2) assert(data1 || !length); data2 = value2->Data; assert(data2 || !value2->Leng); - return ((length == value2->Leng) && !(length && memcmp(data1, data2, length))); + return ((length == value2->Leng) && !(length && memcmp(data1, data2, length*sizeof *data1))); } static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) @@ -68,6 +68,7 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) size_t size = sizeof *v; CompactMemVal *result; uLong length; + unsigned char const *data; assert(!!output); if (!v) @@ -77,6 +78,8 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) } length = v->Leng; + data = v->Data; + assert(data || !length); if (length > (num_bytes_in_Data / size_of_element)) { if (length > ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)) @@ -88,7 +91,7 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) if (result) { result->Leng = length; - memcpy(result->Data,v->Data,(length*size_of_element)); + memcpy(result->Data,data,(length*size_of_element)); output->object_pointer = result; return 0; } diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index dcef99f162..1ccf4328cf 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -57,25 +57,30 @@ static int EqualMemoryValue(dhtKey v1, dhtKey v2) assert(data1 || !length); data2 = value2->Data; assert(data2 || !value2->Leng); - return ((length == value2->Leng) && !(length && memcmp(data1, data2, length))); + return ((length == value2->Leng) && !(length && memcmp(data1, data2, length*sizeof *data1))); } static int DupMemoryValue(dhtValue kv, dhtValue *output) { MemVal const *v= (MemVal const *)kv.object_pointer; + unsigned char const *data; + unsigned long length; MemVal *mv; assert(!!output); if (!v) { output->object_pointer = NilMemVal; return 0; } + length= v->Leng; + data= v->Data; + assert(data || !length); mv= NewMemVal; if (mv) { - mv->Leng= v->Leng; - if (mv->Leng) { - if (mv->Leng <= (((size_t)-1)/(sizeof mv->Data[0]))) { - mv->Data= (uChar *)fxfAlloc(mv->Leng*sizeof mv->Data[0]); + mv->Leng= length; + if (length) { + if (length <= (((size_t)-1)/(sizeof *data))) { + mv->Data= (uChar *)fxfAlloc(length*sizeof *data); if (mv->Data) { - memcpy(mv->Data, v->Data, mv->Leng*sizeof mv->Data[0]); + memcpy(mv->Data, data, length*sizeof *data); output->object_pointer = mv; return 0; } else { From 313cb97d1308af4a836e924499d4280c3181b134 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 11 Sep 2023 21:34:14 -0400 Subject: [PATCH 043/111] Hmm, those assertions aren't necessary. --- DHT/dhtbcmem.c | 5 +---- DHT/dhtcmem.c | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index ab39dfc310..074efe9bf6 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -73,7 +73,6 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) BCMemValue *result; unsigned short length; size_t size = sizeof *original; - unsigned char const *data; assert(!!output); if (!original) @@ -83,8 +82,6 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) } length = original->Leng; - data = original->Data; - assert(data || !length); if (length > (num_bytes_in_Data / size_of_element)) { if (length > ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)) @@ -96,7 +93,7 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) if (result) { result->Leng = length; - memcpy(result->Data,data,(length*size_of_element)); + memcpy(result->Data,original->Data,(length*size_of_element)); output->object_pointer = result; return 0; } diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 6a233ef43b..7c7429c474 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -68,7 +68,6 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) size_t size = sizeof *v; CompactMemVal *result; uLong length; - unsigned char const *data; assert(!!output); if (!v) @@ -78,8 +77,6 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) } length = v->Leng; - data = v->Data; - assert(data || !length); if (length > (num_bytes_in_Data / size_of_element)) { if (length > ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)) @@ -91,7 +88,7 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) if (result) { result->Leng = length; - memcpy(result->Data,data,(length*size_of_element)); + memcpy(result->Data,v->Data,(length*size_of_element)); output->object_pointer = result; return 0; } From 3d00a93a81ca91ac486dfa7cefa5e0cba46991e7 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 12 Sep 2023 20:50:38 -0400 Subject: [PATCH 044/111] Our elements are all unsigned chars, so no reason to complicate things. --- DHT/dhtbcmem.c | 40 ++++++++++++++++++---------------------- DHT/dhtcmem.c | 30 +++++++++++++----------------- DHT/dhtmem.c | 37 +++++++++++++++++++------------------ 3 files changed, 50 insertions(+), 57 deletions(-) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index 074efe9bf6..bd4ddc2a7d 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -23,18 +23,22 @@ #include "dhtbcmem.h" #include "dht.h" +enum { + ENSURE_SIZE_OF_ELEMENT_IS_ONE = 1/(1 == sizeof ((BCMemValue const *)NULL)->Data[0]) +}; + static dhtHashValue ConvertBCMemValue(dhtKey m) { BCMemValue const * const toBeConverted = (BCMemValue const *)m.value.object_pointer; - unsigned int leng; + unsigned short leng; unsigned char const *s; assert(!!toBeConverted); leng = toBeConverted->Leng; s = toBeConverted->Data; dhtHashValue hash = 0; - unsigned int i; - for (i=0; iLeng; - data1 = value1->Data; - assert(data1 || !length); - data2 = value2->Data; - assert(data2 || !value2->Leng); - return ((length == value2->Leng) && !(length && memcmp(data1, data2, length*sizeof *data1))); + return ((length == value2->Leng) && !memcmp(value1->Data, value2->Data, length)); } static int DupBCMemValue(dhtValue kv, dhtValue *output) { BCMemValue const *original = (BCMemValue const *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *original) - offsetof(BCMemValue, Data)); - size_t const size_of_element = sizeof original->Data[0]; BCMemValue *result; unsigned short length; size_t size = sizeof *original; @@ -82,18 +79,18 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) } length = original->Leng; - if (length > (num_bytes_in_Data / size_of_element)) + if (length > num_bytes_in_Data) { - if (length > ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)) + if (length > (((size_t)-1) - size + num_bytes_in_Data)) return 1; - size += ((length * size_of_element) - num_bytes_in_Data); + size += (length - num_bytes_in_Data); } result = (BCMemValue *)fxfAlloc(size); if (result) { result->Leng = length; - memcpy(result->Data,original->Data,(length*size_of_element)); + memcpy(result->Data,original->Data,length); output->object_pointer = result; return 0; } @@ -105,15 +102,14 @@ static void FreeBCMemValue(dhtValue kv) { BCMemValue *freed = (BCMemValue *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *freed) - offsetof(BCMemValue, Data)); - size_t const size_of_element = sizeof freed->Data[0]; if (freed) { size_t size = sizeof *freed; unsigned short length = freed->Leng; - if (length > (num_bytes_in_Data / size_of_element)) + if (length > num_bytes_in_Data) { - assert(length <= ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)); - size += ((length * size_of_element) - num_bytes_in_Data); + assert(length <= (((size_t)-1) - size + num_bytes_in_Data)); + size += (length - num_bytes_in_Data); } fxfFree(freed,size); } @@ -122,8 +118,8 @@ static void FreeBCMemValue(dhtValue kv) static void DumpBCMemValue(dhtValue kv, FILE *f) { BCMemValue const *toBeDumped = (BCMemValue const *)kv.object_pointer; - unsigned int length; - unsigned int i; + unsigned short length; + unsigned short i; assert(toBeDumped && f); diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 7c7429c474..84d09c8b56 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -23,6 +23,10 @@ typedef unsigned long uLong; typedef unsigned char uChar; +enum { + ENSURE_SIZE_OF_ELEMENT_IS_ONE = 1/(1 == sizeof NilCompactMemVal->Data[0]) +}; + static dhtHashValue ConvertCompactMemoryValue(dhtKey m) { CompactMemVal const * const toBeConverted = (CompactMemVal const *)m.value.object_pointer; @@ -48,23 +52,16 @@ static int EqualCompactMemoryValue(dhtKey v1, dhtKey v2) { CompactMemVal const * const value1 = (CompactMemVal const *)v1.value.object_pointer; CompactMemVal const * const value2 = (CompactMemVal const *)v2.value.object_pointer; - unsigned long length; - unsigned char const *data1; - unsigned char const *data2; + uLong length; assert(value1 && value2); length = value1->Leng; - data1 = value1->Data; - assert(data1 || !length); - data2 = value2->Data; - assert(data2 || !value2->Leng); - return ((length == value2->Leng) && !(length && memcmp(data1, data2, length*sizeof *data1))); + return ((length == value2->Leng) && !memcmp(value1->Data, value2->Data, length)); } static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) { CompactMemVal const *v = (CompactMemVal const *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); - size_t const size_of_element = sizeof v->Data[0]; size_t size = sizeof *v; CompactMemVal *result; uLong length; @@ -77,18 +74,18 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) } length = v->Leng; - if (length > (num_bytes_in_Data / size_of_element)) + if (length > num_bytes_in_Data) { - if (length > ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)) + if (length > (((size_t)-1) - size + num_bytes_in_Data)) return 1; - size += ((length * size_of_element) - num_bytes_in_Data); + size += (length - num_bytes_in_Data); } result = (CompactMemVal *)fxfAlloc(size); if (result) { result->Leng = length; - memcpy(result->Data,v->Data,(length*size_of_element)); + memcpy(result->Data,v->Data,length); output->object_pointer = result; return 0; } @@ -100,15 +97,14 @@ static void FreeCompactMemoryValue(dhtValue kv) { CompactMemVal *v = (CompactMemVal *)kv.object_pointer; size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); - size_t const size_of_element = sizeof v->Data[0]; if (v) { size_t size = sizeof *v; uLong length = v->Leng; - if (length > (num_bytes_in_Data / size_of_element)) + if (length > num_bytes_in_Data) { - assert(length <= ((((size_t)-1) - size + num_bytes_in_Data) / size_of_element)); - size += ((length * size_of_element) - num_bytes_in_Data); + assert(length <= (((size_t)-1) - size + num_bytes_in_Data)); + size += (length - num_bytes_in_Data); } fxfFree(v,size); } diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 1ccf4328cf..cb84d8f4c9 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -24,6 +24,10 @@ typedef unsigned long uLong; typedef unsigned char uChar; +enum { + ENSURE_SIZE_OF_ELEMENT_IS_ONE = 1/(1 == sizeof NilMemVal->Data[0]) +}; + static dhtHashValue HashMemoryValue(dhtKey k) { MemVal const * toBeHashed = (MemVal const *)k.value.object_pointer; @@ -47,9 +51,9 @@ static int EqualMemoryValue(dhtKey v1, dhtKey v2) { MemVal const * value1 = (MemVal const *)v1.value.object_pointer; MemVal const * value2 = (MemVal const *)v2.value.object_pointer; - unsigned long length; - unsigned char const *data1; - unsigned char const *data2; + uLong length; + uChar const *data1; + uChar const *data2; assert(value1 && value2); length = value1->Leng; @@ -57,13 +61,13 @@ static int EqualMemoryValue(dhtKey v1, dhtKey v2) assert(data1 || !length); data2 = value2->Data; assert(data2 || !value2->Leng); - return ((length == value2->Leng) && !(length && memcmp(data1, data2, length*sizeof *data1))); + return ((length == value2->Leng) && !(length && memcmp(data1, data2, length))); } static int DupMemoryValue(dhtValue kv, dhtValue *output) { MemVal const *v= (MemVal const *)kv.object_pointer; - unsigned char const *data; - unsigned long length; + uChar const *data; + uLong length; MemVal *mv; assert(!!output); if (!v) { @@ -77,16 +81,14 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) if (mv) { mv->Leng= length; if (length) { - if (length <= (((size_t)-1)/(sizeof *data))) { - mv->Data= (uChar *)fxfAlloc(length*sizeof *data); - if (mv->Data) { - memcpy(mv->Data, data, length*sizeof *data); - output->object_pointer = mv; - return 0; - } else { - FreeMemVal(mv); - return 1; - } + mv->Data= (uChar *)fxfAlloc(length); + if (mv->Data) { + memcpy(mv->Data, data, length); + output->object_pointer = mv; + return 0; + } else { + FreeMemVal(mv); + return 1; } } else { mv->Data= NULL; // NULL is a valid pointer if Leng == 0 @@ -100,8 +102,7 @@ static void FreeMemoryValue(dhtValue kv) { MemVal *v= (MemVal *)kv.object_pointer; if (v) { - assert(v->Leng <= (((size_t)-1)/(sizeof v->Data[0]))); - fxfFree(v->Data, v->Leng*sizeof v->Data[0]); + fxfFree(v->Data, v->Leng); fxfFree(v, sizeof *v); } } From 5e94a63666b5a2419fed5bfc73e7790200783905 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 12 Sep 2023 21:19:05 -0400 Subject: [PATCH 045/111] Personal preference for ++i. --- DHT/dhtcmem.c | 2 +- DHT/dhtmem.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 84d09c8b56..00d92c730c 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -37,7 +37,7 @@ static dhtHashValue ConvertCompactMemoryValue(dhtKey m) s= toBeConverted->Data; dhtHashValue hash= 0; uLong i; - for (i=0; i> 6; diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index cb84d8f4c9..28051fb92e 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -37,7 +37,7 @@ static dhtHashValue HashMemoryValue(dhtKey k) uChar const *s= toBeHashed->Data; dhtHashValue hash= 0; uLong i; - for (i=0; i> 6; From c5103a614d5e8c2f6b8233ebd7ef5ccf08feb1f4 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 13 Sep 2023 21:02:19 -0400 Subject: [PATCH 046/111] Might as well properly handle cases where our lengths are > SIZE_MAX. It's not clear that any such systems exist or are in use, but if the code is really unnecessary then I expect the compiler to remove it. --- DHT/dhtbcmem.c | 17 ++++++++++++++++- DHT/dhtcmem.c | 16 +++++++++++++++- DHT/dhtmem.c | 15 ++++++++++++++- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index bd4ddc2a7d..dede2285ea 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -57,10 +57,25 @@ static int EqualBCMemValue(dhtKey v1, dhtKey v2) BCMemValue const * const value1 = (BCMemValue const *)v1.value.object_pointer; BCMemValue const * const value2 = (BCMemValue const *)v2.value.object_pointer; unsigned short length; + unsigned char const *data1; + unsigned char const *data2; assert(value1 && value2); length = value1->Leng; - return ((length == value2->Leng) && !memcmp(value1->Data, value2->Data, length)); + if (length != value2->Leng) + return 0; + + data1 = value1->Data; + data2 = value2->Data; + while (length > ((size_t)-1)) + { + if (memcmp(data1, data2, ((size_t)-1))) + return 0; + data1 += ((size_t)-1); + data2 += ((size_t)-1); + length -= ((size_t)-1); + } + return !memcmp(data1, data2, length); } static int DupBCMemValue(dhtValue kv, dhtValue *output) diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 00d92c730c..b94930322c 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -53,9 +53,23 @@ static int EqualCompactMemoryValue(dhtKey v1, dhtKey v2) CompactMemVal const * const value1 = (CompactMemVal const *)v1.value.object_pointer; CompactMemVal const * const value2 = (CompactMemVal const *)v2.value.object_pointer; uLong length; + unsigned char const *data1; + unsigned char const *data2; assert(value1 && value2); length = value1->Leng; - return ((length == value2->Leng) && !memcmp(value1->Data, value2->Data, length)); + if (length != value2->Leng) + return 0; + data1 = value1->Data; + data2 = value2->Data; + while (length > ((size_t)-1)) + { + if (memcmp(data1, data2, ((size_t)-1))) + return 0; + data1 += ((size_t)-1); + data2 += ((size_t)-1); + length -= ((size_t)-1); + } + return !memcmp(data1, data2, length); } static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 28051fb92e..511e0268d6 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -61,7 +61,20 @@ static int EqualMemoryValue(dhtKey v1, dhtKey v2) assert(data1 || !length); data2 = value2->Data; assert(data2 || !value2->Leng); - return ((length == value2->Leng) && !(length && memcmp(data1, data2, length))); + + if (length != value2->Leng) + return 0; + if (!length) + return 1; + while (length > ((size_t)-1)) + { + if (memcmp(data1, data2, ((size_t)-1))) + return 0; + data1+= ((size_t)-1); + data2+= ((size_t)-1); + length-= ((size_t)-1); + } + return !memcmp(data1, data2, length); } static int DupMemoryValue(dhtValue kv, dhtValue *output) { From 724edb0199f5a6a0f6e62486676fd76dcc1aefab Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 13 Sep 2023 21:04:16 -0400 Subject: [PATCH 047/111] Explaining some code that is required here but not required elsewhere. --- DHT/dhtmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 511e0268d6..680c3ddd9e 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -64,7 +64,7 @@ static int EqualMemoryValue(dhtKey v1, dhtKey v2) if (length != value2->Leng) return 0; - if (!length) + if (!length) // We'll check this here to avoid (maybe) passing NULL as an argument to memcmp. return 1; while (length > ((size_t)-1)) { From 7ec20143c5a70378d646b00d42f8bbff7a8e925b Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 13 Sep 2023 22:08:41 -0400 Subject: [PATCH 048/111] No reason to load the value twice on each loop iteration. --- DHT/dhtstrng.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 8921abc525..1f79b28c76 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -26,12 +26,14 @@ static dhtHashValue ConvertString(dhtKey k) */ unsigned char const *s= (unsigned char const *)k.value.object_pointer; unsigned long hash; + unsigned char tmp; assert(!!s); hash= 0; - while (*s) { - hash+= *s++; + while ((tmp= *s)) { + hash+= tmp; hash+= hash << 10; hash^= hash >> 6; + ++s; } hash+= hash << 3; hash^= hash >> 11; From 1cb6ae47e0d79654685edc2c3a8091805a92f0f7 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 13 Sep 2023 22:22:30 -0400 Subject: [PATCH 049/111] Adding another check for SIZE_MAX overflow. --- DHT/dhtmem.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 680c3ddd9e..4366aca5f8 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -80,6 +80,7 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) { MemVal const *v= (MemVal const *)kv.object_pointer; uChar const *data; + uChar *newBuffer; uLong length; MemVal *mv; assert(!!output); @@ -90,13 +91,16 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) length= v->Leng; data= v->Data; assert(data || !length); + if (length > ((size_t)-1)) + return 1; mv= NewMemVal; if (mv) { mv->Leng= length; if (length) { - mv->Data= (uChar *)fxfAlloc(length); - if (mv->Data) { - memcpy(mv->Data, data, length); + newBuffer= (uChar *)fxfAlloc(length); + if (newBuffer) { + memcpy(newBuffer, data, length); + mv->Data = newBuffer; output->object_pointer = mv; return 0; } else { From 9266dc0c007a50ca7410cf93ec3fb4b3dcfcd780 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 14 Sep 2023 07:01:09 -0400 Subject: [PATCH 050/111] No need to cast until the final assignment. --- DHT/dhtmem.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 4366aca5f8..1ff150ae04 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -80,7 +80,7 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) { MemVal const *v= (MemVal const *)kv.object_pointer; uChar const *data; - uChar *newBuffer; + void *newBuffer; uLong length; MemVal *mv; assert(!!output); @@ -97,10 +97,10 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) if (mv) { mv->Leng= length; if (length) { - newBuffer= (uChar *)fxfAlloc(length); + newBuffer= fxfAlloc(length); if (newBuffer) { memcpy(newBuffer, data, length); - mv->Data = newBuffer; + mv->Data = (uChar *)newBuffer; output->object_pointer = mv; return 0; } else { From 3a55615b8a64372a203dd9717a39db81eb470718 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 17 Sep 2023 07:16:47 -0400 Subject: [PATCH 051/111] Trying to minimize the memory allocations by allowing for small allocations without rounding. We can always add "rounding to MAX_ALIGNMENT" on top of these changes, but that can be an independent decision. --- DHT/fxf.c | 197 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 149 insertions(+), 48 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index d7c63ebc1a..0c1c29b9b6 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -19,18 +19,26 @@ #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || /* >= C99 -- We have printf ptrdiff_t/size_t specifiers. */ \ (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 -- We have printf ptrdiff_t/size_t specifiers. */ # include +# include typedef ptrdiff_t ptrdiff_t_printf_type; typedef size_t size_t_printf_type; +# ifdef UINTPTR_MAX + typedef uintptr_t pointer_to_int_type; +# else + typedef uintmax_t pointer_to_int_type; +# endif # define PTRDIFF_T_PRINTF_SPECIFIER "td" # define SIZE_T_PRINTF_SPECIFIER "zu" #elif defined(LLONG_MAX) /* We have long long integer types. */ typedef long long int ptrdiff_t_printf_type; typedef unsigned long long int size_t_printf_type; + typedef unsigned long long int pointer_to_int_type; # define PTRDIFF_T_PRINTF_SPECIFIER "lld" # define SIZE_T_PRINTF_SPECIFIER "llu" #else /* We don't have long long integer types. */ typedef long int ptrdiff_t_printf_type; typedef unsigned long int size_t_printf_type; + typedef unsigned long int pointer_to_int_type; # define PTRDIFF_T_PRINTF_SPECIFIER "ld" # define SIZE_T_PRINTF_SPECIFIER "lu" #endif @@ -59,13 +67,14 @@ union MAX_ALIGNED_TYPE { #endif enum { - ENSURE_MAX_ALIGNMENT_IS_POWER_OF_TWO = 1/(!(MAX_ALIGNMENT & (MAX_ALIGNMENT - 1U))) + ENSURE_MAX_ALIGNMENT_IS_POWER_OF_TWO = 1/(!(MAX_ALIGNMENT & (MAX_ALIGNMENT - 1U))), }; -#if !defined(Nil) && !defined(New) && !defined(nNew) /* TODO: Is this the correct check for all of the below lines? */ +#if !defined(Nil) && !defined(New) && !defined(nNew) && !defined(nNewCallc) /* TODO: Is this the correct check for all of the below lines? */ # define Nil(type) ((type *)0) # define New(type) ((type *)malloc(sizeof(type))) # define nNew(n, type) ((type *)nNewImpl(n,sizeof(type))) +# define nNewCalloc(n, type) ((type *)calloc(n,sizeof(type))) static inline void * nNewImpl(size_t const nmemb, size_t const size) { return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : malloc(nmemb*size)); } @@ -140,10 +149,7 @@ enum fxfMINSIZE = sizeof(size_t) }; -/* Below we'll ensure that allocation sizes are all multiples of MAX_ALIGNMENT in the range - [MAX_ALIGNMENT, fxfMAXSIZE]. - If we divide by MAX_ALIGNMENT and subtract 1, we get the range 0, 1, 2, ..., fxfMAXSIZE/MAX_ALIGNMENT - 1. */ -static SizeHead SizeData[fxfMAXSIZE/MAX_ALIGNMENT]; +static SizeHead SizeData[fxfMAXSIZE - 1]; #if defined(SEGMENTED) /* #define ARENA_SEG_SIZE 32000 */ @@ -311,11 +317,7 @@ size_t fxfInit(size_t Size) { Size = ((Size+31)>>5); } - FreeMap= nNew(Size, FreeMapType); /* TODO: Can/Should we replace this allocation+memset with a call to calloc? */ - if (FreeMap) - { - memset(FreeMap, '\0', Size*(sizeof *FreeMap)); - } + FreeMap= nNewCalloc(Size, FreeMapType); #endif /*FREEMAP*/ #endif /*SEGMENTED*/ @@ -395,12 +397,8 @@ void fxfReset(void) * Intel *86 type of CPU, but also there, aligned access is faster. */ #define PTRMASK (MAX_ALIGNMENT-1U) -#define ALIGNED_MINSIZE (MAX_ALIGNMENT+PTRMASK) -#define ALIGN_SIZE_T(s) (((s)+PTRMASK) & (~PTRMASK)) -#define ALIGN(ptr) ((void *)ALIGN_SIZE_T((size_t)ptr)) -#define GetNextPtr(ptr) (*(char **)ALIGN(ptr)) -#define PutNextPtr(dst, ptr) *(char **)ALIGN(dst)= (ptr) +#define GetNextPtr(ptr) (*(void **)(ptr)) #define TMDBG(x) if (0) x @@ -414,6 +412,9 @@ void *fxfAlloc(size_t size) { TMDBG(printf("fxfAlloc - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); DBG((stderr, "%s(%" SIZE_T_PRINTF_SPECIFIER ") =", myname, (size_t_printf_type)size)); + if (!size) + return Nil(void); + if (sizeFreeHead) { +#ifdef SEGMENTED + int ptrSegment; +#endif ptr= (char *)sh->FreeHead; - sh->FreeHead= GetNextPtr(ptr); + sh->FreeHead= ((size < sizeof(void *)) ? Nil(void) : GetNextPtr(ptr)); sh->FreeCount--; sh->MallocCount++; +#ifdef SEGMENTED + for (ptrSegment= CurrentSeg; ptrSegment > 0; --ptrSegment) { + pointer_to_int_type tmp= (pointer_to_int_type)ptr; + pointer_to_int_type segment_begin= (pointer_to_int_type)Arena[ptrSegment]; + if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) + break; + } + ClrRange((char *)ptr-Arena[ptrSegnemtn], size); + TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)(ptr-Arena[ptrSegment]),sh->MallocCount)); +#else ClrRange((char *)ptr-Arena, size); -#if !defined(SEGMENTED) /* TODO: What should we output in the SEGMENTED case? */ TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)(ptr-Arena),sh->MallocCount)); #endif } else { /* we have to allocate a new piece */ - size_t const sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); + size_t sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); TMDBG(printf(" sizeCurrentSeg:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)sizeCurrentSeg)); if (sizeCurrentSeg>=size) { if (size&PTRMASK) { - /* not aligned */ + /* not fully aligned */ + size_t needed_alignment_mask= PTRMASK; + while (needed_alignment_mask >= size) + needed_alignment_mask>>= 1; + pointer_to_int_type curBottomInt= (pointer_to_int_type)BotFreePtr; + if (curBottomInt & needed_alignment_mask) { + size_t const numBytesToAdd= (needed_alignment_mask - (curBottomInt & needed_alignment_mask)) + 1U; + if (numBytesToAdd > (sizeCurrentSeg-size)) + goto NEXT_SEGMENT; + do { + pointer_to_int_type const cur_alignment= (curBottomInt & -curBottomInt); + SizeHead *cur_sh= &SizeData[cur_alignment - 1]; +#ifdef SEGMENTED + SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_alignment); +#else + SetRange((char *)BotFreePtr-Arena,cur_alignment); +#endif + if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { + if (cur_alignment >= sizeof(void *)) + *(void **)BotFreePtr= cur_sh->FreeHead; + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + } + BotFreePtr+= cur_alignment; + curBottomInt+= cur_alignment; + } while (curBottomInt & needed_alignment_mask); + } ptr= BotFreePtr; BotFreePtr+= size; } else { - /* aligned */ + /* fully aligned */ ptr= TopFreePtr-= size; } sh->MallocCount++; -#if !defined(SEGMENTED) /* TODO: What should we output in the SEGMENTED case? */ +#ifdef SEGMENTED + TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,(ptrdiff_t_printf_type)(ptr-Arena[CurrentSeg]),sh->MallocCount)); +#else TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",(ptrdiff_t_printf_type)(ptr-Arena),sh->MallocCount)); #endif } else { +NEXT_SEGMENT: #if defined(SEGMENTED) - if ((CurrentSeg+1) < ArenaSegCnt) { + if (CurrentSeg < (ArenaSegCnt-1)) { + pointer_to_int_type curBottomInt= (pointer_to_int_type)BotFreePtr; + while (curBottomInt & PTRMASK) { + pointer_to_int_type const cur_alignment= (curBottomInt & -curBottomInt); + SizeHead *cur_sh= &SizeData[cur_alignment - 1]; +#ifdef SEGMENTED + SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_alignment); +#else + SetRange((char *)BotFreePtr-Arena,cur_alignment); +#endif + if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { + if (cur_alignment >= sizeof(void *)) + *(void **)BotFreePtr= cur_sh->FreeHead; + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + } + BotFreePtr+= cur_alignment; + curBottomInt+= cur_alignment; + } + if (BotFreePtr < TopFreePtr) { + size_t cur_size= (size_t)(TopFreePtr-BotFreePtr); + SizeHead *cur_sh= &SizeData[cur_size - 1]; +#ifdef SEGMENTED + SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_size); +#else + SetRange((char *)BotFreePtr-Arena,cur_size); +#endif + if ((cur_size >= sizeof(void *)) || !cur_sh->FreeCount) { + if (cur_size >= sizeof(void *)) + *(void **)BotFreePtr= cur_sh->FreeHead; + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + } + } TMDBG(fputs(" next seg", stdout)); ++CurrentSeg; BotFreePtr= Arena[CurrentSeg]; @@ -485,10 +561,22 @@ void fxfFree(void *ptr, size_t size) static char const * const myname= "fxfFree"; SizeHead *sh; -#if !defined(SEGMENTED) /* TODO: What should we output in the SEGMENTED case? */ +#ifdef SEGMENTED + int ptrSegment; + for (ptrSegment= CurrentSeg; ptrSegment > 0; --ptrSegment) { + pointer_to_int_type tmp= (pointer_to_int_type)ptr; + pointer_to_int_type segment_begin= (pointer_to_int_type)Arena[ptrSegment]; + if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) + break; + } + TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,(ptrdiff_t_printf_type)(((char const*)ptr)-Arena[ptrSegment]),(size_t_printf_type)size)); +#else TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(((char const*)ptr)-Arena),(size_t_printf_type)size)); #endif DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *) ptr, (size_t_printf_type) size)); + assert(ptr || !size); + if (!ptr) + return; if (size < fxfMINSIZE) size= fxfMINSIZE; if (size > fxfMAXSIZE) { @@ -496,27 +584,33 @@ void fxfFree(void *ptr, size_t size) myname, (size_t_printf_type) size, (size_t_printf_type) fxfMAXSIZE); exit(-5); } - size= ALIGN_SIZE_T(size); - sh= &SizeData[(size/MAX_ALIGNMENT)-1]; + sh= &SizeData[size - 1]; if (size&PTRMASK) { - /* unaligned size */ + /* not fully aligned size */ TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(BotFreePtr-(char const*)ptr))); - if ((char *)ALIGN((char *)ptr+size) == BotFreePtr) { + if (((char *)ptr+size) == BotFreePtr) { BotFreePtr= (char *)ptr; TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(TopFreePtr-BotFreePtr))); --sh->MallocCount; } else { +#ifdef SEGMENTED + SetRange((char *)ptr-Arena[ptrSegment],size); +#else SetRange((char *)ptr-Arena,size); - *(char **)ALIGN(ptr)= (char *)sh->FreeHead; - sh->FreeHead= ptr; - ++sh->FreeCount; - --sh->MallocCount; - TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); +#endif + if ((size >= sizeof(void *)) || !sh->FreeHead) { + if (size >= sizeof(void *)) + *(void **)ptr= sh->FreeHead; + sh->FreeHead= ptr; + ++sh->FreeCount; + --sh->MallocCount; + TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); + } } } else { - /* aligned size */ + /* fully aligned size */ TMDBG(printf(" ptr-TopFreePtr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(((char const*)ptr)-TopFreePtr))); if ((char *)ptr == TopFreePtr) { TopFreePtr+= size; @@ -524,12 +618,19 @@ void fxfFree(void *ptr, size_t size) --sh->MallocCount; } else { +#ifdef SEGMENTED + SetRange((char *)ptr-Arena[ptrSegment],size); +#else SetRange((char *)ptr-Arena,size); - *(char **)ptr= (char *)sh->FreeHead; - sh->FreeHead= ptr; - ++sh->FreeCount; - --sh->MallocCount; - TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); +#endif + if ((size >= sizeof(void *)) || !sh->FreeCount) { + if (size >= sizeof(void *)) + *(void **)ptr= sh->FreeHead; + sh->FreeHead= ptr; + ++sh->FreeCount; + --sh->MallocCount; + TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); + } } } TMDBG(printf(" MallocCount:%lu",sh->MallocCount)); @@ -559,8 +660,8 @@ size_t fxfTotal(void) { size_t i; for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - UsedBytes+= hd->MallocCount*(i+1U)*MAX_ALIGNMENT; - FreeBytes+= hd->FreeCount*(i+1U)*MAX_ALIGNMENT; + UsedBytes+= hd->MallocCount*(i+1U); + FreeBytes+= hd->FreeCount*(i+1U); } } @@ -595,11 +696,11 @@ void fxfInfo(FILE *f) { fprintf(f, "%12s %10s%10s\n", "Size", "MallocCnt", "FreeCnt"); for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - fprintf(f, "%12zu %10lu%10lu\n", (i+1U)*MAX_ALIGNMENT, hd->MallocCount, hd->FreeCount); + fprintf(f, "%12zu %10lu%10lu\n", i+1U, hd->MallocCount, hd->FreeCount); nrUsed+= hd->MallocCount; - UsedBytes+= hd->MallocCount*(i+1U)*MAX_ALIGNMENT; + UsedBytes+= hd->MallocCount*(i+1U); nrFree+= hd->FreeCount; - FreeBytes+= hd->FreeCount*(i+1U)*MAX_ALIGNMENT; + FreeBytes+= hd->FreeCount*(i+1U); } } fprintf(f, "%12s %10lu%10lu\n", "Total:", nrUsed, nrFree); From ef5596417ad1a0ef4594b60f93164e950c2048e2 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 18 Sep 2023 19:31:41 -0400 Subject: [PATCH 052/111] Rounding allocations to the alignment required for fxfMINSIZE since we can't make use of smaller chunks. --- DHT/fxf.c | 87 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 0c1c29b9b6..07fefcaa15 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -63,13 +63,9 @@ union MAX_ALIGNED_TYPE { void (*function_pointer)(void); long double floating_point; }; -# define MAX_ALIGNMENT sizeof(union MAX_ALIGNED_TYPE) +# define MAX_ALIGNMENT (sizeof(union MAX_ALIGNED_TYPE) & -sizeof(union MAX_ALIGNED_TYPE)) #endif -enum { - ENSURE_MAX_ALIGNMENT_IS_POWER_OF_TWO = 1/(!(MAX_ALIGNMENT & (MAX_ALIGNMENT - 1U))), -}; - #if !defined(Nil) && !defined(New) && !defined(nNew) && !defined(nNewCallc) /* TODO: Is this the correct check for all of the below lines? */ # define Nil(type) ((type *)0) # define New(type) ((type *)malloc(sizeof(type))) @@ -146,10 +142,17 @@ typedef struct { /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ enum { - fxfMINSIZE = sizeof(size_t) + fxfMINSIZE = sizeof(void *) }; +#define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) +#define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. + (Of course, they're probably the same.) + TODO: Can we compute what we want at compile time and just use it? */ +static size_t MIN_ALIGNMENT= MIN_ALIGNMENT_UNDERESTIMATE; /* for now */ -static SizeHead SizeData[fxfMAXSIZE - 1]; +static SizeHead SizeData[1 + (fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). + Maximum allocation is fxfMAXSIZE. + All allocations will be multiples of MIN_ALIGNMENT_UNDERESTIMATE. */ #if defined(SEGMENTED) /* #define ARENA_SEG_SIZE 32000 */ @@ -323,6 +326,10 @@ size_t fxfInit(size_t Size) { memset(SizeData, '\0', sizeof(SizeData)); + MIN_ALIGNMENT= MAX_ALIGNMENT; + while (MIN_ALIGNMENT > fxfMINSIZE) + MIN_ALIGNMENT>>= 1; + return GlobalSize; } @@ -397,6 +404,7 @@ void fxfReset(void) * Intel *86 type of CPU, but also there, aligned access is faster. */ #define PTRMASK (MAX_ALIGNMENT-1U) +#define ALIGN_TO_MINIMUM(s) (((s) + (MIN_ALIGNMENT - 1U)) & ~(MIN_ALIGNMENT - 1U)) #define GetNextPtr(ptr) (*(void **)(ptr)) @@ -427,7 +435,10 @@ void *fxfAlloc(size_t size) { return Nil(void); } - sh= &SizeData[size - 1]; + // Round up to a multiple of MIN_ALIGNMENT + size= ALIGN_TO_MINIMUM(size); + + sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if (sh->FreeHead) { #ifdef SEGMENTED int ptrSegment; @@ -467,18 +478,20 @@ void *fxfAlloc(size_t size) { goto NEXT_SEGMENT; do { pointer_to_int_type const cur_alignment= (curBottomInt & -curBottomInt); - SizeHead *cur_sh= &SizeData[cur_alignment - 1]; #ifdef SEGMENTED SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_alignment); #else SetRange((char *)BotFreePtr-Arena,cur_alignment); #endif - if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { - if (cur_alignment >= sizeof(void *)) - *(void **)BotFreePtr= cur_sh->FreeHead; - cur_sh->FreeHead= BotFreePtr; - ++cur_sh->FreeCount; - TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + if (cur_alignment >= fxfMINSIZE) { + SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { + if (cur_alignment >= sizeof(void *)) + *(void **)BotFreePtr= cur_sh->FreeHead; + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + } } BotFreePtr+= cur_alignment; curBottomInt+= cur_alignment; @@ -506,36 +519,40 @@ void *fxfAlloc(size_t size) { pointer_to_int_type curBottomInt= (pointer_to_int_type)BotFreePtr; while (curBottomInt & PTRMASK) { pointer_to_int_type const cur_alignment= (curBottomInt & -curBottomInt); - SizeHead *cur_sh= &SizeData[cur_alignment - 1]; #ifdef SEGMENTED SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_alignment); #else SetRange((char *)BotFreePtr-Arena,cur_alignment); #endif - if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { - if (cur_alignment >= sizeof(void *)) - *(void **)BotFreePtr= cur_sh->FreeHead; - cur_sh->FreeHead= BotFreePtr; - ++cur_sh->FreeCount; - TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + if (cur_alignment >= fxfMINSIZE) { + SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { + if (cur_alignment >= sizeof(void *)) + *(void **)BotFreePtr= cur_sh->FreeHead; + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + } } BotFreePtr+= cur_alignment; curBottomInt+= cur_alignment; } if (BotFreePtr < TopFreePtr) { size_t cur_size= (size_t)(TopFreePtr-BotFreePtr); - SizeHead *cur_sh= &SizeData[cur_size - 1]; #ifdef SEGMENTED SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_size); #else SetRange((char *)BotFreePtr-Arena,cur_size); #endif - if ((cur_size >= sizeof(void *)) || !cur_sh->FreeCount) { - if (cur_size >= sizeof(void *)) - *(void **)BotFreePtr= cur_sh->FreeHead; - cur_sh->FreeHead= BotFreePtr; - ++cur_sh->FreeCount; - TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + if (cur_size >= fxfMINSIZE) { + SizeHead *cur_sh= &SizeData[(cur_size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + if ((cur_size >= sizeof(void *)) || !cur_sh->FreeCount) { + if (cur_size >= sizeof(void *)) + *(void **)BotFreePtr= cur_sh->FreeHead; + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + } } } TMDBG(fputs(" next seg", stdout)); @@ -584,7 +601,8 @@ void fxfFree(void *ptr, size_t size) myname, (size_t_printf_type) size, (size_t_printf_type) fxfMAXSIZE); exit(-5); } - sh= &SizeData[size - 1]; + size= ALIGN_TO_MINIMUM(size); + sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if (size&PTRMASK) { /* not fully aligned size */ TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(BotFreePtr-(char const*)ptr))); @@ -652,6 +670,9 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { return nptr; } +#define SIZEDATA_INDEX_TO_SIZE(x) ((size_t)(((x) * MIN_ALIGNMENT_UNDERESTIMATE) + \ + ((fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U)))) + size_t fxfTotal(void) { SizeHead const *hd = SizeData; size_t UsedBytes = 0; @@ -660,8 +681,8 @@ size_t fxfTotal(void) { size_t i; for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - UsedBytes+= hd->MallocCount*(i+1U); - FreeBytes+= hd->FreeCount*(i+1U); + UsedBytes+= hd->MallocCount*SIZEDATA_INDEX_TO_SIZE(i); + FreeBytes+= hd->FreeCount*SIZEDATA_INDEX_TO_SIZE(i); } } @@ -696,7 +717,7 @@ void fxfInfo(FILE *f) { fprintf(f, "%12s %10s%10s\n", "Size", "MallocCnt", "FreeCnt"); for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - fprintf(f, "%12zu %10lu%10lu\n", i+1U, hd->MallocCount, hd->FreeCount); + fprintf(f, "%12zu %10lu%10lu\n", SIZEDATA_INDEX_TO_SIZE(i), hd->MallocCount, hd->FreeCount); nrUsed+= hd->MallocCount; UsedBytes+= hd->MallocCount*(i+1U); nrFree+= hd->FreeCount; From 6817cf79ec7b69d04dcdf8488d470f4ce4374993 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 18 Sep 2023 20:16:47 -0400 Subject: [PATCH 053/111] Avoiding casting pointers to integers except when (apparently) absolutely necessary. --- DHT/fxf.c | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 07fefcaa15..949e0cecbd 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -471,18 +471,18 @@ void *fxfAlloc(size_t size) { size_t needed_alignment_mask= PTRMASK; while (needed_alignment_mask >= size) needed_alignment_mask>>= 1; - pointer_to_int_type curBottomInt= (pointer_to_int_type)BotFreePtr; - if (curBottomInt & needed_alignment_mask) { - size_t const numBytesToAdd= (needed_alignment_mask - (curBottomInt & needed_alignment_mask)) + 1U; - if (numBytesToAdd > (sizeCurrentSeg-size)) - goto NEXT_SEGMENT; - do { - pointer_to_int_type const cur_alignment= (curBottomInt & -curBottomInt); #ifdef SEGMENTED - SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_alignment); + size_t curBottomIndex= ((char *)BotFreePtr - Arena[CurrentSeg]); #else - SetRange((char *)BotFreePtr-Arena,cur_alignment); + size_t curBottomIndex= ((char *)BotFreePtr - Arena); #endif + if (curBottomIndex & needed_alignment_mask) { + size_t const numBytesToAdd= (needed_alignment_mask - (curBottomIndex & needed_alignment_mask)) + 1U; + if (numBytesToAdd > (sizeCurrentSeg-size)) + goto NEXT_SEGMENT; + do { + size_t const cur_alignment= (curBottomIndex & -curBottomIndex); + SetRange(curBottomIndex,cur_alignment); if (cur_alignment >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { @@ -494,8 +494,8 @@ void *fxfAlloc(size_t size) { } } BotFreePtr+= cur_alignment; - curBottomInt+= cur_alignment; - } while (curBottomInt & needed_alignment_mask); + curBottomIndex+= cur_alignment; + } while (curBottomIndex & needed_alignment_mask); } ptr= BotFreePtr; BotFreePtr+= size; @@ -516,14 +516,10 @@ void *fxfAlloc(size_t size) { NEXT_SEGMENT: #if defined(SEGMENTED) if (CurrentSeg < (ArenaSegCnt-1)) { - pointer_to_int_type curBottomInt= (pointer_to_int_type)BotFreePtr; - while (curBottomInt & PTRMASK) { - pointer_to_int_type const cur_alignment= (curBottomInt & -curBottomInt); -#ifdef SEGMENTED - SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_alignment); -#else - SetRange((char *)BotFreePtr-Arena,cur_alignment); -#endif + size_t curBottomIndex= ((char *)BotFreePtr - Arena[CurrentSeg]); + while (curBottomIndex & PTRMASK) { + size_t const cur_alignment= (curBottomIndex & -curBottomIndex); + SetRange(curBottomIndex,cur_alignment); if (cur_alignment >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { @@ -535,15 +531,11 @@ void *fxfAlloc(size_t size) { } } BotFreePtr+= cur_alignment; - curBottomInt+= cur_alignment; + curBottomIndex+= cur_alignment; } if (BotFreePtr < TopFreePtr) { size_t cur_size= (size_t)(TopFreePtr-BotFreePtr); -#ifdef SEGMENTED SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_size); -#else - SetRange((char *)BotFreePtr-Arena,cur_size); -#endif if (cur_size >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if ((cur_size >= sizeof(void *)) || !cur_sh->FreeCount) { From eac38ddc344c679f545e93998e40445226b38cd6 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 18 Sep 2023 21:53:31 -0400 Subject: [PATCH 054/111] We don't need to restart the whole function. --- DHT/fxf.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 949e0cecbd..47e0671207 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -437,7 +437,6 @@ void *fxfAlloc(size_t size) { // Round up to a multiple of MIN_ALIGNMENT size= ALIGN_TO_MINIMUM(size); - sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if (sh->FreeHead) { #ifdef SEGMENTED @@ -463,7 +462,11 @@ void *fxfAlloc(size_t size) { } else { /* we have to allocate a new piece */ - size_t sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); + size_t sizeCurrentSeg; +#ifdef SEGMENTED +START_LOOKING_FOR_CHUNK: +#endif + sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); TMDBG(printf(" sizeCurrentSeg:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)sizeCurrentSeg)); if (sizeCurrentSeg>=size) { if (size&PTRMASK) { @@ -551,7 +554,7 @@ void *fxfAlloc(size_t size) { ++CurrentSeg; BotFreePtr= Arena[CurrentSeg]; TopFreePtr= Arena[CurrentSeg]+ARENA_SEG_SIZE; - ptr= fxfAlloc(size); + goto START_LOOKING_FOR_CHUNK; } else ptr= Nil(char); From bcf843481446bfdb2e9c9540ac9131b665661aa0 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 19 Sep 2023 21:31:53 -0400 Subject: [PATCH 055/111] Making it possible to compile without FXF. --- optimisations/hash.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index 5dd94d0eb1..256bad5a9c 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -42,7 +42,7 @@ ** for more details. Two procedures are used: ** dhtLookupElement: This procedure delivers ** a nil pointer, when the given position is not in the hashtable, - ** or a pointer to a hashelement. + ** or a pointer to a hashElement. ** dhtEnterElement: This procedure enters an encoded position ** with its values into the hashtable. ** @@ -56,7 +56,7 @@ ** in 5 moves", since the former can be recomputed faster. For the other ** type of information ("solvable") the comparison is the other way round. ** The compression of the table is an expensive operation, in a lot - ** of exeperiments it has shown to be quite effective in keeping the + ** of experiments it has shown to be quite effective in keeping the ** most valuable information, and speeds up the computation time ** considerably. But to be of any use, there must be enough memory to ** to store more than 800 positions. @@ -64,7 +64,7 @@ ** There seems to be no real penalty in using hashing, even if the ** hit ratio is very small and only about 5%, it speeds up the ** computation time by 30%. - ** I changed the output of hashstat, since its really informative + ** I changed the output of hashstat, since it's really informative ** to see the hit rate. ** ** inithash() @@ -1774,8 +1774,12 @@ static boolean is_proofgame(slice_index si) */ boolean is_hashtable_allocated(void) { +#ifdef FXF return !!fxfInitialised(); /* !! just in case fxfInitialised returns a nonzero value other than 1 and boolean is some type that won't automatically convert it to 1. */ +#else + return hashtable_kilos>0; +#endif } /* Initialise the hashing machinery for the current stipulation From 68c696a24c3dbb4bfb0cb05367ce284adc37efff Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 19 Sep 2023 22:39:10 -0400 Subject: [PATCH 056/111] Several changes. --- DHT/dhtcmem.c | 1 + DHT/dhtexam1.c | 2 +- DHT/dhtmem.c | 3 +-- DHT/dhtvalue.h | 12 ++++++++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index b94930322c..a6a486403a 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "debugging/assert.h" #include "dhtvalue.h" diff --git a/DHT/dhtexam1.c b/DHT/dhtexam1.c index ef61a25e9d..5c0cc27b1d 100644 --- a/DHT/dhtexam1.c +++ b/DHT/dhtexam1.c @@ -62,7 +62,7 @@ uLong inet_addr(char *cp) { return addr; } -struct hostent *gethent(char *file) +struct hostent *gethent(char const *file) { static struct hostent host; char *p; diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 1ff150ae04..92ee473338 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -80,7 +80,6 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) { MemVal const *v= (MemVal const *)kv.object_pointer; uChar const *data; - void *newBuffer; uLong length; MemVal *mv; assert(!!output); @@ -97,7 +96,7 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) if (mv) { mv->Leng= length; if (length) { - newBuffer= fxfAlloc(length); + void *newBuffer= fxfAlloc(length); if (newBuffer) { memcpy(newBuffer, data, length); mv->Data = (uChar *)newBuffer; diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 6a7104bf4f..3f483d91fd 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -14,11 +14,15 @@ #ifdef __cplusplus # if __cplusplus >= 201103L # include +# else +# include # endif # include #elif defined(__STDC_VERSION__) # if __STDC_VERSION__ >= 199901L # include +# else +# include # endif # include #endif @@ -68,6 +72,9 @@ typedef union { # if __cplusplus >= 201103L ::std::uintmax_t unsigned_integer; ::std::intmax_t signed_integer; +# elif defined(LLONG_MAX) + unsigned long long int unsigned_integer; + long long int signed_integer; # else unsigned long int unsigned_integer; long int signed_integer; @@ -80,8 +87,13 @@ typedef union { intmax_t signed_integer; _Bool boolean; # else +# if defined(LLONG_MAX) + unsigned long long int unsigned_integer; + long long int signed_integer; +# else unsigned long int unsigned_integer; long int signed_integer; +# endif int boolean; // What else? # endif sig_atomic_t atomic_integer; From 70973008b2b48d9c3d5acb249657eaff8ef29679 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 20 Sep 2023 07:03:16 -0400 Subject: [PATCH 057/111] Switching to memcpy. Hopefully this will avoid angering the strict-aliasing gods. --- DHT/fxf.c | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 47e0671207..cfd0d0ab16 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -4,6 +4,7 @@ #include #include #include +#include #if defined(__TURBOC__) # include @@ -406,8 +407,6 @@ void fxfReset(void) #define PTRMASK (MAX_ALIGNMENT-1U) #define ALIGN_TO_MINIMUM(s) (((s) + (MIN_ALIGNMENT - 1U)) & ~(MIN_ALIGNMENT - 1U)) -#define GetNextPtr(ptr) (*(void **)(ptr)) - #define TMDBG(x) if (0) x void *fxfAlloc(size_t size) { @@ -443,7 +442,10 @@ void *fxfAlloc(size_t size) { int ptrSegment; #endif ptr= (char *)sh->FreeHead; - sh->FreeHead= ((size < sizeof(void *)) ? Nil(void) : GetNextPtr(ptr)); + if (size < sizeof sh->FreeHead) + sh->FreeHead= Nil(void); + else + memcpy(&sh->FreeHead, ptr, sizeof sh->FreeHead); sh->FreeCount--; sh->MallocCount++; #ifdef SEGMENTED @@ -453,7 +455,7 @@ void *fxfAlloc(size_t size) { if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) break; } - ClrRange((char *)ptr-Arena[ptrSegnemtn], size); + ClrRange((char *)ptr-Arena[ptrSegment], size); TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)(ptr-Arena[ptrSegment]),sh->MallocCount)); #else ClrRange((char *)ptr-Arena, size); @@ -488,9 +490,9 @@ void *fxfAlloc(size_t size) { SetRange(curBottomIndex,cur_alignment); if (cur_alignment >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; - if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { - if (cur_alignment >= sizeof(void *)) - *(void **)BotFreePtr= cur_sh->FreeHead; + if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if (cur_alignment >= sizeof cur_sh->FreeHead) + memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); cur_sh->FreeHead= BotFreePtr; ++cur_sh->FreeCount; TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); @@ -525,9 +527,9 @@ void *fxfAlloc(size_t size) { SetRange(curBottomIndex,cur_alignment); if (cur_alignment >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; - if ((cur_alignment >= sizeof(void *)) || !cur_sh->FreeCount) { - if (cur_alignment >= sizeof(void *)) - *(void **)BotFreePtr= cur_sh->FreeHead; + if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if (cur_alignment >= sizeof cur_sh->FreeHead) + memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); cur_sh->FreeHead= BotFreePtr; ++cur_sh->FreeCount; TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); @@ -541,9 +543,9 @@ void *fxfAlloc(size_t size) { SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_size); if (cur_size >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; - if ((cur_size >= sizeof(void *)) || !cur_sh->FreeCount) { - if (cur_size >= sizeof(void *)) - *(void **)BotFreePtr= cur_sh->FreeHead; + if ((cur_size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if (cur_size >= sizeof cur_sh->FreeHead) + memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); cur_sh->FreeHead= BotFreePtr; ++cur_sh->FreeCount; TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); @@ -612,9 +614,9 @@ void fxfFree(void *ptr, size_t size) #else SetRange((char *)ptr-Arena,size); #endif - if ((size >= sizeof(void *)) || !sh->FreeHead) { - if (size >= sizeof(void *)) - *(void **)ptr= sh->FreeHead; + if ((size >= sizeof sh->FreeHead) || !sh->FreeHead) { + if (size >= sizeof sh->FreeHead) + memcpy(ptr, &sh->FreeHead, sizeof sh->FreeHead); sh->FreeHead= ptr; ++sh->FreeCount; --sh->MallocCount; @@ -636,9 +638,9 @@ void fxfFree(void *ptr, size_t size) #else SetRange((char *)ptr-Arena,size); #endif - if ((size >= sizeof(void *)) || !sh->FreeCount) { - if (size >= sizeof(void *)) - *(void **)ptr= sh->FreeHead; + if ((size >= sizeof sh->FreeHead) || !sh->FreeCount) { + if (size >= sizeof sh->FreeHead) + memcpy(ptr, &sh->FreeHead, sizeof sh->FreeHead); sh->FreeHead= ptr; ++sh->FreeCount; --sh->MallocCount; From 1dc98ef51467cd7cb8940bc8998d623744046450 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 20 Sep 2023 07:14:23 -0400 Subject: [PATCH 058/111] Removing some unnecessary casts. --- DHT/fxf.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index cfd0d0ab16..bede4565fe 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -455,10 +455,10 @@ void *fxfAlloc(size_t size) { if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) break; } - ClrRange((char *)ptr-Arena[ptrSegment], size); + ClrRange(ptr-Arena[ptrSegment], size); TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)(ptr-Arena[ptrSegment]),sh->MallocCount)); #else - ClrRange((char *)ptr-Arena, size); + ClrRange(ptr-Arena, size); TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)(ptr-Arena),sh->MallocCount)); #endif } @@ -477,9 +477,9 @@ void *fxfAlloc(size_t size) { while (needed_alignment_mask >= size) needed_alignment_mask>>= 1; #ifdef SEGMENTED - size_t curBottomIndex= ((char *)BotFreePtr - Arena[CurrentSeg]); + size_t curBottomIndex= (BotFreePtr - Arena[CurrentSeg]); #else - size_t curBottomIndex= ((char *)BotFreePtr - Arena); + size_t curBottomIndex= (BotFreePtr - Arena); #endif if (curBottomIndex & needed_alignment_mask) { size_t const numBytesToAdd= (needed_alignment_mask - (curBottomIndex & needed_alignment_mask)) + 1U; @@ -521,7 +521,7 @@ void *fxfAlloc(size_t size) { NEXT_SEGMENT: #if defined(SEGMENTED) if (CurrentSeg < (ArenaSegCnt-1)) { - size_t curBottomIndex= ((char *)BotFreePtr - Arena[CurrentSeg]); + size_t curBottomIndex= (BotFreePtr - Arena[CurrentSeg]); while (curBottomIndex & PTRMASK) { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); SetRange(curBottomIndex,cur_alignment); @@ -540,7 +540,7 @@ void *fxfAlloc(size_t size) { } if (BotFreePtr < TopFreePtr) { size_t cur_size= (size_t)(TopFreePtr-BotFreePtr); - SetRange((char *)BotFreePtr-Arena[CurrentSeg],cur_size); + SetRange(BotFreePtr-Arena[CurrentSeg],cur_size); if (cur_size >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if ((cur_size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { @@ -566,7 +566,7 @@ void *fxfAlloc(size_t size) { TMDBG(printf(" ptr:%p\n",(void *)ptr)); } } - DBG((df, "%p\n", (void *) ptr)); + DBG((df, "%p\n", (void *)ptr)); return ptr; } @@ -587,7 +587,7 @@ void fxfFree(void *ptr, size_t size) #else TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(((char const*)ptr)-Arena),(size_t_printf_type)size)); #endif - DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *) ptr, (size_t_printf_type) size)); + DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *)ptr, (size_t_printf_type) size)); assert(ptr || !size); if (!ptr) return; From 9e70b8b79f8d3e12c0155f8b4b02e2774a8ab18b Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 20 Sep 2023 07:30:50 -0400 Subject: [PATCH 059/111] Trying to avoid assigning a type to the pointers we return. --- DHT/fxf.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index bede4565fe..a05ea1b03f 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -414,7 +414,7 @@ void *fxfAlloc(size_t size) { static char const * const myname= "fxfAlloc"; #endif SizeHead *sh; - char *ptr= Nil(char); + void *ptr= Nil(void); TMDBG(printf("fxfAlloc - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); DBG((stderr, "%s(%" SIZE_T_PRINTF_SPECIFIER ") =", myname, (size_t_printf_type)size)); @@ -441,7 +441,7 @@ void *fxfAlloc(size_t size) { #ifdef SEGMENTED int ptrSegment; #endif - ptr= (char *)sh->FreeHead; + ptr= sh->FreeHead; if (size < sizeof sh->FreeHead) sh->FreeHead= Nil(void); else @@ -455,11 +455,11 @@ void *fxfAlloc(size_t size) { if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) break; } - ClrRange(ptr-Arena[ptrSegment], size); - TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)(ptr-Arena[ptrSegment]),sh->MallocCount)); + ClrRange((char *)ptr-Arena[ptrSegment], size); + TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)((char *)ptr-Arena[ptrSegment]),sh->MallocCount)); #else - ClrRange(ptr-Arena, size); - TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)(ptr-Arena),sh->MallocCount)); + ClrRange((char *)ptr-Arena, size); + TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)((char *)ptr-Arena),sh->MallocCount)); #endif } else { @@ -507,13 +507,13 @@ void *fxfAlloc(size_t size) { } else { /* fully aligned */ - ptr= TopFreePtr-= size; + ptr= (TopFreePtr-= size); } sh->MallocCount++; #ifdef SEGMENTED - TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,(ptrdiff_t_printf_type)(ptr-Arena[CurrentSeg]),sh->MallocCount)); + TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,(ptrdiff_t_printf_type)((char *)ptr-Arena[CurrentSeg]),sh->MallocCount)); #else - TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",(ptrdiff_t_printf_type)(ptr-Arena),sh->MallocCount)); + TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",(ptrdiff_t_printf_type)((char *)ptr-Arena),sh->MallocCount)); #endif } else @@ -559,14 +559,14 @@ void *fxfAlloc(size_t size) { goto START_LOOKING_FOR_CHUNK; } else - ptr= Nil(char); + ptr= Nil(void); #else /*SEGMENTED*/ - ptr= Nil(char); + ptr= Nil(void); #endif /*!SEGMENTED*/ - TMDBG(printf(" ptr:%p\n",(void *)ptr)); + TMDBG(printf(" ptr:%p\n", ptr)); } } - DBG((df, "%p\n", (void *)ptr)); + DBG((df, "%p\n", ptr)); return ptr; } From fec6d93252896340ff45afde7f2eba3fad24169c Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 20 Sep 2023 08:33:48 -0400 Subject: [PATCH 060/111] Let's stop pretending that SetRange and ClrRange are meaningful in the SEGMENTED case. --- DHT/fxf.c | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index a05ea1b03f..8f57bf5c99 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -23,7 +23,7 @@ # include typedef ptrdiff_t ptrdiff_t_printf_type; typedef size_t size_t_printf_type; -# ifdef UINTPTR_MAX +# if defined(UINTPTR_MAX) typedef uintptr_t pointer_to_int_type; # else typedef uintmax_t pointer_to_int_type; @@ -255,9 +255,6 @@ void PrintFreeMap(FILE *f) { fputc('?', f); } } -#else -static void SetRange(size_t x, size_t l) { (void) x; (void) l; } -static void ClrRange(size_t x, size_t l) { (void) x; (void) l; } #endif /*FREEMAP, !SEGMENTED*/ size_t fxfInit(size_t Size) { @@ -438,7 +435,7 @@ void *fxfAlloc(size_t size) { size= ALIGN_TO_MINIMUM(size); sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if (sh->FreeHead) { -#ifdef SEGMENTED +#if defined(SEGMENTED) int ptrSegment; #endif ptr= sh->FreeHead; @@ -448,24 +445,25 @@ void *fxfAlloc(size_t size) { memcpy(&sh->FreeHead, ptr, sizeof sh->FreeHead); sh->FreeCount--; sh->MallocCount++; -#ifdef SEGMENTED +#if defined(SEGMENTED) for (ptrSegment= CurrentSeg; ptrSegment > 0; --ptrSegment) { pointer_to_int_type tmp= (pointer_to_int_type)ptr; pointer_to_int_type segment_begin= (pointer_to_int_type)Arena[ptrSegment]; if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) break; } - ClrRange((char *)ptr-Arena[ptrSegment], size); TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)((char *)ptr-Arena[ptrSegment]),sh->MallocCount)); #else +# if defined(FREEMAP) ClrRange((char *)ptr-Arena, size); +# endif TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)((char *)ptr-Arena),sh->MallocCount)); #endif } else { /* we have to allocate a new piece */ size_t sizeCurrentSeg; -#ifdef SEGMENTED +#if defined(SEGMENTED) START_LOOKING_FOR_CHUNK: #endif sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); @@ -476,7 +474,7 @@ void *fxfAlloc(size_t size) { size_t needed_alignment_mask= PTRMASK; while (needed_alignment_mask >= size) needed_alignment_mask>>= 1; -#ifdef SEGMENTED +#if defined(SEGMENTED) size_t curBottomIndex= (BotFreePtr - Arena[CurrentSeg]); #else size_t curBottomIndex= (BotFreePtr - Arena); @@ -487,7 +485,9 @@ void *fxfAlloc(size_t size) { goto NEXT_SEGMENT; do { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); +#if defined(FREEMAP) && !defined(SEGMENTED) SetRange(curBottomIndex,cur_alignment); +#endif if (cur_alignment >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { @@ -510,7 +510,7 @@ void *fxfAlloc(size_t size) { ptr= (TopFreePtr-= size); } sh->MallocCount++; -#ifdef SEGMENTED +#if defined(SEGMENTED) TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,(ptrdiff_t_printf_type)((char *)ptr-Arena[CurrentSeg]),sh->MallocCount)); #else TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",(ptrdiff_t_printf_type)((char *)ptr-Arena),sh->MallocCount)); @@ -524,7 +524,6 @@ void *fxfAlloc(size_t size) { size_t curBottomIndex= (BotFreePtr - Arena[CurrentSeg]); while (curBottomIndex & PTRMASK) { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); - SetRange(curBottomIndex,cur_alignment); if (cur_alignment >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { @@ -540,7 +539,6 @@ void *fxfAlloc(size_t size) { } if (BotFreePtr < TopFreePtr) { size_t cur_size= (size_t)(TopFreePtr-BotFreePtr); - SetRange(BotFreePtr-Arena[CurrentSeg],cur_size); if (cur_size >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[(cur_size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if ((cur_size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { @@ -575,7 +573,7 @@ void fxfFree(void *ptr, size_t size) static char const * const myname= "fxfFree"; SizeHead *sh; -#ifdef SEGMENTED +#if defined(SEGMENTED) int ptrSegment; for (ptrSegment= CurrentSeg; ptrSegment > 0; --ptrSegment) { pointer_to_int_type tmp= (pointer_to_int_type)ptr; @@ -609,9 +607,7 @@ void fxfFree(void *ptr, size_t size) --sh->MallocCount; } else { -#ifdef SEGMENTED - SetRange((char *)ptr-Arena[ptrSegment],size); -#else +#if defined(FREEMAP) && !defined(SEGMENTED) SetRange((char *)ptr-Arena,size); #endif if ((size >= sizeof sh->FreeHead) || !sh->FreeHead) { @@ -633,9 +629,7 @@ void fxfFree(void *ptr, size_t size) --sh->MallocCount; } else { -#ifdef SEGMENTED - SetRange((char *)ptr-Arena[ptrSegment],size); -#else +#if defined(FREEMAP) && !defined(SEGMENTED) SetRange((char *)ptr-Arena,size); #endif if ((size >= sizeof sh->FreeHead) || !sh->FreeCount) { From 2908674755aaf4946ad7ed0cd802522f6ca345c4 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 20 Sep 2023 08:42:03 -0400 Subject: [PATCH 061/111] Saving the result of the arithmetic. Even if we're wrong about the pointer being in that array, what we print will at least be reasonable. --- DHT/fxf.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 8f57bf5c99..eaf03cb51f 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -437,6 +437,7 @@ void *fxfAlloc(size_t size) { if (sh->FreeHead) { #if defined(SEGMENTED) int ptrSegment; + ptrdiff_t_printf_type ptrIndex= -1; #endif ptr= sh->FreeHead; if (size < sizeof sh->FreeHead) @@ -446,13 +447,15 @@ void *fxfAlloc(size_t size) { sh->FreeCount--; sh->MallocCount++; #if defined(SEGMENTED) - for (ptrSegment= CurrentSeg; ptrSegment > 0; --ptrSegment) { + for (ptrSegment= CurrentSeg; ptrSegment >= 0; --ptrSegment) { pointer_to_int_type tmp= (pointer_to_int_type)ptr; pointer_to_int_type segment_begin= (pointer_to_int_type)Arena[ptrSegment]; - if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) + if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { + ptrIndex= (tmp - segment_begin); break; + } } - TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)((char *)ptr-Arena[ptrSegment]),sh->MallocCount)); + TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,ptrIndex,sh->MallocCount)); #else # if defined(FREEMAP) ClrRange((char *)ptr-Arena, size); @@ -575,13 +578,16 @@ void fxfFree(void *ptr, size_t size) #if defined(SEGMENTED) int ptrSegment; - for (ptrSegment= CurrentSeg; ptrSegment > 0; --ptrSegment) { + ptrdiff_t_printf_type ptrIndex= -1; + for (ptrSegment= CurrentSeg; ptrSegment >= 0; --ptrSegment) { pointer_to_int_type tmp= (pointer_to_int_type)ptr; pointer_to_int_type segment_begin= (pointer_to_int_type)Arena[ptrSegment]; - if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) + if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { + ptrIndex= (tmp - segment_begin); break; + } } - TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,(ptrdiff_t_printf_type)(((char const*)ptr)-Arena[ptrSegment]),(size_t_printf_type)size)); + TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,ptrIndex,(size_t_printf_type)size)); #else TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(((char const*)ptr)-Arena),(size_t_printf_type)size)); #endif From 1ecde2d8b17afb5bd03810da4bd29c3491402c34 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 20 Sep 2023 21:59:22 -0400 Subject: [PATCH 062/111] Lots of changes, attempting to avoid violating strict aliasing in any observable way. --- DHT/fxf.c | 142 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 86 insertions(+), 56 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index eaf03cb51f..055cf1e114 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -23,6 +23,7 @@ # include typedef ptrdiff_t ptrdiff_t_printf_type; typedef size_t size_t_printf_type; +# define MAX_HASHTABLE_SIZE PTRDIFF_MAX # if defined(UINTPTR_MAX) typedef uintptr_t pointer_to_int_type; # else @@ -34,12 +35,14 @@ typedef long long int ptrdiff_t_printf_type; typedef unsigned long long int size_t_printf_type; typedef unsigned long long int pointer_to_int_type; +# define MAX_HASHTABLE_SIZE LLONG_MAX # define PTRDIFF_T_PRINTF_SPECIFIER "lld" # define SIZE_T_PRINTF_SPECIFIER "llu" #else /* We don't have long long integer types. */ typedef long int ptrdiff_t_printf_type; typedef unsigned long int size_t_printf_type; typedef unsigned long int pointer_to_int_type; +# define MAX_HASHTABLE_SIZE LONG_MAX # define PTRDIFF_T_PRINTF_SPECIFIER "ld" # define SIZE_T_PRINTF_SPECIFIER "lu" #endif @@ -67,11 +70,11 @@ union MAX_ALIGNED_TYPE { # define MAX_ALIGNMENT (sizeof(union MAX_ALIGNED_TYPE) & -sizeof(union MAX_ALIGNED_TYPE)) #endif -#if !defined(Nil) && !defined(New) && !defined(nNew) && !defined(nNewCallc) /* TODO: Is this the correct check for all of the below lines? */ +#if !defined(Nil) && !defined(New) && !defined(nNewUntyped) && !defined(nNewCallocUntyped) /* TODO: Is this the correct check for all of the below lines? */ # define Nil(type) ((type *)0) # define New(type) ((type *)malloc(sizeof(type))) -# define nNew(n, type) ((type *)nNewImpl(n,sizeof(type))) -# define nNewCalloc(n, type) ((type *)calloc(n,sizeof(type))) +# define nNewUntyped(n, type) nNewImpl(n,sizeof(type)) +# define nNewCallocUntyped(n, type) calloc(n,sizeof(type)) static inline void * nNewImpl(size_t const nmemb, size_t const size) { return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : malloc(nmemb*size)); } @@ -158,16 +161,16 @@ static SizeHead SizeData[1 + (fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMA #if defined(SEGMENTED) /* #define ARENA_SEG_SIZE 32000 */ /* #define ARENA_SEG_COUNT ((1024*1024)/ARENA_SEG_SIZE) */ -static char *Arena[ARENA_SEG_COUNT] = { Nil(char) }; +static void *Arena[ARENA_SEG_COUNT]= { Nil(void) }; static int ArenaSegCnt= 0; static int CurrentSeg= 0; #else -static char *Arena= Nil(char); +static void *Arena= Nil(void); #endif /*SEGMENTED*/ static size_t GlobalSize; -static char *BotFreePtr; -static char *TopFreePtr; +static void *BotFreePtr; +static void *TopFreePtr; #undef FREEMAP @@ -257,21 +260,34 @@ void PrintFreeMap(FILE *f) { } #endif /*FREEMAP, !SEGMENTED*/ +static ptrdiff_t_printf_type pointerDifference(void const *ptr1, void const *ptr2) { + return (((char const *)ptr1) - ((char const *)ptr2)); +} + +static void * stepPointer(void *ptr, ptrdiff_t_printf_type step) { + return (void *)(((char *)ptr) + step); +} + size_t fxfInit(size_t Size) { #if defined(LOG) static char const * const myname= "fxfInit"; #endif #if defined(SEGMENTED) - size_t maxSegCnt= (Size ? (1 + ((Size - 1) / ARENA_SEG_SIZE)) : 0); + size_t maxSegCnt; +#endif + if (Size > MAX_HASHTABLE_SIZE) + Size = MAX_HASHTABLE_SIZE; +#if defined(SEGMENTED) + maxSegCnt= (Size / ARENA_SEG_SIZE); if (maxSegCnt > ARENA_SEG_COUNT) maxSegCnt= ARENA_SEG_COUNT; while (ArenaSegCnt > maxSegCnt) { --ArenaSegCnt; free(Arena[ArenaSegCnt]); - Arena[ArenaSegCnt]= Nil(char); + Arena[ArenaSegCnt]= Nil(void); } while (ArenaSegCnt < maxSegCnt) { - if ((Arena[ArenaSegCnt]= nNew(ARENA_SEG_SIZE, char)) == Nil(char)) + if (!(Arena[ArenaSegCnt]= nNewUntyped(ARENA_SEG_SIZE, char))) break; ++ArenaSegCnt; } @@ -279,14 +295,14 @@ size_t fxfInit(size_t Size) { BotFreePtr= Arena[CurrentSeg]; TopFreePtr= Arena[CurrentSeg]; if (TopFreePtr) - TopFreePtr+= ARENA_SEG_SIZE; + TopFreePtr= stepPointer(TopFreePtr, ARENA_SEG_SIZE); GlobalSize= ArenaSegCnt*ARENA_SEG_SIZE; #else #if defined(FREEMAP) if (FreeMap) { free(FreeMap); - FreeMap = Nil(FreeMapType); + FreeMap= Nil(FreeMapType); } #endif if (Arena) @@ -295,7 +311,7 @@ size_t fxfInit(size_t Size) { Size= MAX_ALIGNMENT; else Size&= ~(MAX_ALIGNMENT - 1U); - if ((Arena=nNew(Size, char)) == Nil(char)) { + if (!(Arena=nNewUntyped(Size, char))) { ERROR_LOG2("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " bytes\n", myname, (size_t_printf_type) Size); BotFreePtr= Arena; @@ -304,7 +320,7 @@ size_t fxfInit(size_t Size) { return GlobalSize; } BotFreePtr= Arena; - TopFreePtr= Arena+Size; + TopFreePtr= stepPointer(Arena, (ptrdiff_t_printf_type)Size); GlobalSize= Size; #if defined(FREEMAP) @@ -318,11 +334,11 @@ size_t fxfInit(size_t Size) { Size = ((Size+31)>>5); } - FreeMap= nNewCalloc(Size, FreeMapType); + FreeMap= (FreeMapType *)nNewCallocUntyped(Size, FreeMapType); #endif /*FREEMAP*/ #endif /*SEGMENTED*/ - memset(SizeData, '\0', sizeof(SizeData)); + memset(SizeData, '\0', sizeof SizeData); MIN_ALIGNMENT= MAX_ALIGNMENT; while (MIN_ALIGNMENT > fxfMINSIZE) @@ -338,29 +354,29 @@ void fxfTeardown(void) { --ArenaSegCnt; free(Arena[ArenaSegCnt]); - Arena[ArenaSegCnt] = Nil(char); + Arena[ArenaSegCnt]= Nil(void); } CurrentSeg= 0; #else #if defined(FREEMAP) free(FreeMap); - FreeMap= Nil(unsigned int); + FreeMap= Nil(FreeMapType); #endif /*FREEMAP*/ free(Arena); - Arena= Nil(char); + Arena= Nil(void); #endif /*SEGMENTED*/ - memset(SizeData, '\0', sizeof(SizeData)); + memset(SizeData, '\0', sizeof SizeData); GlobalSize= 0; - TopFreePtr= Nil(char); - BotFreePtr= Nil(char); + TopFreePtr= Nil(void); + BotFreePtr= Nil(void); } int fxfInitialised(void) { #if defined(SEGMENTED) - return Arena[0]!=0; + return !!Arena[0]; #else - return Arena!=0; + return !!Arena; #endif } @@ -371,12 +387,12 @@ void fxfReset(void) BotFreePtr= Arena[CurrentSeg]; TopFreePtr= Arena[CurrentSeg]; if (TopFreePtr) - TopFreePtr+= ARENA_SEG_SIZE; + TopFreePtr= stepPointer(TopFreePtr, ARENA_SEG_SIZE); #else BotFreePtr= Arena; TopFreePtr= Arena; if (TopFreePtr) - TopFreePtr+= GlobalSize; + TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t_printf_type)GlobalSize); #if defined(FREEMAP) if (FreeMap) @@ -458,9 +474,9 @@ void *fxfAlloc(size_t size) { TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,ptrIndex,sh->MallocCount)); #else # if defined(FREEMAP) - ClrRange((char *)ptr-Arena, size); + ClrRange(pointerDifference(ptr, Arena), size); # endif - TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)((char *)ptr-Arena),sh->MallocCount)); + TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,pointerDifference(ptr, Arena),sh->MallocCount)); #endif } else { @@ -469,7 +485,7 @@ void *fxfAlloc(size_t size) { #if defined(SEGMENTED) START_LOOKING_FOR_CHUNK: #endif - sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); + sizeCurrentSeg = (size_t)pointerDifference(TopFreePtr,BotFreePtr); TMDBG(printf(" sizeCurrentSeg:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)sizeCurrentSeg)); if (sizeCurrentSeg>=size) { if (size&PTRMASK) { @@ -478,9 +494,9 @@ void *fxfAlloc(size_t size) { while (needed_alignment_mask >= size) needed_alignment_mask>>= 1; #if defined(SEGMENTED) - size_t curBottomIndex= (BotFreePtr - Arena[CurrentSeg]); + size_t curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena[CurrentSeg]); #else - size_t curBottomIndex= (BotFreePtr - Arena); + size_t curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena); #endif if (curBottomIndex & needed_alignment_mask) { size_t const numBytesToAdd= (needed_alignment_mask - (curBottomIndex & needed_alignment_mask)) + 1U; @@ -501,22 +517,22 @@ void *fxfAlloc(size_t size) { TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); } } - BotFreePtr+= cur_alignment; + BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t_printf_type)cur_alignment); curBottomIndex+= cur_alignment; } while (curBottomIndex & needed_alignment_mask); } ptr= BotFreePtr; - BotFreePtr+= size; + BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t_printf_type)size); } else { /* fully aligned */ - ptr= (TopFreePtr-= size); + ptr= (TopFreePtr= stepPointer(TopFreePtr, -(ptrdiff_t_printf_type)size)); } sh->MallocCount++; #if defined(SEGMENTED) - TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,(ptrdiff_t_printf_type)((char *)ptr-Arena[CurrentSeg]),sh->MallocCount)); + TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,pointerDifference(ptr, Arena[CurrentSeg]),sh->MallocCount)); #else - TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",(ptrdiff_t_printf_type)((char *)ptr-Arena),sh->MallocCount)); + TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",pointerDifference(ptr,Arena),sh->MallocCount)); #endif } else @@ -537,7 +553,7 @@ void *fxfAlloc(size_t size) { TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); } } - BotFreePtr+= cur_alignment; + BotFreePtr= stepPointer(BotFreePtr, cur_alignment); curBottomIndex+= cur_alignment; } if (BotFreePtr < TopFreePtr) { @@ -556,7 +572,7 @@ void *fxfAlloc(size_t size) { TMDBG(fputs(" next seg", stdout)); ++CurrentSeg; BotFreePtr= Arena[CurrentSeg]; - TopFreePtr= Arena[CurrentSeg]+ARENA_SEG_SIZE; + TopFreePtr= stepPointer(Arena[CurrentSeg], ARENA_SEG_SIZE); goto START_LOOKING_FOR_CHUNK; } else @@ -576,25 +592,31 @@ void fxfFree(void *ptr, size_t size) static char const * const myname= "fxfFree"; SizeHead *sh; + ptrdiff_t_printf_type ptrIndex; #if defined(SEGMENTED) int ptrSegment; - ptrdiff_t_printf_type ptrIndex= -1; +#endif + assert(ptr || !size); + if (!ptr) + return; +#if defined(SEGMENTED) for (ptrSegment= CurrentSeg; ptrSegment >= 0; --ptrSegment) { pointer_to_int_type tmp= (pointer_to_int_type)ptr; pointer_to_int_type segment_begin= (pointer_to_int_type)Arena[ptrSegment]; - if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { + if (tmp >= segment_begin) { ptrIndex= (tmp - segment_begin); - break; + if (ptrIndex < ARENA_SEG_SIZE) + goto FOUND_PUTATIVE_SEGMENT; } } + ptrIndex= -1; +FOUND_PUTATIVE_SEGMENT: TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,ptrIndex,(size_t_printf_type)size)); #else - TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(((char const*)ptr)-Arena),(size_t_printf_type)size)); + ptrIndex= pointerDifference(ptr,Arena); + TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrIndex,(size_t_printf_type)size)); #endif DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *)ptr, (size_t_printf_type) size)); - assert(ptr || !size); - if (!ptr) - return; if (size < fxfMINSIZE) size= fxfMINSIZE; if (size > fxfMAXSIZE) { @@ -603,18 +625,26 @@ void fxfFree(void *ptr, size_t size) exit(-5); } size= ALIGN_TO_MINIMUM(size); +#if !defined(NDEBUG) + if (ptrIndex > 0) { + size_t needed_alignment= MAX_ALIGNMENT; + while (needed_alignment > size) + needed_alignment>>= 1; + assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); + } +#endif sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if (size&PTRMASK) { /* not fully aligned size */ - TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(BotFreePtr-(char const*)ptr))); - if (((char *)ptr+size) == BotFreePtr) { - BotFreePtr= (char *)ptr; - TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(TopFreePtr-BotFreePtr))); + TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,pointerDifference(BotFreePtr,ptr))); + if (stepPointer(ptr, (ptrdiff_t_printf_type)size) == BotFreePtr) { + BotFreePtr= ptr; + TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,pointerDifference(TopFreePtr,BotFreePtr))); --sh->MallocCount; } else { #if defined(FREEMAP) && !defined(SEGMENTED) - SetRange((char *)ptr-Arena,size); + SetRange((pointerDifference(ptr,Arena),size); #endif if ((size >= sizeof sh->FreeHead) || !sh->FreeHead) { if (size >= sizeof sh->FreeHead) @@ -628,15 +658,15 @@ void fxfFree(void *ptr, size_t size) } else { /* fully aligned size */ - TMDBG(printf(" ptr-TopFreePtr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(((char const*)ptr)-TopFreePtr))); - if ((char *)ptr == TopFreePtr) { - TopFreePtr+= size; - TMDBG(printf(" TopFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(TopFreePtr-BotFreePtr))); + TMDBG(printf(" ptr-TopFreePtr:%" PTRDIFF_T_PRINTF_SPECIFIER,pointerDifference(ptr,TopFreePtr))); + if (ptr == TopFreePtr) { + TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t_printf_type)size); + TMDBG(printf(" TopFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,pointerDifference(TopFreePtr,BotFreePtr))); --sh->MallocCount; } else { #if defined(FREEMAP) && !defined(SEGMENTED) - SetRange((char *)ptr-Arena,size); + SetRange(pointerDifference(ptr,Arena),size); #endif if ((size >= sizeof sh->FreeHead) || !sh->FreeCount) { if (size >= sizeof sh->FreeHead) @@ -688,7 +718,7 @@ size_t fxfTotal(void) { void fxfInfo(FILE *f) { size_t const one_kilo = 1<<10; - size_t const sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); + size_t const sizeCurrentSeg = (size_t)pointerDifference(TopFreePtr,BotFreePtr); size_t const sizeArenaUsed = GlobalSize-sizeCurrentSeg #if defined(SEGMENTED) From 28eb69f9a0f137211c2f481491309e09867e5f52 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 20 Sep 2023 22:48:30 -0400 Subject: [PATCH 063/111] We don't need that check. --- DHT/fxf.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 055cf1e114..458290de36 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -23,7 +23,6 @@ # include typedef ptrdiff_t ptrdiff_t_printf_type; typedef size_t size_t_printf_type; -# define MAX_HASHTABLE_SIZE PTRDIFF_MAX # if defined(UINTPTR_MAX) typedef uintptr_t pointer_to_int_type; # else @@ -35,14 +34,12 @@ typedef long long int ptrdiff_t_printf_type; typedef unsigned long long int size_t_printf_type; typedef unsigned long long int pointer_to_int_type; -# define MAX_HASHTABLE_SIZE LLONG_MAX # define PTRDIFF_T_PRINTF_SPECIFIER "lld" # define SIZE_T_PRINTF_SPECIFIER "llu" #else /* We don't have long long integer types. */ typedef long int ptrdiff_t_printf_type; typedef unsigned long int size_t_printf_type; typedef unsigned long int pointer_to_int_type; -# define MAX_HASHTABLE_SIZE LONG_MAX # define PTRDIFF_T_PRINTF_SPECIFIER "ld" # define SIZE_T_PRINTF_SPECIFIER "lu" #endif @@ -273,12 +270,7 @@ size_t fxfInit(size_t Size) { static char const * const myname= "fxfInit"; #endif #if defined(SEGMENTED) - size_t maxSegCnt; -#endif - if (Size > MAX_HASHTABLE_SIZE) - Size = MAX_HASHTABLE_SIZE; -#if defined(SEGMENTED) - maxSegCnt= (Size / ARENA_SEG_SIZE); + size_t maxSegCnt= (Size / ARENA_SEG_SIZE); if (maxSegCnt > ARENA_SEG_COUNT) maxSegCnt= ARENA_SEG_COUNT; while (ArenaSegCnt > maxSegCnt) { From 231273a8743d1221d03343dce653e3872448a987 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 21 Sep 2023 06:15:26 -0400 Subject: [PATCH 064/111] We don't need that assertion. If people try to allocate and fail, they should be able to free with the same parameters. --- DHT/fxf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 458290de36..03142784f1 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -588,7 +588,6 @@ void fxfFree(void *ptr, size_t size) #if defined(SEGMENTED) int ptrSegment; #endif - assert(ptr || !size); if (!ptr) return; #if defined(SEGMENTED) From a1cd0a9026d2d8722d85574ed1309380fea5140d Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 21 Sep 2023 14:11:25 -0400 Subject: [PATCH 065/111] We don't need those #includes since they're in fxf.h. --- DHT/fxf.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 03142784f1..057f22c70d 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -3,7 +3,6 @@ #include "debugging/assert.h" #include #include -#include #include #if defined(__TURBOC__) @@ -19,7 +18,6 @@ #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || /* >= C99 -- We have printf ptrdiff_t/size_t specifiers. */ \ (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 -- We have printf ptrdiff_t/size_t specifiers. */ -# include # include typedef ptrdiff_t ptrdiff_t_printf_type; typedef size_t size_t_printf_type; From 8a524feb4eaee6753b03695b46a5a543527087a2 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 21 Sep 2023 16:42:47 -0400 Subject: [PATCH 066/111] Small tweaks. --- DHT/fxf.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 057f22c70d..9111e07dcd 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -255,11 +255,11 @@ void PrintFreeMap(FILE *f) { } #endif /*FREEMAP, !SEGMENTED*/ -static ptrdiff_t_printf_type pointerDifference(void const *ptr1, void const *ptr2) { +static inline ptrdiff_t_printf_type pointerDifference(void const *ptr1, void const *ptr2) { return (((char const *)ptr1) - ((char const *)ptr2)); } -static void * stepPointer(void *ptr, ptrdiff_t_printf_type step) { +static inline void * stepPointer(void *ptr, ptrdiff_t_printf_type step) { return (void *)(((char *)ptr) + step); } @@ -277,7 +277,8 @@ size_t fxfInit(size_t Size) { Arena[ArenaSegCnt]= Nil(void); } while (ArenaSegCnt < maxSegCnt) { - if (!(Arena[ArenaSegCnt]= nNewUntyped(ARENA_SEG_SIZE, char))) + Arena[ArenaSegCnt]= nNewUntyped(ARENA_SEG_SIZE, char); + if (!Arena[ArenaSegCnt]) break; ++ArenaSegCnt; } @@ -301,7 +302,8 @@ size_t fxfInit(size_t Size) { Size= MAX_ALIGNMENT; else Size&= ~(MAX_ALIGNMENT - 1U); - if (!(Arena=nNewUntyped(Size, char))) { + Arena= nNewUntyped(Size, char); + if (!Arena) { ERROR_LOG2("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " bytes\n", myname, (size_t_printf_type) Size); BotFreePtr= Arena; From ad2ed9097274d15f55299c636c4067c4576963df Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 21 Sep 2023 20:02:19 -0400 Subject: [PATCH 067/111] Never allocate more than the user requested. --- DHT/fxf.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 9111e07dcd..c161268173 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -298,10 +298,7 @@ size_t fxfInit(size_t Size) { #endif if (Arena) free(Arena); - if (Size < MAX_ALIGNMENT) - Size= MAX_ALIGNMENT; - else - Size&= ~(MAX_ALIGNMENT - 1U); + Size&= ~(MAX_ALIGNMENT - 1U); Arena= nNewUntyped(Size, char); if (!Arena) { ERROR_LOG2("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " bytes\n", From ac621ce413744c627d316772fe58e5cb5acdea57 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 25 Sep 2023 17:50:32 -0400 Subject: [PATCH 068/111] We need pointer differences to be accurate, so we can't allocate too much. --- DHT/fxf.c | 105 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 47 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index c161268173..1b4c0d4c2b 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -21,25 +21,29 @@ # include typedef ptrdiff_t ptrdiff_t_printf_type; typedef size_t size_t_printf_type; -# if defined(UINTPTR_MAX) - typedef uintptr_t pointer_to_int_type; -# else - typedef uintmax_t pointer_to_int_type; -# endif +# define MAX_POINTER_DIFFERENCE PTRDIFF_MAX +# if defined(UINTPTR_MAX) + typedef uintptr_t convert_pointer_to_int_type; +# else + typedef uintmax_t convert_pointer_to_int_type; +# endif # define PTRDIFF_T_PRINTF_SPECIFIER "td" # define SIZE_T_PRINTF_SPECIFIER "zu" -#elif defined(LLONG_MAX) /* We have long long integer types. */ +#else +# define MAX_POINTER_DIFFERENCE (((size_t)-1)>>1) /* just a guess */ +# if defined(LLONG_MAX) /* We have long long integer types. */ typedef long long int ptrdiff_t_printf_type; typedef unsigned long long int size_t_printf_type; - typedef unsigned long long int pointer_to_int_type; -# define PTRDIFF_T_PRINTF_SPECIFIER "lld" -# define SIZE_T_PRINTF_SPECIFIER "llu" -#else /* We don't have long long integer types. */ + typedef unsigned long long int convert_pointer_to_int_type; +# define PTRDIFF_T_PRINTF_SPECIFIER "lld" +# define SIZE_T_PRINTF_SPECIFIER "llu" +# else /* We don't have long long integer types. */ typedef long int ptrdiff_t_printf_type; typedef unsigned long int size_t_printf_type; - typedef unsigned long int pointer_to_int_type; -# define PTRDIFF_T_PRINTF_SPECIFIER "ld" -# define SIZE_T_PRINTF_SPECIFIER "lu" + typedef unsigned long int convert_pointer_to_int_type; +# define PTRDIFF_T_PRINTF_SPECIFIER "ld" +# define SIZE_T_PRINTF_SPECIFIER "lu" +# endif #endif #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) @@ -114,10 +118,12 @@ typedef struct { void * FreeHead; } SizeHead; +#define CLIP_TO_MAX_POINTER_DIFFERENCE(x) (((x) > MAX_POINTER_DIFFERENCE) ? MAX_POINTER_DIFFERENCE : (x)) + #if defined(DOS) /* MSDOS 16 Bit support (maxmemory <= 1 MB) */ #define SEGMENTED -#define ARENA_SEG_SIZE (32000 & ~(MAX_ALIGNMENT - 1U)) +#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(32000) & ~(MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((1024*1024)/ARENA_SEG_SIZE) #define OSNAME "MSDOS" #define OSMAXMEM "1 MB" @@ -125,7 +131,7 @@ typedef struct { /* Win95/Win98/WinME can only allocate chunks up to 255 MB */ /* maxmemory <= 768 MB */ #define SEGMENTED -#define ARENA_SEG_SIZE (1000000 & ~(MAX_ALIGNMENT - 1U)) +#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(1000000) & ~(MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((768*1024*1024)/ARENA_SEG_SIZE) #define OSNAME "Win95/Win98/WinME" #define OSMAXMEM "768 MB" @@ -147,7 +153,7 @@ enum #define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. (Of course, they're probably the same.) TODO: Can we compute what we want at compile time and just use it? */ -static size_t MIN_ALIGNMENT= MIN_ALIGNMENT_UNDERESTIMATE; /* for now */ +static size_t min_alignment= 0; /* for now */ static SizeHead SizeData[1 + (fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). Maximum allocation is fxfMAXSIZE. @@ -255,11 +261,11 @@ void PrintFreeMap(FILE *f) { } #endif /*FREEMAP, !SEGMENTED*/ -static inline ptrdiff_t_printf_type pointerDifference(void const *ptr1, void const *ptr2) { +static inline ptrdiff_t pointerDifference(void const *ptr1, void const *ptr2) { return (((char const *)ptr1) - ((char const *)ptr2)); } -static inline void * stepPointer(void *ptr, ptrdiff_t_printf_type step) { +static inline void * stepPointer(void *ptr, ptrdiff_t step) { return (void *)(((char *)ptr) + step); } @@ -298,18 +304,20 @@ size_t fxfInit(size_t Size) { #endif if (Arena) free(Arena); + if (Size > MAX_POINTER_DIFFERENCE) + Size= MAX_POINTER_DIFFERENCE; Size&= ~(MAX_ALIGNMENT - 1U); Arena= nNewUntyped(Size, char); if (!Arena) { ERROR_LOG2("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " bytes\n", - myname, (size_t_printf_type) Size); + myname, (size_t_printf_type)Size); BotFreePtr= Arena; TopFreePtr= Arena; GlobalSize= 0; return GlobalSize; } BotFreePtr= Arena; - TopFreePtr= stepPointer(Arena, (ptrdiff_t_printf_type)Size); + TopFreePtr= stepPointer(Arena, (ptrdiff_t)Size); GlobalSize= Size; #if defined(FREEMAP) @@ -329,9 +337,12 @@ size_t fxfInit(size_t Size) { memset(SizeData, '\0', sizeof SizeData); - MIN_ALIGNMENT= MAX_ALIGNMENT; - while (MIN_ALIGNMENT > fxfMINSIZE) - MIN_ALIGNMENT>>= 1; + if (!min_alignment) + { + min_alignment= MAX_ALIGNMENT; + while (min_alignment > fxfMINSIZE) + min_alignment>>= 1; + } return GlobalSize; } @@ -381,7 +392,7 @@ void fxfReset(void) BotFreePtr= Arena; TopFreePtr= Arena; if (TopFreePtr) - TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t_printf_type)GlobalSize); + TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t)GlobalSize); #if defined(FREEMAP) if (FreeMap) @@ -407,7 +418,7 @@ void fxfReset(void) * Intel *86 type of CPU, but also there, aligned access is faster. */ #define PTRMASK (MAX_ALIGNMENT-1U) -#define ALIGN_TO_MINIMUM(s) (((s) + (MIN_ALIGNMENT - 1U)) & ~(MIN_ALIGNMENT - 1U)) +#define ALIGN_TO_MINIMUM(s) (((s) + (min_alignment - 1U)) & ~(min_alignment - 1U)) #define TMDBG(x) if (0) x @@ -436,13 +447,13 @@ void *fxfAlloc(size_t size) { return Nil(void); } - // Round up to a multiple of MIN_ALIGNMENT + // Round up to a multiple of min_alignment size= ALIGN_TO_MINIMUM(size); sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if (sh->FreeHead) { #if defined(SEGMENTED) int ptrSegment; - ptrdiff_t_printf_type ptrIndex= -1; + ptrdiff_t ptrIndex= -1; #endif ptr= sh->FreeHead; if (size < sizeof sh->FreeHead) @@ -453,19 +464,19 @@ void *fxfAlloc(size_t size) { sh->MallocCount++; #if defined(SEGMENTED) for (ptrSegment= CurrentSeg; ptrSegment >= 0; --ptrSegment) { - pointer_to_int_type tmp= (pointer_to_int_type)ptr; - pointer_to_int_type segment_begin= (pointer_to_int_type)Arena[ptrSegment]; + convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; + convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { ptrIndex= (tmp - segment_begin); break; } } - TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,ptrIndex,sh->MallocCount)); + TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,sh->MallocCount)); #else # if defined(FREEMAP) ClrRange(pointerDifference(ptr, Arena), size); # endif - TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,pointerDifference(ptr, Arena),sh->MallocCount)); + TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)pointerDifference(ptr, Arena),sh->MallocCount)); #endif } else { @@ -506,22 +517,22 @@ void *fxfAlloc(size_t size) { TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); } } - BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t_printf_type)cur_alignment); + BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)cur_alignment); curBottomIndex+= cur_alignment; } while (curBottomIndex & needed_alignment_mask); } ptr= BotFreePtr; - BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t_printf_type)size); + BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)size); } else { /* fully aligned */ - ptr= (TopFreePtr= stepPointer(TopFreePtr, -(ptrdiff_t_printf_type)size)); + ptr= (TopFreePtr= stepPointer(TopFreePtr, -(ptrdiff_t)size)); } sh->MallocCount++; #if defined(SEGMENTED) - TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,pointerDifference(ptr, Arena[CurrentSeg]),sh->MallocCount)); + TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,(ptrdiff_t_printf_type)pointerDifference(ptr, Arena[CurrentSeg]),sh->MallocCount)); #else - TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",pointerDifference(ptr,Arena),sh->MallocCount)); + TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",(ptrdiff_t_printf_type)pointerDifference(ptr,Arena),sh->MallocCount)); #endif } else @@ -581,7 +592,7 @@ void fxfFree(void *ptr, size_t size) static char const * const myname= "fxfFree"; SizeHead *sh; - ptrdiff_t_printf_type ptrIndex; + ptrdiff_t ptrIndex; #if defined(SEGMENTED) int ptrSegment; #endif @@ -589,8 +600,8 @@ void fxfFree(void *ptr, size_t size) return; #if defined(SEGMENTED) for (ptrSegment= CurrentSeg; ptrSegment >= 0; --ptrSegment) { - pointer_to_int_type tmp= (pointer_to_int_type)ptr; - pointer_to_int_type segment_begin= (pointer_to_int_type)Arena[ptrSegment]; + convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; + convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; if (tmp >= segment_begin) { ptrIndex= (tmp - segment_begin); if (ptrIndex < ARENA_SEG_SIZE) @@ -599,10 +610,10 @@ void fxfFree(void *ptr, size_t size) } ptrIndex= -1; FOUND_PUTATIVE_SEGMENT: - TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,ptrIndex,(size_t_printf_type)size)); + TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); #else ptrIndex= pointerDifference(ptr,Arena); - TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrIndex,(size_t_printf_type)size)); + TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); #endif DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *)ptr, (size_t_printf_type) size)); if (size < fxfMINSIZE) @@ -624,10 +635,10 @@ void fxfFree(void *ptr, size_t size) sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if (size&PTRMASK) { /* not fully aligned size */ - TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,pointerDifference(BotFreePtr,ptr))); - if (stepPointer(ptr, (ptrdiff_t_printf_type)size) == BotFreePtr) { + TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(BotFreePtr,ptr))); + if (stepPointer(ptr, (ptrdiff_t)size) == BotFreePtr) { BotFreePtr= ptr; - TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,pointerDifference(TopFreePtr,BotFreePtr))); + TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(TopFreePtr,BotFreePtr))); --sh->MallocCount; } else { @@ -646,10 +657,10 @@ void fxfFree(void *ptr, size_t size) } else { /* fully aligned size */ - TMDBG(printf(" ptr-TopFreePtr:%" PTRDIFF_T_PRINTF_SPECIFIER,pointerDifference(ptr,TopFreePtr))); + TMDBG(printf(" ptr-TopFreePtr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(ptr,TopFreePtr))); if (ptr == TopFreePtr) { - TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t_printf_type)size); - TMDBG(printf(" TopFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,pointerDifference(TopFreePtr,BotFreePtr))); + TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t)size); + TMDBG(printf(" TopFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(TopFreePtr,BotFreePtr))); --sh->MallocCount; } else { From b10c9c6b1fa57e690dfb417e5985c5e6f6439037 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 25 Sep 2023 21:54:37 -0400 Subject: [PATCH 069/111] Outputting the amount requested as well as the amount we actually tried to allocate. Also, we shouldn't assume that (void *)NULL is represented by all 0x00s. --- DHT/fxf.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 1b4c0d4c2b..5dc7f39555 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -301,6 +301,9 @@ size_t fxfInit(size_t Size) { free(FreeMap); FreeMap= Nil(FreeMapType); } +#endif +#if defined(LOG) + size_t const orig_Size= Size; #endif if (Arena) free(Arena); @@ -309,8 +312,8 @@ size_t fxfInit(size_t Size) { Size&= ~(MAX_ALIGNMENT - 1U); Arena= nNewUntyped(Size, char); if (!Arena) { - ERROR_LOG2("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " bytes\n", - myname, (size_t_printf_type)Size); + ERROR_LOG3("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " <= %" SIZE_T_PRINTF_SPECIFIER " bytes\n", + myname, (size_t_printf_type)Size, (size_t_printf_type)orig_Size); BotFreePtr= Arena; TopFreePtr= Arena; GlobalSize= 0; @@ -335,7 +338,12 @@ size_t fxfInit(size_t Size) { #endif /*FREEMAP*/ #endif /*SEGMENTED*/ - memset(SizeData, '\0', sizeof SizeData); + for (Size= 0; Size < ((sizeof SizeData)/(sizeof *SizeData)); ++Size) + { + SizeData[Size].MallocCount= 0; + SizeData[Size].FreeCount= 0; + SizeData[Size].FreeHead= Nil(void); + } if (!min_alignment) { @@ -365,7 +373,15 @@ void fxfTeardown(void) free(Arena); Arena= Nil(void); #endif /*SEGMENTED*/ - memset(SizeData, '\0', sizeof SizeData); + { + size_t i; + for (i= 0; i < ((sizeof SizeData)/(sizeof *SizeData)); ++i) + { + SizeData[i].MallocCount= 0; + SizeData[i].FreeCount= 0; + SizeData[i].FreeHead= Nil(void); + } + } GlobalSize= 0; TopFreePtr= Nil(void); BotFreePtr= Nil(void); @@ -408,7 +424,15 @@ void fxfReset(void) } #endif - memset(SizeData, '\0', sizeof SizeData); + { + size_t i; + for (i= 0; i < ((sizeof SizeData)/(sizeof *SizeData)); ++i) + { + SizeData[i].MallocCount= 0; + SizeData[i].FreeCount= 0; + SizeData[i].FreeHead= Nil(void); + } + } } /* we have to define the following, since some architectures cannot From 6af5725c66c518389b064fa7248a502d230a8aed Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 28 Sep 2023 19:31:42 -0400 Subject: [PATCH 070/111] Ensuring that these values are correct even if we later change fxfMINSIZE to 0. --- DHT/fxf.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 5dc7f39555..bc2dfc1cb3 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -150,14 +150,16 @@ enum fxfMINSIZE = sizeof(void *) }; #define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) -#define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. - (Of course, they're probably the same.) - TODO: Can we compute what we want at compile time and just use it? */ -static size_t min_alignment= 0; /* for now */ - -static SizeHead SizeData[1 + (fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). - Maximum allocation is fxfMAXSIZE. - All allocations will be multiples of MIN_ALIGNMENT_UNDERESTIMATE. */ +#define MIN_ALIGNMENT_UNDERESTIMATE ((fxfMINSIZE > 0) ? \ + ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) : \ + 1) /* We'd prefer the top bit, but we'll compute that during fxfInit. + (Of course, they're probably the same.) + TODO: Can we compute what we want at compile time and just use it? */ +static size_t min_alignment= (fxfMINSIZE <= 0); /* for now */ + +static SizeHead SizeData[1 + (fxfMAXSIZE - ((fxfMINSIZE > 0) ? fxfMINSIZE : 0))/MIN_ALIGNMENT_UNDERESTIMATE]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). + Maximum allocation is fxfMAXSIZE. + All allocations will be multiples of MIN_ALIGNMENT_UNDERESTIMATE. */ #if defined(SEGMENTED) /* #define ARENA_SEG_SIZE 32000 */ From 33066e4712861f8c49d9771fa84990c575080f1a Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 29 Sep 2023 17:52:12 -0400 Subject: [PATCH 071/111] Ensuring values are sensible. --- DHT/fxf.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/DHT/fxf.c b/DHT/fxf.c index bc2dfc1cb3..3b4f48874e 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -149,6 +149,13 @@ enum { fxfMINSIZE = sizeof(void *) }; + +enum { + ENSURE_FXFMINSIZE_GE_0 = 1/(fxfMINSIZE >= 0), + ENSURE_FXFMAXSIZE_GT_0 = 1/(fxfMAXSIZE > 0), + ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE) +}; + #define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) #define MIN_ALIGNMENT_UNDERESTIMATE ((fxfMINSIZE > 0) ? \ ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) : \ From 9a432091b9f7ee473b7729c66617a290400e0d12 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 29 Sep 2023 21:21:15 -0400 Subject: [PATCH 072/111] We're requiring the sizes to be nonnegative, so we can simplify the conditionals. --- DHT/fxf.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 3b4f48874e..fe3806fa5c 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -157,16 +157,16 @@ enum { }; #define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) -#define MIN_ALIGNMENT_UNDERESTIMATE ((fxfMINSIZE > 0) ? \ +#define MIN_ALIGNMENT_UNDERESTIMATE (fxfMINSIZE ? \ ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) : \ 1) /* We'd prefer the top bit, but we'll compute that during fxfInit. (Of course, they're probably the same.) TODO: Can we compute what we want at compile time and just use it? */ -static size_t min_alignment= (fxfMINSIZE <= 0); /* for now */ +static size_t min_alignment= !fxfMINSIZE; /* for now */ -static SizeHead SizeData[1 + (fxfMAXSIZE - ((fxfMINSIZE > 0) ? fxfMINSIZE : 0))/MIN_ALIGNMENT_UNDERESTIMATE]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). - Maximum allocation is fxfMAXSIZE. - All allocations will be multiples of MIN_ALIGNMENT_UNDERESTIMATE. */ +static SizeHead SizeData[1 + ((fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE)]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). + Maximum allocation is fxfMAXSIZE. + All allocations will be multiples of MIN_ALIGNMENT_UNDERESTIMATE. */ #if defined(SEGMENTED) /* #define ARENA_SEG_SIZE 32000 */ From b07e77d985aef7ad78600ad756d3815f8b36668e Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 6 Oct 2023 22:34:36 -0400 Subject: [PATCH 073/111] Going with what is apparently the canonical way to get the maximum alignment. --- DHT/fxf.c | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index fe3806fa5c..576bbd6d5b 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -54,19 +54,22 @@ # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # include # endif -union MAX_ALIGNED_TYPE { +struct GET_MAX_ALIGNMENT_TYPE { + unsigned char c; + union { # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) - uintmax_t unsigned_integer; + uintmax_t unsigned_integer; # elif defined(LLONG_MAX) /* We have long long integer types. */ - unsigned long long int unsigned_integer; + unsigned long long int unsigned_integer; # else - unsigned long int unsigned_integer; + unsigned long int unsigned_integer; # endif - const volatile void * object_pointer; - void (*function_pointer)(void); - long double floating_point; + const volatile void * object_pointer; + void (*function_pointer)(void); + long double floating_point; + } max_aligned_union; }; -# define MAX_ALIGNMENT (sizeof(union MAX_ALIGNED_TYPE) & -sizeof(union MAX_ALIGNED_TYPE)) +# define MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union) #endif #if !defined(Nil) && !defined(New) && !defined(nNewUntyped) && !defined(nNewCallocUntyped) /* TODO: Is this the correct check for all of the below lines? */ @@ -496,14 +499,18 @@ void *fxfAlloc(size_t size) { sh->FreeCount--; sh->MallocCount++; #if defined(SEGMENTED) - for (ptrSegment= CurrentSeg; ptrSegment >= 0; --ptrSegment) { - convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; - convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; - if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { - ptrIndex= (tmp - segment_begin); - break; - } - } + ptrSegment= CurrentSeg; + if (ptrSegment) + do { + convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; + convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; + if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { + ptrIndex= (tmp - segment_begin); + break; + } + } while (--ptrSegment >= 0); + else + ptrIndex= pointerDifference(ptr, Arena[ptrSegment]); TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,sh->MallocCount)); #else # if defined(FREEMAP) @@ -717,11 +724,18 @@ void fxfFree(void *ptr, size_t size) void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { void *nptr; if (!ptr) + { + assert(!OldSize); return fxfAlloc(NewSize); + } if (!NewSize) + { fxfFree(ptr, OldSize); + return Nil(void); + } + /* TODO: return ptr if the block is large enough and we can safely fxfFree unused space */ nptr= fxfAlloc(NewSize); - if (NewSize && nptr) + if (nptr) { memcpy(nptr, ptr, ((NewSize < OldSize) ? NewSize : OldSize)); fxfFree(ptr, OldSize); From 314acfe4adf330b4f450ab149a27c259a400cf46 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 6 Oct 2023 22:54:36 -0400 Subject: [PATCH 074/111] Small tweaks. --- DHT/fxf.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 576bbd6d5b..8d314ed6a4 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -489,7 +489,7 @@ void *fxfAlloc(size_t size) { if (sh->FreeHead) { #if defined(SEGMENTED) int ptrSegment; - ptrdiff_t ptrIndex= -1; + ptrdiff_t ptrIndex; #endif ptr= sh->FreeHead; if (size < sizeof sh->FreeHead) @@ -500,17 +500,20 @@ void *fxfAlloc(size_t size) { sh->MallocCount++; #if defined(SEGMENTED) ptrSegment= CurrentSeg; - if (ptrSegment) + if (CurrentSeg) { do { convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { ptrIndex= (tmp - segment_begin); - break; + goto FOUND_PUTATIVE_SEGMENT; + } - } while (--ptrSegment >= 0); - else - ptrIndex= pointerDifference(ptr, Arena[ptrSegment]); + } while (0 <= --ptrSegment); + ptrIndex= -1; + } else + ptrIndex= pointerDifference(ptr, Arena[0]); +FOUND_PUTATIVE_SEGMENT: TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,sh->MallocCount)); #else # if defined(FREEMAP) @@ -639,16 +642,20 @@ void fxfFree(void *ptr, size_t size) if (!ptr) return; #if defined(SEGMENTED) - for (ptrSegment= CurrentSeg; ptrSegment >= 0; --ptrSegment) { - convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; - convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; - if (tmp >= segment_begin) { - ptrIndex= (tmp - segment_begin); - if (ptrIndex < ARENA_SEG_SIZE) - goto FOUND_PUTATIVE_SEGMENT; - } - } - ptrIndex= -1; + ptrSegment= CurrentSeg; + if (CurrentSeg) { + do { + convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; + convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; + if (tmp >= segment_begin) { + ptrIndex= (tmp - segment_begin); + if (ptrIndex < ARENA_SEG_SIZE) + goto FOUND_PUTATIVE_SEGMENT; + } + } while (0 <= --ptrSegment); + ptrIndex= -1; + } else + ptrIndex= pointerDifference(ptr, Arena[0]); FOUND_PUTATIVE_SEGMENT: TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); #else From 213662db00a6f627ac41422df07b10e1fde6e35b Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 6 Oct 2023 22:56:09 -0400 Subject: [PATCH 075/111] We only need to compute that value once. --- DHT/fxf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 8d314ed6a4..a31e28e9bb 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -501,8 +501,8 @@ void *fxfAlloc(size_t size) { #if defined(SEGMENTED) ptrSegment= CurrentSeg; if (CurrentSeg) { + convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; do { - convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { ptrIndex= (tmp - segment_begin); @@ -644,8 +644,8 @@ void fxfFree(void *ptr, size_t size) #if defined(SEGMENTED) ptrSegment= CurrentSeg; if (CurrentSeg) { + convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; do { - convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; if (tmp >= segment_begin) { ptrIndex= (tmp - segment_begin); From 99f9193330c9000d8d7fb4a6abec171cac73ba12 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 9 Oct 2023 20:52:43 -0400 Subject: [PATCH 076/111] Adding some asserts. --- DHT/fxf.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index a31e28e9bb..81a69c9efa 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -654,12 +654,15 @@ void fxfFree(void *ptr, size_t size) } } while (0 <= --ptrSegment); ptrIndex= -1; - } else + } else { ptrIndex= pointerDifference(ptr, Arena[0]); + assert((ptrIndex >= 0) && (ptrIndex < ARENA_SEG_SIZE)); + } FOUND_PUTATIVE_SEGMENT: TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); #else ptrIndex= pointerDifference(ptr,Arena); + assert((ptrIndex >= 0) && (ptrIndex < GlobalSize)); TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); #endif DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *)ptr, (size_t_printf_type) size)); @@ -672,11 +675,22 @@ void fxfFree(void *ptr, size_t size) } size= ALIGN_TO_MINIMUM(size); #if !defined(NDEBUG) - if (ptrIndex > 0) { - size_t needed_alignment= MAX_ALIGNMENT; - while (needed_alignment > size) - needed_alignment>>= 1; - assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); +# if defined(SEGMENTED) + if (!CurrentSeg) /* Otherwise we'd be relying on converting to convert_pointer_to_int_type, + and such calculations aren't guaranteed to provide exactly what we need. */ + { + assert(size <= (ARENA_SEG_SIZE - ptrIndex)) +# else + { + assert(size <= (GlobalSize - ptrIndex)) +#endif + if (ptrIndex > 0) + { + size_t needed_alignment= MAX_ALIGNMENT; + while (needed_alignment > size) + needed_alignment>>= 1; + assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); + } } #endif sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; @@ -735,6 +749,39 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { assert(!OldSize); return fxfAlloc(NewSize); } +#if !defined(NDEBUG) +# if defined(SEGMENTED) + if (!CurrentSeg) /* Otherwise we'd be relying on converting to convert_pointer_to_int_type, + and such calculations aren't guaranteed to provide exactly what we need. */ + { + ptrdiff_t const ptrIndex= pointerDifference(ptr,Arena[0]); + assert(ptrIndex < ARENA_SEG_SIZE); +# else + { + ptrdiff_t const ptrIndex= pointerDifference(ptr,Arena); + assert(ptrIndex < GlobalSize); +# endif + assert(ptrIndex >= 0); + if (ptrIndex > 0) + { + size_t allocatedSize= OldSize; + size_t needed_alignment; + if (allocatedSize < fxfMINSIZE) + allocatedSize= fxfMINSIZE; + assert(allocatedSize <= fxfMAXSIZE); + allocatedSize= ALIGN_TO_MINIMUM(allocatedSize); +# if defined(SEGMENTED) + assert(allocatedSize <= (ARENA_SEG_SIZE - ptrIndex)); +# else + assert(allocatedSize <= (GlobalSize - ptrIndex)); +# endif + needed_alignment= MAX_ALIGNMENT; + while (needed_alignment > allocatedSize) + needed_alignment>>= 1; + assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); + } + } +#endif if (!NewSize) { fxfFree(ptr, OldSize); From ecddab6770e751d3713659fe99c29cfb8e302100 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 9 Oct 2023 22:32:25 -0400 Subject: [PATCH 077/111] Adding more asserts. --- DHT/fxf.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 81a69c9efa..cebc753916 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -274,10 +274,12 @@ void PrintFreeMap(FILE *f) { #endif /*FREEMAP, !SEGMENTED*/ static inline ptrdiff_t pointerDifference(void const *ptr1, void const *ptr2) { + assert(ptr1 && ptr2); return (((char const *)ptr1) - ((char const *)ptr2)); } static inline void * stepPointer(void *ptr, ptrdiff_t step) { + assert(!!ptr); return (void *)(((char *)ptr) + step); } @@ -507,7 +509,6 @@ void *fxfAlloc(size_t size) { if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { ptrIndex= (tmp - segment_begin); goto FOUND_PUTATIVE_SEGMENT; - } } while (0 <= --ptrSegment); ptrIndex= -1; @@ -533,13 +534,14 @@ void *fxfAlloc(size_t size) { if (sizeCurrentSeg>=size) { if (size&PTRMASK) { /* not fully aligned */ + size_t curBottomIndex; size_t needed_alignment_mask= PTRMASK; while (needed_alignment_mask >= size) needed_alignment_mask>>= 1; #if defined(SEGMENTED) - size_t curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena[CurrentSeg]); + curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena[CurrentSeg]); #else - size_t curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena); + curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena); #endif if (curBottomIndex & needed_alignment_mask) { size_t const numBytesToAdd= (needed_alignment_mask - (curBottomIndex & needed_alignment_mask)) + 1U; From 988659cbe7958d2e7451987e5f5f89446f3b7e18 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 9 Oct 2023 22:39:53 -0400 Subject: [PATCH 078/111] Oops, forgot some semicolons. --- DHT/fxf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index cebc753916..fd7d571d40 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -681,10 +681,10 @@ void fxfFree(void *ptr, size_t size) if (!CurrentSeg) /* Otherwise we'd be relying on converting to convert_pointer_to_int_type, and such calculations aren't guaranteed to provide exactly what we need. */ { - assert(size <= (ARENA_SEG_SIZE - ptrIndex)) + assert(size <= (ARENA_SEG_SIZE - ptrIndex)); # else { - assert(size <= (GlobalSize - ptrIndex)) + assert(size <= (GlobalSize - ptrIndex)); #endif if (ptrIndex > 0) { From c251c5dff1dc273e3485b0f9b090627b39415e50 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 12 Oct 2023 20:57:56 -0400 Subject: [PATCH 079/111] Expanding on the comment. --- DHT/fxf.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index fd7d571d40..a8ffb95ef6 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -789,7 +789,14 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { fxfFree(ptr, OldSize); return Nil(void); } - /* TODO: return ptr if the block is large enough and we can safely fxfFree unused space */ + /* TODO: It may be worth trying to return ptr if ALIGN_TO_MINIMUM(OldSize) >= NewSize. + To go along with this, we'd have to carefully add any excess to the free store. + In the !defined(SEGMENTED) case this is likely easy, but in the defined(SEGMENTED) + case it may be difficult. Regardless, the computations to set this up -- or even + determine if it's possible -- are kind of annoying, and they'd only be worthwhile + if we hit this possibility frequently (and if the alternative below is expensive + or proves impossible). This all would need to be investigated. + */ nptr= fxfAlloc(NewSize); if (nptr) { From 3ba6c2f393f76e0e029e1574b177dc57939c07f0 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 19 Oct 2023 20:44:19 -0400 Subject: [PATCH 080/111] Making Cppcheck a bit happier. --- optimisations/hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index 256bad5a9c..c52123ce42 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -357,7 +357,7 @@ static void slice_property_offset_shifter(slice_index si, * @param delta indicates how much to shift the value offsets */ static void shift_offsets(slice_index si, - stip_structure_traversal *st, + stip_structure_traversal const *st, unsigned int delta) { unsigned int i; From 35256e017a8a591bdf12d30f6c12eaf144015dbe Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 20 Oct 2023 21:23:28 -0400 Subject: [PATCH 081/111] Small tweak, suggested by Cppcheck. --- debugging/trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debugging/trace.c b/debugging/trace.c index 21c5ebf128..f6c9c96079 100644 --- a/debugging/trace.c +++ b/debugging/trace.c @@ -521,7 +521,7 @@ static void trace_link(char const *prefix, slice_index si, char const *suffix) static char const context_shortcuts[] = { 'I', 'A', 'D', 'H', 'M', 'T', 'N' }; static char const level_shortcuts[] = { 'T', 'S', 'N' }; -static void trace_common(slice_index si, stip_structure_traversal *st) +static void trace_common(slice_index si, stip_structure_traversal const *st) { if (do_trace) { From d48c3f07f4d5d0911cf1b3a1aa3c40003ab10727 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 21 Oct 2023 23:10:49 -0400 Subject: [PATCH 082/111] If we aren't using FXF, anyone who includes this file needs to know where malloc is. --- DHT/dhtvalue.h | 1 + 1 file changed, 1 insertion(+) diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 3f483d91fd..cf1f9ee857 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -30,6 +30,7 @@ #if defined(FXF) #include "fxf.h" #else +#include #define fxfAlloc(x) malloc(x) #define fxfFree(x,n) free(x) #endif /*FXF*/ From caddc531621276d5cf3dae3a37e064bdef32adcd Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 26 Oct 2023 07:31:08 -0400 Subject: [PATCH 083/111] Adding a sanity check. --- DHT/dhtsimpl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DHT/dhtsimpl.c b/DHT/dhtsimpl.c index 253fee9264..15289d8e01 100644 --- a/DHT/dhtsimpl.c +++ b/DHT/dhtsimpl.c @@ -37,6 +37,9 @@ static unsigned long ConvertSimpleValue(dhtKey k) unsigned long long a, b, c; c = 0x9e3779b97f4a7c13LLU; # else +# if !((((-1LU) >> 31) >> 31) >> 1) +# error "ERROR: unable to build dhtsimpl.c due to lack of a sufficiently-large integer type." +# endif unsigned long a, b, c; c = 0x9e3779b97f4a7c13LU; # endif From 0444b32acd60792388c9b511b670126b648be4d3 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 7 Nov 2023 06:46:37 -0500 Subject: [PATCH 084/111] Adding some more asserts. --- DHT/fxf.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index a8ffb95ef6..6101317ae2 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -634,7 +634,9 @@ void *fxfAlloc(size_t size) { void fxfFree(void *ptr, size_t size) { +#if defined(LOG) || defined(DEBUG) static char const * const myname= "fxfFree"; +#endif SizeHead *sh; ptrdiff_t ptrIndex; @@ -643,6 +645,7 @@ void fxfFree(void *ptr, size_t size) #endif if (!ptr) return; + assert(!!size); #if defined(SEGMENTED) ptrSegment= CurrentSeg; if (CurrentSeg) { @@ -657,7 +660,7 @@ void fxfFree(void *ptr, size_t size) } while (0 <= --ptrSegment); ptrIndex= -1; } else { - ptrIndex= pointerDifference(ptr, Arena[0]); + ptrIndex= pointerDifference(ptr,Arena[0]); assert((ptrIndex >= 0) && (ptrIndex < ARENA_SEG_SIZE)); } FOUND_PUTATIVE_SEGMENT: @@ -670,11 +673,7 @@ void fxfFree(void *ptr, size_t size) DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *)ptr, (size_t_printf_type) size)); if (size < fxfMINSIZE) size= fxfMINSIZE; - if (size > fxfMAXSIZE) { - fprintf(stderr, "%s: size=%" SIZE_T_PRINTF_SPECIFIER " >= %" SIZE_T_PRINTF_SPECIFIER "\n", - myname, (size_t_printf_type) size, (size_t_printf_type) fxfMAXSIZE); - exit(-5); - } + assert(size <= fxfMAXSIZE); size= ALIGN_TO_MINIMUM(size); #if !defined(NDEBUG) # if defined(SEGMENTED) @@ -682,9 +681,11 @@ void fxfFree(void *ptr, size_t size) and such calculations aren't guaranteed to provide exactly what we need. */ { assert(size <= (ARENA_SEG_SIZE - ptrIndex)); + assert(((ptrIndex + size) <= pointerDifference(BotFreePtr,Arena[0])) || (ptr >= TopFreePtr)); # else { assert(size <= (GlobalSize - ptrIndex)); + assert(((ptrIndex + size) <= pointerDifference(BotFreePtr,Arena)) || (ptr >= TopFreePtr)); #endif if (ptrIndex > 0) { From 829790f3f26d4a418cbb656c748d8ec618781876 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 14 Nov 2023 21:36:00 -0500 Subject: [PATCH 085/111] Noting something else that we may want to consider doing. --- DHT/dhtvalue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index cf1f9ee857..177a53b581 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -31,7 +31,7 @@ #include "fxf.h" #else #include -#define fxfAlloc(x) malloc(x) +#define fxfAlloc(x) malloc(x) /* TODO: Should we track allocations to ensure that we never allocate more than GlobalSize total byte(s)? */ #define fxfFree(x,n) free(x) #endif /*FXF*/ From 3d6eac8592a134e8fee32c4d76b4881ff0502280 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 17 Nov 2023 18:15:31 -0500 Subject: [PATCH 086/111] Preferring this syntax, as used throughout the rest of the codebase. --- optimisations/hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index c52123ce42..0aa31d10a3 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1774,7 +1774,7 @@ static boolean is_proofgame(slice_index si) */ boolean is_hashtable_allocated(void) { -#ifdef FXF +#if defined(FXF) return !!fxfInitialised(); /* !! just in case fxfInitialised returns a nonzero value other than 1 and boolean is some type that won't automatically convert it to 1. */ #else From 3e11bfc836218793cb179167e7bad47da7b004be Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 17 Nov 2023 18:35:53 -0500 Subject: [PATCH 087/111] Correcting the comment to use the appropriate example value. --- DHT/dhtvalue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 177a53b581..0558801a80 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -31,7 +31,7 @@ #include "fxf.h" #else #include -#define fxfAlloc(x) malloc(x) /* TODO: Should we track allocations to ensure that we never allocate more than GlobalSize total byte(s)? */ +#define fxfAlloc(x) malloc(x) /* TODO: Should we track allocations to ensure that we never allocate more than some chosen number (e.g., hashtable_kilos*1024) of total byte(s)? */ #define fxfFree(x,n) free(x) #endif /*FXF*/ From 32c0df46022b6565366ab5be96a64e8e1cab493c Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 21 Nov 2023 21:18:38 -0500 Subject: [PATCH 088/111] Finally removing the old code. --- optimisations/hash.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/optimisations/hash.h b/optimisations/hash.h index 95dc21369e..bc7b3a79d3 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -136,17 +136,6 @@ enum hashbuf_length = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) + offsetof(BCMemValue, Data) }; -#if 0 -/* Previously hashbuf_length was the below constant. - MAX_LENGTH_OF_ENCODING wasn't computed or used but should have been. */ -enum -{ - hashbuf_length = 256, - MAX_LENGTH_OF_ENCODING = (hashbuf_length - offsetof(BCMemValue, Data))/sizeof(byte), - ENSURE_MAX_LENGTH_FITS_IN_UNSIGNED_SHORT = 1/(MAX_LENGTH_OF_ENCODING <= USHRT_MAX), -}; -#endif - typedef union { BCMemValue cmv; From 35fa6828d6410d82a1ca6c41caad9600fff75150 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 22 Nov 2023 20:26:48 -0500 Subject: [PATCH 089/111] Saving 1, and better explaining where the bound comes from. --- optimisations/hash.h | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/optimisations/hash.h b/optimisations/hash.h index bc7b3a79d3..672de0fbf8 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -98,15 +98,22 @@ typedef unsigned char byte; - nr_ghosts <= underworld_capacity - bytes_per_spec <= 4 - being_solved.number_of_imitators <= maxinum - The most annoying quantity to bound is en_passant_top[nbply] - en_passant_top[nbply-1]. We must have - - en_passant_top[nbply] - en_passant_top[nbply-1] <= maxply + 1 - but it's likely that tighter bounds are possible. + The most annoying quantity to bound is en_passant_top[nbply] - en_passant_top[nbply-1]. There are + loops like + for (i = en_passant_top[nbply-1]+1; i<=en_passant_top[nbply]; ++i) + *bp++ = (byte)(en_passant_multistep_over[i] - square_a1); + and for these to be correct we must have + en_passant_top[nbply] < number of elements in en_passant_multistep_over = maxply + 1 (at this time). + Meanwhile, en_passant_top[nbply-1] is an unsigned int, hence is >= 0. From these we infer + - en_passant_top[nbply] - en_passant_top[nbply] <= maxply + but it's likely that a tighter bound is possible, preferably one that doesn't rely on knowing the + number of elements in en_passant_multistep_over. With all of the above, we can determine the maximum bytes that an encoding should take up. We'll let the compiler perform the arithmetic. */ enum { - MAX_EN_PASSANT_TOP_DIFFERENCE = maxply + 1, // TODO: Improve this! + MAX_EN_PASSANT_TOP_DIFFERENCE = maxply, /* TODO: Improve this! */ COMMONENCODE_MAX = 16 + sizeof BGL_values[White] + sizeof BGL_values[Black] + maxinum From 9f202a43fc59cb9ddb3e39352df094b594b4b131 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 25 Nov 2023 20:19:53 -0500 Subject: [PATCH 090/111] Probably improving this bound. --- optimisations/hash.h | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/optimisations/hash.h b/optimisations/hash.h index 672de0fbf8..40e906319c 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -98,22 +98,17 @@ typedef unsigned char byte; - nr_ghosts <= underworld_capacity - bytes_per_spec <= 4 - being_solved.number_of_imitators <= maxinum - The most annoying quantity to bound is en_passant_top[nbply] - en_passant_top[nbply-1]. There are - loops like - for (i = en_passant_top[nbply-1]+1; i<=en_passant_top[nbply]; ++i) - *bp++ = (byte)(en_passant_multistep_over[i] - square_a1); - and for these to be correct we must have - en_passant_top[nbply] < number of elements in en_passant_multistep_over = maxply + 1 (at this time). - Meanwhile, en_passant_top[nbply-1] is an unsigned int, hence is >= 0. From these we infer - - en_passant_top[nbply] - en_passant_top[nbply] <= maxply - but it's likely that a tighter bound is possible, preferably one that doesn't rely on knowing the - number of elements in en_passant_multistep_over. + en_passant_top[nbply] - en_passant_top[nbply-1] requires a bit more thought. It should be an upper + bound on the number of en passant squares stored per ply. For now we'll assume that + - en_passant_top[nbply] - en_passant_top[nbply-1] <= nr_rows_on_board - 2 + as (nr_rows_on_board - 2) is the number of squares that a pawn jumping from the first rank to the + last rank would skip over. With all of the above, we can determine the maximum bytes that an encoding should take up. We'll let the compiler perform the arithmetic. */ enum { - MAX_EN_PASSANT_TOP_DIFFERENCE = maxply, /* TODO: Improve this! */ + MAX_EN_PASSANT_TOP_DIFFERENCE = nr_squares - 2, /* TODO: Is this a safe maximum? Can we get away with a smaller value? */ COMMONENCODE_MAX = 16 + sizeof BGL_values[White] + sizeof BGL_values[Black] + maxinum From 7be78f37b345ded8c6dd5beb85e0cbceef3e6448 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 26 Nov 2023 12:18:51 -0500 Subject: [PATCH 091/111] Asserting on one of our assumptions. --- optimisations/hash.c | 1 + 1 file changed, 1 insertion(+) diff --git a/optimisations/hash.c b/optimisations/hash.c index 0aa31d10a3..76a6f292d8 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1304,6 +1304,7 @@ static byte *CommonEncode(byte *bp, { unsigned int i; + assert((en_passant_top[nbply]-en_passant_top[nbply-1])<=MAX_EN_PASSANT_TOP_DIFFERENCE); for (i = en_passant_top[nbply-1]+1; i<=en_passant_top[nbply]; ++i) *bp++ = (byte)(en_passant_multistep_over[i] - square_a1); } From facb5e743292041b53361aa816b44480d5329e2e Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 26 Nov 2023 13:01:06 -0500 Subject: [PATCH 092/111] Correcting the constant's name. --- optimisations/hash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimisations/hash.h b/optimisations/hash.h index 40e906319c..3b8b1ffd8a 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -108,7 +108,7 @@ typedef unsigned char byte; */ enum { - MAX_EN_PASSANT_TOP_DIFFERENCE = nr_squares - 2, /* TODO: Is this a safe maximum? Can we get away with a smaller value? */ + MAX_EN_PASSANT_TOP_DIFFERENCE = nr_rows_on_board - 2, /* TODO: Is this a safe maximum? Can we get away with a smaller value? */ COMMONENCODE_MAX = 16 + sizeof BGL_values[White] + sizeof BGL_values[Black] + maxinum From bece5909a91a81c2faeafff80da60659ddfe7080 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 26 Nov 2023 15:53:56 -0500 Subject: [PATCH 093/111] Ensuring we have the constants we need. --- optimisations/hash.h | 1 + 1 file changed, 1 insertion(+) diff --git a/optimisations/hash.h b/optimisations/hash.h index 3b8b1ffd8a..f725e467a2 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -14,6 +14,7 @@ #include "solving/machinery/solve.h" #include "position/underworld.h" #include "position/position.h" +#include "position/board.h" #include "solving/ply.h" #include "conditions/bgl.h" #include From 23005b1e8e292172915574229d98ab32df3048b0 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 26 Nov 2023 20:20:50 -0500 Subject: [PATCH 094/111] Shifting the sanity check to the .c file. --- optimisations/hash.c | 4 ++++ optimisations/hash.h | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index 76a6f292d8..4c4ffeb722 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -120,6 +120,10 @@ #include "platform/timer.h" #endif +enum { + ENSURE_MAX_LENGTH_FITS_IN_UNSIGNED_SHORT = 1/(MAX_LENGTH_OF_ENCODING <= USHRT_MAX) +}; + unsigned long hash_max_number_storable_positions = ULONG_MAX; typedef unsigned int hash_value_type; diff --git a/optimisations/hash.h b/optimisations/hash.h index f725e467a2..256c411545 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -17,7 +17,6 @@ #include "position/board.h" #include "solving/ply.h" #include "conditions/bgl.h" -#include #if defined(TESTHASH) # if !defined(HASHRATE) @@ -135,7 +134,6 @@ enum { MAX_LENGTH_OF_ENCODING = ((SMALLENCODE_MAX > LARGEENCODE_MAX) ? ((SMALLENCODE_MAX > PROOFENCODE_MAX) ? SMALLENCODE_MAX : PROOFENCODE_MAX) : ((LARGEENCODE_MAX > PROOFENCODE_MAX) ? LARGEENCODE_MAX : PROOFENCODE_MAX)), - ENSURE_MAX_LENGTH_FITS_IN_UNSIGNED_SHORT = 1/(MAX_LENGTH_OF_ENCODING <= USHRT_MAX), hashbuf_length = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) + offsetof(BCMemValue, Data) }; From ec2e09cfa8609dc59ed6bf59399bf45f6f286d8e Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 26 Nov 2023 20:21:34 -0500 Subject: [PATCH 095/111] Changing this isn't part of the branch. --- DHT/depend | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/DHT/depend b/DHT/depend index 8518b65096..ce88a9dce0 100644 --- a/DHT/depend +++ b/DHT/depend @@ -1,47 +1,70 @@ DHT/dht$(OBJ_SUFFIX): DHT/dht.c debugging/assert.h utilities/boolean.h \ DHT/dhtvalue.h DHT/fxf.h DHT/dht.h debugging/trace.h + debugging/assert.h: + utilities/boolean.h: + DHT/dhtvalue.h: + DHT/fxf.h: + DHT/dht.h: + debugging/trace.h: -DHT/dhtcmem$(OBJ_SUFFIX): DHT/dhtcmem.c debugging/assert.h DHT/dhtvalue.h DHT/fxf.h \ - DHT/dhtcmem.h DHT/dht.h -debugging/assert.h: +DHT/dhtcmem$(OBJ_SUFFIX): DHT/dhtcmem.c DHT/dhtvalue.h DHT/fxf.h DHT/dhtcmem.h \ + DHT/dht.h + DHT/dhtvalue.h: + DHT/fxf.h: + DHT/dhtcmem.h: + DHT/dht.h: -DHT/dhtmem$(OBJ_SUFFIX): DHT/dhtmem.c debugging/assert.h DHT/dhtvalue.h DHT/fxf.h \ - DHT/dhtmem.h DHT/dht.h -debugging/assert.h: +DHT/dhtmem$(OBJ_SUFFIX): DHT/dhtmem.c DHT/dhtvalue.h DHT/fxf.h DHT/dhtmem.h \ + DHT/dht.h + DHT/dhtvalue.h: + DHT/fxf.h: + DHT/dhtmem.h: + DHT/dht.h: -DHT/dhtsimpl$(OBJ_SUFFIX): DHT/dhtsimpl.c debugging/assert.h DHT/dhtvalue.h \ - DHT/fxf.h -debugging/assert.h: +DHT/dhtsimpl$(OBJ_SUFFIX): DHT/dhtsimpl.c DHT/dhtvalue.h DHT/fxf.h + DHT/dhtvalue.h: + DHT/fxf.h: -DHT/dhtstrng$(OBJ_SUFFIX): DHT/dhtstrng.c debugging/assert.h DHT/dhtvalue.h \ - DHT/fxf.h DHT/dht.h -debugging/assert.h: +DHT/dhtstrng$(OBJ_SUFFIX): DHT/dhtstrng.c DHT/dhtvalue.h DHT/fxf.h DHT/dht.h + DHT/dhtvalue.h: + DHT/fxf.h: + DHT/dht.h: DHT/dhtvalue$(OBJ_SUFFIX): DHT/dhtvalue.c DHT/dhtvalue.h DHT/fxf.h DHT/dht.h + DHT/dhtvalue.h: + DHT/fxf.h: + DHT/dht.h: DHT/dhtbcmem$(OBJ_SUFFIX): DHT/dhtbcmem.c debugging/assert.h DHT/dhtvalue.h \ DHT/fxf.h DHT/dhtbcmem.h DHT/dht.h + debugging/assert.h: + DHT/dhtvalue.h: + DHT/fxf.h: + DHT/dhtbcmem.h: + DHT/dht.h: DHT/fxf$(OBJ_SUFFIX): DHT/fxf.c debugging/assert.h DHT/fxf.h + debugging/assert.h: + DHT/fxf.h: From 5c885c3d4d00d596f7213c95aadbb312fb4f3368 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 7 Dec 2023 20:48:59 -0500 Subject: [PATCH 096/111] Adding a TODO. --- DHT/fxf.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DHT/fxf.c b/DHT/fxf.c index 6101317ae2..1766439e6a 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -141,6 +141,10 @@ typedef struct { #endif /* The maximum size an fxfAlloc can handle */ +/* TODO: It's awkward that this limit seems to be based on what's needed for an external type. + Ignoring that, do these macros really accurately determine the maximum we need? + Can we instead compute it as an expression involving, say, sizeof(void *) and any other + relevant system properties? */ #if defined(SEGMENTED) || defined(__TURBOC__) #define fxfMAXSIZE (((size_t)1024) & ~(MAX_ALIGNMENT - 1U)) #else From 454665c14b4dc38563e1f6529bef8d918abce085 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 9 Dec 2023 17:04:05 -0500 Subject: [PATCH 097/111] Adding some useful comments, making fxfMAXSIZE an enum (to match fxfMINSIZE), allowing access to that value from outside FXF, and carefully storing whatever we can before moving on to the next segment. --- DHT/fxf.c | 51 +++++++++++++++++++++++++++----------------- DHT/fxf.h | 1 + optimisations/hash.c | 7 ++---- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 1766439e6a..ede8b32005 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -141,20 +141,28 @@ typedef struct { #endif /* The maximum size an fxfAlloc can handle */ -/* TODO: It's awkward that this limit seems to be based on what's needed for an external type. - Ignoring that, do these macros really accurately determine the maximum we need? - Can we instead compute it as an expression involving, say, sizeof(void *) and any other - relevant system properties? */ +/* TODO: Do the macros really accurately determine the maximum we need (apparently 1024 or 2048)? + Can we instead compute the needed value(s) with expressions involving, say, sizeof(void *) + and any other system properties we have access to? +*/ +enum +{ + fxfMAXSIZE = #if defined(SEGMENTED) || defined(__TURBOC__) -#define fxfMAXSIZE (((size_t)1024) & ~(MAX_ALIGNMENT - 1U)) +# if defined(ARENA_SEG_SIZE) + ((((1024 > ARENA_SEG_SIZE) ? ARENA_SEG_SIZE : ((size_t)1024)) +# else + ((((size_t)1024) +# endif #else -#define fxfMAXSIZE (((size_t)2048) & ~(MAX_ALIGNMENT - 1U)) /* this is needed only when sizeof(void*)==8 */ + ((((size_t)2048) /* This is needed only when sizeof(void*)==8. */ #endif + + (MAX_ALIGNMENT - 1U)) & ~(MAX_ALIGNMENT - 1U)) /* Round up if necessary. */ +}; -/* Different size of fxfMINSIZE for 32-/64/Bit compilation */ enum { - fxfMINSIZE = sizeof(void *) + fxfMINSIZE = sizeof(void *) /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ }; enum { @@ -287,6 +295,10 @@ static inline void * stepPointer(void *ptr, ptrdiff_t step) { return (void *)(((char *)ptr) + step); } +size_t fxfMaxAllocation(void) { + return fxfMAXSIZE; +} + size_t fxfInit(size_t Size) { #if defined(LOG) static char const * const myname= "fxfInit"; @@ -605,17 +617,18 @@ void *fxfAlloc(size_t size) { BotFreePtr= stepPointer(BotFreePtr, cur_alignment); curBottomIndex+= cur_alignment; } - if (BotFreePtr < TopFreePtr) { - size_t cur_size= (size_t)(TopFreePtr-BotFreePtr); - if (cur_size >= fxfMINSIZE) { - SizeHead *cur_sh= &SizeData[(cur_size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; - if ((cur_size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { - if (cur_size >= sizeof cur_sh->FreeHead) - memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); - cur_sh->FreeHead= BotFreePtr; - ++cur_sh->FreeCount; - TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); - } + curBottomIndex= (size_t)(TopFreePtr-BotFreePtr); + while ((curBottomIndex >= fxfMINSIZE) && curBottomIndex) { + size_t next_chunk_size= ((curBottomIndex > fxfMAXSIZE) ? fxfMAXSIZE : curBottomIndex); + SizeHead *cur_sh= &SizeData[(next_chunk_size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + if ((next_chunk_size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if (next_chunk_size >= sizeof cur_sh->FreeHead) + memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + BotFreePtr= stepPointer(BotFreePtr, next_chunk_size); + curBottomIndex-= next_chunk_size; } } TMDBG(fputs(" next seg", stdout)); diff --git a/DHT/fxf.h b/DHT/fxf.h index 7f430c948d..d56cbcfb3e 100644 --- a/DHT/fxf.h +++ b/DHT/fxf.h @@ -11,6 +11,7 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize); void fxfFree(void *ptr, size_t size); void fxfInfo(FILE *); size_t fxfTotal(void); +size_t fxfMaxAllocation(void); /* Reset the internal data structures to the state that was reached * after the latest call to fxfInit() */ diff --git a/optimisations/hash.c b/optimisations/hash.c index 4c4ffeb722..5ee65beaf6 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -2570,14 +2570,11 @@ void hash_opener_solve(slice_index si) TraceFunctionResultEnd(); } -/* assert()s below this line must remain active even in "productive" - * executables. */ -#undef NDEBUG -#include - /* Check assumptions made in the hashing module. Abort if one of them * isn't met. * This is called from checkGlobalAssumptions() once at program start. + * NOTE: Currently these are all compile-time checks, but we reserve + * the right to add run-time checks in the future. */ void check_hash_assumptions(void) { From 4cbce9e482ce4fb1b3d3efa19f8d43659d092e73 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 9 Dec 2023 17:29:53 -0500 Subject: [PATCH 098/111] Always including the corresponding header, eliminating a warning about an empty translation unit that we get when not using FXF. --- DHT/fxf.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DHT/fxf.c b/DHT/fxf.c index ede8b32005..84c6fd5c0e 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -14,8 +14,12 @@ # endif /*DBMALLOC*/ #endif /*__TURBOC__*/ +#endif + #include "fxf.h" +#if defined (FXF) + #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || /* >= C99 -- We have printf ptrdiff_t/size_t specifiers. */ \ (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 -- We have printf ptrdiff_t/size_t specifiers. */ # include From 8b56652b58b55cedb539db0da8501e098561b830 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 9 Dec 2023 17:38:25 -0500 Subject: [PATCH 099/111] We don't need a loop since we already know that we're < fxfMAXSIZE (as otherwise size would have necessarily fit). --- DHT/fxf.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 84c6fd5c0e..be158ce27d 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -622,17 +622,14 @@ void *fxfAlloc(size_t size) { curBottomIndex+= cur_alignment; } curBottomIndex= (size_t)(TopFreePtr-BotFreePtr); - while ((curBottomIndex >= fxfMINSIZE) && curBottomIndex) { - size_t next_chunk_size= ((curBottomIndex > fxfMAXSIZE) ? fxfMAXSIZE : curBottomIndex); - SizeHead *cur_sh= &SizeData[(next_chunk_size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; - if ((next_chunk_size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { - if (next_chunk_size >= sizeof cur_sh->FreeHead) + if ((curBottomIndex >= fxfMINSIZE) && curBottomIndex) { + SizeHead *cur_sh= &SizeData[(curBottomIndex - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + if ((curBottomIndex >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if (curBottomIndex >= sizeof cur_sh->FreeHead) memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); cur_sh->FreeHead= BotFreePtr; ++cur_sh->FreeCount; TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); - BotFreePtr= stepPointer(BotFreePtr, next_chunk_size); - curBottomIndex-= next_chunk_size; } } TMDBG(fputs(" next seg", stdout)); From fe4a949200dfb3c4baa9db43c160313eccc68a89 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 9 Dec 2023 21:37:39 -0500 Subject: [PATCH 100/111] Replacing two checks with a single check against a compile-time constant. --- DHT/fxf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index be158ce27d..aae82d3ab9 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -622,7 +622,7 @@ void *fxfAlloc(size_t size) { curBottomIndex+= cur_alignment; } curBottomIndex= (size_t)(TopFreePtr-BotFreePtr); - if ((curBottomIndex >= fxfMINSIZE) && curBottomIndex) { + if (curBottomIndex >= ((fxfMINSIZE > 0) ? fxfMINSIZE : 1)) { SizeHead *cur_sh= &SizeData[(curBottomIndex - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; if ((curBottomIndex >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { if (curBottomIndex >= sizeof cur_sh->FreeHead) From 39e282750b06eacc39a7dfb44d04083aef1daf65 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 10 Dec 2023 07:08:46 -0500 Subject: [PATCH 101/111] Using a different hack to eliminate the warning. --- DHT/fxf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index aae82d3ab9..d587ea994e 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -14,12 +14,8 @@ # endif /*DBMALLOC*/ #endif /*__TURBOC__*/ -#endif - #include "fxf.h" -#if defined (FXF) - #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || /* >= C99 -- We have printf ptrdiff_t/size_t specifiers. */ \ (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 -- We have printf ptrdiff_t/size_t specifiers. */ # include @@ -888,4 +884,8 @@ void fxfInfo(FILE *f) { } } +#else /*FXF*/ + +extern unsigned char FXF_C_NONEMPTY_TRANSLATION_UNIT; + #endif /*FXF*/ From bcab92b8bba41babae5e1f0671f542b413c39150 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 11 Dec 2023 21:19:39 -0500 Subject: [PATCH 102/111] Replacing a complicated expression with a macro. --- DHT/fxf.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index d587ea994e..57acebf2f7 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -182,6 +182,9 @@ static size_t min_alignment= !fxfMINSIZE; /* for now */ static SizeHead SizeData[1 + ((fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE)]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). Maximum allocation is fxfMAXSIZE. All allocations will be multiples of MIN_ALIGNMENT_UNDERESTIMATE. */ +#define SIZEDATA_SIZE_TO_INDEX(s) (((s) - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE) +#define SIZEDATA_INDEX_TO_SIZE(x) ((size_t)(((x) * MIN_ALIGNMENT_UNDERESTIMATE) + \ + ((fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U)))) #if defined(SEGMENTED) /* #define ARENA_SEG_SIZE 32000 */ @@ -503,7 +506,7 @@ void *fxfAlloc(size_t size) { // Round up to a multiple of min_alignment size= ALIGN_TO_MINIMUM(size); - sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; if (sh->FreeHead) { #if defined(SEGMENTED) int ptrSegment; @@ -569,7 +572,7 @@ void *fxfAlloc(size_t size) { SetRange(curBottomIndex,cur_alignment); #endif if (cur_alignment >= fxfMINSIZE) { - SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { if (cur_alignment >= sizeof cur_sh->FreeHead) memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); @@ -605,7 +608,7 @@ void *fxfAlloc(size_t size) { while (curBottomIndex & PTRMASK) { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); if (cur_alignment >= fxfMINSIZE) { - SizeHead *cur_sh= &SizeData[(cur_alignment - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { if (cur_alignment >= sizeof cur_sh->FreeHead) memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); @@ -619,7 +622,7 @@ void *fxfAlloc(size_t size) { } curBottomIndex= (size_t)(TopFreePtr-BotFreePtr); if (curBottomIndex >= ((fxfMINSIZE > 0) ? fxfMINSIZE : 1)) { - SizeHead *cur_sh= &SizeData[(curBottomIndex - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(curBottomIndex)]; if ((curBottomIndex >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { if (curBottomIndex >= sizeof cur_sh->FreeHead) memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); @@ -710,7 +713,7 @@ void fxfFree(void *ptr, size_t size) } } #endif - sh= &SizeData[(size - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE]; + sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; if (size&PTRMASK) { /* not fully aligned size */ TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(BotFreePtr,ptr))); @@ -821,9 +824,6 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { return nptr; } -#define SIZEDATA_INDEX_TO_SIZE(x) ((size_t)(((x) * MIN_ALIGNMENT_UNDERESTIMATE) + \ - ((fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U)))) - size_t fxfTotal(void) { SizeHead const *hd = SizeData; size_t UsedBytes = 0; From 02c90db407ec18b8aebf615ad9cd4eea4a7b45c1 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 12 Dec 2023 21:52:34 -0500 Subject: [PATCH 103/111] Ensuring that these values stay aligned. --- position/pieceid.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/position/pieceid.h b/position/pieceid.h index f90153555a..bd770ed4c4 100644 --- a/position/pieceid.h +++ b/position/pieceid.h @@ -21,6 +21,11 @@ typedef unsigned long PieceIdType; #define GetPieceId(spec) ((spec) >> PieceIdOffset) #define ClearPieceId(spec) SetPieceId(spec,NullPieceId) +enum +{ + ENSURE_PIECEIDWIDTH_IS_LARGE_ENOUGH=1/!((MaxPieceId>>(PieceIdWidth-1u))>>1) +}; + extern square PiecePositionsInDiagram[MaxPieceId+1]; #define GetPositionInDiagram(spec) PiecePositionsInDiagram[GetPieceId(spec)] From 7f6c53aadd38f8157ffb4624f1fdb17db23d33df Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 12 Dec 2023 21:58:06 -0500 Subject: [PATCH 104/111] Merging the enums to eliminate a compiler warning. --- DHT/fxf.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 57acebf2f7..75ccd7eeb4 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -147,6 +147,7 @@ typedef struct { */ enum { + fxfMINSIZE = sizeof(void *), /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ fxfMAXSIZE = #if defined(SEGMENTED) || defined(__TURBOC__) # if defined(ARENA_SEG_SIZE) @@ -160,11 +161,6 @@ enum + (MAX_ALIGNMENT - 1U)) & ~(MAX_ALIGNMENT - 1U)) /* Round up if necessary. */ }; -enum -{ - fxfMINSIZE = sizeof(void *) /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ -}; - enum { ENSURE_FXFMINSIZE_GE_0 = 1/(fxfMINSIZE >= 0), ENSURE_FXFMAXSIZE_GT_0 = 1/(fxfMAXSIZE > 0), From 8f7a9002549238b8413ff28612acec2b48fc3c47 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 13 Dec 2023 19:50:32 -0500 Subject: [PATCH 105/111] Things simplify if we assume that fxfMINSIZE > 0, so let's do that. --- DHT/fxf.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 75ccd7eeb4..2bd484fd93 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -162,18 +162,15 @@ enum }; enum { - ENSURE_FXFMINSIZE_GE_0 = 1/(fxfMINSIZE >= 0), - ENSURE_FXFMAXSIZE_GT_0 = 1/(fxfMAXSIZE > 0), + ENSURE_FXFMINSIZE_GT_0 = 1/(fxfMINSIZE > 0), ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE) }; #define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) -#define MIN_ALIGNMENT_UNDERESTIMATE (fxfMINSIZE ? \ - ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) : \ - 1) /* We'd prefer the top bit, but we'll compute that during fxfInit. - (Of course, they're probably the same.) - TODO: Can we compute what we want at compile time and just use it? */ -static size_t min_alignment= !fxfMINSIZE; /* for now */ +#define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. + (Of course, they're probably the same.) + TODO: Can we compute what we want at compile time and just use it? */ +static size_t min_alignment= 0; /* for now */ static SizeHead SizeData[1 + ((fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE)]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). Maximum allocation is fxfMAXSIZE. @@ -617,7 +614,7 @@ void *fxfAlloc(size_t size) { curBottomIndex+= cur_alignment; } curBottomIndex= (size_t)(TopFreePtr-BotFreePtr); - if (curBottomIndex >= ((fxfMINSIZE > 0) ? fxfMINSIZE : 1)) { + if (curBottomIndex >= fxfMINSIZE) { SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(curBottomIndex)]; if ((curBottomIndex >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { if (curBottomIndex >= sizeof cur_sh->FreeHead) From 3eaa9d1ef732026c922cfaabb5d18fe37844b99f Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 13 Dec 2023 20:19:07 -0500 Subject: [PATCH 106/111] Adding the buffer option. --- DHT/dhtvalue.h | 1 + 1 file changed, 1 insertion(+) diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 0558801a80..cc389da97d 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -105,6 +105,7 @@ typedef union { #ifdef DHTVALUE_NEEDS_FLOATING_POINT long double floating_point; #endif + unsigned char buffer[1]; /* treat as having sizeof(dhtValue) elements */ } dhtValue; typedef struct { From ab293f8a09ec02f70b700713dbcccc26833143ba Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 14 Jan 2024 15:47:43 -0500 Subject: [PATCH 107/111] Choosing a more descriptive name. --- DHT/dht.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index b64e6de4ca..f8a5d5a5b6 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -398,7 +398,7 @@ typedef struct { dhtHashValue (*Hash)(dhtKey); int (*Equal)(dhtKey, dhtKey); - int (*DupKey)(dhtValue, dhtValue *); + int (*DupKeyValue)(dhtValue, dhtValue *); int (*DupData)(dhtValue, dhtValue *); void (*FreeKey)(dhtValue); void (*FreeData)(dhtValue); @@ -512,12 +512,12 @@ dht *dhtCreate(dhtValueType KeyType, dhtValuePolicy KeyPolicy, if (KeyPolicy==dhtNoCopy) { - ht->procs.DupKey= dhtProcedures[dhtSimpleValue]->Dup; + ht->procs.DupKeyValue= dhtProcedures[dhtSimpleValue]->Dup; ht->procs.FreeKey= dhtProcedures[dhtSimpleValue]->Free; } else if (KeyPolicy==dhtCopy) { - ht->procs.DupKey= dhtProcedures[KeyType]->Dup; + ht->procs.DupKeyValue= dhtProcedures[KeyType]->Dup; ht->procs.FreeKey= dhtProcedures[KeyType]->Free; } @@ -902,7 +902,7 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) #endif TraceFunctionParamListEnd(); - if ((ht->procs.DupKey)(key.value, &KeyV.value)) + if ((ht->procs.DupKeyValue)(key.value, &KeyV.value)) { TraceText("key duplication failed\n"); TraceFunctionExit(__func__); From 8d849586a1e4b6d3dedf0583cb68e64938b89d8c Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 14 Jan 2024 15:51:11 -0500 Subject: [PATCH 108/111] We operate on dhtKeys through their values, so make that clear. --- DHT/dht.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index f8a5d5a5b6..20d71e9299 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -400,10 +400,10 @@ typedef struct int (*Equal)(dhtKey, dhtKey); int (*DupKeyValue)(dhtValue, dhtValue *); int (*DupData)(dhtValue, dhtValue *); - void (*FreeKey)(dhtValue); + void (*FreeKeyValue)(dhtValue); void (*FreeData)(dhtValue); void (*DumpData)(dhtValue,FILE *); - void (*DumpKey)(dhtValue,FILE *); + void (*DumpKeyValue)(dhtValue,FILE *); } Procedures; typedef struct dht { @@ -508,17 +508,17 @@ dht *dhtCreate(dhtValueType KeyType, dhtValuePolicy KeyPolicy, ht->procs.Hash= dhtProcedures[KeyType]->Hash; ht->procs.Equal= dhtProcedures[KeyType]->Equal; ht->procs.DumpData= dhtProcedures[DtaType]->Dump; - ht->procs.DumpKey= dhtProcedures[KeyType]->Dump; + ht->procs.DumpKeyValue= dhtProcedures[KeyType]->Dump; if (KeyPolicy==dhtNoCopy) { ht->procs.DupKeyValue= dhtProcedures[dhtSimpleValue]->Dup; - ht->procs.FreeKey= dhtProcedures[dhtSimpleValue]->Free; + ht->procs.FreeKeyValue= dhtProcedures[dhtSimpleValue]->Free; } else if (KeyPolicy==dhtCopy) { ht->procs.DupKeyValue= dhtProcedures[KeyType]->Dup; - ht->procs.FreeKey= dhtProcedures[KeyType]->Free; + ht->procs.FreeKeyValue= dhtProcedures[KeyType]->Free; } if (DataPolicy==dhtNoCopy) @@ -559,7 +559,7 @@ void dhtDestroy(HashTable *ht) while (b) { InternHsElement *tmp= b; - (ht->procs.FreeKey)(b->HsEl.Key.value); + (ht->procs.FreeKeyValue)(b->HsEl.Key.value); (ht->procs.FreeData)(b->HsEl.Data); b= b->Next; FreeInternHsElement(tmp); @@ -599,7 +599,7 @@ void dhtDumpIndented(int ind, HashTable *ht, FILE *f) while (b) { fprintf(f, "%*s ", ind, ""); - (ht->procs.DumpKey)(b->HsEl.Key.value, f); + (ht->procs.DumpKeyValue)(b->HsEl.Key.value, f); fputs("->", f); (ht->procs.DumpData)(b->HsEl.Data, f); b= b->Next; @@ -858,7 +858,7 @@ void dhtRemoveElement(HashTable *ht, dhtKey key) *phe= he->Next; (ht->procs.FreeData)(he->HsEl.Data); - (ht->procs.FreeKey)(he->HsEl.Key.value); + (ht->procs.FreeKeyValue)(he->HsEl.Key.value); FreeInternHsElement(he); ht->KeyCount--; if (ActualLoadFactor(ht) < ht->MinLoadFactor) @@ -913,7 +913,7 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) if ((ht->procs.DupData)(data, &DataV)) { - (ht->procs.FreeKey)(KeyV.value); + (ht->procs.FreeKeyValue)(KeyV.value); TraceText("data duplication failed\n"); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void *)dhtNilElement); @@ -933,7 +933,7 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) TraceEOL(); if (he==0) { - (ht->procs.FreeKey)(KeyV.value); + (ht->procs.FreeKeyValue)(KeyV.value); (ht->procs.FreeData)(DataV); TraceText("allocation of new intern Hs element failed\n"); TraceFunctionExit(__func__); @@ -953,7 +953,7 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) if (ht->DtaPolicy == dhtCopy) (ht->procs.FreeData)(he->HsEl.Data); if (ht->KeyPolicy == dhtCopy) - (ht->procs.FreeKey)(he->HsEl.Key.value); + (ht->procs.FreeKeyValue)(he->HsEl.Key.value); } he->HsEl.Key = KeyV; From bcc1958cc06ae90d7abf9972644da338293dfaf3 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 28 Jan 2024 15:23:57 -0500 Subject: [PATCH 109/111] Restoring an assert. --- DHT/dht.c | 1 + 1 file changed, 1 insertion(+) diff --git a/DHT/dht.c b/DHT/dht.c index 20d71e9299..14b850cf5b 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -902,6 +902,7 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) #endif TraceFunctionParamListEnd(); + assert(!!key.value.object_pointer); if ((ht->procs.DupKeyValue)(key.value, &KeyV.value)) { TraceText("key duplication failed\n"); From be528c5da9f05cc38ca57d1f13cb35ba53cd448d Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 28 Jan 2024 15:25:01 -0500 Subject: [PATCH 110/111] Keeping the original format of the assert. --- DHT/dht.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/dht.c b/DHT/dht.c index 14b850cf5b..cf7f8e5b2b 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -902,7 +902,7 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) #endif TraceFunctionParamListEnd(); - assert(!!key.value.object_pointer); + assert(key.value.object_pointer!=0); /* Here we assume that object_pointer is the active member. */ if ((ht->procs.DupKeyValue)(key.value, &KeyV.value)) { TraceText("key duplication failed\n"); From 61f32331613d995361dc82b19493b3f50ca2b629 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 28 Jan 2024 15:33:10 -0500 Subject: [PATCH 111/111] Marking the new assert as a TODO. --- DHT/dht.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DHT/dht.c b/DHT/dht.c index cf7f8e5b2b..7c0fc8b265 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -902,7 +902,8 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) #endif TraceFunctionParamListEnd(); - assert(key.value.object_pointer!=0); /* Here we assume that object_pointer is the active member. */ + assert(key.value.object_pointer!=0); /* TODO: This assert assumes that object_pointer is the active member. + Is there a more generic test we could do? Do we need one? */ if ((ht->procs.DupKeyValue)(key.value, &KeyV.value)) { TraceText("key duplication failed\n");