From 69cac7bea56581944428b30cffe1255d945e0f99 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 16 May 2024 20:42:46 -0400 Subject: [PATCH 01/53] Creating a new version of feature/tighten_hash_usage based off develop commit 5ffb472 --- DHT/dht.c | 88 ++++--- DHT/dht.h | 36 +-- DHT/dhtbcmem.c | 127 +++++++--- DHT/dhtbcmem.h | 6 +- DHT/dhtcmem.c | 111 +++++++-- DHT/dhtcmem.h | 10 +- DHT/dhtexam1.c | 40 +-- DHT/dhtexam2.c | 8 +- DHT/dhtmanex.c | 20 +- DHT/dhtmem.c | 114 ++++++--- DHT/dhtmem.h | 12 +- DHT/dhtsimpl.c | 70 ++++-- DHT/dhtstrng.c | 95 +++++--- DHT/dhtvalue.c | 2 +- DHT/dhtvalue.h | 78 +++++- DHT/fxf.c | 562 +++++++++++++++++++++++++++++++++---------- DHT/fxf.h | 4 +- debugging/trace.c | 4 +- optimisations/hash.c | 269 ++++++++++----------- optimisations/hash.h | 131 +++++++++- position/pieceid.h | 7 +- 21 files changed, 1259 insertions(+), 535 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index 90103df1a2..7c0fc8b265 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -396,14 +396,14 @@ static InternHsElement *stepDirTable(dirEnumerate *enumeration) typedef struct { - dhtHashValue (*Hash)(dhtConstValue); - int (*Equal)(dhtConstValue, dhtConstValue); - dhtConstValue (*DupKey)(dhtConstValue); - dhtConstValue (*DupData)(dhtConstValue); - void (*FreeKey)(dhtValue); + dhtHashValue (*Hash)(dhtKey); + int (*Equal)(dhtKey, dhtKey); + int (*DupKeyValue)(dhtValue, dhtValue *); + int (*DupData)(dhtValue, dhtValue *); + void (*FreeKeyValue)(dhtValue); void (*FreeData)(dhtValue); - void (*DumpData)(dhtConstValue,FILE *); - void (*DumpKey)(dhtConstValue,FILE *); + void (*DumpData)(dhtValue,FILE *); + void (*DumpKeyValue)(dhtValue,FILE *); } Procedures; typedef struct dht { @@ -436,7 +436,7 @@ typedef struct dht { #define ActualLoadFactor(h) (((h)->KeyCount*100)/(h)->DirTab.count) #endif /*OVERFLOW_SAVE*/ -unsigned long dhtKeyCount(dht *h) +unsigned long dhtKeyCount(dht const *h) { return h->KeyCount; } @@ -508,17 +508,17 @@ dht *dhtCreate(dhtValueType KeyType, dhtValuePolicy KeyPolicy, ht->procs.Hash= dhtProcedures[KeyType]->Hash; ht->procs.Equal= dhtProcedures[KeyType]->Equal; ht->procs.DumpData= dhtProcedures[DtaType]->Dump; - ht->procs.DumpKey= dhtProcedures[KeyType]->Dump; + ht->procs.DumpKeyValue= dhtProcedures[KeyType]->Dump; if (KeyPolicy==dhtNoCopy) { - ht->procs.DupKey= dhtProcedures[dhtSimpleValue]->Dup; - ht->procs.FreeKey= dhtProcedures[dhtSimpleValue]->Free; + ht->procs.DupKeyValue= dhtProcedures[dhtSimpleValue]->Dup; + ht->procs.FreeKeyValue= dhtProcedures[dhtSimpleValue]->Free; } else if (KeyPolicy==dhtCopy) { - ht->procs.DupKey= dhtProcedures[KeyType]->Dup; - ht->procs.FreeKey= dhtProcedures[KeyType]->Free; + ht->procs.DupKeyValue= dhtProcedures[KeyType]->Dup; + ht->procs.FreeKeyValue= dhtProcedures[KeyType]->Free; } if (DataPolicy==dhtNoCopy) @@ -559,8 +559,8 @@ void dhtDestroy(HashTable *ht) while (b) { InternHsElement *tmp= b; - (ht->procs.FreeKey)((dhtValue)b->HsEl.Key); - (ht->procs.FreeData)((dhtValue)b->HsEl.Data); + (ht->procs.FreeKeyValue)(b->HsEl.Key.value); + (ht->procs.FreeData)(b->HsEl.Data); b= b->Next; FreeInternHsElement(tmp); } @@ -599,7 +599,7 @@ void dhtDumpIndented(int ind, HashTable *ht, FILE *f) while (b) { fprintf(f, "%*s ", ind, ""); - (ht->procs.DumpKey)(b->HsEl.Key, f); + (ht->procs.DumpKeyValue)(b->HsEl.Key.value, f); fputs("->", f); (ht->procs.DumpData)(b->HsEl.Data, f); b= b->Next; @@ -796,14 +796,18 @@ LOCAL void ShrinkHashTable(HashTable *ht) shrinkDirTable(&ht->DirTab); } -LOCAL InternHsElement **LookupInternHsElement(HashTable *ht, dhtConstValue key) +LOCAL InternHsElement **LookupInternHsElement(HashTable *ht, dhtKey key) { uLong h; InternHsElement **phe; TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); - TraceFunctionParam("%p",(void const *)key); +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) + TraceFunctionParam("%jx",key.value.unsigned_integer); +#else + TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); +#endif TraceFunctionParamListEnd(); h = DynamicHash(ht->p, ht->maxp, (ht->procs.Hash)(key)); @@ -827,14 +831,18 @@ LOCAL InternHsElement **LookupInternHsElement(HashTable *ht, dhtConstValue key) return phe; } -void dhtRemoveElement(HashTable *ht, dhtConstValue key) +void dhtRemoveElement(HashTable *ht, dhtKey key) { MYNAME(dhtRemoveElement) InternHsElement **phe, *he; TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); - TraceFunctionParam("%p",(void const *)key); +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) + TraceFunctionParam("%jx",key.value.unsigned_integer); +#else + TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); +#endif TraceFunctionParamListEnd(); phe= LookupInternHsElement(ht, key); @@ -849,8 +857,8 @@ void dhtRemoveElement(HashTable *ht, dhtConstValue key) ht->NextStep= ht->NextStep->Next; *phe= he->Next; - (ht->procs.FreeData)((dhtValue)he->HsEl.Data); - (ht->procs.FreeKey)((dhtValue)he->HsEl.Key); + (ht->procs.FreeData)(he->HsEl.Data); + (ht->procs.FreeKeyValue)(he->HsEl.Key.value); FreeInternHsElement(he); ht->KeyCount--; if (ActualLoadFactor(ht) < ht->MinLoadFactor) @@ -877,21 +885,26 @@ void dhtRemoveElement(HashTable *ht, dhtConstValue key) TraceFunctionResultEnd(); } -dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data) +dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) { InternHsElement **phe, *he; - dhtConstValue KeyV; - dhtConstValue DataV; + dhtKey KeyV; + dhtValue DataV; TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); - TraceFunctionParam("%p",(void const *)key); - TraceFunctionParam("%p",(void const *)data); +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) + TraceFunctionParam("%jx",key.value.unsigned_integer); + TraceFunctionParam("%jx",data.unsigned_integer); +#else + TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); + TraceFunctionParam("%lx",(unsigned long)data.unsigned_integer); +#endif TraceFunctionParamListEnd(); - assert(key!=0); - KeyV = (ht->procs.DupKey)(key); - if (KeyV==0) + assert(key.value.object_pointer!=0); /* TODO: This assert assumes that object_pointer is the active member. + Is there a more generic test we could do? Do we need one? */ + if ((ht->procs.DupKeyValue)(key.value, &KeyV.value)) { TraceText("key duplication failed\n"); TraceFunctionExit(__func__); @@ -900,10 +913,9 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data return dhtNilElement; } - DataV = data==0 ? 0 : (ht->procs.DupData)(data); - if (data!=0 && DataV==0) + if ((ht->procs.DupData)(data, &DataV)) { - (ht->procs.FreeKey)((dhtValue)KeyV); + (ht->procs.FreeKeyValue)(KeyV.value); TraceText("data duplication failed\n"); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void *)dhtNilElement); @@ -923,8 +935,8 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data TraceEOL(); if (he==0) { - (ht->procs.FreeKey)((dhtValue)KeyV); - (ht->procs.FreeData)((dhtValue)DataV); + (ht->procs.FreeKeyValue)(KeyV.value); + (ht->procs.FreeData)(DataV); TraceText("allocation of new intern Hs element failed\n"); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void *)dhtNilElement); @@ -941,9 +953,9 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data else { if (ht->DtaPolicy == dhtCopy) - (ht->procs.FreeData)((dhtValue)he->HsEl.Data); + (ht->procs.FreeData)(he->HsEl.Data); if (ht->KeyPolicy == dhtCopy) - (ht->procs.FreeKey)((dhtValue)he->HsEl.Key); + (ht->procs.FreeKeyValue)(he->HsEl.Key.value); } he->HsEl.Key = KeyV; @@ -975,7 +987,7 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtConstValue key, dhtConstValue data return &he->HsEl; } -dhtElement *dhtLookupElement(HashTable *ht, dhtConstValue key) +dhtElement *dhtLookupElement(HashTable *ht, dhtKey key) { InternHsElement **phe; dhtElement *result; diff --git a/DHT/dht.h b/DHT/dht.h index a2a50c07bd..da48e683da 100644 --- a/DHT/dht.h +++ b/DHT/dht.h @@ -6,7 +6,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ typedef enum { @@ -15,28 +15,28 @@ typedef enum { /* Now finally this is the HashElement */ typedef struct { - dhtConstValue Key; - dhtConstValue Data; + dhtKey Key; + dhtValue Data; } dhtElement; -#define dhtNilElement ((dhtElement *)0) +#define dhtNilElement ((dhtElement *)0) struct dht; -#define dhtNilHashTable ((struct dht *)0) +#define dhtNilHashTable ((struct dht *)0) /* procedures */ -struct dht *dhtCreate(dhtValueType KeyType, dhtValuePolicy KeyPolicy, - dhtValueType DtaType, dhtValuePolicy DataPolicy); -dhtElement *dhtEnterElement(struct dht *, dhtConstValue key, dhtConstValue data); -unsigned int dhtBucketStat (struct dht *, unsigned int *counter, unsigned int n); -void dhtDestroy (struct dht *); -void dhtDump (struct dht *, FILE *); -void dhtDumpIndented (int i, struct dht *, FILE *); -void dhtRemoveElement (struct dht *, dhtConstValue key); -dhtElement *dhtLookupElement (struct dht *, dhtConstValue key); -dhtElement *dhtGetFirstElement(struct dht *); -dhtElement *dhtGetNextElement (struct dht *); -unsigned long dhtKeyCount (struct dht *); -char const *dhtErrorMsg (void); +struct dht *dhtCreate(dhtValueType KeyType, dhtValuePolicy KeyPolicy, + dhtValueType DtaType, dhtValuePolicy DataPolicy); +dhtElement *dhtEnterElement(struct dht *, dhtKey key, dhtValue data); +unsigned int dhtBucketStat(struct dht *, unsigned int *counter, unsigned int n); +void dhtDestroy(struct dht *); +void dhtDump(struct dht *, FILE *); +void dhtDumpIndented(int ind, struct dht *, FILE *); +void dhtRemoveElement(struct dht *, dhtKey key); +dhtElement *dhtLookupElement(struct dht *, dhtKey key); +dhtElement *dhtGetFirstElement(struct dht *); +dhtElement *dhtGetNextElement(struct dht *); +unsigned long dhtKeyCount(struct dht const *); +char const *dhtErrorMsg(void); extern char dhtError[]; diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index 54c474c91a..dede2285ea 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -4,33 +4,41 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ -#include "debugging/assert.h" #include #include #include #include +#include #if defined(__BORLANDC__) #include #endif /*__BORLANDC__*/ +#include "debugging/assert.h" #include "dhtvalue.h" #include "dhtbcmem.h" #include "dht.h" -static dhtHashValue ConvertBCMemValue(dhtConstValue m) +enum { + ENSURE_SIZE_OF_ELEMENT_IS_ONE = 1/(1 == sizeof ((BCMemValue const *)NULL)->Data[0]) +}; + +static dhtHashValue ConvertBCMemValue(dhtKey m) { - BCMemValue const * const toBeConverted = (BCMemValue const *)m; - unsigned int leng = toBeConverted->Leng; - unsigned char const *s = toBeConverted->Data; + BCMemValue const * const toBeConverted = (BCMemValue const *)m.value.object_pointer; + unsigned short leng; + unsigned char const *s; + assert(!!toBeConverted); + leng = toBeConverted->Leng; + s = toBeConverted->Data; dhtHashValue hash = 0; - unsigned int i; - for (i=0; iData + value1->Leng; - - return memcmp(value1,value2,size)==0; + BCMemValue const * const value1 = (BCMemValue const *)v1.value.object_pointer; + BCMemValue const * const value2 = (BCMemValue const *)v2.value.object_pointer; + unsigned short length; + unsigned char const *data1; + unsigned char const *data2; + assert(value1 && value2); + length = value1->Leng; + + if (length != value2->Leng) + return 0; + + data1 = value1->Data; + data2 = value2->Data; + while (length > ((size_t)-1)) + { + if (memcmp(data1, data2, ((size_t)-1))) + return 0; + data1 += ((size_t)-1); + data2 += ((size_t)-1); + length -= ((size_t)-1); + } + return !memcmp(data1, data2, length); } -static dhtConstValue DupBCMemValue(dhtConstValue v) +static int DupBCMemValue(dhtValue kv, dhtValue *output) { - BCMemValue const * const original = (BCMemValue const *)v; - size_t const size = (sizeof *original - - sizeof original->Data - + original->Leng); + BCMemValue const *original = (BCMemValue const *)kv.object_pointer; + size_t const num_bytes_in_Data = ((sizeof *original) - offsetof(BCMemValue, Data)); + BCMemValue *result; + unsigned short length; + size_t size = sizeof *original; + + assert(!!output); + if (!original) + { + output->object_pointer = NULL; + return 0; + } - BCMemValue * const result = fxfAlloc(size); - if (result!=0) - memcpy(result,original,size); + length = original->Leng; + if (length > num_bytes_in_Data) + { + if (length > (((size_t)-1) - size + num_bytes_in_Data)) + return 1; + size += (length - num_bytes_in_Data); + } + + result = (BCMemValue *)fxfAlloc(size); + if (result) + { + result->Leng = length; + memcpy(result->Data,original->Data,length); + output->object_pointer = result; + return 0; + } - return (dhtConstValue)result; + return 1; } -static void FreeBCMemVal(dhtValue v) +static void FreeBCMemValue(dhtValue kv) { - BCMemValue * const freed = (BCMemValue *)v; - size_t const size = sizeof *freed - sizeof freed->Data + freed->Leng; - fxfFree(freed,size); + BCMemValue *freed = (BCMemValue *)kv.object_pointer; + size_t const num_bytes_in_Data = ((sizeof *freed) - offsetof(BCMemValue, Data)); + if (freed) + { + size_t size = sizeof *freed; + unsigned short length = freed->Leng; + if (length > num_bytes_in_Data) + { + assert(length <= (((size_t)-1) - size + num_bytes_in_Data)); + size += (length - num_bytes_in_Data); + } + fxfFree(freed,size); + } } -static void DumpBCMemValue(dhtConstValue v, FILE *f) +static void DumpBCMemValue(dhtValue kv, FILE *f) { - BCMemValue const * const toBeDumped = (BCMemValue const *)v; - unsigned int const length = toBeDumped->Leng; - unsigned int i; + BCMemValue const *toBeDumped = (BCMemValue const *)kv.object_pointer; + unsigned short length; + unsigned short i; + + assert(toBeDumped && f); - fprintf(f, "(%d)", toBeDumped->Leng); + length = toBeDumped->Leng; + fprintf(f, "(%u)", length); for (i=0; iData[i] & 0xff); + fprintf(f, "%02x", (toBeDumped->Data[i] & 0xffU)); } dhtValueProcedures dhtBCMemoryProcs = @@ -90,6 +149,6 @@ dhtValueProcedures dhtBCMemoryProcs = ConvertBCMemValue, EqualBCMemValue, DupBCMemValue, - FreeBCMemVal, + FreeBCMemValue, DumpBCMemValue }; diff --git a/DHT/dhtbcmem.h b/DHT/dhtbcmem.h index 30f015648a..55bd635f36 100644 --- a/DHT/dhtbcmem.h +++ b/DHT/dhtbcmem.h @@ -6,7 +6,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept + * comment with the above copyright notice is kept * intact and in place. */ @@ -16,8 +16,8 @@ */ typedef struct BCMemValue { - unsigned char Leng; - unsigned char Data[1]; + unsigned short Leng; + unsigned char Data[1]; } BCMemValue; #endif /*DHTBCMEM_INCLUDED*/ diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 9353418cac..a6a486403a 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -4,13 +4,15 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ #include #include #include +#include +#include "debugging/assert.h" #include "dhtvalue.h" #include "dhtcmem.h" #include "dht.h" @@ -22,13 +24,21 @@ typedef unsigned long uLong; typedef unsigned char uChar; -static dhtHashValue ConvertCompactMemoryValue(dhtConstValue m) +enum { + ENSURE_SIZE_OF_ELEMENT_IS_ONE = 1/(1 == sizeof NilCompactMemVal->Data[0]) +}; + +static dhtHashValue ConvertCompactMemoryValue(dhtKey m) { - uLong leng= ((CompactMemVal const *)m)->Leng; - uChar const *s= ((CompactMemVal const *)m)->Data; + CompactMemVal const * const toBeConverted = (CompactMemVal const *)m.value.object_pointer; + uLong leng; + uChar const *s; + assert(!!toBeConverted); + leng= toBeConverted->Leng; + s= toBeConverted->Data; dhtHashValue hash= 0; uLong i; - for (i=0; i> 6; @@ -39,39 +49,90 @@ static dhtHashValue ConvertCompactMemoryValue(dhtConstValue m) return hash; } -static int EqualCompactMemoryValue(dhtConstValue v1, dhtConstValue v2) +static int EqualCompactMemoryValue(dhtKey v1, dhtKey v2) { - if (((CompactMemVal const *)v1)->Leng != ((CompactMemVal const *)v2)->Leng) + CompactMemVal const * const value1 = (CompactMemVal const *)v1.value.object_pointer; + CompactMemVal const * const value2 = (CompactMemVal const *)v2.value.object_pointer; + uLong length; + unsigned char const *data1; + unsigned char const *data2; + assert(value1 && value2); + length = value1->Leng; + if (length != value2->Leng) return 0; - if (memcmp(((CompactMemVal const *)v1)->Data, - ((CompactMemVal const *)v2)->Data, ((CompactMemVal const *)v1)->Leng)) - return 0; - else - return 1; + data1 = value1->Data; + data2 = value2->Data; + while (length > ((size_t)-1)) + { + if (memcmp(data1, data2, ((size_t)-1))) + return 0; + data1 += ((size_t)-1); + data2 += ((size_t)-1); + length -= ((size_t)-1); + } + return !memcmp(data1, data2, length); } -static dhtConstValue DupCompactMemoryValue(dhtConstValue v) +static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) { - CompactMemVal *cm= NewCompactMemVal(((CompactMemVal const *)v)->Leng); - if (cm) { - cm->Leng= ((CompactMemVal const *)v)->Leng; - memcpy(cm->Data, ((CompactMemVal const *)v)->Data, cm->Leng); - return (dhtValue)cm; + CompactMemVal const *v = (CompactMemVal const *)kv.object_pointer; + size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); + size_t size = sizeof *v; + CompactMemVal *result; + uLong length; + + assert(!!output); + if (!v) + { + output->object_pointer = NilCompactMemVal; + return 0; } - return (dhtValue)cm; + + length = v->Leng; + if (length > num_bytes_in_Data) + { + if (length > (((size_t)-1) - size + num_bytes_in_Data)) + return 1; + size += (length - num_bytes_in_Data); + } + + result = (CompactMemVal *)fxfAlloc(size); + if (result) + { + result->Leng = length; + memcpy(result->Data,v->Data,length); + output->object_pointer = result; + return 0; + } + + return 1; } -static void FreeCompactMemoryValue(dhtValue v) +static void FreeCompactMemoryValue(dhtValue kv) { - FreeCompactMemVal(v); + CompactMemVal *v = (CompactMemVal *)kv.object_pointer; + size_t const num_bytes_in_Data = ((sizeof *v) - offsetof(CompactMemVal, Data)); + if (v) + { + size_t size = sizeof *v; + uLong length = v->Leng; + if (length > num_bytes_in_Data) + { + assert(length <= (((size_t)-1) - size + num_bytes_in_Data)); + size += (length - num_bytes_in_Data); + } + fxfFree(v,size); + } } -static void DumpCompactMemoryValue(dhtConstValue v, FILE *f) +static void DumpCompactMemoryValue(dhtValue kv, FILE *f) { + CompactMemVal const *v = (CompactMemVal const *)kv.object_pointer; uLong i; - fprintf(f, "(%lu)", ((CompactMemVal const *)v)->Leng); - for (i=0; i<((CompactMemVal const *)v)->Leng; i++) - fprintf(f, "%02x", ((CompactMemVal const *)v)->Data[i] & 0xff); + assert(v && f); + fprintf(f, "(%lu)", v->Leng); + for (i=0; iLeng; i++) + fprintf(f, "%02x", (v->Data[i] & 0xffU)); } dhtValueProcedures dhtCompactMemoryProcs = { diff --git a/DHT/dhtcmem.h b/DHT/dhtcmem.h index 7e1a6c85d8..85b24a1c4a 100644 --- a/DHT/dhtcmem.h +++ b/DHT/dhtcmem.h @@ -6,7 +6,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept + * comment with the above copyright notice is kept * intact and in place. */ @@ -14,10 +14,10 @@ * It is allocated via one malloc call. */ typedef struct CompactMemVal { - unsigned long Leng; - unsigned char Data[1]; + unsigned long Leng; + unsigned char Data[1]; } CompactMemVal; -#define NilCompactMemVal (CompactMemVal *)0 -#define NewCompactMemVal(n) (CompactMemVal *)fxfAlloc(sizeof(CompactMemVal)+(n)*sizeof(uChar)) +#define NilCompactMemVal ((CompactMemVal *)0) +#define NewCompactMemVal(n) ((CompactMemVal *)fxfAlloc(sizeof(CompactMemVal)+(n)*sizeof(uChar))) #define FreeCompactMemVal(v) fxfFree(v, sizeof(CompactMemVal)+((CompactMemVal const *)(v))->Leng*sizeof(uChar)) #endif /*DHTCMEM_INCLUDED*/ diff --git a/DHT/dhtexam1.c b/DHT/dhtexam1.c index 7fb3e9adf2..5c0cc27b1d 100644 --- a/DHT/dhtexam1.c +++ b/DHT/dhtexam1.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ @@ -62,7 +62,7 @@ uLong inet_addr(char *cp) { return addr; } -struct hostent *gethent(char *file) +struct hostent *gethent(char const *file) { static struct hostent host; char *p; @@ -115,6 +115,8 @@ int main(int argc, char *argv[]) { struct hostent *host; int i; char *hostsfile; + dhtKey k; + dhtValue v; if (argc > 1) { @@ -194,14 +196,18 @@ int main(int argc, char *argv[]) { */ InetAddr= *(uLong *)h; - if (dhtLookupElement(NameToInet, (dhtValue)HostName)) { + k.value.object_pointer = HostName; + if (dhtLookupElement(NameToInet, k)) { fprintf(stderr, "Hostname %s already entered\n", host->h_name); } - dhtEnterElement(NameToInet, (dhtValue)HostName, (dhtValue)InetAddr); - if (dhtLookupElement(InetToName, (dhtValue)InetAddr)) { + v.unsigned_integer = InetAddr; + dhtEnterElement(NameToInet, k, v); + k.value.unsigned_integer = InetAddr; + if (dhtLookupElement(InetToName, k)) { fprintf(stderr, "InetAddr 0x%08lx already entered\n", InetAddr); } - dhtEnterElement(InetToName, (dhtValue)InetAddr, (dhtValue)HostName); + v.object_pointer = HostName; + dhtEnterElement(InetToName, k, v); } #if defined(FXF) fputs("fxf-Info after filling the hash tables\n",stderr); @@ -219,7 +225,7 @@ int main(int argc, char *argv[]) { fDumpMallinfo(stderr); */ - fputs("Testing if we get alle entries from NameToInet via GetFirst and GetNext...\n",stderr); + fputs("Testing if we get all entries from NameToInet via GetFirst and GetNext...\n",stderr); he= dhtGetFirstElement(NameToInet); i= 0; while (he) { @@ -239,33 +245,35 @@ int main(int argc, char *argv[]) { he= dhtGetFirstElement(NameToInet); while (he) { - uLong adr= (uLong)he->Data; + uLong adr= (uLong)he->Data.unsigned_integer; char *Name; #if defined(USE_MEMVAL) strncpy(str, - (char *)((MemVal *)he->Key)->Data, - ((MemVal *)he->Key)->Leng); - str[((MemVal *)he->Key)->Leng]='\0'; + (char *)((MemVal *)he->Key.value.object_pointer)->Data, + ((MemVal *)he->Key.value.object_pointer)->Leng); + str[((MemVal *)he->Key.value.object_pointer)->Leng]='\0'; Name= str; #else - Name= (char *)he->Key; + Name= (char *)he->Key.value.object_pointer; #endif /*USE_MEMVAL*/ printf("%3u.%3u.%3u.%3u = %s ", BYT(adr), BYT(adr>>8), BYT(adr>>16), BYT(adr>>24), Name); dhtRemoveElement(NameToInet, he->Key); - dhtRemoveElement(InetToName, he->Data); + k.value = he->Data; + dhtRemoveElement(InetToName, k); printf(" Deleting and checking consistency (Load=%lu... ", dhtKeyCount(NameToInet)); i=0; hhe= dhtGetFirstElement(NameToInet); fputs(" ", stdout); fflush(stdout); while (hhe) { - dhtElement *he1= dhtLookupElement(InetToName, hhe->Data); - if (strcmp((char *)he1->Data, (char *)hhe->Key) != 0) { + k.value = hhe->Data; + dhtElement *he1= dhtLookupElement(InetToName, k); + if (strcmp((char const *)he1->Data.object_pointer, (char const *)hhe->Key.value.object_pointer) != 0) { puts("\nSorry, Mismatch"); exit(1); } - if (he1->Key != hhe->Data) { + if (he1->Key.value.unsigned_integer != hhe->Data.unsigned_integer) { puts("\nSorry, Mismatch"); exit(2); } diff --git a/DHT/dhtexam2.c b/DHT/dhtexam2.c index 58a9229e8e..4f3f6dbf23 100644 --- a/DHT/dhtexam2.c +++ b/DHT/dhtexam2.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ @@ -23,6 +23,8 @@ int main(int argc, char *argv[]) { int cnt, EntryCnt= 0; long int tmp; + dhtKey k; + dhtValue v; struct dummy { long l1; @@ -82,7 +84,9 @@ int main(int argc, char *argv[]) { for (cnt=0; cntage= Age; info_to_enter->room= Room; /* the string will be duplicated the info not */ - dhtEnterElement(OurTable, - (dhtValue)string_to_enter, (dhtValue)info_to_enter); + k.value.object_pointer = string_to_enter; + v.object_pointer = info_to_enter; + dhtEnterElement(OurTable, k, v); } /* access table */ while (scanf("%127s", name_to_find) != EOF) { - he= dhtLookupElement(OurTable, (dhtValue)name_to_find); + k.value.object_pointer = name_to_find; + he= dhtLookupElement(OurTable, k); if (he != dhtNilElement) { /* if item is in the table */ (void)printf("found %s, age = %d, room = %d\n", - (char *)he->Key, - ((struct info *)he->Data)->age, - ((struct info *)he->Data)->room); + (char *)he->Key.value.object_pointer, + ((struct info *)he->Data.object_pointer)->age, + ((struct info *)he->Data.object_pointer)->room); } else { (void)printf("no such employee %s\n", name_to_find); } @@ -79,7 +83,7 @@ int main( ) /* now delete all struct info in the table */ he= dhtGetFirstElement(OurTable); while (he) { - free((dhtValue)he->Data); + free(he->Data.object_pointer); he= dhtGetNextElement(OurTable); } /* now destroy the whole table */ diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 7f573748ff..92ee473338 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ @@ -16,6 +16,7 @@ # include #endif /*__BORLANDC__*/ +#include "debugging/assert.h" #include "dhtvalue.h" #include "dhtmem.h" #include "dht.h" @@ -23,59 +24,112 @@ typedef unsigned long uLong; typedef unsigned char uChar; -static dhtHashValue HashMemoryValue(dhtConstValue v) +enum { + ENSURE_SIZE_OF_ELEMENT_IS_ONE = 1/(1 == sizeof NilMemVal->Data[0]) +}; + +static dhtHashValue HashMemoryValue(dhtKey k) { - uLong leng= ((MemVal const *)v)->Leng; - uChar const *s= ((MemVal const *)v)->Data; + MemVal const * toBeHashed = (MemVal const *)k.value.object_pointer; + assert(!!toBeHashed); + assert(toBeHashed->Data || !toBeHashed->Leng); + uLong leng= toBeHashed->Leng; + uChar const *s= toBeHashed->Data; dhtHashValue hash= 0; uLong i; - for (i=0; i> 6; + for (i=0; i> 6; } hash+= hash << 3; hash^= hash >> 11; hash+= hash << 15; return hash; } -static int EqualMemoryValue(dhtConstValue v1, dhtConstValue v2) +static int EqualMemoryValue(dhtKey v1, dhtKey v2) { - if (((MemVal const *)v1)->Leng != ((MemVal const *)v2)->Leng) - return 0; - if (memcmp(((MemVal const *)v1)->Data, ((MemVal const *)v2)->Data, ((MemVal const *)v1)->Leng)) + MemVal const * value1 = (MemVal const *)v1.value.object_pointer; + MemVal const * value2 = (MemVal const *)v2.value.object_pointer; + uLong length; + uChar const *data1; + uChar const *data2; + + assert(value1 && value2); + length = value1->Leng; + data1 = value1->Data; + assert(data1 || !length); + data2 = value2->Data; + assert(data2 || !value2->Leng); + + if (length != value2->Leng) return 0; - else + if (!length) // We'll check this here to avoid (maybe) passing NULL as an argument to memcmp. return 1; + while (length > ((size_t)-1)) + { + if (memcmp(data1, data2, ((size_t)-1))) + return 0; + data1+= ((size_t)-1); + data2+= ((size_t)-1); + length-= ((size_t)-1); + } + return !memcmp(data1, data2, length); } - -static dhtConstValue DupMemoryValue(dhtConstValue v) +static int DupMemoryValue(dhtValue kv, dhtValue *output) { + MemVal const *v= (MemVal const *)kv.object_pointer; + uChar const *data; + uLong length; MemVal *mv; - + assert(!!output); + if (!v) { + output->object_pointer = NilMemVal; + return 0; + } + length= v->Leng; + data= v->Data; + assert(data || !length); + if (length > ((size_t)-1)) + return 1; mv= NewMemVal; if (mv) { - mv->Data= (unsigned char *)fxfAlloc(((MemVal const *)v)->Leng); - if (mv->Data) { - mv->Leng= ((MemVal const *)v)->Leng; - memcpy(mv->Data, ((MemVal const *)v)->Data, mv->Leng); - return (dhtValue)mv; + mv->Leng= length; + if (length) { + void *newBuffer= fxfAlloc(length); + if (newBuffer) { + memcpy(newBuffer, data, length); + mv->Data = (uChar *)newBuffer; + output->object_pointer = mv; + return 0; + } else { + FreeMemVal(mv); + return 1; + } } else { - FreeMemVal(mv); - mv = NilMemVal; + mv->Data= NULL; // NULL is a valid pointer if Leng == 0 + output->object_pointer= mv; + return 0; } } - return (dhtValue)mv; + return 1; } -static void FreeMemoryValue(dhtValue v) +static void FreeMemoryValue(dhtValue kv) { - DeleteMemVal(v); + MemVal *v= (MemVal *)kv.object_pointer; + if (v) { + fxfFree(v->Data, v->Leng); + fxfFree(v, sizeof *v); + } } -static void DumpMemoryValue(dhtConstValue v, FILE *f) { +static void DumpMemoryValue(dhtValue kv, FILE *f) { + MemVal const * v= (MemVal *)kv.object_pointer; uLong i; - fprintf(f, "(%lu)", ((MemVal const *)v)->Leng); - for (i=0; i<((MemVal const *)v)->Leng; i++) - fprintf(f, "%02x", ((MemVal const *)v)->Data[i] & 0xff); + assert(v && f); + assert(v->Data || !v->Leng); + fprintf(f, "(%lu)", v->Leng); + for (i=0; iLeng; i++) + fprintf(f, "%02x", (v->Data[i] & 0xffU)); } dhtValueProcedures dhtMemoryProcs = { diff --git a/DHT/dhtmem.h b/DHT/dhtmem.h index 57774b3538..c7640bfa77 100644 --- a/DHT/dhtmem.h +++ b/DHT/dhtmem.h @@ -6,7 +6,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ @@ -16,12 +16,12 @@ * hash-table is needed. */ typedef struct MemVal { - unsigned long Leng; - unsigned char *Data; + unsigned long Leng; + unsigned char *Data; } MemVal; -#define NilMemVal (MemVal *)0 -#define NewMemVal (MemVal *)fxfAlloc(sizeof(MemVal)) +#define NilMemVal ((MemVal *)0) +#define NewMemVal ((MemVal *)fxfAlloc(sizeof(MemVal))) #define FreeMemVal(v) fxfFree(v, sizeof(MemVal)) -#define DeleteMemVal(v) if (((MemVal const *)(v))!=NilMemVal) fxfFree(((MemVal const *)(v))->Data, ((MemVal const *)(v))->Leng), FreeMemVal(v) +#define DeleteMemVal(v) do {if (((MemVal const *)(v))!=NilMemVal) fxfFree(((MemVal const *)(v))->Data, ((MemVal const *)(v))->Leng), FreeMemVal(v);} while (0) #endif /*DHTMEM_INCLUDED*/ diff --git a/DHT/dhtsimpl.c b/DHT/dhtsimpl.c index 7ee2c4aba5..15289d8e01 100644 --- a/DHT/dhtsimpl.c +++ b/DHT/dhtsimpl.c @@ -4,9 +4,10 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ +#include "debugging/assert.h" #include "dhtvalue.h" #if defined(ARCH64) @@ -30,52 +31,69 @@ b -= c; b -= a; b ^= (a<<18); \ c -= a; c -= b; c ^= (b>>22); \ } -static unsigned long ConvertSimpleValue(dhtConstValue v) +static unsigned long ConvertSimpleValue(dhtKey k) { +# if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) + unsigned long long a, b, c; + c = 0x9e3779b97f4a7c13LLU; +# else +# if !((((-1LU) >> 31) >> 31) >> 1) +# error "ERROR: unable to build dhtsimpl.c due to lack of a sufficiently-large integer type." +# endif unsigned long a, b, c; - c = 0x9e3779b97f4a7c13LL; - a = v<<1; - b = v; + c = 0x9e3779b97f4a7c13LU; +# endif + a = k.value.unsigned_integer<<1; + b = k.value.unsigned_integer; mix(a,b,c); - return c; + return (unsigned long)c; } #else -static unsigned long ConvertSimpleValue(dhtConstValue v) +static unsigned long ConvertSimpleValue(dhtKey k) { - size_t c = (size_t)v; - size_t a = 0; - size_t b = 0x9e3779b9; + unsigned long c = k.value.unsigned_integer; + unsigned long a = 0; + unsigned long b = 0x9e3779b9U; a -= c; a ^= c >> 13; - b -= c; b -= a; b ^= a << 8; - c -= a; c -= b; c ^= b >> 13; - a -= b; a -= c; a ^= c >> 12; - b -= c; b -= a; b ^= a << 16; - c -= a; c -= b; c ^= b >> 5; - a -= b; a -= c; a ^= c >> 3; - b -= c; b -= a; b ^= a << 10; - c -= a; c -= b; c ^= b >> 15; - return (unsigned long)c; + b -= c; b -= a; b ^= (a << 8); + c -= a; c -= b; c ^= (b >> 13); + a -= b; a -= c; a ^= (c >> 12); + b -= c; b -= a; b ^= (a << 16); + c -= a; c -= b; c ^= (b >> 5); + a -= b; a -= c; a ^= (c >> 3); + b -= c; b -= a; b ^= (a << 10); + c -= a; c -= b; c ^= (b >> 15); + return c; } #endif /*ARCH64*/ -static int EqualSimpleValue(dhtConstValue v1, dhtConstValue v2) +static int EqualSimpleValue(dhtKey k1, dhtKey k2) { - return v1 == v2; + return (k1.value.unsigned_integer == k2.value.unsigned_integer); } -static dhtConstValue DupSimpleValue(dhtConstValue v) +static int DupSimpleValue(dhtValue kv, dhtValue *output) { - return v; + assert(!!output); + *output = kv; + return 0; } -static void FreeSimpleValue(dhtValue v) +static void FreeSimpleValue(dhtValue kv) { + (void)kv; } -static void DumpSimpleValue(dhtConstValue v, FILE *f) +static void DumpSimpleValue(dhtValue kv, FILE *f) { - fprintf(f, "%08lx", (unsigned long)(size_t)v); + assert(!!f); +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) + fprintf(f, "%08jx", kv.unsigned_integer); +#else + fprintf(f, "%08lx", kv.unsigned_integer); +#endif } dhtValueProcedures dhtSimpleProcs = diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 24d6431056..1f79b28c76 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -4,64 +4,91 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ #include #include #include +#include "debugging/assert.h" #include "dhtvalue.h" #include "dht.h" -static unsigned long ConvertString(dhtConstValue v) +static dhtHashValue ConvertString(dhtKey k) { - /* I found this hash function on - * http://ourworld.compuserve.com/homepages/bob_jenkins/doobs.htm - * There are other functions, but this one is has some advantages: - * - its small - * - independant from word sizes - * - needs no initialisation - */ - unsigned char const *s= (unsigned char const *)v; - unsigned long hash= 0; - while (*s) { - hash+= *s++; - hash+= hash << 10; - hash^= hash >> 6; - } - hash+= hash << 3; - hash^= hash >> 11; - hash+= hash << 15; - return hash; + /* I found this hash function on + * http://ourworld.compuserve.com/homepages/bob_jenkins/doobs.htm + * There are other functions, but this one has some advantages: + * - it's small + * - independent from word sizes + * - needs no initialisation + */ + unsigned char const *s= (unsigned char const *)k.value.object_pointer; + unsigned long hash; + unsigned char tmp; + assert(!!s); + hash= 0; + while ((tmp= *s)) { + hash+= tmp; + hash+= hash << 10; + hash^= hash >> 6; + ++s; + } + hash+= hash << 3; + hash^= hash >> 11; + hash+= hash << 15; + return (dhtHashValue)hash; } -static int EqualString(dhtConstValue v1, dhtConstValue v2) +static int EqualString(dhtKey v1, dhtKey v2) { - if (strcmp((char const *)v1, (char const *)v2)) - return 0; - else - return 1; + char const *s1= (char const *)v1.value.object_pointer; + char const *s2= (char const *)v2.value.object_pointer; + assert(s1 && s2); + return !strcmp(s1, s2); } -static dhtConstValue DupString(dhtConstValue v) +static int DupString(dhtValue v, dhtValue *output) { char *nv; - nv= (char *)fxfAlloc(strlen((char const *)v)+1); - if (nv!=0) - strcpy(nv, (char const *)v); - return (dhtConstValue)nv; + size_t len; + char const *original= (char const *)v.object_pointer; + assert(!!output); + if (!original) { + output->object_pointer= NULL; + return 0; + } + len= strlen(original); + if ((len < ((size_t)-1)) && !original[len]) { + ++len; + nv= (char *)fxfAlloc(len); + if (nv) { + memcpy(nv, original, len); + output->object_pointer= nv; + return 0; + } + } + return 1; } static void FreeString(dhtValue v) { - fxfFree(v, strlen((char const *)v)+1); + char *s= (char *)v.object_pointer; + if (s) + { + size_t const len= strlen(s); + assert((len < ((size_t)-1)) && !s[len]); + fxfFree(s, len+1); + } } -static void DumpString(dhtConstValue v, FILE *f) +static void DumpString(dhtValue v, FILE *f) { - fputs((char const *)v,f); + char const *s= (char const *)v.object_pointer; + assert(s && f); + fputs(s,f); } -dhtValueProcedures dhtStringProcs = { +dhtValueProcedures dhtStringProcs= { ConvertString, EqualString, DupString, diff --git a/DHT/dhtvalue.c b/DHT/dhtvalue.c index ef5761d598..321a08f8fc 100644 --- a/DHT/dhtvalue.c +++ b/DHT/dhtvalue.c @@ -4,7 +4,7 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ #include "dhtvalue.h" diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index befd63b4df..cc389da97d 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -7,15 +7,31 @@ * Institut fuer Informatik, TU Muenchen, Germany * bartel@informatik.tu-muenchen.de * You may use this code as you wish, as long as this - * comment with the above copyright notice is keept intact + * comment with the above copyright notice is kept intact * and in place. */ #include +#ifdef __cplusplus +# if __cplusplus >= 201103L +# include +# else +# include +# endif +# include +#elif defined(__STDC_VERSION__) +# if __STDC_VERSION__ >= 199901L +# include +# else +# include +# endif +# include +#endif #if defined(FXF) #include "fxf.h" #else -#define fxfAlloc(x) malloc(x) +#include +#define fxfAlloc(x) malloc(x) /* TODO: Should we track allocations to ensure that we never allocate more than some chosen number (e.g., hashtable_kilos*1024) of total byte(s)? */ #define fxfFree(x,n) free(x) #endif /*FXF*/ @@ -52,19 +68,63 @@ typedef enum { extern char const *dhtValueTypeToString[dhtValueTypeCnt]; -typedef void *dhtValue; -typedef void const *dhtConstValue; +typedef union { +#ifdef __cplusplus +# if __cplusplus >= 201103L + ::std::uintmax_t unsigned_integer; + ::std::intmax_t signed_integer; +# elif defined(LLONG_MAX) + unsigned long long int unsigned_integer; + long long int signed_integer; +# else + unsigned long int unsigned_integer; + long int signed_integer; +# endif + bool boolean; + ::std::sig_atomic_t atomic_integer; +#else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + uintmax_t unsigned_integer; + intmax_t signed_integer; + _Bool boolean; +# else +# if defined(LLONG_MAX) + unsigned long long int unsigned_integer; + long long int signed_integer; +# else + unsigned long int unsigned_integer; + long int signed_integer; +# endif + int boolean; // What else? +# endif + sig_atomic_t atomic_integer; +#endif + const volatile void * object_pointer; + char character; + void (*function_pointer)(void); +#ifdef DHTVALUE_NEEDS_FLOATING_POINT + long double floating_point; +#endif + unsigned char buffer[1]; /* treat as having sizeof(dhtValue) elements */ +} dhtValue; + +typedef struct { + dhtValue value; +} dhtKey; + typedef unsigned long dhtHashValue; typedef struct { - dhtHashValue (*Hash)(dhtConstValue); - int (*Equal)(dhtConstValue, dhtConstValue); - dhtConstValue (*Dup)(dhtConstValue); - void (*Free)(dhtValue); - void (*Dump)(dhtConstValue, FILE *); + dhtHashValue (*Hash)(dhtKey); + int (*Equal)(dhtKey, dhtKey); + int (*Dup)(dhtValue, dhtValue *); // should return 0 on success (and store the copied value at the second argument) and nonzero on error + void (*Free)(dhtValue); + void (*Dump)(dhtValue, FILE *); } dhtValueProcedures; #if defined(REGISTER_SIMPLE) +/* dhtSimple verifies equality by comparing unsigned_integer members; + keys should be stored to unsigned_integer members, cast if needed */ extern dhtValueProcedures dhtSimpleProcs; #endif /*REGISTER_SIMPLE*/ #if defined(REGISTER_STRING) diff --git a/DHT/fxf.c b/DHT/fxf.c index 7c7e285c34..2bd484fd93 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -3,7 +3,7 @@ #include "debugging/assert.h" #include #include -#include +#include #if defined(__TURBOC__) # include @@ -18,27 +18,65 @@ #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || /* >= C99 -- We have printf ptrdiff_t/size_t specifiers. */ \ (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 -- We have printf ptrdiff_t/size_t specifiers. */ -# include +# include typedef ptrdiff_t ptrdiff_t_printf_type; typedef size_t size_t_printf_type; +# define MAX_POINTER_DIFFERENCE PTRDIFF_MAX +# if defined(UINTPTR_MAX) + typedef uintptr_t convert_pointer_to_int_type; +# else + typedef uintmax_t convert_pointer_to_int_type; +# endif # define PTRDIFF_T_PRINTF_SPECIFIER "td" # define SIZE_T_PRINTF_SPECIFIER "zu" -#elif defined(LLONG_MAX) /* We have long long integer types. */ +#else +# define MAX_POINTER_DIFFERENCE (((size_t)-1)>>1) /* just a guess */ +# if defined(LLONG_MAX) /* We have long long integer types. */ typedef long long int ptrdiff_t_printf_type; typedef unsigned long long int size_t_printf_type; -# define PTRDIFF_T_PRINTF_SPECIFIER "lld" -# define SIZE_T_PRINTF_SPECIFIER "llu" -#else /* We don't have long long integer types. */ + typedef unsigned long long int convert_pointer_to_int_type; +# define PTRDIFF_T_PRINTF_SPECIFIER "lld" +# define SIZE_T_PRINTF_SPECIFIER "llu" +# else /* We don't have long long integer types. */ typedef long int ptrdiff_t_printf_type; typedef unsigned long int size_t_printf_type; -# define PTRDIFF_T_PRINTF_SPECIFIER "ld" -# define SIZE_T_PRINTF_SPECIFIER "lu" + typedef unsigned long int convert_pointer_to_int_type; +# define PTRDIFF_T_PRINTF_SPECIFIER "ld" +# define SIZE_T_PRINTF_SPECIFIER "lu" +# endif +#endif + +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) +# define MAX_ALIGNMENT _Alignof(max_align_t) +#elif (defined(__cplusplus) && (__cplusplus >= 201103L)) +# define MAX_ALIGNMENT alignof(max_align_t) +#else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# include +# endif +struct GET_MAX_ALIGNMENT_TYPE { + unsigned char c; + union { +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + uintmax_t unsigned_integer; +# elif defined(LLONG_MAX) /* We have long long integer types. */ + unsigned long long int unsigned_integer; +# else + unsigned long int unsigned_integer; +# endif + const volatile void * object_pointer; + void (*function_pointer)(void); + long double floating_point; + } max_aligned_union; +}; +# define MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union) #endif -#if !defined(Nil) && !defined(New) && !defined(nNew) /* TODO: Is this the correct check for all of the below lines? */ +#if !defined(Nil) && !defined(New) && !defined(nNewUntyped) && !defined(nNewCallocUntyped) /* TODO: Is this the correct check for all of the below lines? */ # define Nil(type) ((type *)0) # define New(type) ((type *)malloc(sizeof(type))) -# define nNew(n, type) ((type *)nNewImpl(n,sizeof(type))) +# define nNewUntyped(n, type) nNewImpl(n,sizeof(type)) +# define nNewCallocUntyped(n, type) calloc(n,sizeof(type)) static inline void * nNewImpl(size_t const nmemb, size_t const size) { return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : malloc(nmemb*size)); } @@ -68,7 +106,7 @@ static inline void * nNewImpl(size_t const nmemb, size_t const size) { /* FiXed and Fast malloc, free * As the name tells: this code implements on top of traditional * malloc/realloc/free a fast version, that relies on a lot of - * allocation/delallocation of fixed sized blocks of memory. For + * allocation/deallocation of fixed sized blocks of memory. For * each size of memory we keep a head pointer and all freed chunks * of memory is threaded on this list. If memory of this size * is requested, we drag it from the list, otherwise we carve it @@ -80,13 +118,15 @@ static inline void * nNewImpl(size_t const nmemb, size_t const size) { typedef struct { unsigned long MallocCount; unsigned long FreeCount; - char * FreeHead; + void * FreeHead; } SizeHead; +#define CLIP_TO_MAX_POINTER_DIFFERENCE(x) (((x) > MAX_POINTER_DIFFERENCE) ? MAX_POINTER_DIFFERENCE : (x)) + #if defined(DOS) /* MSDOS 16 Bit support (maxmemory <= 1 MB) */ #define SEGMENTED -#define ARENA_SEG_SIZE 32000 +#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(32000) & ~(MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((1024*1024)/ARENA_SEG_SIZE) #define OSNAME "MSDOS" #define OSMAXMEM "1 MB" @@ -94,40 +134,64 @@ typedef struct { /* Win95/Win98/WinME can only allocate chunks up to 255 MB */ /* maxmemory <= 768 MB */ #define SEGMENTED -#define ARENA_SEG_SIZE 1000000 +#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(1000000) & ~(MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((768*1024*1024)/ARENA_SEG_SIZE) #define OSNAME "Win95/Win98/WinME" #define OSMAXMEM "768 MB" #endif /* The maximum size an fxfAlloc can handle */ +/* TODO: Do the macros really accurately determine the maximum we need (apparently 1024 or 2048)? + Can we instead compute the needed value(s) with expressions involving, say, sizeof(void *) + and any other system properties we have access to? +*/ +enum +{ + fxfMINSIZE = sizeof(void *), /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ + fxfMAXSIZE = #if defined(SEGMENTED) || defined(__TURBOC__) -#define fxfMAXSIZE ((size_t)1024) +# if defined(ARENA_SEG_SIZE) + ((((1024 > ARENA_SEG_SIZE) ? ARENA_SEG_SIZE : ((size_t)1024)) +# else + ((((size_t)1024) +# endif #else -#define fxfMAXSIZE ((size_t)2048) /* this is needed only when sizeof(void*)==8 */ + ((((size_t)2048) /* This is needed only when sizeof(void*)==8. */ #endif + + (MAX_ALIGNMENT - 1U)) & ~(MAX_ALIGNMENT - 1U)) /* Round up if necessary. */ +}; -/* Different size of fxfMINSIZE for 32-/64/Bit compilation */ -enum -{ - fxfMINSIZE = sizeof(size_t) +enum { + ENSURE_FXFMINSIZE_GT_0 = 1/(fxfMINSIZE > 0), + ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE) }; -static SizeHead SizeData[fxfMAXSIZE+1]; +#define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) +#define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. + (Of course, they're probably the same.) + TODO: Can we compute what we want at compile time and just use it? */ +static size_t min_alignment= 0; /* for now */ + +static SizeHead SizeData[1 + ((fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE)]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). + Maximum allocation is fxfMAXSIZE. + All allocations will be multiples of MIN_ALIGNMENT_UNDERESTIMATE. */ +#define SIZEDATA_SIZE_TO_INDEX(s) (((s) - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE) +#define SIZEDATA_INDEX_TO_SIZE(x) ((size_t)(((x) * MIN_ALIGNMENT_UNDERESTIMATE) + \ + ((fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U)))) #if defined(SEGMENTED) /* #define ARENA_SEG_SIZE 32000 */ /* #define ARENA_SEG_COUNT ((1024*1024)/ARENA_SEG_SIZE) */ -static char *Arena[ARENA_SEG_COUNT] = { Nil(char) }; +static void *Arena[ARENA_SEG_COUNT]= { Nil(void) }; static int ArenaSegCnt= 0; static int CurrentSeg= 0; #else -static char *Arena= Nil(char); +static void *Arena= Nil(void); #endif /*SEGMENTED*/ static size_t GlobalSize; -static char *BotFreePtr; -static char *TopFreePtr; +static void *BotFreePtr; +static void *TopFreePtr; #undef FREEMAP @@ -215,26 +279,38 @@ void PrintFreeMap(FILE *f) { fputc('?', f); } } -#else -static void SetRange(size_t x, size_t l) { (void) x; (void) l; } -static void ClrRange(size_t x, size_t l) { (void) x; (void) l; } #endif /*FREEMAP, !SEGMENTED*/ +static inline ptrdiff_t pointerDifference(void const *ptr1, void const *ptr2) { + assert(ptr1 && ptr2); + return (((char const *)ptr1) - ((char const *)ptr2)); +} + +static inline void * stepPointer(void *ptr, ptrdiff_t step) { + assert(!!ptr); + return (void *)(((char *)ptr) + step); +} + +size_t fxfMaxAllocation(void) { + return fxfMAXSIZE; +} + size_t fxfInit(size_t Size) { #if defined(LOG) static char const * const myname= "fxfInit"; #endif #if defined(SEGMENTED) - size_t maxSegCnt= (Size ? (1 + ((Size - 1) / ARENA_SEG_SIZE)) : 0); + size_t maxSegCnt= (Size / ARENA_SEG_SIZE); if (maxSegCnt > ARENA_SEG_COUNT) maxSegCnt= ARENA_SEG_COUNT; while (ArenaSegCnt > maxSegCnt) { --ArenaSegCnt; free(Arena[ArenaSegCnt]); - Arena[ArenaSegCnt]= Nil(char); + Arena[ArenaSegCnt]= Nil(void); } while (ArenaSegCnt < maxSegCnt) { - if ((Arena[ArenaSegCnt]= nNew(ARENA_SEG_SIZE, char)) == Nil(char)) + Arena[ArenaSegCnt]= nNewUntyped(ARENA_SEG_SIZE, char); + if (!Arena[ArenaSegCnt]) break; ++ArenaSegCnt; } @@ -242,28 +318,35 @@ size_t fxfInit(size_t Size) { BotFreePtr= Arena[CurrentSeg]; TopFreePtr= Arena[CurrentSeg]; if (TopFreePtr) - TopFreePtr+= ARENA_SEG_SIZE; + TopFreePtr= stepPointer(TopFreePtr, ARENA_SEG_SIZE); GlobalSize= ArenaSegCnt*ARENA_SEG_SIZE; #else #if defined(FREEMAP) if (FreeMap) { free(FreeMap); - FreeMap = Nil(FreeMapType); + FreeMap= Nil(FreeMapType); } +#endif +#if defined(LOG) + size_t const orig_Size= Size; #endif if (Arena) free(Arena); - if ((Arena=nNew(Size, char)) == Nil(char)) { - ERROR_LOG2("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " bytes\n", - myname, (size_t_printf_type) Size); + if (Size > MAX_POINTER_DIFFERENCE) + Size= MAX_POINTER_DIFFERENCE; + Size&= ~(MAX_ALIGNMENT - 1U); + Arena= nNewUntyped(Size, char); + if (!Arena) { + ERROR_LOG3("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " <= %" SIZE_T_PRINTF_SPECIFIER " bytes\n", + myname, (size_t_printf_type)Size, (size_t_printf_type)orig_Size); BotFreePtr= Arena; TopFreePtr= Arena; GlobalSize= 0; return GlobalSize; } BotFreePtr= Arena; - TopFreePtr= Arena+Size; + TopFreePtr= stepPointer(Arena, (ptrdiff_t)Size); GlobalSize= Size; #if defined(FREEMAP) @@ -277,15 +360,23 @@ size_t fxfInit(size_t Size) { Size = ((Size+31)>>5); } - FreeMap= nNew(Size, FreeMapType); /* TODO: Can/Should we replace this allocation+memset with a call to calloc? */ - if (FreeMap) - { - memset(FreeMap, '\0', Size*(sizeof *FreeMap)); - } + FreeMap= (FreeMapType *)nNewCallocUntyped(Size, FreeMapType); #endif /*FREEMAP*/ #endif /*SEGMENTED*/ - memset(SizeData, '\0', sizeof(SizeData)); + for (Size= 0; Size < ((sizeof SizeData)/(sizeof *SizeData)); ++Size) + { + SizeData[Size].MallocCount= 0; + SizeData[Size].FreeCount= 0; + SizeData[Size].FreeHead= Nil(void); + } + + if (!min_alignment) + { + min_alignment= MAX_ALIGNMENT; + while (min_alignment > fxfMINSIZE) + min_alignment>>= 1; + } return GlobalSize; } @@ -297,29 +388,37 @@ void fxfTeardown(void) { --ArenaSegCnt; free(Arena[ArenaSegCnt]); - Arena[ArenaSegCnt] = Nil(char); + Arena[ArenaSegCnt]= Nil(void); } CurrentSeg= 0; #else #if defined(FREEMAP) free(FreeMap); - FreeMap= Nil(unsigned int); + FreeMap= Nil(FreeMapType); #endif /*FREEMAP*/ free(Arena); - Arena= Nil(char); + Arena= Nil(void); #endif /*SEGMENTED*/ - memset(SizeData, '\0', sizeof(SizeData)); + { + size_t i; + for (i= 0; i < ((sizeof SizeData)/(sizeof *SizeData)); ++i) + { + SizeData[i].MallocCount= 0; + SizeData[i].FreeCount= 0; + SizeData[i].FreeHead= Nil(void); + } + } GlobalSize= 0; - TopFreePtr= Nil(char); - BotFreePtr= Nil(char); + TopFreePtr= Nil(void); + BotFreePtr= Nil(void); } int fxfInitialised(void) { #if defined(SEGMENTED) - return Arena[0]!=0; + return !!Arena[0]; #else - return Arena!=0; + return !!Arena; #endif } @@ -330,12 +429,12 @@ void fxfReset(void) BotFreePtr= Arena[CurrentSeg]; TopFreePtr= Arena[CurrentSeg]; if (TopFreePtr) - TopFreePtr+= ARENA_SEG_SIZE; + TopFreePtr= stepPointer(TopFreePtr, ARENA_SEG_SIZE); #else BotFreePtr= Arena; TopFreePtr= Arena; if (TopFreePtr) - TopFreePtr+= GlobalSize; + TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t)GlobalSize); #if defined(FREEMAP) if (FreeMap) @@ -345,13 +444,21 @@ void fxfReset(void) #if !defined(NDEBUG) { - unsigned int i; - for (i = 1; i<=50; ++i) + size_t i; + for (i = 0; i<((sizeof SizeData)/(sizeof *SizeData)); ++i) assert(SizeData[i].MallocCount==0); } #endif - memset(SizeData, '\0', sizeof SizeData); + { + size_t i; + for (i= 0; i < ((sizeof SizeData)/(sizeof *SizeData)); ++i) + { + SizeData[i].MallocCount= 0; + SizeData[i].FreeCount= 0; + SizeData[i].FreeHead= Nil(void); + } + } } /* we have to define the following, since some architectures cannot @@ -360,12 +467,8 @@ void fxfReset(void) * SPARC, HPPA, MIPS. We wouldn't need this when running on an * Intel *86 type of CPU, but also there, aligned access is faster. */ -#define PTRMASK (sizeof(char *)-1) -#define ALIGNED_MINSIZE (sizeof(char *)+PTRMASK) -#define ALIGN(ptr) (((size_t)ptr+PTRMASK) & (~PTRMASK)) - -#define GetNextPtr(ptr) (*(char **)ALIGN(ptr)) -#define PutNextPtr(dst, ptr) *(char **)ALIGN(dst)= ptr +#define PTRMASK (MAX_ALIGNMENT-1U) +#define ALIGN_TO_MINIMUM(s) (((s) + (min_alignment - 1U)) & ~(min_alignment - 1U)) #define TMDBG(x) if (0) x @@ -374,13 +477,16 @@ void *fxfAlloc(size_t size) { static char const * const myname= "fxfAlloc"; #endif SizeHead *sh; - char *ptr; + void *ptr= Nil(void); TMDBG(printf("fxfAlloc - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); DBG((stderr, "%s(%" SIZE_T_PRINTF_SPECIFIER ") =", myname, (size_t_printf_type)size)); + if (!size) + return Nil(void); + if (sizefxfMAXSIZE) { @@ -388,114 +494,261 @@ void *fxfAlloc(size_t size) { myname, (size_t_printf_type) size, (size_t_printf_type) fxfMAXSIZE); - return Nil(char); + return Nil(void); } - if ( (size&PTRMASK) && sizeFreeHead) { +#if defined(SEGMENTED) + int ptrSegment; + ptrdiff_t ptrIndex; +#endif ptr= sh->FreeHead; - sh->FreeHead= GetNextPtr(ptr); + if (size < sizeof sh->FreeHead) + sh->FreeHead= Nil(void); + else + memcpy(&sh->FreeHead, ptr, sizeof sh->FreeHead); sh->FreeCount--; sh->MallocCount++; - ClrRange((char *)ptr-Arena, size); -#if !defined(SEGMENTED) /* TODO: What should we output in the SEGMENTED case? */ - TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)(ptr-Arena),sh->MallocCount)); +#if defined(SEGMENTED) + ptrSegment= CurrentSeg; + if (CurrentSeg) { + convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; + do { + convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; + if ((tmp >= segment_begin) && ((tmp - segment_begin) < ARENA_SEG_SIZE)) { + ptrIndex= (tmp - segment_begin); + goto FOUND_PUTATIVE_SEGMENT; + } + } while (0 <= --ptrSegment); + ptrIndex= -1; + } else + ptrIndex= pointerDifference(ptr, Arena[0]); +FOUND_PUTATIVE_SEGMENT: + TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,sh->MallocCount)); +#else +# if defined(FREEMAP) + ClrRange(pointerDifference(ptr, Arena), size); +# endif + TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)pointerDifference(ptr, Arena),sh->MallocCount)); #endif } else { /* we have to allocate a new piece */ - size_t const sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); + size_t sizeCurrentSeg; +#if defined(SEGMENTED) +START_LOOKING_FOR_CHUNK: +#endif + sizeCurrentSeg = (size_t)pointerDifference(TopFreePtr,BotFreePtr); TMDBG(printf(" sizeCurrentSeg:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)sizeCurrentSeg)); if (sizeCurrentSeg>=size) { if (size&PTRMASK) { - /* not aligned */ + /* not fully aligned */ + size_t curBottomIndex; + size_t needed_alignment_mask= PTRMASK; + while (needed_alignment_mask >= size) + needed_alignment_mask>>= 1; +#if defined(SEGMENTED) + curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena[CurrentSeg]); +#else + curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena); +#endif + if (curBottomIndex & needed_alignment_mask) { + size_t const numBytesToAdd= (needed_alignment_mask - (curBottomIndex & needed_alignment_mask)) + 1U; + if (numBytesToAdd > (sizeCurrentSeg-size)) + goto NEXT_SEGMENT; + do { + size_t const cur_alignment= (curBottomIndex & -curBottomIndex); +#if defined(FREEMAP) && !defined(SEGMENTED) + SetRange(curBottomIndex,cur_alignment); +#endif + if (cur_alignment >= fxfMINSIZE) { + SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; + if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if (cur_alignment >= sizeof cur_sh->FreeHead) + memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + } + } + BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)cur_alignment); + curBottomIndex+= cur_alignment; + } while (curBottomIndex & needed_alignment_mask); + } ptr= BotFreePtr; - BotFreePtr+= size; + BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)size); } else { - /* aligned */ - ptr= TopFreePtr-= size; + /* fully aligned */ + ptr= (TopFreePtr= stepPointer(TopFreePtr, -(ptrdiff_t)size)); } sh->MallocCount++; -#if !defined(SEGMENTED) /* TODO: What should we output in the SEGMENTED case? */ - TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",(ptrdiff_t_printf_type)(ptr-Arena),sh->MallocCount)); +#if defined(SEGMENTED) + TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,(ptrdiff_t_printf_type)pointerDifference(ptr, Arena[CurrentSeg]),sh->MallocCount)); +#else + TMDBG(printf(" current seg ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",(ptrdiff_t_printf_type)pointerDifference(ptr,Arena),sh->MallocCount)); #endif } else { +NEXT_SEGMENT: #if defined(SEGMENTED) - if ((CurrentSeg+1) < ArenaSegCnt) { + if (CurrentSeg < (ArenaSegCnt-1)) { + size_t curBottomIndex= (BotFreePtr - Arena[CurrentSeg]); + while (curBottomIndex & PTRMASK) { + size_t const cur_alignment= (curBottomIndex & -curBottomIndex); + if (cur_alignment >= fxfMINSIZE) { + SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; + if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if (cur_alignment >= sizeof cur_sh->FreeHead) + memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + } + } + BotFreePtr= stepPointer(BotFreePtr, cur_alignment); + curBottomIndex+= cur_alignment; + } + curBottomIndex= (size_t)(TopFreePtr-BotFreePtr); + if (curBottomIndex >= fxfMINSIZE) { + SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(curBottomIndex)]; + if ((curBottomIndex >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if (curBottomIndex >= sizeof cur_sh->FreeHead) + memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); + cur_sh->FreeHead= BotFreePtr; + ++cur_sh->FreeCount; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + } + } TMDBG(fputs(" next seg", stdout)); ++CurrentSeg; BotFreePtr= Arena[CurrentSeg]; - TopFreePtr= Arena[CurrentSeg]+ARENA_SEG_SIZE; - ptr= fxfAlloc(size); + TopFreePtr= stepPointer(Arena[CurrentSeg], ARENA_SEG_SIZE); + goto START_LOOKING_FOR_CHUNK; } else - ptr= Nil(char); + ptr= Nil(void); #else /*SEGMENTED*/ - ptr= Nil(char); + ptr= Nil(void); #endif /*!SEGMENTED*/ - TMDBG(printf(" ptr:%p\n",(void *)ptr)); + TMDBG(printf(" ptr:%p\n", ptr)); } } - DBG((df, "%p\n", (void *) ptr)); + DBG((df, "%p\n", ptr)); return ptr; } void fxfFree(void *ptr, size_t size) { +#if defined(LOG) || defined(DEBUG) static char const * const myname= "fxfFree"; +#endif SizeHead *sh; -#if !defined(SEGMENTED) /* TODO: What should we output in the SEGMENTED case? */ - TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(((char const*)ptr)-Arena),(size_t_printf_type)size)); + ptrdiff_t ptrIndex; +#if defined(SEGMENTED) + int ptrSegment; #endif - DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *) ptr, (size_t_printf_type) size)); - if (size > fxfMAXSIZE) { - fprintf(stderr, "%s: size=%" SIZE_T_PRINTF_SPECIFIER " >= %" SIZE_T_PRINTF_SPECIFIER "\n", - myname, (size_t_printf_type) size, (size_t_printf_type) fxfMAXSIZE); - exit(-5); + if (!ptr) + return; + assert(!!size); +#if defined(SEGMENTED) + ptrSegment= CurrentSeg; + if (CurrentSeg) { + convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; + do { + convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; + if (tmp >= segment_begin) { + ptrIndex= (tmp - segment_begin); + if (ptrIndex < ARENA_SEG_SIZE) + goto FOUND_PUTATIVE_SEGMENT; + } + } while (0 <= --ptrSegment); + ptrIndex= -1; + } else { + ptrIndex= pointerDifference(ptr,Arena[0]); + assert((ptrIndex >= 0) && (ptrIndex < ARENA_SEG_SIZE)); } +FOUND_PUTATIVE_SEGMENT: + TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); +#else + ptrIndex= pointerDifference(ptr,Arena); + assert((ptrIndex >= 0) && (ptrIndex < GlobalSize)); + TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); +#endif + DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *)ptr, (size_t_printf_type) size)); if (size < fxfMINSIZE) size= fxfMINSIZE; - if ((size&PTRMASK) && size= TopFreePtr)); +# else + { + assert(size <= (GlobalSize - ptrIndex)); + assert(((ptrIndex + size) <= pointerDifference(BotFreePtr,Arena)) || (ptr >= TopFreePtr)); +#endif + if (ptrIndex > 0) + { + size_t needed_alignment= MAX_ALIGNMENT; + while (needed_alignment > size) + needed_alignment>>= 1; + assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); + } + } +#endif + sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; if (size&PTRMASK) { - /* unaligned size */ - TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(BotFreePtr-(char const*)ptr))); - if ((char *)ptr+size == BotFreePtr) { - BotFreePtr-= size; - TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(TopFreePtr-BotFreePtr))); + /* not fully aligned size */ + TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(BotFreePtr,ptr))); + if (stepPointer(ptr, (ptrdiff_t)size) == BotFreePtr) { + BotFreePtr= ptr; + TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(TopFreePtr,BotFreePtr))); --sh->MallocCount; } else { - SetRange((char *)ptr-Arena,size); - *(char **)ALIGN(ptr)= sh->FreeHead; - sh->FreeHead= ptr; - ++sh->FreeCount; - --sh->MallocCount; - TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); +#if defined(FREEMAP) && !defined(SEGMENTED) + SetRange((pointerDifference(ptr,Arena),size); +#endif + if ((size >= sizeof sh->FreeHead) || !sh->FreeHead) { + if (size >= sizeof sh->FreeHead) + memcpy(ptr, &sh->FreeHead, sizeof sh->FreeHead); + sh->FreeHead= ptr; + ++sh->FreeCount; + --sh->MallocCount; + TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); + } } } else { - /* aligned size */ - TMDBG(printf(" ptr-TopFreePtr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(((char const*)ptr)-TopFreePtr))); - if ((char *)ptr == TopFreePtr) { - TopFreePtr+= size; - TMDBG(printf(" TopFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)(TopFreePtr-BotFreePtr))); + /* fully aligned size */ + TMDBG(printf(" ptr-TopFreePtr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(ptr,TopFreePtr))); + if (ptr == TopFreePtr) { + TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t)size); + TMDBG(printf(" TopFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(TopFreePtr,BotFreePtr))); --sh->MallocCount; } else { - SetRange((char *)ptr-Arena,size); - *(char **)ptr= sh->FreeHead; - sh->FreeHead= ptr; - ++sh->FreeCount; - --sh->MallocCount; - TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); +#if defined(FREEMAP) && !defined(SEGMENTED) + SetRange(pointerDifference(ptr,Arena),size); +#endif + if ((size >= sizeof sh->FreeHead) || !sh->FreeCount) { + if (size >= sizeof sh->FreeHead) + memcpy(ptr, &sh->FreeHead, sizeof sh->FreeHead); + sh->FreeHead= ptr; + ++sh->FreeCount; + --sh->MallocCount; + TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); + } } } TMDBG(printf(" MallocCount:%lu",sh->MallocCount)); @@ -505,11 +758,58 @@ void fxfFree(void *ptr, size_t size) void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { void *nptr; if (!ptr) + { + assert(!OldSize); return fxfAlloc(NewSize); + } +#if !defined(NDEBUG) +# if defined(SEGMENTED) + if (!CurrentSeg) /* Otherwise we'd be relying on converting to convert_pointer_to_int_type, + and such calculations aren't guaranteed to provide exactly what we need. */ + { + ptrdiff_t const ptrIndex= pointerDifference(ptr,Arena[0]); + assert(ptrIndex < ARENA_SEG_SIZE); +# else + { + ptrdiff_t const ptrIndex= pointerDifference(ptr,Arena); + assert(ptrIndex < GlobalSize); +# endif + assert(ptrIndex >= 0); + if (ptrIndex > 0) + { + size_t allocatedSize= OldSize; + size_t needed_alignment; + if (allocatedSize < fxfMINSIZE) + allocatedSize= fxfMINSIZE; + assert(allocatedSize <= fxfMAXSIZE); + allocatedSize= ALIGN_TO_MINIMUM(allocatedSize); +# if defined(SEGMENTED) + assert(allocatedSize <= (ARENA_SEG_SIZE - ptrIndex)); +# else + assert(allocatedSize <= (GlobalSize - ptrIndex)); +# endif + needed_alignment= MAX_ALIGNMENT; + while (needed_alignment > allocatedSize) + needed_alignment>>= 1; + assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); + } + } +#endif if (!NewSize) + { fxfFree(ptr, OldSize); + return Nil(void); + } + /* TODO: It may be worth trying to return ptr if ALIGN_TO_MINIMUM(OldSize) >= NewSize. + To go along with this, we'd have to carefully add any excess to the free store. + In the !defined(SEGMENTED) case this is likely easy, but in the defined(SEGMENTED) + case it may be difficult. Regardless, the computations to set this up -- or even + determine if it's possible -- are kind of annoying, and they'd only be worthwhile + if we hit this possibility frequently (and if the alternative below is expensive + or proves impossible). This all would need to be investigated. + */ nptr= fxfAlloc(NewSize); - if (NewSize && nptr) + if (nptr) { memcpy(nptr, ptr, ((NewSize < OldSize) ? NewSize : OldSize)); fxfFree(ptr, OldSize); @@ -522,11 +822,11 @@ size_t fxfTotal(void) { size_t UsedBytes = 0; size_t FreeBytes = 0; - unsigned int i; - for (i=0; i<=fxfMAXSIZE; i++,hd++) { + size_t i; + for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - UsedBytes+= hd->MallocCount*i; - FreeBytes+= hd->FreeCount*i; + UsedBytes+= hd->MallocCount*SIZEDATA_INDEX_TO_SIZE(i); + FreeBytes+= hd->FreeCount*SIZEDATA_INDEX_TO_SIZE(i); } } @@ -535,7 +835,7 @@ size_t fxfTotal(void) { void fxfInfo(FILE *f) { size_t const one_kilo = 1<<10; - size_t const sizeCurrentSeg = (size_t)(TopFreePtr-BotFreePtr); + size_t const sizeCurrentSeg = (size_t)pointerDifference(TopFreePtr,BotFreePtr); size_t const sizeArenaUsed = GlobalSize-sizeCurrentSeg #if defined(SEGMENTED) @@ -557,15 +857,15 @@ void fxfInfo(FILE *f) { size_t UsedBytes = 0; size_t FreeBytes = 0; - unsigned int i; + size_t i; fprintf(f, "%12s %10s%10s\n", "Size", "MallocCnt", "FreeCnt"); - for (i=0; i<=fxfMAXSIZE; i++,hd++) { + for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount+hd->FreeCount>0) { - fprintf(f, "%12u %10lu%10lu\n", i, hd->MallocCount, hd->FreeCount); + fprintf(f, "%12zu %10lu%10lu\n", SIZEDATA_INDEX_TO_SIZE(i), hd->MallocCount, hd->FreeCount); nrUsed+= hd->MallocCount; - UsedBytes+= hd->MallocCount*i; + UsedBytes+= hd->MallocCount*(i+1U); nrFree+= hd->FreeCount; - FreeBytes+= hd->FreeCount*i; + FreeBytes+= hd->FreeCount*(i+1U); } } fprintf(f, "%12s %10lu%10lu\n", "Total:", nrUsed, nrFree); @@ -577,4 +877,8 @@ void fxfInfo(FILE *f) { } } +#else /*FXF*/ + +extern unsigned char FXF_C_NONEMPTY_TRANSLATION_UNIT; + #endif /*FXF*/ diff --git a/DHT/fxf.h b/DHT/fxf.h index 401dfc5dfb..d56cbcfb3e 100644 --- a/DHT/fxf.h +++ b/DHT/fxf.h @@ -2,14 +2,16 @@ #define FXF_H #include +#include -size_t fxfInit(size_t GlobalSize); /* returns the number of bytes actually allocated */ +size_t fxfInit(size_t Size); /* returns the number of bytes actually allocated */ int fxfInitialised(void); void *fxfAlloc(size_t size); void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize); void fxfFree(void *ptr, size_t size); void fxfInfo(FILE *); size_t fxfTotal(void); +size_t fxfMaxAllocation(void); /* Reset the internal data structures to the state that was reached * after the latest call to fxfInit() */ diff --git a/debugging/trace.c b/debugging/trace.c index e24e42e4c7..f6c9c96079 100644 --- a/debugging/trace.c +++ b/debugging/trace.c @@ -473,7 +473,7 @@ void TraceCurrentHashBuffer(void) HashBuffer const *hb = &hashBuffers[nbply]; unsigned int i; - printf(" #%lu nbply:%u Leng:%u ",level,nbply,hb->cmv.Leng); + printf(" #%lu nbply:%u Leng:%hu ",level,nbply,hb->cmv.Leng); for (i = 0; icmv.Leng; ++i) printf("%02x ",(unsigned int)hb->cmv.Data[i]); putchar('\n'); @@ -521,7 +521,7 @@ static void trace_link(char const *prefix, slice_index si, char const *suffix) static char const context_shortcuts[] = { 'I', 'A', 'D', 'H', 'M', 'T', 'N' }; static char const level_shortcuts[] = { 'T', 'S', 'N' }; -static void trace_common(slice_index si, stip_structure_traversal *st) +static void trace_common(slice_index si, stip_structure_traversal const *st) { if (do_trace) { diff --git a/optimisations/hash.c b/optimisations/hash.c index a16fa2b104..e1fd9fd3ff 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -42,7 +42,7 @@ ** for more details. Two procedures are used: ** dhtLookupElement: This procedure delivers ** a nil pointer, when the given position is not in the hashtable, - ** or a pointer to a hashelement. + ** or a pointer to a hashElement. ** dhtEnterElement: This procedure enters an encoded position ** with its values into the hashtable. ** @@ -56,7 +56,7 @@ ** in 5 moves", since the former can be recomputed faster. For the other ** type of information ("solvable") the comparison is the other way round. ** The compression of the table is an expensive operation, in a lot - ** of exeperiments it has shown to be quite effective in keeping the + ** of experiments it has shown to be quite effective in keeping the ** most valuable information, and speeds up the computation time ** considerably. But to be of any use, there must be enough memory to ** to store more than 800 positions. @@ -64,7 +64,7 @@ ** There seems to be no real penalty in using hashing, even if the ** hit ratio is very small and only about 5%, it speeds up the ** computation time by 30%. - ** I changed the output of hashstat, since its really informative + ** I changed the output of hashstat, since it's really informative ** to see the hit rate. ** ** inithash() @@ -96,7 +96,7 @@ #include "conditions/duellists.h" #include "conditions/haunted_chess.h" #include "conditions/imitator.h" -#include "conditions/fuddled_men.h" +//#include "conditions/fuddled_men.h" #include "options/nontrivial.h" #include "solving/avoid_unsolvable.h" #include "solving/castling.h" @@ -121,6 +121,10 @@ #include "platform/timer.h" #endif +enum { + ENSURE_MAX_LENGTH_FITS_IN_UNSIGNED_SHORT = 1/(MAX_LENGTH_OF_ENCODING <= USHRT_MAX) +}; + #if defined(FXF) unsigned long hash_max_kilo_storable_positions = ULONG_MAX; #endif @@ -151,6 +155,11 @@ static hash_value_type minimalElementValueAfterCompression; static unsigned int nr_hash_slices; static slice_index hash_slices[max_nr_slices]; +enum +{ + NUM_ELEMENTS_IN_HASHBUFFER = ((sizeof(HashBuffer) - offsetof(BCMemValue, Data))/sizeof(byte)), + ENSURE_HASHBUFFER_DATA_HAS_AT_LEAST_NR_ROWS_ON_BOARD_ENTRIES = 1/(NUM_ELEMENTS_IN_HASHBUFFER >= nr_rows_on_board) +}; HashBuffer hashBuffers[maxply+1]; @@ -249,26 +258,6 @@ static void (*encode)(stip_length_type min_length, typedef unsigned int data_type; -/* hash table element type defining the data member as we use it in - * this module - */ -typedef struct -{ - dhtValue Key; - data_type data; -} element_t; - -/* Grand union of "element" type and the generic one used by the hash - * table implementation. - * Using this union type rather than casting frm dhtElement * to - * element_t * avoids aliasing issues. - */ -typedef union -{ - dhtElement d; - element_t e; -} hashElement_union_t; - /* Hashing properties of stipulation slices */ typedef struct @@ -375,7 +364,7 @@ static void slice_property_offset_shifter(slice_index si, * @param delta indicates how much to shift the value offsets */ static void shift_offsets(slice_index si, - stip_structure_traversal *st, + stip_structure_traversal const *st, unsigned int delta) { unsigned int i; @@ -654,17 +643,17 @@ static void init_slice_properties(slice_index si) /* Pseudo hash table element - template for fast initialization of * newly created actual table elements */ -static hashElement_union_t template_element; +static dhtElement template_element; -static void set_value_attack_nosuccess(hashElement_union_t *hue, +static void set_value_attack_nosuccess(dhtElement *e, slice_index si, hash_value_type val) { unsigned int const offset = slice_properties[si].u.d.offsetNoSucc; unsigned int const bits = ((offset < (CHAR_BIT * (sizeof val))) ? (val << offset) : 0); unsigned int const mask = slice_properties[si].u.d.maskNoSucc; - element_t * const e = &hue->e; + data_type tmp; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceFunctionParam("%u",val); @@ -672,27 +661,29 @@ static void set_value_attack_nosuccess(hashElement_union_t *hue, TraceValue("%u",slice_properties[si].size); TraceValue("%u",offset); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->data); - TraceValue("pre:%08x ",e->data); + TraceValue("%p",(void *)&e->Data.unsigned_integer); + TraceValue("pre:%08x ",(unsigned int)(data_type)e->Data.unsigned_integer); TraceValue("%08x",bits); TraceEOL(); assert((bits&mask)==bits); - e->data &= ~mask; - e->data |= bits; - TraceValue("post:%08x",e->data); + tmp = (data_type)e->Data.unsigned_integer; + tmp &= ~mask; + tmp |= bits; + e->Data.unsigned_integer = tmp; + TraceValue("post:%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); TraceFunctionResultEnd(); } -static void set_value_attack_success(hashElement_union_t *hue, +static void set_value_attack_success(dhtElement *e, slice_index si, hash_value_type val) { unsigned int const offset = slice_properties[si].u.d.offsetSucc; unsigned int const bits = ((offset < (CHAR_BIT * (sizeof val))) ? (val << offset) : 0); unsigned int const mask = slice_properties[si].u.d.maskSucc; - element_t * const e = &hue->e; + data_type tmp; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); @@ -702,28 +693,30 @@ static void set_value_attack_success(hashElement_union_t *hue, TraceValue("%u",slice_properties[si].size); TraceValue("%u",offset); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->data); - TraceValue("pre:%08x ",e->data); + TraceValue("%p",(void *)&e->Data.unsigned_integer); + TraceValue("pre:%08x ",(unsigned int)(data_type)e->Data.unsigned_integer); TraceValue("%08x",bits); TraceEOL(); assert((bits&mask)==bits); - e->data &= ~mask; - e->data |= bits; - TraceValue("post:%08x",e->data); + tmp = (data_type)e->Data.unsigned_integer; + tmp &= ~mask; + tmp |= bits; + e->Data.unsigned_integer = tmp; + TraceValue("post:%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); TraceFunctionResultEnd(); } -static void set_value_help(hashElement_union_t *hue, +static void set_value_help(dhtElement *e, slice_index si, hash_value_type val) { unsigned int const offset = slice_properties[si].u.h.offsetNoSucc; unsigned int const bits = val << offset; unsigned int const mask = slice_properties[si].u.h.maskNoSucc; - element_t * const e = &hue->e; + data_type tmp; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceFunctionParam("%u",val); @@ -731,31 +724,32 @@ static void set_value_help(hashElement_union_t *hue, TraceValue("%u",slice_properties[si].size); TraceValue("%u",offset); TraceValue("0x%08x ",mask); - TraceValue("%p ",(void *)&e->data); - TraceValue("pre:0x%08x ",e->data); + TraceValue("%p ",(void *)&e->Data.unsigned_integer); + TraceValue("pre:0x%08x ",(unsigned int)(data_type)e->Data.unsigned_integer); TraceValue("0x%08x",bits); TraceEOL(); assert((bits&mask)==bits); - e->data &= ~mask; - e->data |= bits; - TraceValue("post:0x%08x",e->data); + tmp = (data_type)e->Data.unsigned_integer; + tmp &= ~mask; + tmp |= bits; + e->Data.unsigned_integer = tmp; + TraceValue("post:0x%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); TraceFunctionResultEnd(); } -static hash_value_type get_value_attack_success(hashElement_union_t const *hue, +static hash_value_type get_value_attack_success(dhtElement const *e, slice_index si) { unsigned int const offset = slice_properties[si].u.d.offsetSucc; unsigned int const mask = slice_properties[si].u.d.maskSucc; - element_t const * const e = &hue->e; - data_type const result = (e->data & mask) >> offset; + data_type const result = (((data_type)e->Data.unsigned_integer) & mask) >> offset; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->data); - TraceValue("%08x",e->data); + TraceValue("%p",(void const *)&e->Data.unsigned_integer); + TraceValue("%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -764,18 +758,17 @@ static hash_value_type get_value_attack_success(hashElement_union_t const *hue, return result; } -static hash_value_type get_value_attack_nosuccess(hashElement_union_t const *hue, +static hash_value_type get_value_attack_nosuccess(dhtElement const *e, slice_index si) { unsigned int const offset = slice_properties[si].u.d.offsetNoSucc; unsigned int const mask = slice_properties[si].u.d.maskNoSucc; - element_t const * const e = &hue->e; - data_type const result = (e->data & mask) >> offset; + data_type const result = (((data_type)e->Data.unsigned_integer) & mask) >> offset; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceValue("%08x ",mask); - TraceValue("%p",(void *)&e->data); - TraceValue("%08x",e->data); + TraceValue("%p",(void const *)&e->Data.unsigned_integer); + TraceValue("%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -784,19 +777,18 @@ static hash_value_type get_value_attack_nosuccess(hashElement_union_t const *hue return result; } -static hash_value_type get_value_help(hashElement_union_t const *hue, +static hash_value_type get_value_help(dhtElement const *e, slice_index si) { unsigned int const offset = slice_properties[si].u.h.offsetNoSucc; unsigned int const mask = slice_properties[si].u.h.maskNoSucc; - element_t const * const e = &hue->e; - data_type const result = (e->data & mask) >> offset; + data_type const result = (((data_type)e->Data.unsigned_integer) & mask) >> offset; TraceFunctionEntry(__func__); TraceFunctionParam("%u",si); TraceValue("%u",offset); TraceValue("0x%08x ",mask); - TraceValue("%p ",(void *)&e->data); - TraceValue("0x%08x",e->data); + TraceValue("%p ",(void const *)&e->Data.unsigned_integer); + TraceValue("0x%08x",(unsigned int)(data_type)e->Data.unsigned_integer); TraceEOL(); TraceFunctionExit(__func__); @@ -811,7 +803,7 @@ static hash_value_type get_value_help(hashElement_union_t const *hue, * @param si slice index of slice * @return value of contribution of slice si to *he's value */ -static hash_value_type own_value_of_data_solve(hashElement_union_t const *hue, +static hash_value_type own_value_of_data_solve(dhtElement const *hue, slice_index si) { stip_length_type const length = SLICE_U(si).branch.length; @@ -821,7 +813,7 @@ static hash_value_type own_value_of_data_solve(hashElement_union_t const *hue, hash_value_type success_neg; TraceFunctionEntry(__func__); - TraceFunctionParam("%p",hue); + TraceFunctionParam("%p",(void const *)hue); TraceFunctionParam("%u",si); TraceFunctionParamListEnd(); @@ -845,14 +837,14 @@ static hash_value_type own_value_of_data_solve(hashElement_union_t const *hue, * @param si slice index of help slice * @return value of contribution of slice si to *he's value */ -static hash_value_type own_value_of_data_help(hashElement_union_t const *hue, +static hash_value_type own_value_of_data_help(dhtElement const *hue, slice_index si) { unsigned int const parity = slice_properties[si].u.h.parity; hash_value_type result; TraceFunctionEntry(__func__); - TraceFunctionParam("%p",hue); + TraceFunctionParam("%p",(void const *)hue); TraceFunctionParam("%u",si); TraceFunctionParamListEnd(); @@ -872,14 +864,14 @@ static hash_value_type own_value_of_data_help(hashElement_union_t const *hue, * @param si slice index * @return value of contribuation of the slice to *he's value */ -static hash_value_type value_of_data_from_slice(hashElement_union_t const *hue, +static hash_value_type value_of_data_from_slice(dhtElement const *hue, slice_index si) { hash_value_type result; unsigned int const offset = slice_properties[si].valueOffset; TraceFunctionEntry(__func__); - TraceFunctionParam("%p",hue); + TraceFunctionParam("%p",(void const *)hue); TraceFunctionParam("%u",si); TraceFunctionParamListEnd(); @@ -915,13 +907,13 @@ static hash_value_type value_of_data_from_slice(hashElement_union_t const *hue, * @param he address of hash table element to determine value of * @return value of *he */ -static hash_value_type value_of_data(hashElement_union_t const *hue) +static hash_value_type value_of_data(dhtElement const *hue) { hash_value_type result = 0; unsigned int i; TraceFunctionEntry(__func__); - TraceFunctionParam("%p",hue); + TraceFunctionParam("%p",(void const *)hue); TraceFunctionParamListEnd(); for (i = 0; iKey); #if defined(TESTHASH) if (nrElementsRemovedInCompression + dhtKeyCount(pyhash) != nrElementsAtStartOfCompression) @@ -1198,7 +1184,7 @@ static void ProofSmallEncodePiece(byte **bp, encoded |= 1 << black_bit; if (*even) { - **bp += encoded<<(CHAR_BIT/2); + **bp += (byte)(encoded<<(CHAR_BIT/2)); ++*bp; } else @@ -1313,7 +1299,7 @@ static byte *CommonEncode(byte *bp, if (move_effect_journal[capture].type==move_effect_piece_removal) ply_last_capture = nbply; else - ply_last_capture = find_last_capture(); + ply_last_capture = (ply)find_last_capture(); if (ply_last_capture!=ply_nil) { @@ -1340,7 +1326,8 @@ static byte *CommonEncode(byte *bp, if (min_length>slack_length+1) { - assert(validity_value<=(1<<2*CHAR_BIT)); + assert((((validity_value>>(CHAR_BIT-1))>>(CHAR_BIT-1))>>2)<2); /* Equivalent to assert(validity_value<=(1<<2*CHAR_BIT) but we don't have to worry about the shift overflowing. + TODO: Is this the right assertion? validity_value == 0x10000 would pass. */ *bp++ = (byte)(validity_value); *bp++ = (byte)(validity_value>>CHAR_BIT); } @@ -1348,6 +1335,7 @@ static byte *CommonEncode(byte *bp, { unsigned int i; + assert((en_passant_top[nbply]-en_passant_top[nbply-1])<=MAX_EN_PASSANT_TOP_DIFFERENCE); for (i = en_passant_top[nbply-1]+1; i<=en_passant_top[nbply]; ++i) *bp++ = (byte)(en_passant_multistep_over[i] - square_a1); } @@ -1416,7 +1404,7 @@ static void ProofEncode(stip_length_type min_length, stip_length_type validity_v byte *bp = position+nr_rows_on_board; /* clear the bits for storing the position of pieces */ - memset(position, 0, nr_rows_on_board); + memset(position, 0, nr_rows_on_board * sizeof *position); { boolean even = false; @@ -1463,8 +1451,8 @@ static void ProofEncode(stip_length_type min_length, stip_length_type validity_v /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=UCHAR_MAX); - hb->cmv.Leng = (unsigned char)(bp-hb->cmv.Data); + assert((bp - hb->cmv.Data) <= MAX_LENGTH_OF_ENCODING); + hb->cmv.Leng = (bp - hb->cmv.Data); } static unsigned int TellCommonEncodePosLeng(unsigned int len, @@ -1625,7 +1613,7 @@ static void LargeEncode(stip_length_type min_length, TraceFunctionParamListEnd(); /* clear the bits for storing the position of pieces */ - memset(position,0,nr_rows_on_board); + memset(position,0,nr_rows_on_board * sizeof *position); for (row=0; rowcmv.Data<=UCHAR_MAX); - hb->cmv.Leng = (unsigned char)(bp-hb->cmv.Data); + assert((bp - hb->cmv.Data) <= MAX_LENGTH_OF_ENCODING); + hb->cmv.Leng = (bp - hb->cmv.Data); TraceFunctionExit(__func__); TraceFunctionResultEnd(); @@ -1706,8 +1694,8 @@ static void SmallEncode(stip_length_type min_length, /* Now the rest of the party */ bp = CommonEncode(bp,min_length,validity_value); - assert(bp-hb->cmv.Data<=UCHAR_MAX); - hb->cmv.Leng = (unsigned char)(bp-hb->cmv.Data); + assert((bp - hb->cmv.Data) <= MAX_LENGTH_OF_ENCODING); + hb->cmv.Leng = (bp - hb->cmv.Data); TraceFunctionExit(__func__); TraceFunctionResultEnd(); @@ -1717,7 +1705,7 @@ static void SmallEncode(stip_length_type min_length, * element's data field with null values * @param he address of hash table element */ -static void init_elements(hashElement_union_t *hue) +static void init_elements(dhtElement *hue) { unsigned int i; @@ -1762,9 +1750,9 @@ static unsigned long allocCounter = 0; * @param hb has value (basis for calculation of key) * @return address of element */ -static dhtElement *allocDHTelement(dhtConstValue hb) +static dhtElement *allocDHTelement(dhtKey hb) { - dhtElement *result = dhtEnterElement(pyhash,hb,template_element.d.Data); + dhtElement *result = dhtEnterElement(pyhash,hb,template_element.Data); while (result==dhtNilElement) { unsigned long const nrKeysBeforeCompression = dhtKeyCount(pyhash); @@ -1781,11 +1769,11 @@ static dhtElement *allocDHTelement(dhtConstValue hb) fprintf(stderr, "\nOUT OF SPACE: Unable to create hash table in %s in %s -- aborting.\n", __func__, __FILE__); exit(2); /* TODO: Do we have to exit here? */ } - result = dhtEnterElement(pyhash,hb,template_element.d.Data); + result = dhtEnterElement(pyhash,hb,template_element.Data); break; } else - result = dhtEnterElement(pyhash,hb,template_element.d.Data); + result = dhtEnterElement(pyhash,hb,template_element.Data); } #if defined(FXF) @@ -1888,7 +1876,8 @@ static boolean is_proofgame(slice_index si) boolean is_hashtable_allocated(void) { #if defined(FXF) - return fxfInitialised(); + return !!fxfInitialised(); /* !! just in case fxfInitialised returns a nonzero value other than 1 + and boolean is some type that won't automatically convert it to 1. */ #else return hashtable_kilos>0; #endif @@ -1915,7 +1904,7 @@ static void inithash(slice_index si) init_slice_properties(si); - template_element.d.Data = 0; + template_element.Data.unsigned_integer = 0; init_elements(&template_element); is_table_uncompressed = true; /* V3.60 TLi */ @@ -2316,7 +2305,7 @@ static void addtohash_battle_nosuccess(slice_index si, stip_length_type n, stip_length_type min_length_adjusted) { - HashBuffer const * const hb = &hashBuffers[nbply]; + dhtKey hb; hash_value_type const val = (n+1-min_length_adjusted)/2; dhtElement *he; @@ -2325,18 +2314,16 @@ static void addtohash_battle_nosuccess(slice_index si, TraceFunctionParam("%u",min_length_adjusted); TraceFunctionParamListEnd(); + hb.value.object_pointer = &hashBuffers[nbply].cmv; he = dhtLookupElement(pyhash,hb); if (he==dhtNilElement) { - hashElement_union_t * const hue = (hashElement_union_t *)allocDHTelement(hb); - set_value_attack_nosuccess(hue,si,val); + he = allocDHTelement(hb); + set_value_attack_nosuccess(he,si,val); } else - { - hashElement_union_t * const hue = (hashElement_union_t *)he; - if (get_value_attack_nosuccess(hue,si)val) - set_value_attack_success(hue,si,val); - } + if (get_value_attack_success(he,si)>val) + set_value_attack_success(he,si,val); TraceFunctionExit(__func__); TraceFunctionResultEnd(); @@ -2431,6 +2415,7 @@ stip_length_type delegate_can_attack_in_n(slice_index si, void attack_hashed_tester_solve(slice_index si) { dhtElement const *he; + dhtKey k; slice_index const base = SLICE_U(si).derived_pipe.base; stip_length_type const min_length = SLICE_U(base).branch.min_length; stip_length_type const played = SLICE_U(base).branch.length-solve_nr_remaining; @@ -2448,23 +2433,23 @@ void attack_hashed_tester_solve(slice_index si) (*encode)(min_length,validity_value); - he = dhtLookupElement(pyhash,&hashBuffers[nbply]); + k.value.object_pointer = &hashBuffers[nbply].cmv; + he = dhtLookupElement(pyhash,k); if (he==dhtNilElement) solve_result = delegate_can_attack_in_n(si,min_length_adjusted); else { - hashElement_union_t const * const hue = (hashElement_union_t const *)he; stip_length_type const parity = (solve_nr_remaining-min_length_adjusted)%2; /* It is more likely that a position has no solution. */ /* Therefore let's check for "no solution" first. TLi */ - hash_value_type const val_nosuccess = get_value_attack_nosuccess(hue,base); + hash_value_type const val_nosuccess = get_value_attack_nosuccess(he,base); stip_length_type const n_nosuccess = 2*val_nosuccess + min_length_adjusted-parity; if (n_nosuccess>=MOVE_HAS_SOLVED_LENGTH()) solve_result = MOVE_HAS_NOT_SOLVED_LENGTH(); else { - hash_value_type const val_success = get_value_attack_success(hue,base); + hash_value_type const val_success = get_value_attack_success(he,base); stip_length_type const n_success = 2*val_success + min_length_adjusted+2-parity; if (n_success<=MOVE_HAS_SOLVED_LENGTH()) solve_result = n_success; @@ -2489,20 +2474,19 @@ void attack_hashed_tester_solve(slice_index si) TraceFunctionResultEnd(); } -static hashElement_union_t *find_or_add_help_elmt(HashBuffer const *hb) +static dhtElement *find_or_add_help_elmt(HashBuffer const *hb) { TraceFunctionEntry(__func__); TraceFunctionParam("%p",hb); TraceFunctionParamListEnd(); { - dhtElement * const he = dhtLookupElement(pyhash,hb); - hashElement_union_t * const result = (hashElement_union_t *)(he==dhtNilElement - ? allocDHTelement(hb) - : he); - + dhtKey k; + dhtElement *result; + k.value.object_pointer = &hb->cmv; + result = dhtLookupElement(pyhash,k); TraceFunctionExit(__func__); - TraceFunctionResult("%p",result); + TraceFunctionResult("%p",(void const *)result); TraceFunctionResultEnd(); return result; } @@ -2579,7 +2563,7 @@ static void addtohash_help_solved(slice_index si) #endif /*HASHRATE*/ } -static hashElement_union_t const *find_help_elmt_solved(HashBuffer *hb) +static dhtElement const *find_help_elmt_solved(HashBuffer *hb) { TraceFunctionEntry(__func__); TraceFunctionParam("%p",hb); @@ -2590,10 +2574,10 @@ static hashElement_union_t const *find_help_elmt_solved(HashBuffer *hb) hb->cmv.Data[hb->cmv.Leng-1] += (byte)2; { - dhtElement const * const he = dhtLookupElement(pyhash,hb); - hashElement_union_t const * const result = (he==dhtNilElement - ? 0 - : (hashElement_union_t const *)he); + dhtKey k; + dhtElement const *result; + k.value.object_pointer = &hb->cmv; + result = dhtLookupElement(pyhash,k); hb->cmv.Data[hb->cmv.Leng-1] -= (byte)2; @@ -2607,7 +2591,7 @@ static hashElement_union_t const *find_help_elmt_solved(HashBuffer *hb) static boolean is_position_known_to_be_solved(HashBuffer *hb, slice_index si) { boolean result; - hashElement_union_t const *hue_solved; + dhtElement const *hue_solved; TraceFunctionEntry(__func__); TraceFunctionParam("%p",hb); @@ -2660,20 +2644,19 @@ static void addtohash_help_not_solved(slice_index si) #endif /*HASHRATE*/ } -static hashElement_union_t const *find_help_elmt_not_solved(HashBuffer *hb) +static dhtElement const *find_help_elmt_not_solved(HashBuffer *hb) { TraceFunctionEntry(__func__); TraceFunctionParam("%p",hb); TraceFunctionParamListEnd(); { - dhtElement const *he = dhtLookupElement(pyhash,hb); - hashElement_union_t const * const result = (he==dhtNilElement - ? 0 - : (hashElement_union_t const *)he); - + dhtKey k; + dhtElement const *result; + k.value.object_pointer = &hb->cmv; + result = dhtLookupElement(pyhash,k); TraceFunctionExit(__func__); - TraceFunctionResult("%p",result); + TraceFunctionResult("%p",(void const *)result->object_pointer); TraceFunctionResultEnd(); return result; } @@ -2682,7 +2665,7 @@ static hashElement_union_t const *find_help_elmt_not_solved(HashBuffer *hb) static boolean is_position_known_not_to_be_solved(HashBuffer *hb, slice_index si) { boolean result; - hashElement_union_t const *hue_not_solved; + dhtElement const *hue_not_solved; TraceFunctionEntry(__func__); TraceFunctionParam("%p",hb); @@ -2715,7 +2698,7 @@ static boolean is_position_known_not_to_be_solved(HashBuffer *hb, slice_index si static boolean inhash_help(slice_index si) { boolean result; - HashBuffer * const hb = &hashBuffers[nbply]; + HashBuffer *hb; stip_length_type const validity_value = (solve_nr_remaining-1)/2+1; stip_length_type const min_length = SLICE_U(si).branch.min_length; @@ -2729,9 +2712,9 @@ static boolean inhash_help(slice_index si) /* Create a difference between odd and even numbers of moves. * A solution for h#n isn't necessarily a solution for h#n.5 */ - assert(hb->cmv.Lengcmv.Data[hashBuffers[nbply].cmv.Leng] = (byte)(min_length%2); - ++hb->cmv.Leng; + hb = &hashBuffers[nbply]; + assert(hb->cmv.Lengcmv.Data[hb->cmv.Leng++] = (byte)(min_length%2); ifHASHRATE(use_all++); diff --git a/optimisations/hash.h b/optimisations/hash.h index d370c87e0a..12a6b940e2 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -12,7 +12,12 @@ #include "DHT/dhtbcmem.h" #include "pieces/pieces.h" #include "solving/machinery/solve.h" +#include "position/underworld.h" +#include "position/position.h" +#include "position/board.h" #include "solving/ply.h" +#include "conditions/bgl.h" +#include "conditions/fuddled_men.h" #if defined(TESTHASH) # if !defined(HASHRATE) @@ -23,9 +28,129 @@ /* typedefs */ typedef unsigned char byte; +/* Time for some math -- let's figure out the maximum number of bytes a position encoding may take up. + If it's a proofgame then ProofEncode will be used. Otherwise, SmallEncode or LargeEncode will be chosen, + the former if TellSmallEncodePosLeng <= TellLargeEncodePosLeng, the latter otherwise. + - ProofEncode <= nr_rows_on_board + + "num nonempty squares (found by looping over nr_rows_on_board and nr_files_on_board)" * max(ProofLargeEncodePiece, ProofSmallEncodePiece) + + 1 + + nr_ghosts * SmallEncodePiece + + CommonEncode + - LargeEncode = nr_rows_on_board + + "num nonempty squares (found by looping over nr_rows_on_board and nr_files_on_board)" * LargeEncodePiece + + nr_ghosts * (2 + bytes_per_spec) + + CommonEncode + - SmallEncode = "num nonempty squares (found by looping over nr_rows_on_board and nr_files_on_board)" * SmallEncodePiece + + nr_ghosts * SmallEncodePiece + + CommonEncode + - TellLargeEncodePosLeng = TellCommonEncodePosLeng(len, nbr_p) + with + len = 8 + + "num nonempty squares (found by looping boardnum until 0 is found)" * bytes_per_piece + + !!CondFlag[BGL] * (sizeof BGL_values[White] + sizeof BGL_values[Black]) + + nr_ghosts * bytes_per_piece + nbr_p = "num nonempty squares (found by looping boardnum until 0 is found)" + - TellSmallEncodePosLeng = TellCommonEncodePosLeng(len, nbr_p) + with + len = "num nonempty squares (found by looping boardnum until 0 is found)" * (1 + bytes_per_piece) + + nr_ghosts * bytes_per_piece + nbr_p = "num nonempty squares (found by looping boardnum until 0 is found)" + TellCommonEncodePosLeng(len, np) is a strictly increasing function of len. SmallEncode will therefore be chosen iff + "num nonempty squares" <= 8 + !!CondFlag[BGL] * (sizeof BGL_values[White] + sizeof BGL_values[Black]). + Therefore: + - If SmallEncode is chosen then "num nonempty squares" <= min((nr_rows_on_board * nr_files_on_board), (8 + sizeof BGL_values[White] + sizeof BGL_values[Black])). + - If LargeEncode is chosen then 8 < "num nonempty squares" <= (nr_rows_on_board * nr_files_on_board). + Substituting we find that + - SmallEncode <= min((nr_rows_on_board * nr_files_on_board), (8 + sizeof BGL_values[White] + sizeof BGL_values[Black])) * SmallEncodePiece + + nr_ghosts * SmallEncodePiece + + CommonEncode + - LargeEncode <= nr_rows_on_board + + (nr_rows_on_board * nr_files_on_board) * LargeEncodePiece + + nr_ghosts * (2 + bytes_per_spec) + + CommonEncode + - ProofEncode <= nr_rows_on_board + + (nr_rows_on_board * nr_files_on_board) * max(ProofLargeEncodePiece, ProofSmallEncodePiece) + + 1 + + nr_ghosts * SmallEncodePiece + + CommonEncode + Meanwhile, inspection reveals that: + - ProofLargeEncodePiece = 2 + - ProofSmallEncodePiece <= 1 + - SmallEncodePiece <= 1 + 1 + bytes_per_spec + - LargeEncodePiece <= 1 + bytes_per_spec + - CommonEncode <= 2 + 2 + 1 + 1 + 1 + + being_solved.number_of_imitators + + 1 + 1 + 3 + 1 + + bytes_per_spec + + 1 + 2 + + en_passant_top[nbply] - en_passant_top[nbply-1] + + 1 + + sizeof BGL_values[White] + sizeof BGL_values[Black] + + 2 + + sizeof fuddled + Substituting yields: + - SmallEncode <= min((nr_rows_on_board * nr_files_on_board), (8 + sizeof BGL_values[White] + sizeof BGL_values[Black])) * (1 + 1 + bytes_per_spec) + + nr_ghosts * (1 + 1 + bytes_per_spec) + + CommonEncode + - LargeEncode <= nr_rows_on_board + + (nr_rows_on_board * nr_files_on_board) * (1 + bytes_per_spec) + + nr_ghosts * (2 + bytes_per_spec) + + CommonEncode + - ProofEncode <= nr_rows_on_board + + (nr_rows_on_board * nr_files_on_board) * 2 + + 1 + + nr_ghosts * (2 + bytes_per_spec) + + CommonEncode + We need to bound the remaining non-constants. We have + - nr_ghosts <= underworld_capacity + - bytes_per_spec <= 4 + - being_solved.number_of_imitators <= maxinum + en_passant_top[nbply] - en_passant_top[nbply-1] requires a bit more thought. It should be an upper + bound on the number of en passant squares stored per ply. For now we'll assume that + - en_passant_top[nbply] - en_passant_top[nbply-1] <= nr_rows_on_board - 2 + as (nr_rows_on_board - 2) is the number of squares that a pawn jumping from the first rank to the + last rank would skip over. + With all of the above, we can determine the maximum bytes that an encoding should take up. We'll + let the compiler perform the arithmetic. +*/ + +enum { + MAX_NR_GHOSTS = underworld_capacity, + MAX_BYTES_PER_SPEC = 4, + MAX_NUMBER_OF_IMITATORS = maxinum, + MAX_EN_PASSANT_TOP_DIFFERENCE = nr_rows_on_board - 2, /* TODO: Is this a safe maximum? Can we get away with a smaller value? */ + COMMONENCODE_MAX = 2 + 2 + 1 + 1 + 1 + + MAX_NUMBER_OF_IMITATORS + + 1 + 1 + 3 + 1 + + MAX_BYTES_PER_SPEC + + 1 + 2 + + MAX_EN_PASSANT_TOP_DIFFERENCE + + 1 + + sizeof BGL_values[White] + sizeof BGL_values[Black] + + 2 + + sizeof fuddled, + NUM_NONEMPTY_SMALL_FIRST = (nr_rows_on_board * nr_files_on_board), + NUM_NONEMPTY_SMALL_SECOND = (8 + sizeof BGL_values[White] + sizeof BGL_values[Black]), + NUM_NONEMPTY_SMALL = ((NUM_NONEMPTY_SMALL_FIRST < NUM_NONEMPTY_SMALL_SECOND) ? NUM_NONEMPTY_SMALL_FIRST : NUM_NONEMPTY_SMALL_SECOND), + SMALLENCODE_MAX = (NUM_NONEMPTY_SMALL * (1 + 1 + MAX_BYTES_PER_SPEC)) + + (MAX_NR_GHOSTS * (1 + 1 + MAX_BYTES_PER_SPEC)) + + COMMONENCODE_MAX, + LARGEENCODE_MAX = nr_rows_on_board + + ((nr_rows_on_board * nr_files_on_board) * (1 + MAX_BYTES_PER_SPEC)) + + (MAX_NR_GHOSTS * (1 + 1 + MAX_BYTES_PER_SPEC)) + + COMMONENCODE_MAX, + PROOFENCODE_MAX = nr_rows_on_board + + ((nr_rows_on_board * nr_files_on_board) * 2) + + 1 + + (MAX_NR_GHOSTS * (1 + 1 + MAX_BYTES_PER_SPEC)) + + COMMONENCODE_MAX +}; + enum { - hashbuf_length = 256 + MAX_LENGTH_OF_ENCODING = ((SMALLENCODE_MAX > LARGEENCODE_MAX) ? ((SMALLENCODE_MAX > PROOFENCODE_MAX) ? SMALLENCODE_MAX : PROOFENCODE_MAX) + : ((LARGEENCODE_MAX > PROOFENCODE_MAX) ? LARGEENCODE_MAX : PROOFENCODE_MAX)), + hashbuf_length = (MAX_LENGTH_OF_ENCODING * sizeof(byte)) + offsetof(BCMemValue, Data) }; typedef union @@ -36,9 +161,7 @@ typedef union extern HashBuffer hashBuffers[maxply+1]; -#if defined(FXF) -extern unsigned long hash_max_kilo_storable_positions; -#endif +extern unsigned long hash_max_number_storable_positions; /* exported functions */ void check_hash_assumptions(void); diff --git a/position/pieceid.h b/position/pieceid.h index e402f0ddd5..bd770ed4c4 100644 --- a/position/pieceid.h +++ b/position/pieceid.h @@ -8,7 +8,7 @@ enum { NullPieceId = 0, MinPieceId = 1, - MaxPieceId = 63 + MaxPieceId = 64 }; typedef unsigned long PieceIdType; @@ -21,6 +21,11 @@ typedef unsigned long PieceIdType; #define GetPieceId(spec) ((spec) >> PieceIdOffset) #define ClearPieceId(spec) SetPieceId(spec,NullPieceId) +enum +{ + ENSURE_PIECEIDWIDTH_IS_LARGE_ENOUGH=1/!((MaxPieceId>>(PieceIdWidth-1u))>>1) +}; + extern square PiecePositionsInDiagram[MaxPieceId+1]; #define GetPositionInDiagram(spec) PiecePositionsInDiagram[GetPieceId(spec)] From dec54dddc9c9b3d0fc219cae0ff887d2d8eb8524 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 16 May 2024 21:13:29 -0400 Subject: [PATCH 02/53] Updating comment. --- optimisations/hash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimisations/hash.h b/optimisations/hash.h index 12a6b940e2..43bdf51b07 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -161,7 +161,7 @@ typedef union extern HashBuffer hashBuffers[maxply+1]; -extern unsigned long hash_max_number_storable_positions; +extern unsigned long hash_max_kilo_storable_positions; /* exported functions */ void check_hash_assumptions(void); From cf8b6bc974c576685ea742d203666f46441b6a9c Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 16 May 2024 22:11:29 -0400 Subject: [PATCH 03/53] Correcting this comment. --- optimisations/hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index e1fd9fd3ff..d65d43afd1 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -46,7 +46,7 @@ ** dhtEnterElement: This procedure enters an encoded position ** with its values into the hashtable. ** - ** When there is no more memory, or more than hash_max_number_storable_positions positions + ** When there is no more memory, or more than hash_max_kilo_storable_positions*1000 positions ** are stored in the hash-table, then some positions are removed ** from the table. This is done in the compress procedure. ** This procedure uses a little improved scheme introduced by Torsten. From 7150d8af666b6ea25bdfbf2df77d9a4fe6b4afe6 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 16 May 2024 22:18:37 -0400 Subject: [PATCH 04/53] No reason not to include these even though they're already needed in the header file. --- optimisations/hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index d65d43afd1..721b4cde6b 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -96,7 +96,7 @@ #include "conditions/duellists.h" #include "conditions/haunted_chess.h" #include "conditions/imitator.h" -//#include "conditions/fuddled_men.h" +#include "conditions/fuddled_men.h" #include "options/nontrivial.h" #include "solving/avoid_unsolvable.h" #include "solving/castling.h" From 29bc885decb38abc26011275bda68d9c0a2cb587 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 17 May 2024 07:22:09 -0400 Subject: [PATCH 05/53] Adding a useful comment. --- optimisations/hash.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/optimisations/hash.c b/optimisations/hash.c index 721b4cde6b..f52c1619ec 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -2865,6 +2865,8 @@ void hash_opener_solve(slice_index si) /* Check assumptions made in the hashing module. Abort if one of them * isn't met. * This is called from checkGlobalAssumptions() once at program start. + * NOTE: Currently these are all compile-time checks, but we reserve + * the right to add run-time checks in the future. */ void check_hash_assumptions(void) { From 881c20cf1228aeaaf8db62746ad0721dc4283b71 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 17 May 2024 19:15:02 -0400 Subject: [PATCH 06/53] Removing some unnecessary changes. --- optimisations/hash.c | 8 ++++---- optimisations/hash.h | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index f52c1619ec..109340a581 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -1184,7 +1184,7 @@ static void ProofSmallEncodePiece(byte **bp, encoded |= 1 << black_bit; if (*even) { - **bp += (byte)(encoded<<(CHAR_BIT/2)); + **bp += encoded<<(CHAR_BIT/2); ++*bp; } else @@ -1299,7 +1299,7 @@ static byte *CommonEncode(byte *bp, if (move_effect_journal[capture].type==move_effect_piece_removal) ply_last_capture = nbply; else - ply_last_capture = (ply)find_last_capture(); + ply_last_capture = find_last_capture(); if (ply_last_capture!=ply_nil) { @@ -1979,7 +1979,7 @@ static void inithash(slice_index si) if (hashtable_kilos>0 && hash_max_kilo_storable_positions==ULONG_MAX) hash_max_kilo_storable_positions= hashtable_kilos/(Large+sizeof(char *)+1); #endif - } + } } #if defined(FXF) @@ -2656,7 +2656,7 @@ static dhtElement const *find_help_elmt_not_solved(HashBuffer *hb) k.value.object_pointer = &hb->cmv; result = dhtLookupElement(pyhash,k); TraceFunctionExit(__func__); - TraceFunctionResult("%p",(void const *)result->object_pointer); + TraceFunctionResult("%p",(void const *)result); TraceFunctionResultEnd(); return result; } diff --git a/optimisations/hash.h b/optimisations/hash.h index 43bdf51b07..238d434c14 100644 --- a/optimisations/hash.h +++ b/optimisations/hash.h @@ -60,7 +60,7 @@ typedef unsigned char byte; Therefore: - If SmallEncode is chosen then "num nonempty squares" <= min((nr_rows_on_board * nr_files_on_board), (8 + sizeof BGL_values[White] + sizeof BGL_values[Black])). - If LargeEncode is chosen then 8 < "num nonempty squares" <= (nr_rows_on_board * nr_files_on_board). - Substituting we find that + Substituting we find that when chosen: - SmallEncode <= min((nr_rows_on_board * nr_files_on_board), (8 + sizeof BGL_values[White] + sizeof BGL_values[Black])) * SmallEncodePiece + nr_ghosts * SmallEncodePiece + CommonEncode @@ -161,7 +161,9 @@ typedef union extern HashBuffer hashBuffers[maxply+1]; +#if defined(FXF) extern unsigned long hash_max_kilo_storable_positions; +#endif /* exported functions */ void check_hash_assumptions(void); From c85b0de165740ec4aee80c1bec5529dff28022f9 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 18 May 2024 07:10:02 -0400 Subject: [PATCH 07/53] Oops, we need to initialize the element here. --- optimisations/hash.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/optimisations/hash.c b/optimisations/hash.c index 109340a581..dc51f4569b 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -2485,6 +2485,8 @@ static dhtElement *find_or_add_help_elmt(HashBuffer const *hb) dhtElement *result; k.value.object_pointer = &hb->cmv; result = dhtLookupElement(pyhash,k); + if (!result) + result = allocDHTelement(k); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void const *)result); TraceFunctionResultEnd(); From bcbbe7381c27742ddbe3cece726b6e2d59be52bb Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 18 May 2024 08:55:25 -0400 Subject: [PATCH 08/53] Switching to the idiomatic expression. --- optimisations/hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimisations/hash.c b/optimisations/hash.c index dc51f4569b..87c59e1e77 100644 --- a/optimisations/hash.c +++ b/optimisations/hash.c @@ -2485,7 +2485,7 @@ static dhtElement *find_or_add_help_elmt(HashBuffer const *hb) dhtElement *result; k.value.object_pointer = &hb->cmv; result = dhtLookupElement(pyhash,k); - if (!result) + if (result==dhtNilElement) result = allocDHTelement(k); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void const *)result); From b3159bd5b2c66b95abaf9cf31984b0331252be80 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 22 May 2024 21:00:22 -0400 Subject: [PATCH 09/53] sh->MallocCount should be the number of allocated blocks. We should therefore always decrement, just like we always increment it when allocating. --- DHT/fxf.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 2bd484fd93..c04de344cd 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -713,7 +713,6 @@ void fxfFree(void *ptr, size_t size) if (stepPointer(ptr, (ptrdiff_t)size) == BotFreePtr) { BotFreePtr= ptr; TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(TopFreePtr,BotFreePtr))); - --sh->MallocCount; } else { #if defined(FREEMAP) && !defined(SEGMENTED) @@ -724,7 +723,6 @@ void fxfFree(void *ptr, size_t size) memcpy(ptr, &sh->FreeHead, sizeof sh->FreeHead); sh->FreeHead= ptr; ++sh->FreeCount; - --sh->MallocCount; TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); } } @@ -735,7 +733,6 @@ void fxfFree(void *ptr, size_t size) if (ptr == TopFreePtr) { TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t)size); TMDBG(printf(" TopFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(TopFreePtr,BotFreePtr))); - --sh->MallocCount; } else { #if defined(FREEMAP) && !defined(SEGMENTED) @@ -746,11 +743,11 @@ void fxfFree(void *ptr, size_t size) memcpy(ptr, &sh->FreeHead, sizeof sh->FreeHead); sh->FreeHead= ptr; ++sh->FreeCount; - --sh->MallocCount; TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); } } } + --sh->MallocCount; TMDBG(printf(" MallocCount:%lu",sh->MallocCount)); TMDBG(putchar('\n')); } From a32691cc457629b5034f420817b8e4f0d3034c3b Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 27 May 2024 16:07:43 -0400 Subject: [PATCH 10/53] Oops, forgot to update a couple of computations. --- DHT/fxf.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index c04de344cd..469f110fe2 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -565,7 +565,7 @@ void *fxfAlloc(size_t size) { SetRange(curBottomIndex,cur_alignment); #endif if (cur_alignment >= fxfMINSIZE) { - SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; + SizeHead * const cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { if (cur_alignment >= sizeof cur_sh->FreeHead) memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); @@ -601,7 +601,7 @@ void *fxfAlloc(size_t size) { while (curBottomIndex & PTRMASK) { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); if (cur_alignment >= fxfMINSIZE) { - SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; + SizeHead * const cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { if (cur_alignment >= sizeof cur_sh->FreeHead) memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); @@ -615,7 +615,7 @@ void *fxfAlloc(size_t size) { } curBottomIndex= (size_t)(TopFreePtr-BotFreePtr); if (curBottomIndex >= fxfMINSIZE) { - SizeHead *cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(curBottomIndex)]; + SizeHead * const cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(curBottomIndex)]; if ((curBottomIndex >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { if (curBottomIndex >= sizeof cur_sh->FreeHead) memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); @@ -635,6 +635,11 @@ void *fxfAlloc(size_t size) { #else /*SEGMENTED*/ ptr= Nil(void); #endif /*!SEGMENTED*/ + /* TODO: Instead of returning Nil(void), should we try to satisfy the request with a larger + block from the free store? + ADVANTAGE: may not have to return Nil(void) + DISADVANTAGE: the extra-large nature of the block will be forgotten when/if + the block is returned to the free store */ TMDBG(printf(" ptr:%p\n", ptr)); } } @@ -860,9 +865,9 @@ void fxfInfo(FILE *f) { if (hd->MallocCount+hd->FreeCount>0) { fprintf(f, "%12zu %10lu%10lu\n", SIZEDATA_INDEX_TO_SIZE(i), hd->MallocCount, hd->FreeCount); nrUsed+= hd->MallocCount; - UsedBytes+= hd->MallocCount*(i+1U); + UsedBytes+= hd->MallocCount*SIZEDATA_INDEX_TO_SIZE(i); nrFree+= hd->FreeCount; - FreeBytes+= hd->FreeCount*(i+1U); + FreeBytes+= hd->FreeCount*SIZEDATA_INDEX_TO_SIZE(i); } } fprintf(f, "%12s %10lu%10lu\n", "Total:", nrUsed, nrFree); From 3d2071f311cc7066534dfc30a29ccbe481f21530 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 27 May 2024 20:00:07 -0400 Subject: [PATCH 11/53] The FXF code must be generic. However, on some systems, with our precise usage, we may be able to work with a smaller maximum alignment than what the system would require in general. This update allows us to select such a value as a compile-time flag. This, of course, would have to be carefully tested for each such system. --- DHT/fxf.c | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 469f110fe2..333f88278a 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -46,31 +46,33 @@ # endif #endif -#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) -# define MAX_ALIGNMENT _Alignof(max_align_t) -#elif (defined(__cplusplus) && (__cplusplus >= 201103L)) -# define MAX_ALIGNMENT alignof(max_align_t) -#else -# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) -# include -# endif -struct GET_MAX_ALIGNMENT_TYPE { - unsigned char c; - union { -# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) - uintmax_t unsigned_integer; -# elif defined(LLONG_MAX) /* We have long long integer types. */ - unsigned long long int unsigned_integer; +#if !defined(MAX_ALIGNMENT) +# if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) +# define MAX_ALIGNMENT _Alignof(max_align_t) +# elif (defined(__cplusplus) && (__cplusplus >= 201103L)) +# define MAX_ALIGNMENT alignof(max_align_t) # else - unsigned long int unsigned_integer; +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# include +# endif + struct GET_MAX_ALIGNMENT_TYPE { + unsigned char c; + union { +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + uintmax_t unsigned_integer; +# elif defined(LLONG_MAX) /* We have long long integer types. */ + unsigned long long int unsigned_integer; +# else + unsigned long int unsigned_integer; +# endif + const volatile void * object_pointer; + void (*function_pointer)(void); + long double floating_point; + } max_aligned_union; + }; +# define MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union) # endif - const volatile void * object_pointer; - void (*function_pointer)(void); - long double floating_point; - } max_aligned_union; -}; -# define MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union) -#endif +#endif /*MAX_ALIGNMENT*/ #if !defined(Nil) && !defined(New) && !defined(nNewUntyped) && !defined(nNewCallocUntyped) /* TODO: Is this the correct check for all of the below lines? */ # define Nil(type) ((type *)0) From 6a27c3e57beaea3732fdbc540470382d660542d1 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 28 May 2024 09:05:42 -0400 Subject: [PATCH 12/53] Making the comment a bit more accurate. --- DHT/fxf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 333f88278a..5d07ba4631 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -637,11 +637,11 @@ void *fxfAlloc(size_t size) { #else /*SEGMENTED*/ ptr= Nil(void); #endif /*!SEGMENTED*/ - /* TODO: Instead of returning Nil(void), should we try to satisfy the request with a larger - block from the free store? + /* TODO: Instead of returning Nil(void), should we try to satisfy the request by breaking + apart a larger block from the free store? ADVANTAGE: may not have to return Nil(void) - DISADVANTAGE: the extra-large nature of the block will be forgotten when/if - the block is returned to the free store */ + DISADVANTAGE: we would no longer ever be able to use that contiguous memory to + satisfy a large request */ TMDBG(printf(" ptr:%p\n", ptr)); } } From cc8b444e85e8efbd388cc35bd3fe83b5183ad18b Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 1 Jun 2024 14:07:10 -0400 Subject: [PATCH 13/53] Hoping to simplify the logic by moving the boilerplate into functions. Also, implementing my TODO as a compile-time option. Primarily pushing now so that I can pull this into my testing infrastructure. --- DHT/fxf.c | 277 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 168 insertions(+), 109 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 5d07ba4631..a9d9b465fb 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -1,4 +1,4 @@ -#if defined (FXF) +#if defined(FXF) #include "debugging/assert.h" #include @@ -46,11 +46,20 @@ # endif #endif -#if !defined(MAX_ALIGNMENT) +#if defined(FXF_MAX_ALIGNMENT) +enum { + ENSURE_FXF_MAX_ALIGNMENT_IS_REASONABLE_POWER_OF_TWO = 1/((FXF_MAX_ALIGNMENT > 0) && + (FXF_MAX_ALIGNMENT <= (size_t)-1) && + !(FXF_MAX_ALIGNMENT & (FXF_MAX_ALIGNMENT - 1U))) +}; +# if ((FXF_MAX_ALIGNMENT <= 0) || (FXF_MAX_ALIGNMENT & (FXF_MAX_ALIGNMENT - 1U))) +# #error "ERROR: MAX_ALIGNMENT must be a power of 2." +# endif +#else # if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) -# define MAX_ALIGNMENT _Alignof(max_align_t) +# define FXF_MAX_ALIGNMENT _Alignof(max_align_t) # elif (defined(__cplusplus) && (__cplusplus >= 201103L)) -# define MAX_ALIGNMENT alignof(max_align_t) +# define FXF_MAX_ALIGNMENT alignof(max_align_t) # else # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # include @@ -70,9 +79,9 @@ long double floating_point; } max_aligned_union; }; -# define MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union) +# define FXF_MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union) # endif -#endif /*MAX_ALIGNMENT*/ +#endif /*FXF_MAX_ALIGNMENT*/ #if !defined(Nil) && !defined(New) && !defined(nNewUntyped) && !defined(nNewCallocUntyped) /* TODO: Is this the correct check for all of the below lines? */ # define Nil(type) ((type *)0) @@ -128,7 +137,7 @@ typedef struct { #if defined(DOS) /* MSDOS 16 Bit support (maxmemory <= 1 MB) */ #define SEGMENTED -#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(32000) & ~(MAX_ALIGNMENT - 1U)) +#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(32000) & ~(FXF_MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((1024*1024)/ARENA_SEG_SIZE) #define OSNAME "MSDOS" #define OSMAXMEM "1 MB" @@ -136,7 +145,7 @@ typedef struct { /* Win95/Win98/WinME can only allocate chunks up to 255 MB */ /* maxmemory <= 768 MB */ #define SEGMENTED -#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(1000000) & ~(MAX_ALIGNMENT - 1U)) +#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(1000000) & ~(FXF_MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((768*1024*1024)/ARENA_SEG_SIZE) #define OSNAME "Win95/Win98/WinME" #define OSMAXMEM "768 MB" @@ -160,7 +169,7 @@ enum #else ((((size_t)2048) /* This is needed only when sizeof(void*)==8. */ #endif - + (MAX_ALIGNMENT - 1U)) & ~(MAX_ALIGNMENT - 1U)) /* Round up if necessary. */ + + (FXF_MAX_ALIGNMENT - 1U)) & ~(FXF_MAX_ALIGNMENT - 1U)) /* Round up if necessary. */ }; enum { @@ -169,9 +178,9 @@ enum { }; #define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) -#define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > MAX_ALIGNMENT) ? MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. - (Of course, they're probably the same.) - TODO: Can we compute what we want at compile time and just use it? */ +#define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > FXF_MAX_ALIGNMENT) ? FXF_MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. + (Of course, they're probably the same.) + TODO: Can we compute what we want at compile time and just use it? */ static size_t min_alignment= 0; /* for now */ static SizeHead SizeData[1 + ((fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE)]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). @@ -337,7 +346,7 @@ size_t fxfInit(size_t Size) { free(Arena); if (Size > MAX_POINTER_DIFFERENCE) Size= MAX_POINTER_DIFFERENCE; - Size&= ~(MAX_ALIGNMENT - 1U); + Size&= ~(FXF_MAX_ALIGNMENT - 1U); Arena= nNewUntyped(Size, char); if (!Arena) { ERROR_LOG3("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " <= %" SIZE_T_PRINTF_SPECIFIER " bytes\n", @@ -375,7 +384,7 @@ size_t fxfInit(size_t Size) { if (!min_alignment) { - min_alignment= MAX_ALIGNMENT; + min_alignment= FXF_MAX_ALIGNMENT; while (min_alignment > fxfMINSIZE) min_alignment>>= 1; } @@ -469,16 +478,54 @@ void fxfReset(void) * SPARC, HPPA, MIPS. We wouldn't need this when running on an * Intel *86 type of CPU, but also there, aligned access is faster. */ -#define PTRMASK (MAX_ALIGNMENT-1U) +#define PTRMASK (FXF_MAX_ALIGNMENT-1U) #define ALIGN_TO_MINIMUM(s) (((s) + (min_alignment - 1U)) & ~(min_alignment - 1U)) -#define TMDBG(x) if (0) x +#if defined(FXF_ENABLE_TMDBG) +# define TMDBG(x) do {x;} while (0) +#else /*FXF_ENABLE_TMDBG*/ +# define TMDBG(x) do {;} while (0) +#endif /*!FXF_ENABLE_TMDBG*/ + +static int pushOntoFreeStore(void * const ptr, size_t const size) { + SizeHead *cur_sh; + assert((size >= fxfMINSIZE) && + (size <= fxfMAXSIZE) && + !(size & (min_alignment - 1U))); + cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; + if ((size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if (size >= sizeof cur_sh->FreeHead) + memcpy(ptr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); + cur_sh->FreeHead= ptr; + cur_sh->FreeCount++; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); + return 1; + } + return 0; +} + +static void * popOffFreeStore(size_t const size) { + SizeHead *cur_sh; + void *ptr; + assert((size >= fxfMINSIZE) && + (size <= fxfMAXSIZE) && + !(size & (min_alignment - 1U))); + cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; + ptr= cur_sh->FreeHead; + if (ptr) { + cur_sh->FreeCount--; + if (size < sizeof cur_sh->FreeHead) + cur_sh->FreeHead= Nil(void); + else + memcpy(&cur_sh->FreeHead, ptr, sizeof cur_sh->FreeHead); + } + return ptr; +} void *fxfAlloc(size_t size) { #if defined(LOG) || defined(DEBUG) static char const * const myname= "fxfAlloc"; #endif - SizeHead *sh; void *ptr= Nil(void); TMDBG(printf("fxfAlloc - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); @@ -501,21 +548,17 @@ void *fxfAlloc(size_t size) { // Round up to a multiple of min_alignment size= ALIGN_TO_MINIMUM(size); - sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; - if (sh->FreeHead) { -#if defined(SEGMENTED) - int ptrSegment; - ptrdiff_t ptrIndex; + ptr= popOffFreeStore(size); + + if (ptr) { + SizeHead *const sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; +#if defined(SEGMENTED) && defined(FXF_ENABLE_TMDBG) + ptrdiff_t ptrIndex; + int ptrSegment= CurrentSeg; #endif - ptr= sh->FreeHead; - if (size < sizeof sh->FreeHead) - sh->FreeHead= Nil(void); - else - memcpy(&sh->FreeHead, ptr, sizeof sh->FreeHead); - sh->FreeCount--; sh->MallocCount++; #if defined(SEGMENTED) - ptrSegment= CurrentSeg; +# if defined(FXF_ENABLE_TMDBG) if (CurrentSeg) { convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; do { @@ -530,12 +573,13 @@ void *fxfAlloc(size_t size) { ptrIndex= pointerDifference(ptr, Arena[0]); FOUND_PUTATIVE_SEGMENT: TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,sh->MallocCount)); -#else +# endif /*FXF_ENABLE_TMDBG*/ +#else /*SEGMENTED*/ # if defined(FREEMAP) ClrRange(pointerDifference(ptr, Arena), size); # endif TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)pointerDifference(ptr, Arena),sh->MallocCount)); -#endif +#endif /*!SEGMENTED*/ } else { /* we have to allocate a new piece */ @@ -546,36 +590,28 @@ void *fxfAlloc(size_t size) { sizeCurrentSeg = (size_t)pointerDifference(TopFreePtr,BotFreePtr); TMDBG(printf(" sizeCurrentSeg:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)sizeCurrentSeg)); if (sizeCurrentSeg>=size) { + SizeHead *sh; if (size&PTRMASK) { /* not fully aligned */ - size_t curBottomIndex; - size_t needed_alignment_mask= PTRMASK; - while (needed_alignment_mask >= size) - needed_alignment_mask>>= 1; + size_t curBottomIndex= #if defined(SEGMENTED) - curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena[CurrentSeg]); + (size_t)pointerDifference(BotFreePtr,Arena[CurrentSeg]); #else - curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena); + (size_t)pointerDifference(BotFreePtr,Arena); #endif + size_t needed_alignment_mask= PTRMASK; + while (needed_alignment_mask >= size) + needed_alignment_mask>>= 1; if (curBottomIndex & needed_alignment_mask) { - size_t const numBytesToAdd= (needed_alignment_mask - (curBottomIndex & needed_alignment_mask)) + 1U; - if (numBytesToAdd > (sizeCurrentSeg-size)) + if ((needed_alignment_mask - (curBottomIndex & needed_alignment_mask)) >= (sizeCurrentSeg-size)) goto NEXT_SEGMENT; do { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); #if defined(FREEMAP) && !defined(SEGMENTED) SetRange(curBottomIndex,cur_alignment); #endif - if (cur_alignment >= fxfMINSIZE) { - SizeHead * const cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; - if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { - if (cur_alignment >= sizeof cur_sh->FreeHead) - memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); - cur_sh->FreeHead= BotFreePtr; - ++cur_sh->FreeCount; - TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); - } - } + if (cur_alignment >= fxfMINSIZE) + pushOntoFreeStore(BotFreePtr, cur_alignment); BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)cur_alignment); curBottomIndex+= cur_alignment; } while (curBottomIndex & needed_alignment_mask); @@ -587,6 +623,7 @@ void *fxfAlloc(size_t size) { /* fully aligned */ ptr= (TopFreePtr= stepPointer(TopFreePtr, -(ptrdiff_t)size)); } + sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; sh->MallocCount++; #if defined(SEGMENTED) TMDBG(printf(" current seg ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",CurrentSeg,(ptrdiff_t_printf_type)pointerDifference(ptr, Arena[CurrentSeg]),sh->MallocCount)); @@ -602,30 +639,14 @@ void *fxfAlloc(size_t size) { size_t curBottomIndex= (BotFreePtr - Arena[CurrentSeg]); while (curBottomIndex & PTRMASK) { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); - if (cur_alignment >= fxfMINSIZE) { - SizeHead * const cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(cur_alignment)]; - if ((cur_alignment >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { - if (cur_alignment >= sizeof cur_sh->FreeHead) - memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); - cur_sh->FreeHead= BotFreePtr; - ++cur_sh->FreeCount; - TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); - } - } + if (cur_alignment >= fxfMINSIZE) + pushOntoFreeStore(BotFreePtr, cur_alignment); BotFreePtr= stepPointer(BotFreePtr, cur_alignment); curBottomIndex+= cur_alignment; } curBottomIndex= (size_t)(TopFreePtr-BotFreePtr); - if (curBottomIndex >= fxfMINSIZE) { - SizeHead * const cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(curBottomIndex)]; - if ((curBottomIndex >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { - if (curBottomIndex >= sizeof cur_sh->FreeHead) - memcpy(BotFreePtr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); - cur_sh->FreeHead= BotFreePtr; - ++cur_sh->FreeCount; - TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); - } - } + if (curBottomIndex >= fxfMINSIZE) + pushOntoFreeStore(BotFreePtr, curBottomIndex); TMDBG(fputs(" next seg", stdout)); ++CurrentSeg; BotFreePtr= Arena[CurrentSeg]; @@ -633,15 +654,34 @@ void *fxfAlloc(size_t size) { goto START_LOOKING_FOR_CHUNK; } else +#endif /*SEGMENTED*/ + { ptr= Nil(void); -#else /*SEGMENTED*/ - ptr= Nil(void); -#endif /*!SEGMENTED*/ - /* TODO: Instead of returning Nil(void), should we try to satisfy the request by breaking - apart a larger block from the free store? - ADVANTAGE: may not have to return Nil(void) - DISADVANTAGE: we would no longer ever be able to use that contiguous memory to - satisfy a large request */ +#if defined(FXF_BREAK_APART_LARGER_CHUNK) + /* See if we can find a larger chunk to pull a piece out of. */ + for (size_t largerSize= size; largerSize < fxfMAXSIZE;) { + largerSize+= min_alignment; + ptr= popOffFreeStore(largerSize); + if (ptr) { + void *newPtr= stepPointer(ptr, size); + size_t cur_alignment= (size & -size); + SizeData[SIZEDATA_SIZE_TO_INDEX(size)].MallocCount++; + largerSize-= size; + while ((cur_alignment < FXF_MAX_ALIGNMENT) && (cur_alignment <= (largerSize>>1))) { + if (cur_alignment >= fxfMINSIZE) + pushOntoFreeStore(newPtr, cur_alignment); + newPtr= stepPointer(newPtr, cur_alignment); + largerSize-= cur_alignment; + size+= cur_alignment; + cur_alignment= (size & -size); + } + if (largerSize >= fxfMINSIZE) + pushOntoFreeStore(newPtr, largerSize); + break; + } + } +#endif /*FXF_BREAK_APART_LARGER_CHUNK*/ + } TMDBG(printf(" ptr:%p\n", ptr)); } } @@ -651,19 +691,19 @@ void *fxfAlloc(size_t size) { void fxfFree(void *ptr, size_t size) { -#if defined(LOG) || defined(DEBUG) - static char const * const myname= "fxfFree"; -#endif SizeHead *sh; +#if defined(FXF_ENABLE_TMDBG) || !defined(NDEBUG) ptrdiff_t ptrIndex; -#if defined(SEGMENTED) +#endif +#if defined(SEGMENTED) && defined(FXF_ENABLE_TMDBG) int ptrSegment; #endif if (!ptr) return; assert(!!size); #if defined(SEGMENTED) +# if defined(FXF_ENABLE_TMDBG) || !defined(NDEBUG) ptrSegment= CurrentSeg; if (CurrentSeg) { convert_pointer_to_int_type tmp= (convert_pointer_to_int_type)ptr; @@ -682,11 +722,14 @@ void fxfFree(void *ptr, size_t size) } FOUND_PUTATIVE_SEGMENT: TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); -#else +# endif /*FXF_ENABLE_TMDBG*/ +#else /*SEGMENTED*/ +# if defined(FXF_ENABLE_TMDBG) || !defined(NDEBUG) ptrIndex= pointerDifference(ptr,Arena); +# endif assert((ptrIndex >= 0) && (ptrIndex < GlobalSize)); TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); -#endif +#endif /*!SEGMENTED*/ DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *)ptr, (size_t_printf_type) size)); if (size < fxfMINSIZE) size= fxfMINSIZE; @@ -706,14 +749,13 @@ void fxfFree(void *ptr, size_t size) #endif if (ptrIndex > 0) { - size_t needed_alignment= MAX_ALIGNMENT; + size_t needed_alignment= FXF_MAX_ALIGNMENT; while (needed_alignment > size) needed_alignment>>= 1; assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); } } #endif - sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; if (size&PTRMASK) { /* not fully aligned size */ TMDBG(printf(" BotFreePtr-ptr:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(BotFreePtr,ptr))); @@ -725,13 +767,7 @@ void fxfFree(void *ptr, size_t size) #if defined(FREEMAP) && !defined(SEGMENTED) SetRange((pointerDifference(ptr,Arena),size); #endif - if ((size >= sizeof sh->FreeHead) || !sh->FreeHead) { - if (size >= sizeof sh->FreeHead) - memcpy(ptr, &sh->FreeHead, sizeof sh->FreeHead); - sh->FreeHead= ptr; - ++sh->FreeCount; - TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); - } + pushOntoFreeStore(ptr, size); } } else { @@ -745,27 +781,24 @@ void fxfFree(void *ptr, size_t size) #if defined(FREEMAP) && !defined(SEGMENTED) SetRange(pointerDifference(ptr,Arena),size); #endif - if ((size >= sizeof sh->FreeHead) || !sh->FreeCount) { - if (size >= sizeof sh->FreeHead) - memcpy(ptr, &sh->FreeHead, sizeof sh->FreeHead); - sh->FreeHead= ptr; - ++sh->FreeCount; - TMDBG(printf(" FreeCount:%lu",sh->FreeCount)); - } + pushOntoFreeStore(ptr, size); } } - --sh->MallocCount; + sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; + sh->MallocCount--; TMDBG(printf(" MallocCount:%lu",sh->MallocCount)); TMDBG(putchar('\n')); } void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { void *nptr; + size_t original_allocation; if (!ptr) { assert(!OldSize); return fxfAlloc(NewSize); } + assert(OldSize && (OldSize <= fxfMAXSIZE)); #if !defined(NDEBUG) # if defined(SEGMENTED) if (!CurrentSeg) /* Otherwise we'd be relying on converting to convert_pointer_to_int_type, @@ -792,7 +825,7 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { # else assert(allocatedSize <= (GlobalSize - ptrIndex)); # endif - needed_alignment= MAX_ALIGNMENT; + needed_alignment= FXF_MAX_ALIGNMENT; while (needed_alignment > allocatedSize) needed_alignment>>= 1; assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); @@ -804,14 +837,40 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { fxfFree(ptr, OldSize); return Nil(void); } - /* TODO: It may be worth trying to return ptr if ALIGN_TO_MINIMUM(OldSize) >= NewSize. - To go along with this, we'd have to carefully add any excess to the free store. - In the !defined(SEGMENTED) case this is likely easy, but in the defined(SEGMENTED) - case it may be difficult. Regardless, the computations to set this up -- or even - determine if it's possible -- are kind of annoying, and they'd only be worthwhile - if we hit this possibility frequently (and if the alternative below is expensive - or proves impossible). This all would need to be investigated. - */ + original_allocation= OldSize; + if (original_allocation < fxfMINSIZE) + original_allocation= fxfMINSIZE; + original_allocation= ALIGN_TO_MINIMUM(original_allocation); + if (NewSize <= original_allocation) + { + size_t needed_allocation= NewSize; + if (needed_allocation < fxfMINSIZE) + needed_allocation= fxfMINSIZE; + needed_allocation= ALIGN_TO_MINIMUM(needed_allocation); + if (needed_allocation == original_allocation) + return ptr; +#if defined(FXF_BREAK_APART_LARGER_CHUNK) + if (needed_allocation < original_allocation) + { + size_t cur_alignment= (needed_allocation & -needed_allocation); + nptr= stepPointer(ptr, needed_allocation); + SizeData[SIZEDATA_SIZE_TO_INDEX(original_allocation)].MallocCount--; + SizeData[SIZEDATA_SIZE_TO_INDEX(needed_allocation)].MallocCount++; + original_allocation-= needed_allocation; + while ((cur_alignment < FXF_MAX_ALIGNMENT) && (cur_alignment <= (original_allocation>>1))) { + if (cur_alignment >= fxfMINSIZE) + pushOntoFreeStore(nptr, cur_alignment); + nptr= stepPointer(nptr, cur_alignment); + original_allocation-= cur_alignment; + needed_allocation+= cur_alignment; + cur_alignment= (needed_allocation & -needed_allocation); + } + if (original_allocation >= fxfMINSIZE) + pushOntoFreeStore(nptr, original_allocation); + return ptr; + } +#endif /*FXF_BREAK_APART_LARGER_CHUNK*/ + } nptr= fxfAlloc(NewSize); if (nptr) { From fe2c881927f876be3b54abb69042acd80d9f71cb Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 1 Jun 2024 19:28:17 -0400 Subject: [PATCH 14/53] Using the write check, not that it should really matter in our uses. --- DHT/fxf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index a9d9b465fb..85b0181a93 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -493,7 +493,7 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { (size <= fxfMAXSIZE) && !(size & (min_alignment - 1U))); cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; - if ((size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeCount) { + if ((size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeHead) { if (size >= sizeof cur_sh->FreeHead) memcpy(ptr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); cur_sh->FreeHead= ptr; From bd7d0b5a72af2181413c7ae2a03100ca6b413b0a Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 7 Jun 2024 07:11:36 -0400 Subject: [PATCH 15/53] Striving to free memory in the reverse order that it was allocated in. This seems to eliminate the remaing discrepancies we aren't worried about alignment, though more testing is necessary. --- DHT/dht.c | 109 ++++++++++++++++++++++++++++++++++-------------------- DHT/fxf.c | 7 +++- 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index 7c0fc8b265..e6b582a9eb 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -559,8 +559,8 @@ void dhtDestroy(HashTable *ht) while (b) { InternHsElement *tmp= b; - (ht->procs.FreeKeyValue)(b->HsEl.Key.value); (ht->procs.FreeData)(b->HsEl.Data); + (ht->procs.FreeKeyValue)(b->HsEl.Key.value); b= b->Next; FreeInternHsElement(tmp); } @@ -803,11 +803,18 @@ LOCAL InternHsElement **LookupInternHsElement(HashTable *ht, dhtKey key) TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); + if (ht->KeyPolicy==dhtNoCopy) + { #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) - TraceFunctionParam("%jx",key.value.unsigned_integer); + TraceFunctionParam("%jx",key.value.unsigned_integer); #else - TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); + TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); #endif + } + else + { + TraceFunctionParam("%p",(void const *)key.value.object_pointer); // TODO: something more generic here? + } TraceFunctionParamListEnd(); h = DynamicHash(ht->p, ht->maxp, (ht->procs.Hash)(key)); @@ -838,11 +845,18 @@ void dhtRemoveElement(HashTable *ht, dhtKey key) TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); + if (ht->KeyPolicy==dhtNoCopy) + { #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) - TraceFunctionParam("%jx",key.value.unsigned_integer); + TraceFunctionParam("%jx",key.value.unsigned_integer); #else - TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); + TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); #endif + } + else + { + TraceFunctionParam("%p",(void const *)key.value.object_pointer); // TODO: something more generic here? + } TraceFunctionParamListEnd(); phe= LookupInternHsElement(ht, key); @@ -888,40 +902,36 @@ void dhtRemoveElement(HashTable *ht, dhtKey key) dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) { InternHsElement **phe, *he; - dhtKey KeyV; - dhtValue DataV; TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); + if (ht->KeyPolicy==dhtNoCopy) + { #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) - TraceFunctionParam("%jx",key.value.unsigned_integer); - TraceFunctionParam("%jx",data.unsigned_integer); + TraceFunctionParam("%jx",key.value.unsigned_integer); #else - TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); - TraceFunctionParam("%lx",(unsigned long)data.unsigned_integer); + TraceFunctionParam("%lx",(unsigned long)key.value.unsigned_integer); #endif - TraceFunctionParamListEnd(); - - assert(key.value.object_pointer!=0); /* TODO: This assert assumes that object_pointer is the active member. - Is there a more generic test we could do? Do we need one? */ - if ((ht->procs.DupKeyValue)(key.value, &KeyV.value)) + } + else { - TraceText("key duplication failed\n"); - TraceFunctionExit(__func__); - TraceFunctionResult("%p",(void *)dhtNilElement); - TraceFunctionResultEnd(); - return dhtNilElement; + TraceFunctionParam("%p",(void const *)key.value.object_pointer); // TODO: something more generic here? } - - if ((ht->procs.DupData)(data, &DataV)) + if (ht->DtaPolicy==dhtNoCopy) { - (ht->procs.FreeKeyValue)(KeyV.value); - TraceText("data duplication failed\n"); - TraceFunctionExit(__func__); - TraceFunctionResult("%p",(void *)dhtNilElement); - TraceFunctionResultEnd(); - return dhtNilElement; +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) + TraceFunctionParam("%jx",data.unsigned_integer); +#else + TraceFunctionParam("%lx",(unsigned long)data.unsigned_integer); +#endif } + else + { + TraceFunctionParam("%p",(void const *)data.object_pointer); // TODO: something more generic here? + } + + assert(key.value.object_pointer!=0); /* TODO: This assert assumes that object_pointer is the active member. + Is there a more generic test we could do? Do we need one? */ phe = LookupInternHsElement(ht,key); TraceValue("%p",(void *)phe); @@ -935,32 +945,51 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) TraceEOL(); if (he==0) { - (ht->procs.FreeKeyValue)(KeyV.value); - (ht->procs.FreeData)(DataV); TraceText("allocation of new intern Hs element failed\n"); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void *)dhtNilElement); TraceFunctionResultEnd(); return dhtNilElement; } - else + if ((ht->procs.DupKeyValue)(key.value, &he->HsEl.Key.value)) + { + FreeInternHsElement(he); + TraceText("key duplication failed\n"); + TraceFunctionExit(__func__); + TraceFunctionResult("%p",(void *)dhtNilElement); + TraceFunctionResultEnd(); + return dhtNilElement; + } + if ((ht->procs.DupData)(data, &he->HsEl.Data)) { - *phe = he; - he->Next = NilInternHsElement; - ht->KeyCount++; + (ht->procs.FreeKeyValue)(he->HsEl.Key.value); + FreeInternHsElement(he); + TraceText("data duplication failed\n"); + TraceFunctionExit(__func__); + TraceFunctionResult("%p",(void *)dhtNilElement); + TraceFunctionResultEnd(); + return dhtNilElement; } + *phe = he; + he->Next = NilInternHsElement; + ht->KeyCount++; } else { + dhtValue DataV; + if ((ht->procs.DupData)(data, &DataV)) + { + TraceText("data duplication failed\n"); + TraceFunctionExit(__func__); + TraceFunctionResult("%p",(void *)dhtNilElement); + TraceFunctionResultEnd(); + return dhtNilElement; + } if (ht->DtaPolicy == dhtCopy) (ht->procs.FreeData)(he->HsEl.Data); - if (ht->KeyPolicy == dhtCopy) - (ht->procs.FreeKeyValue)(he->HsEl.Key.value); + he->HsEl.Data = DataV; } - he->HsEl.Key = KeyV; - he->HsEl.Data = DataV; - if (ActualLoadFactor(ht)>ht->MaxLoadFactor) { /* diff --git a/DHT/fxf.c b/DHT/fxf.c index 85b0181a93..7547d016f0 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -493,6 +493,7 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { (size <= fxfMAXSIZE) && !(size & (min_alignment - 1U))); cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; + assert((!cur_sh->FreeHead) == (!cur_sh->FreeCount)); if ((size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeHead) { if (size >= sizeof cur_sh->FreeHead) memcpy(ptr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); @@ -512,6 +513,7 @@ static void * popOffFreeStore(size_t const size) { !(size & (min_alignment - 1U))); cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; ptr= cur_sh->FreeHead; + assert((!ptr) == (!cur_sh->FreeCount)); if (ptr) { cur_sh->FreeCount--; if (size < sizeof cur_sh->FreeHead) @@ -621,7 +623,8 @@ void *fxfAlloc(size_t size) { } else { /* fully aligned */ - ptr= (TopFreePtr= stepPointer(TopFreePtr, -(ptrdiff_t)size)); + TopFreePtr= stepPointer(TopFreePtr, -(ptrdiff_t)size); + ptr= TopFreePtr; } sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; sh->MallocCount++; @@ -644,7 +647,7 @@ void *fxfAlloc(size_t size) { BotFreePtr= stepPointer(BotFreePtr, cur_alignment); curBottomIndex+= cur_alignment; } - curBottomIndex= (size_t)(TopFreePtr-BotFreePtr); + curBottomIndex= (size_t)pointerDifference(TopFreePtr,BotFreePtr); if (curBottomIndex >= fxfMINSIZE) pushOntoFreeStore(BotFreePtr, curBottomIndex); TMDBG(fputs(" next seg", stdout)); From 527a6970435e9cabaa103900aeb6d26876fb9347 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 7 Jun 2024 20:33:47 -0400 Subject: [PATCH 16/53] Removing an unnecessary check. --- DHT/fxf.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 7547d016f0..64d3133caf 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -52,9 +52,6 @@ enum { (FXF_MAX_ALIGNMENT <= (size_t)-1) && !(FXF_MAX_ALIGNMENT & (FXF_MAX_ALIGNMENT - 1U))) }; -# if ((FXF_MAX_ALIGNMENT <= 0) || (FXF_MAX_ALIGNMENT & (FXF_MAX_ALIGNMENT - 1U))) -# #error "ERROR: MAX_ALIGNMENT must be a power of 2." -# endif #else # if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) # define FXF_MAX_ALIGNMENT _Alignof(max_align_t) From 0c09c827a7bb9cec174405216cd2a15e9afd4dd8 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 8 Jun 2024 13:36:11 -0400 Subject: [PATCH 17/53] Restoring the "on equivalent key" behavior. I'm not sure we need it, and it's something of a pessimization, but now is not the time to figure that out. --- DHT/dht.c | 78 ++++++++++++++++++++++++++++++++++--------------------- DHT/dht.h | 8 ++++++ 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index e6b582a9eb..1da2c6a8f7 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -899,9 +899,21 @@ void dhtRemoveElement(HashTable *ht, dhtKey key) TraceFunctionResultEnd(); } +/* dhtEnterElement adds the key, data pair. If an equivalent key already exists + then the data will be updated and the existing key will be replaced by the + provided key. (The latter can matter if keys contain data that doesn't affect + the equivalence check.) + Ths function allocates copies before freeing any old data, allowing us to bail + on failure while leaving the old data intact. This isn't ideal from a memory- + usage perspective, so if this "strong exception guarantee" is unnecessary then + the code can likely be simplified and improved. */ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) { InternHsElement **phe, *he; + dhtKey KeyV; + dhtValue DataV; + dhtValue *KeyVPtr; + dhtValue *DataVPtr; TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)ht); @@ -951,44 +963,50 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) TraceFunctionResultEnd(); return dhtNilElement; } - if ((ht->procs.DupKeyValue)(key.value, &he->HsEl.Key.value)) - { + KeyVPtr = &he->HsEl.Key.value; + DataVPtr = &he->HsEl.Data; + } + else + { + KeyVPtr = &KeyV.value; + DataVPtr = &DataV; + } + if ((ht->procs.DupKeyValue)(key.value, KeyVPtr)) + { + if (!*phe) FreeInternHsElement(he); - TraceText("key duplication failed\n"); - TraceFunctionExit(__func__); - TraceFunctionResult("%p",(void *)dhtNilElement); - TraceFunctionResultEnd(); - return dhtNilElement; - } - if ((ht->procs.DupData)(data, &he->HsEl.Data)) - { - (ht->procs.FreeKeyValue)(he->HsEl.Key.value); + TraceText("key duplication failed\n"); + TraceFunctionExit(__func__); + TraceFunctionResult("%p",(void *)dhtNilElement); + TraceFunctionResultEnd(); + return dhtNilElement; + } + if ((ht->procs.DupData)(data, DataVPtr)) + { + (ht->procs.FreeKeyValue)(*KeyVPtr); + if (!*phe) FreeInternHsElement(he); - TraceText("data duplication failed\n"); - TraceFunctionExit(__func__); - TraceFunctionResult("%p",(void *)dhtNilElement); - TraceFunctionResultEnd(); - return dhtNilElement; - } - *phe = he; - he->Next = NilInternHsElement; - ht->KeyCount++; + TraceText("data duplication failed\n"); + TraceFunctionExit(__func__); + TraceFunctionResult("%p",(void *)dhtNilElement); + TraceFunctionResultEnd(); + return dhtNilElement; } - else + if (*phe) { - dhtValue DataV; - if ((ht->procs.DupData)(data, &DataV)) - { - TraceText("data duplication failed\n"); - TraceFunctionExit(__func__); - TraceFunctionResult("%p",(void *)dhtNilElement); - TraceFunctionResultEnd(); - return dhtNilElement; - } if (ht->DtaPolicy == dhtCopy) (ht->procs.FreeData)(he->HsEl.Data); + if (ht->KeyPolicy == dhtCopy) + (ht->procs.FreeKeyValue)(he->HsEl.Key.value); + he->HsEl.Key = KeyV; he->HsEl.Data = DataV; } + else + { + *phe = he; + he->Next = NilInternHsElement; + ht->KeyCount++; + } if (ActualLoadFactor(ht)>ht->MaxLoadFactor) { diff --git a/DHT/dht.h b/DHT/dht.h index da48e683da..541424c797 100644 --- a/DHT/dht.h +++ b/DHT/dht.h @@ -26,7 +26,15 @@ struct dht; /* procedures */ struct dht *dhtCreate(dhtValueType KeyType, dhtValuePolicy KeyPolicy, dhtValueType DtaType, dhtValuePolicy DataPolicy); + +/* dhtEnterElement adds the key, data pair. If an equivalent key already exists + then the data will be updated and the existing key will be replaced by the + provided key. (The latter can matter if keys contain data that doesn't affect + the equivalence check.) + TODO: Is the "equivalent key" behavior what we need/want? Otherwise it's + pessimization in comparison to just leaving the original key intact. */ dhtElement *dhtEnterElement(struct dht *, dhtKey key, dhtValue data); + unsigned int dhtBucketStat(struct dht *, unsigned int *counter, unsigned int n); void dhtDestroy(struct dht *); void dhtDump(struct dht *, FILE *); From 3fd0c08b466c4704c22f52c9833ecadc1e23bfda Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 8 Jun 2024 17:23:59 -0400 Subject: [PATCH 18/53] Breaking apart large blocks by default since it seems to be useful; marking part of DHT/fxf.c as worthy of closer examination. --- DHT/dht.c | 13 ++++++++----- DHT/fxf.c | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index 1da2c6a8f7..549ed94527 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -903,7 +903,7 @@ void dhtRemoveElement(HashTable *ht, dhtKey key) then the data will be updated and the existing key will be replaced by the provided key. (The latter can matter if keys contain data that doesn't affect the equivalence check.) - Ths function allocates copies before freeing any old data, allowing us to bail + This function allocates copies before freeing any old data, allowing us to bail on failure while leaving the old data intact. This isn't ideal from a memory- usage perspective, so if this "strong exception guarantee" is unnecessary then the code can likely be simplified and improved. */ @@ -1010,22 +1010,25 @@ dhtElement *dhtEnterElement(HashTable *ht, dhtKey key, dhtValue data) if (ActualLoadFactor(ht)>ht->MaxLoadFactor) { - /* +#if 0 fputs("Dumping Hash-Table before expansion\n",stderr); fDumpHashTable(ht, stderr); - */ +#endif if (ExpandHashTable(ht)!=dhtOkStatus) { + /* TODO: It seems strange to return dhtNilElement AFTER we've added a new entry + or overwritten an old one. Should we return something else here? Should + we check and deal with this issue BEFORE creating or overwriting an entry? */ TraceText("expansion failed\n"); TraceFunctionExit(__func__); TraceFunctionResult("%p",(void *)dhtNilElement); TraceFunctionResultEnd(); return dhtNilElement; } - /* +#if 0 fputs("Dumping Hash-Table after expansion\n",stderr); fDumpHashTable(ht, stderr); - */ +#endif } TraceFunctionExit(__func__); diff --git a/DHT/fxf.c b/DHT/fxf.c index 64d3133caf..8c335f6386 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -657,7 +657,7 @@ void *fxfAlloc(size_t size) { #endif /*SEGMENTED*/ { ptr= Nil(void); -#if defined(FXF_BREAK_APART_LARGER_CHUNK) +#if !defined(FXF_DONT_BREAK_LARGE_CHUNKS) /* See if we can find a larger chunk to pull a piece out of. */ for (size_t largerSize= size; largerSize < fxfMAXSIZE;) { largerSize+= min_alignment; @@ -680,7 +680,7 @@ void *fxfAlloc(size_t size) { break; } } -#endif /*FXF_BREAK_APART_LARGER_CHUNK*/ +#endif /*!FXF_DONT_BREAK_LARGE_CHUNKS*/ } TMDBG(printf(" ptr:%p\n", ptr)); } From 3f7f1b6965aea90674826d8b3f333139fc9b444d Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 11 Aug 2024 21:17:32 -0400 Subject: [PATCH 19/53] First crack at allowing one to set the desired alignment at compile-time. --- DHT/fxf.c | 176 ++++++++++++++++++++++------------------------ makefile.defaults | 4 +- 2 files changed, 88 insertions(+), 92 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 8c335f6386..42822a07d6 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -46,17 +46,23 @@ # endif #endif -#if defined(FXF_MAX_ALIGNMENT) -enum { - ENSURE_FXF_MAX_ALIGNMENT_IS_REASONABLE_POWER_OF_TWO = 1/((FXF_MAX_ALIGNMENT > 0) && - (FXF_MAX_ALIGNMENT <= (size_t)-1) && - !(FXF_MAX_ALIGNMENT & (FXF_MAX_ALIGNMENT - 1U))) -}; -#else +#if defined(FXF_MAX_ALIGNMENT_TYPE) +# if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) +# define MAX_ALIGNMENT _Alignof(FXF_MAX_ALIGNMENT_TYPE) +# elif (defined(__cplusplus) && (__cplusplus >= 201103L)) +# define MAX_ALIGNMENT alignof(FXF_MAX_ALIGNMENT_TYPE) +# else + struct GET_MAX_ALIGNMENT_TYPE { + unsigned char c; + FXF_MAX_ALIGNMENT_TYPE max_alignment_member; + }; +# define MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_alignment_member) +# endif +#else /*FXF_MAX_ALIGNMENT_TYPE*/ # if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) -# define FXF_MAX_ALIGNMENT _Alignof(max_align_t) +# define MAX_ALIGNMENT _Alignof(max_align_t) # elif (defined(__cplusplus) && (__cplusplus >= 201103L)) -# define FXF_MAX_ALIGNMENT alignof(max_align_t) +# define MAX_ALIGNMENT alignof(max_align_t) # else # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # include @@ -76,9 +82,31 @@ enum { long double floating_point; } max_aligned_union; }; -# define FXF_MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union) +# define MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union) # endif -#endif /*FXF_MAX_ALIGNMENT*/ +#endif /*!FXF_MAX_ALIGNMENT_TYPE*/ + +#if defined(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE) +# if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) +# define NOT_MULTIPLE_ALIGNMENT ((_Alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE) > MAX_ALIGNMENT) ? \ + MAX_ALIGNMENT : \ + _Alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE)) +# elif (defined(__cplusplus) && (__cplusplus >= 201103L)) +# define NOT_MULTIPLE_ALIGNMENT ((alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE) > MAX_ALIGNMENT) ? \ + MAX_ALIGNMENT : \ + alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE)) +# else + struct GET_NOT_EVEN_ALIGNMENT_TYPE { + unsigned char c; + FXF_NOT_MULTIPLE_ALIGNMENT_TYPE not_even_alignment_member; + }; +# define NOT_MULTIPLE_ALIGNMENT ((offsetof(struct GET_NOT_EVEN_ALIGNMENT_TYPE, not_even_alignment_member) > MAX_ALIGNMENT) ? \ + MAX_ALIGNMENT : \ + offsetof(struct GET_NOT_EVEN_ALIGNMENT_TYPE, not_even_alignment_member)) +# endif +#else /*FXF_NOT_MULTIPLE_ALIGNMENT_TYPE*/ +# define NOT_MULTIPLE_ALIGNMENT MAX_ALIGNMENT +#endif /*!FXF_NOT_MULTIPLE_ALIGNMENT_TYPE*/ #if !defined(Nil) && !defined(New) && !defined(nNewUntyped) && !defined(nNewCallocUntyped) /* TODO: Is this the correct check for all of the below lines? */ # define Nil(type) ((type *)0) @@ -134,7 +162,7 @@ typedef struct { #if defined(DOS) /* MSDOS 16 Bit support (maxmemory <= 1 MB) */ #define SEGMENTED -#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(32000) & ~(FXF_MAX_ALIGNMENT - 1U)) +#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(32000) & ~(MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((1024*1024)/ARENA_SEG_SIZE) #define OSNAME "MSDOS" #define OSMAXMEM "1 MB" @@ -142,7 +170,7 @@ typedef struct { /* Win95/Win98/WinME can only allocate chunks up to 255 MB */ /* maxmemory <= 768 MB */ #define SEGMENTED -#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(1000000) & ~(FXF_MAX_ALIGNMENT - 1U)) +#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(1000000) & ~(MAX_ALIGNMENT - 1U)) #define ARENA_SEG_COUNT ((768*1024*1024)/ARENA_SEG_SIZE) #define OSNAME "Win95/Win98/WinME" #define OSMAXMEM "768 MB" @@ -166,7 +194,7 @@ enum #else ((((size_t)2048) /* This is needed only when sizeof(void*)==8. */ #endif - + (FXF_MAX_ALIGNMENT - 1U)) & ~(FXF_MAX_ALIGNMENT - 1U)) /* Round up if necessary. */ + + (MAX_ALIGNMENT - 1U)) & ~(MAX_ALIGNMENT - 1U)) /* Round up if necessary. */ }; enum { @@ -175,12 +203,12 @@ enum { }; #define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) -#define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > FXF_MAX_ALIGNMENT) ? FXF_MAX_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. - (Of course, they're probably the same.) - TODO: Can we compute what we want at compile time and just use it? */ -static size_t min_alignment= 0; /* for now */ +#define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > NOT_MULTIPLE_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. + (Of course, they're probably the same.) + TODO: Can we compute what we want at compile time and just use it? */ +static size_t min_alignment= NOT_MULTIPLE_ALIGNMENT; /* for now */ -static SizeHead SizeData[1 + ((fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE)]; /* Minimum allocation is (fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U). +static SizeHead SizeData[1 + ((fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE)]; /* Minimum allocation is fxfMINSIZE. Maximum allocation is fxfMAXSIZE. All allocations will be multiples of MIN_ALIGNMENT_UNDERESTIMATE. */ #define SIZEDATA_SIZE_TO_INDEX(s) (((s) - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE) @@ -303,6 +331,8 @@ size_t fxfMaxAllocation(void) { return fxfMAXSIZE; } +#define ALIGN_TO_MINIMUM(s) ((((s) - 1U) & ~(min_alignment - 1U)) + min_alignment) + size_t fxfInit(size_t Size) { #if defined(LOG) static char const * const myname= "fxfInit"; @@ -343,7 +373,7 @@ size_t fxfInit(size_t Size) { free(Arena); if (Size > MAX_POINTER_DIFFERENCE) Size= MAX_POINTER_DIFFERENCE; - Size&= ~(FXF_MAX_ALIGNMENT - 1U); + Size&= ~(MAX_ALIGNMENT - 1U); Arena= nNewUntyped(Size, char); if (!Arena) { ERROR_LOG3("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " <= %" SIZE_T_PRINTF_SPECIFIER " bytes\n", @@ -379,12 +409,8 @@ size_t fxfInit(size_t Size) { SizeData[Size].FreeHead= Nil(void); } - if (!min_alignment) - { - min_alignment= FXF_MAX_ALIGNMENT; - while (min_alignment > fxfMINSIZE) - min_alignment>>= 1; - } + while ((min_alignment>>1) >= fxfMINSIZE) + min_alignment>>= 1; return GlobalSize; } @@ -475,8 +501,7 @@ void fxfReset(void) * SPARC, HPPA, MIPS. We wouldn't need this when running on an * Intel *86 type of CPU, but also there, aligned access is faster. */ -#define PTRMASK (FXF_MAX_ALIGNMENT-1U) -#define ALIGN_TO_MINIMUM(s) (((s) + (min_alignment - 1U)) & ~(min_alignment - 1U)) +#define PTRMASK (MAX_ALIGNMENT-1U) #if defined(FXF_ENABLE_TMDBG) # define TMDBG(x) do {x;} while (0) @@ -502,7 +527,8 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { return 0; } -static void * popOffFreeStore(size_t const size) { +static void * popOffFreeStore(size_t const size) +{ SizeHead *cur_sh; void *ptr; assert((size >= fxfMINSIZE) && @@ -535,8 +561,7 @@ void *fxfAlloc(size_t size) { if (sizefxfMAXSIZE) + else if (size>fxfMAXSIZE) { ERROR_LOG3("%s: size=%" SIZE_T_PRINTF_SPECIFIER " > %" SIZE_T_PRINTF_SPECIFIER "\n", myname, @@ -547,6 +572,7 @@ void *fxfAlloc(size_t size) { // Round up to a multiple of min_alignment size= ALIGN_TO_MINIMUM(size); + ptr= popOffFreeStore(size); if (ptr) { @@ -598,19 +624,19 @@ void *fxfAlloc(size_t size) { #else (size_t)pointerDifference(BotFreePtr,Arena); #endif - size_t needed_alignment_mask= PTRMASK; + size_t needed_alignment_mask= (NOT_MULTIPLE_ALIGNMENT-1U); while (needed_alignment_mask >= size) needed_alignment_mask>>= 1; - if (curBottomIndex & needed_alignment_mask) { - if ((needed_alignment_mask - (curBottomIndex & needed_alignment_mask)) >= (sizeCurrentSeg-size)) + curBottomIndex&= needed_alignment_mask; + if (curBottomIndex) { + if ((needed_alignment_mask - curBottomIndex) >= (sizeCurrentSeg - size)) goto NEXT_SEGMENT; do { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); #if defined(FREEMAP) && !defined(SEGMENTED) SetRange(curBottomIndex,cur_alignment); #endif - if (cur_alignment >= fxfMINSIZE) - pushOntoFreeStore(BotFreePtr, cur_alignment); + pushOntoFreeStore(BotFreePtr, cur_alignment); BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)cur_alignment); curBottomIndex+= cur_alignment; } while (curBottomIndex & needed_alignment_mask); @@ -635,18 +661,23 @@ void *fxfAlloc(size_t size) { { NEXT_SEGMENT: #if defined(SEGMENTED) - if (CurrentSeg < (ArenaSegCnt-1)) { - size_t curBottomIndex= (BotFreePtr - Arena[CurrentSeg]); - while (curBottomIndex & PTRMASK) { + if ((CurrentSeg+1) < ArenaSegCnt) { + size_t curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena[CurrentSeg]); + while (curBottomIndex & (NOT_MULTIPLE_ALIGNMENT-1U)) + { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); - if (cur_alignment >= fxfMINSIZE) - pushOntoFreeStore(BotFreePtr, cur_alignment); - BotFreePtr= stepPointer(BotFreePtr, cur_alignment); + pushOntoFreeStore(BotFreePtr, cur_alignment); + BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)cur_alignment); curBottomIndex+= cur_alignment; } curBottomIndex= (size_t)pointerDifference(TopFreePtr,BotFreePtr); if (curBottomIndex >= fxfMINSIZE) + { + assert(!(curBottomIndex & (NOT_MULTIPLE_ALIGNMENT - 1U))); pushOntoFreeStore(BotFreePtr, curBottomIndex); + } + else if (curBottomIndex) + TMDBG(printf(" leaking %" SIZE_T_PRINTF_SPECIFIER " byte(s) moving from segment %d to segment %d\n", (size_t_printf_type)curBottomIndex, CurrentSeg, CurrentSeg+1)); TMDBG(fputs(" next seg", stdout)); ++CurrentSeg; BotFreePtr= Arena[CurrentSeg]; @@ -657,30 +688,7 @@ void *fxfAlloc(size_t size) { #endif /*SEGMENTED*/ { ptr= Nil(void); -#if !defined(FXF_DONT_BREAK_LARGE_CHUNKS) - /* See if we can find a larger chunk to pull a piece out of. */ - for (size_t largerSize= size; largerSize < fxfMAXSIZE;) { - largerSize+= min_alignment; - ptr= popOffFreeStore(largerSize); - if (ptr) { - void *newPtr= stepPointer(ptr, size); - size_t cur_alignment= (size & -size); - SizeData[SIZEDATA_SIZE_TO_INDEX(size)].MallocCount++; - largerSize-= size; - while ((cur_alignment < FXF_MAX_ALIGNMENT) && (cur_alignment <= (largerSize>>1))) { - if (cur_alignment >= fxfMINSIZE) - pushOntoFreeStore(newPtr, cur_alignment); - newPtr= stepPointer(newPtr, cur_alignment); - largerSize-= cur_alignment; - size+= cur_alignment; - cur_alignment= (size & -size); - } - if (largerSize >= fxfMINSIZE) - pushOntoFreeStore(newPtr, largerSize); - break; - } - } -#endif /*!FXF_DONT_BREAK_LARGE_CHUNKS*/ + /* TODO: Should we try to break apart a larger chunk? */ } TMDBG(printf(" ptr:%p\n", ptr)); } @@ -733,7 +741,10 @@ void fxfFree(void *ptr, size_t size) DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *)ptr, (size_t_printf_type) size)); if (size < fxfMINSIZE) size= fxfMINSIZE; - assert(size <= fxfMAXSIZE); + else + { + assert(size <= fxfMAXSIZE); + } size= ALIGN_TO_MINIMUM(size); #if !defined(NDEBUG) # if defined(SEGMENTED) @@ -749,7 +760,7 @@ void fxfFree(void *ptr, size_t size) #endif if (ptrIndex > 0) { - size_t needed_alignment= FXF_MAX_ALIGNMENT; + size_t needed_alignment= MAX_ALIGNMENT; while (needed_alignment > size) needed_alignment>>= 1; assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); @@ -818,14 +829,17 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { size_t needed_alignment; if (allocatedSize < fxfMINSIZE) allocatedSize= fxfMINSIZE; - assert(allocatedSize <= fxfMAXSIZE); + else + { + assert(allocatedSize <= fxfMAXSIZE); + } allocatedSize= ALIGN_TO_MINIMUM(allocatedSize); # if defined(SEGMENTED) assert(allocatedSize <= (ARENA_SEG_SIZE - ptrIndex)); # else assert(allocatedSize <= (GlobalSize - ptrIndex)); # endif - needed_alignment= FXF_MAX_ALIGNMENT; + needed_alignment= MAX_ALIGNMENT; while (needed_alignment > allocatedSize) needed_alignment>>= 1; assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); @@ -849,27 +863,7 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { needed_allocation= ALIGN_TO_MINIMUM(needed_allocation); if (needed_allocation == original_allocation) return ptr; -#if defined(FXF_BREAK_APART_LARGER_CHUNK) - if (needed_allocation < original_allocation) - { - size_t cur_alignment= (needed_allocation & -needed_allocation); - nptr= stepPointer(ptr, needed_allocation); - SizeData[SIZEDATA_SIZE_TO_INDEX(original_allocation)].MallocCount--; - SizeData[SIZEDATA_SIZE_TO_INDEX(needed_allocation)].MallocCount++; - original_allocation-= needed_allocation; - while ((cur_alignment < FXF_MAX_ALIGNMENT) && (cur_alignment <= (original_allocation>>1))) { - if (cur_alignment >= fxfMINSIZE) - pushOntoFreeStore(nptr, cur_alignment); - nptr= stepPointer(nptr, cur_alignment); - original_allocation-= cur_alignment; - needed_allocation+= cur_alignment; - cur_alignment= (needed_allocation & -needed_allocation); - } - if (original_allocation >= fxfMINSIZE) - pushOntoFreeStore(nptr, original_allocation); - return ptr; - } -#endif /*FXF_BREAK_APART_LARGER_CHUNK*/ + /* TODO: Should we try to break apart this chunk? */ } nptr= fxfAlloc(NewSize); if (nptr) diff --git a/makefile.defaults b/makefile.defaults index 9e104697b1..f76fad3a64 100644 --- a/makefile.defaults +++ b/makefile.defaults @@ -102,12 +102,14 @@ VERSION=4.90 # Other # ----- -DEFS=$(DEFINEMACRO)SIGNALS $(DEFINEMACRO)MSG_IN_MEM $(DEFINEMACRO)FXF $(DEFINEMACRO)DOMEASURE +DEFS=$(DEFINEMACRO)SIGNALS $(DEFINEMACRO)MSG_IN_MEM $(DEFINEMACRO)FXF $(DEFINEMACRO)DOMEASURE $(DEFINEMACRO)FXF_MAX_ALIGNMENT_TYPE=void* $(DEFINEMACRO)FXF_NOT_MULTIPLE_ALIGNMENT_TYPE=short # $(DEFINEMACRO)DOMEASURE # $(DEFINEMACRO)TESTHASH # $(DEFINEMACRO)NDEBUG # $(DEFINEMACRO)DOTRACE # $(DEFINEMACRO)DOTRACECALLSTACK +# $(DEFINEMACRO)FXF_MAX_ALIGNMENT_TYPE=void* +# $(DEFINEMACRO)FXF_NOT_MULTIPLE_ALIGNMENT_TYPE=short # From 85c1b012ed43fdd32efde4aef22e7baf6b07ab9c Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 12 Aug 2024 20:20:33 -0400 Subject: [PATCH 20/53] Fixing some asserts. --- DHT/fxf.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 42822a07d6..b9b9f2b3cd 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -760,9 +760,15 @@ void fxfFree(void *ptr, size_t size) #endif if (ptrIndex > 0) { - size_t needed_alignment= MAX_ALIGNMENT; - while (needed_alignment > size) - needed_alignment>>= 1; + size_t needed_alignment; + if (size&PTRMASK) + { + needed_alignment= NOT_MULTIPLE_ALIGNMENT; + while (needed_alignment > size) + needed_alignment>>= 1; + } + else + needed_alignment= MAX_ALIGNMENT; assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); } } @@ -839,9 +845,14 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { # else assert(allocatedSize <= (GlobalSize - ptrIndex)); # endif - needed_alignment= MAX_ALIGNMENT; - while (needed_alignment > allocatedSize) - needed_alignment>>= 1; + if (allocatedSize&PTRMASK) + { + needed_alignment= NOT_MULTIPLE_ALIGNMENT; + while (needed_alignment > allocatedSize) + needed_alignment>>= 1; + } + else + needed_alignment= MAX_ALIGNMENT; assert(!(((size_t)ptrIndex) & (needed_alignment - 1U))); } } From ae429e412e8bce578d439ee2a392b3672eed6061 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 12 Aug 2024 23:09:55 -0400 Subject: [PATCH 21/53] Might as well use macros for everything. --- DHT/fxf.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index b9b9f2b3cd..62aa3626e0 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -208,12 +208,10 @@ enum { TODO: Can we compute what we want at compile time and just use it? */ static size_t min_alignment= NOT_MULTIPLE_ALIGNMENT; /* for now */ -static SizeHead SizeData[1 + ((fxfMAXSIZE - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE)]; /* Minimum allocation is fxfMINSIZE. - Maximum allocation is fxfMAXSIZE. - All allocations will be multiples of MIN_ALIGNMENT_UNDERESTIMATE. */ -#define SIZEDATA_SIZE_TO_INDEX(s) (((s) - fxfMINSIZE)/MIN_ALIGNMENT_UNDERESTIMATE) -#define SIZEDATA_INDEX_TO_SIZE(x) ((size_t)(((x) * MIN_ALIGNMENT_UNDERESTIMATE) + \ - ((fxfMINSIZE + (MIN_ALIGNMENT_UNDERESTIMATE - 1U)) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U)))) +#define ROUNDED_MIN_SIZE_UNDERESTIMATE (((fxfMINSIZE - 1U) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U)) + MIN_ALIGNMENT_UNDERESTIMATE) +#define SIZEDATA_SIZE_TO_INDEX(s) (((s) - ROUNDED_MIN_SIZE_UNDERESTIMATE)/MIN_ALIGNMENT_UNDERESTIMATE) +#define SIZEDATA_INDEX_TO_SIZE(x) ((size_t)(((x) * MIN_ALIGNMENT_UNDERESTIMATE) + ROUNDED_MIN_SIZE_UNDERESTIMATE)) +static SizeHead SizeData[1 + SIZEDATA_SIZE_TO_INDEX(fxfMAXSIZE)]; #if defined(SEGMENTED) /* #define ARENA_SEG_SIZE 32000 */ From 0f04b2e7046361f48072d4227759bebdb06d38cc Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 12 Aug 2024 23:27:56 -0400 Subject: [PATCH 22/53] Fixing these macros (maybe). --- DHT/fxf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 62aa3626e0..6cc50a89cf 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -187,14 +187,14 @@ enum fxfMAXSIZE = #if defined(SEGMENTED) || defined(__TURBOC__) # if defined(ARENA_SEG_SIZE) - ((((1024 > ARENA_SEG_SIZE) ? ARENA_SEG_SIZE : ((size_t)1024)) + (((((1024 > (ARENA_SEG_SIZE)) ? (ARENA_SEG_SIZE) : ((size_t)1024)) # else - ((((size_t)1024) + (((((size_t)1024) # endif #else - ((((size_t)2048) /* This is needed only when sizeof(void*)==8. */ + (((((size_t)2048) /* This is needed only when sizeof(void*)==8. */ #endif - + (MAX_ALIGNMENT - 1U)) & ~(MAX_ALIGNMENT - 1U)) /* Round up if necessary. */ + - 1U) & ~(MAX_ALIGNMENT - 1U)) + MAX_ALIGNMENT) /* Round up if necessary. */ }; enum { From 4c7d4b58c75c14f5e6c760b91abe50c55fed0ccc Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 13 Aug 2024 08:13:17 -0400 Subject: [PATCH 23/53] Getting a better compile-time estimate for the minimum alignment, at least in some cases. --- DHT/fxf.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 6cc50a89cf..9b68155aad 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -202,10 +202,11 @@ enum { ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE) }; -#define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) -#define MIN_ALIGNMENT_UNDERESTIMATE ((BOTTOM_BIT_OF_FXFMINSIZE > NOT_MULTIPLE_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : BOTTOM_BIT_OF_FXFMINSIZE) /* We'd prefer the top bit, but we'll compute that during fxfInit. - (Of course, they're probably the same.) - TODO: Can we compute what we want at compile time and just use it? */ +#define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) /* TODO: The top bit would be better below, + but can we determine that at compile time? */ +#define MIN_ALIGNMENT_UNDERESTIMATE ((NOT_MULTIPLE_ALIGNMENT < BOTTOM_BIT_OF_FXFMINSIZE) ? NOT_MULTIPLE_ALIGNMENT : \ + (BOTTOM_BIT_OF_FXFMINSIZE << ((fxfMINSIZE & (fxfMINSIZE - 1U)) && (NOT_MULTIPLE_ALIGNMENT > BOTTOM_BIT_OF_FXFMINSIZE)))) + static size_t min_alignment= NOT_MULTIPLE_ALIGNMENT; /* for now */ #define ROUNDED_MIN_SIZE_UNDERESTIMATE (((fxfMINSIZE - 1U) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U)) + MIN_ALIGNMENT_UNDERESTIMATE) From 6978868f6db60ff11528df14e2598f6b20c66663 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 13 Aug 2024 17:26:50 -0400 Subject: [PATCH 24/53] Doing our best to properly estimate this value under all circumstances. --- DHT/fxf.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 9b68155aad..1021d79322 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -202,11 +202,9 @@ enum { ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE) }; -#define BOTTOM_BIT_OF_FXFMINSIZE ((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE) /* TODO: The top bit would be better below, - but can we determine that at compile time? */ -#define MIN_ALIGNMENT_UNDERESTIMATE ((NOT_MULTIPLE_ALIGNMENT < BOTTOM_BIT_OF_FXFMINSIZE) ? NOT_MULTIPLE_ALIGNMENT : \ - (BOTTOM_BIT_OF_FXFMINSIZE << ((fxfMINSIZE & (fxfMINSIZE - 1U)) && (NOT_MULTIPLE_ALIGNMENT > BOTTOM_BIT_OF_FXFMINSIZE)))) - +#define MIN_ALIGNMENT_UNDERESTIMATE (((NOT_MULTIPLE_ALIGNMENT>>1) < fxfMINSIZE) ? NOT_MULTIPLE_ALIGNMENT : \ + ((fxfMINSIZE & (fxfMINSIZE - 1U)) ? (((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE)<<2) : \ + fxfMINSIZE)) static size_t min_alignment= NOT_MULTIPLE_ALIGNMENT; /* for now */ #define ROUNDED_MIN_SIZE_UNDERESTIMATE (((fxfMINSIZE - 1U) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U)) + MIN_ALIGNMENT_UNDERESTIMATE) From eda84bc7f59b3fa0a0af796f9d048c32789d1467 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 14 Aug 2024 08:15:16 -0400 Subject: [PATCH 25/53] Aiming for portability. --- DHT/fxf.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 1021d79322..3b7bd55301 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -21,6 +21,7 @@ # include typedef ptrdiff_t ptrdiff_t_printf_type; typedef size_t size_t_printf_type; + typedef uintmax_t largest_integer_type; # define MAX_POINTER_DIFFERENCE PTRDIFF_MAX # if defined(UINTPTR_MAX) typedef uintptr_t convert_pointer_to_int_type; @@ -35,12 +36,14 @@ typedef long long int ptrdiff_t_printf_type; typedef unsigned long long int size_t_printf_type; typedef unsigned long long int convert_pointer_to_int_type; + typedef unsigned long long int largest_integer_type; # define PTRDIFF_T_PRINTF_SPECIFIER "lld" # define SIZE_T_PRINTF_SPECIFIER "llu" # else /* We don't have long long integer types. */ typedef long int ptrdiff_t_printf_type; typedef unsigned long int size_t_printf_type; typedef unsigned long int convert_pointer_to_int_type; + typedef unsigned long int largest_integer_type; # define PTRDIFF_T_PRINTF_SPECIFIER "ld" # define SIZE_T_PRINTF_SPECIFIER "lu" # endif @@ -70,13 +73,7 @@ struct GET_MAX_ALIGNMENT_TYPE { unsigned char c; union { -# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) - uintmax_t unsigned_integer; -# elif defined(LLONG_MAX) /* We have long long integer types. */ - unsigned long long int unsigned_integer; -# else - unsigned long int unsigned_integer; -# endif + largest_integer_type unsigned_integer; const volatile void * object_pointer; void (*function_pointer)(void); long double floating_point; @@ -199,11 +196,12 @@ enum enum { ENSURE_FXFMINSIZE_GT_0 = 1/(fxfMINSIZE > 0), - ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE) + ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE), + ENSURE_FXFMAXSIZE_FULLY_ALIGNED = 1/!(fxfMAXSIZE & (MAX_ALIGNMENT - 1U)) }; #define MIN_ALIGNMENT_UNDERESTIMATE (((NOT_MULTIPLE_ALIGNMENT>>1) < fxfMINSIZE) ? NOT_MULTIPLE_ALIGNMENT : \ - ((fxfMINSIZE & (fxfMINSIZE - 1U)) ? (((size_t)fxfMINSIZE & -(size_t)fxfMINSIZE)<<2) : \ + ((fxfMINSIZE & (fxfMINSIZE - 1U)) ? ((fxfMINSIZE & -(largest_integer_type)fxfMINSIZE)<<2) : \ fxfMINSIZE)) static size_t min_alignment= NOT_MULTIPLE_ALIGNMENT; /* for now */ From 5599b891440b4e09fb627c7ff50205ade7c335cb Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 16 Aug 2024 15:30:54 -0400 Subject: [PATCH 26/53] Trying to guarantee exactly what we need. --- DHT/fxf.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 3b7bd55301..2825e6d643 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -197,7 +197,13 @@ enum enum { ENSURE_FXFMINSIZE_GT_0 = 1/(fxfMINSIZE > 0), ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE), - ENSURE_FXFMAXSIZE_FULLY_ALIGNED = 1/!(fxfMAXSIZE & (MAX_ALIGNMENT - 1U)) +#if defined(SEGMENT) + ENSURE_SEGMENTS_ALIGNED = 1/!((ARENA_SEG_SIZE & (((ARENA_SEG_SIZE < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) - 1U)) && + (ARENA_SEG_SIZE & (ARENA_SEG_SIZE - 1U))), +#endif + ENSURE_FXFMAXSIZE_ALIGNED = 1/!((fxfMAXSIZE & (NOT_MULTIPLE_ALIGNMENT - 1U)) && (fxfMAXSIZE & (fxfMAXSIZE - 1U))), + ENSURE_ALIGNMENT_ORDERED = 1/((NOT_MULTIPLE_ALIGNMENT > 0) && (NOT_MULTIPLE_ALIGNMENT <= MAX_ALIGNMENT)), + ENSURE_ALIGNMENTS_POWERS_OF_2 = 1/!((NOT_MULTIPLE_ALIGNMENT & (NOT_MULTIPLE_ALIGNMENT - 1U)) || (MAX_ALIGNMENT & (MAX_ALIGNMENT - 1U))) }; #define MIN_ALIGNMENT_UNDERESTIMATE (((NOT_MULTIPLE_ALIGNMENT>>1) < fxfMINSIZE) ? NOT_MULTIPLE_ALIGNMENT : \ @@ -368,7 +374,13 @@ size_t fxfInit(size_t Size) { free(Arena); if (Size > MAX_POINTER_DIFFERENCE) Size= MAX_POINTER_DIFFERENCE; - Size&= ~(MAX_ALIGNMENT - 1U); + if (Size < NOT_MULTIPLE_ALIGNMENT) + while (Size & (Size - 1U)) + Size&= (Size - 1U); + else if (Size < MAX_ALIGNMENT) + Size&= ~(NOT_MULTIPLE_ALIGNMENT - 1U); + else + Size&= ~(MAX_ALIGNMENT - 1U); Arena= nNewUntyped(Size, char); if (!Arena) { ERROR_LOG3("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " <= %" SIZE_T_PRINTF_SPECIFIER " bytes\n", From d73486836555da31c1aea82bb141750e6b486c5a Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 16 Aug 2024 20:58:39 -0400 Subject: [PATCH 27/53] SEGMENTED code needs to compile. --- DHT/fxf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 2825e6d643..ab165d2806 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -711,7 +711,7 @@ void fxfFree(void *ptr, size_t size) #if defined(FXF_ENABLE_TMDBG) || !defined(NDEBUG) ptrdiff_t ptrIndex; #endif -#if defined(SEGMENTED) && defined(FXF_ENABLE_TMDBG) +#if defined(SEGMENTED) && (defined(FXF_ENABLE_TMDBG) || !defined(NDEBUG)) int ptrSegment; #endif if (!ptr) From 83130b2b33c3e0e59aa2845e3b6569da456dad9b Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 17 Aug 2024 16:56:14 -0400 Subject: [PATCH 28/53] Removing some hard-coded values, hopefully correctly. --- DHT/fxf.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index ab165d2806..308215830a 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -174,24 +174,18 @@ typedef struct { #endif /* The maximum size an fxfAlloc can handle */ -/* TODO: Do the macros really accurately determine the maximum we need (apparently 1024 or 2048)? - Can we instead compute the needed value(s) with expressions involving, say, sizeof(void *) - and any other system properties we have access to? -*/ -enum +#define DESIRED_MAX_ALLOC (256 * sizeof(void *)) +/* TODO: Is the above sufficiently large for all of our needs without being excessive? */ +#define DESIRED_MAX_ALLOC_ALIGNMENT ((DESIRED_MAX_ALLOC < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) +#define ROUNDED_DESIRED_MAXIMUM_ALLOCATION (((DESIRED_MAX_ALLOC - 1U) & ~(DESIRED_MAX_ALLOC_ALIGNMENT - 1U)) + DESIRED_MAX_ALLOC_ALIGNMENT) +enum { fxfMINSIZE = sizeof(void *), /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ - fxfMAXSIZE = -#if defined(SEGMENTED) || defined(__TURBOC__) -# if defined(ARENA_SEG_SIZE) - (((((1024 > (ARENA_SEG_SIZE)) ? (ARENA_SEG_SIZE) : ((size_t)1024)) -# else - (((((size_t)1024) -# endif +#if defined(SEGMENTED) + fxfMAXSIZE = ((ROUNDED_DESIRED_MAXIMUM_ALLOCATION > ARENA_SEG_SIZE) ? ARENA_SEG_SIZE : ROUNDED_DESIRED_MAXIMUM_ALLOCATION) #else - (((((size_t)2048) /* This is needed only when sizeof(void*)==8. */ + fxfMAXSIZE = ROUNDED_DESIRED_MAXIMUM_ALLOCATION #endif - - 1U) & ~(MAX_ALIGNMENT - 1U)) + MAX_ALIGNMENT) /* Round up if necessary. */ }; enum { From e5fae8a3dc1619a274b2e695010747050ea689b9 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 18 Aug 2024 13:41:55 -0400 Subject: [PATCH 29/53] Moving some of the debugging infrastructure inside the free store manipulation functions. --- DHT/fxf.c | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 308215830a..6833f88987 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -192,7 +192,7 @@ enum { ENSURE_FXFMINSIZE_GT_0 = 1/(fxfMINSIZE > 0), ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE), #if defined(SEGMENT) - ENSURE_SEGMENTS_ALIGNED = 1/!((ARENA_SEG_SIZE & (((ARENA_SEG_SIZE < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) - 1U)) && + ENSURE_SEGMENTS_ALIGNED = 1/!((ARENeA_SEG_SIZE & (((ARENA_SEG_SIZE < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) - 1U)) && (ARENA_SEG_SIZE & (ARENA_SEG_SIZE - 1U))), #endif ENSURE_FXFMAXSIZE_ALIGNED = 1/!((fxfMAXSIZE & (NOT_MULTIPLE_ALIGNMENT - 1U)) && (fxfMAXSIZE & (fxfMAXSIZE - 1U))), @@ -517,6 +517,9 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { !(size & (min_alignment - 1U))); cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; assert((!cur_sh->FreeHead) == (!cur_sh->FreeCount)); +#if defined(FREEMAP) && !defined(SEGMENTED) + SetRange(pointerDifference(ptr, Arena), size); +#endif if ((size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeHead) { if (size >= sizeof cur_sh->FreeHead) memcpy(ptr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); @@ -544,6 +547,9 @@ static void * popOffFreeStore(size_t const size) cur_sh->FreeHead= Nil(void); else memcpy(&cur_sh->FreeHead, ptr, sizeof cur_sh->FreeHead); +# if defined(FREEMAP) && !defined(SEGMENTED) + ClrRange(pointerDifference(ptr, Arena), size); +# endif } return ptr; } @@ -601,9 +607,6 @@ void *fxfAlloc(size_t size) { TMDBG(printf(" FreeCount:%lu ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,sh->MallocCount)); # endif /*FXF_ENABLE_TMDBG*/ #else /*SEGMENTED*/ -# if defined(FREEMAP) - ClrRange(pointerDifference(ptr, Arena), size); -# endif TMDBG(printf(" FreeCount:%lu ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " MallocCount:%lu\n",sh->FreeCount,(ptrdiff_t_printf_type)pointerDifference(ptr, Arena),sh->MallocCount)); #endif /*!SEGMENTED*/ } @@ -634,9 +637,6 @@ void *fxfAlloc(size_t size) { goto NEXT_SEGMENT; do { size_t const cur_alignment= (curBottomIndex & -curBottomIndex); -#if defined(FREEMAP) && !defined(SEGMENTED) - SetRange(curBottomIndex,cur_alignment); -#endif pushOntoFreeStore(BotFreePtr, cur_alignment); BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)cur_alignment); curBottomIndex+= cur_alignment; @@ -782,9 +782,6 @@ void fxfFree(void *ptr, size_t size) TMDBG(printf(" BotFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(TopFreePtr,BotFreePtr))); } else { -#if defined(FREEMAP) && !defined(SEGMENTED) - SetRange((pointerDifference(ptr,Arena),size); -#endif pushOntoFreeStore(ptr, size); } } @@ -795,12 +792,8 @@ void fxfFree(void *ptr, size_t size) TopFreePtr= stepPointer(TopFreePtr, (ptrdiff_t)size); TMDBG(printf(" TopFreePtr sizeCurrentSeg:%" PTRDIFF_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)pointerDifference(TopFreePtr,BotFreePtr))); } - else { -#if defined(FREEMAP) && !defined(SEGMENTED) - SetRange(pointerDifference(ptr,Arena),size); -#endif + else pushOntoFreeStore(ptr, size); - } } sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; sh->MallocCount--; From 644aeb28f57b9108a90ba561658a8cdd0cd815c8 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 18 Aug 2024 17:43:36 -0400 Subject: [PATCH 30/53] Adding some compile-time checks that should eliminate the run-time ones. --- DHT/fxf.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 6833f88987..1bef5a49d6 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -520,8 +520,10 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { #if defined(FREEMAP) && !defined(SEGMENTED) SetRange(pointerDifference(ptr, Arena), size); #endif - if ((size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeHead) { - if (size >= sizeof cur_sh->FreeHead) + if ((ROUNDED_MIN_SIZE_UNDERESTIMATE >= sizeof cur_sh->FreeHead) /* compile-time check that's likely true */ || + (size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeHead) { + if ((ROUNDED_MIN_SIZE_UNDERESTIMATE >= sizeof cur_sh->FreeHead) /* compile-time check that's likely true */ || + (size >= sizeof cur_sh->FreeHead)) memcpy(ptr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); cur_sh->FreeHead= ptr; cur_sh->FreeCount++; @@ -543,7 +545,8 @@ static void * popOffFreeStore(size_t const size) assert((!ptr) == (!cur_sh->FreeCount)); if (ptr) { cur_sh->FreeCount--; - if (size < sizeof cur_sh->FreeHead) + if ((ROUNDED_MIN_SIZE_UNDERESTIMATE < sizeof cur_sh->FreeHead) /* compile-time check that's likely false */ && + (size < sizeof cur_sh->FreeHead)) cur_sh->FreeHead= Nil(void); else memcpy(&cur_sh->FreeHead, ptr, sizeof cur_sh->FreeHead); From a97bcc391783f2c7d2226f0526ebbac9f981e930 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 18 Aug 2024 17:45:08 -0400 Subject: [PATCH 31/53] Tightening some assertions. --- DHT/fxf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 1bef5a49d6..fd8a498f5e 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -512,7 +512,7 @@ void fxfReset(void) static int pushOntoFreeStore(void * const ptr, size_t const size) { SizeHead *cur_sh; - assert((size >= fxfMINSIZE) && + assert((size >= ROUNDED_MIN_SIZE_UNDERESTIMATE) && (size <= fxfMAXSIZE) && !(size & (min_alignment - 1U))); cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; @@ -537,7 +537,7 @@ static void * popOffFreeStore(size_t const size) { SizeHead *cur_sh; void *ptr; - assert((size >= fxfMINSIZE) && + assert((size >= ROUNDED_MIN_SIZE_UNDERESTIMATE) && (size <= fxfMAXSIZE) && !(size & (min_alignment - 1U))); cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; From 36dbd69844ae66fb82d2f312da5c34b0987de7a6 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 20 Aug 2024 20:58:10 -0400 Subject: [PATCH 32/53] Noting in the other place where we may leak memory (if we happen to change our minimum allocation). --- DHT/fxf.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index fd8a498f5e..b13c8ef7ac 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -195,7 +195,9 @@ enum { ENSURE_SEGMENTS_ALIGNED = 1/!((ARENeA_SEG_SIZE & (((ARENA_SEG_SIZE < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) - 1U)) && (ARENA_SEG_SIZE & (ARENA_SEG_SIZE - 1U))), #endif - ENSURE_FXFMAXSIZE_ALIGNED = 1/!((fxfMAXSIZE & (NOT_MULTIPLE_ALIGNMENT - 1U)) && (fxfMAXSIZE & (fxfMAXSIZE - 1U))), + ENSURE_FXFMAXSIZE_ALIGNED = 1/((!(fxfMAXSIZE & (fxfMAXSIZE - 1U))) || + ((fxfMAXSIZE < MAX_ALIGNMENT) && !(fxfMAXSIZE & (NOT_MULTIPLE_ALIGNMENT - 1U))) || + !(fxfMAXSIZE & (MAX_ALIGNMENT - 1U))), ENSURE_ALIGNMENT_ORDERED = 1/((NOT_MULTIPLE_ALIGNMENT > 0) && (NOT_MULTIPLE_ALIGNMENT <= MAX_ALIGNMENT)), ENSURE_ALIGNMENTS_POWERS_OF_2 = 1/!((NOT_MULTIPLE_ALIGNMENT & (NOT_MULTIPLE_ALIGNMENT - 1U)) || (MAX_ALIGNMENT & (MAX_ALIGNMENT - 1U))) }; @@ -520,8 +522,12 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { #if defined(FREEMAP) && !defined(SEGMENTED) SetRange(pointerDifference(ptr, Arena), size); #endif - if ((ROUNDED_MIN_SIZE_UNDERESTIMATE >= sizeof cur_sh->FreeHead) /* compile-time check that's likely true */ || - (size >= sizeof cur_sh->FreeHead) || !cur_sh->FreeHead) { + if ((ROUNDED_MIN_SIZE_UNDERESTIMATE < sizeof cur_sh->FreeHead) /* compile-time check that's likely false */ && + (size < sizeof cur_sh->FreeHead) && + cur_sh->FreeHead) + TMDBG(printf(" leaking %" SIZE_T_PRINTF_SPECIFIER " byte(s) instead of freeing them\n", (size_t_printf_type)curBottomIndex)); + else + { if ((ROUNDED_MIN_SIZE_UNDERESTIMATE >= sizeof cur_sh->FreeHead) /* compile-time check that's likely true */ || (size >= sizeof cur_sh->FreeHead)) memcpy(ptr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); @@ -889,10 +895,9 @@ size_t fxfTotal(void) { size_t i; for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { - if (hd->MallocCount+hd->FreeCount>0) { - UsedBytes+= hd->MallocCount*SIZEDATA_INDEX_TO_SIZE(i); - FreeBytes+= hd->FreeCount*SIZEDATA_INDEX_TO_SIZE(i); - } + size_t const cur_size= SIZEDATA_INDEX_TO_SIZE(i); + UsedBytes+= hd->MallocCount*cur_size; + FreeBytes+= hd->FreeCount*cur_size; } return UsedBytes+FreeBytes; @@ -925,7 +930,7 @@ void fxfInfo(FILE *f) { size_t i; fprintf(f, "%12s %10s%10s\n", "Size", "MallocCnt", "FreeCnt"); for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { - if (hd->MallocCount+hd->FreeCount>0) { + if (hd->MallocCount || hd->FreeCount) { fprintf(f, "%12zu %10lu%10lu\n", SIZEDATA_INDEX_TO_SIZE(i), hd->MallocCount, hd->FreeCount); nrUsed+= hd->MallocCount; UsedBytes+= hd->MallocCount*SIZEDATA_INDEX_TO_SIZE(i); From cbce3da4e7709cccd069fd39da349c3535eaec20 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 20 Aug 2024 21:02:08 -0400 Subject: [PATCH 33/53] Only compute that value once. --- DHT/fxf.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index b13c8ef7ac..129ddbcbad 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -931,11 +931,12 @@ void fxfInfo(FILE *f) { fprintf(f, "%12s %10s%10s\n", "Size", "MallocCnt", "FreeCnt"); for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount || hd->FreeCount) { - fprintf(f, "%12zu %10lu%10lu\n", SIZEDATA_INDEX_TO_SIZE(i), hd->MallocCount, hd->FreeCount); + size_t const cur_size= SIZEDATA_INDEX_TO_SIZE(i); + fprintf(f, "%12zu %10lu%10lu\n", cur_size, hd->MallocCount, hd->FreeCount); nrUsed+= hd->MallocCount; - UsedBytes+= hd->MallocCount*SIZEDATA_INDEX_TO_SIZE(i); + UsedBytes+= hd->MallocCount*cur_size; nrFree+= hd->FreeCount; - FreeBytes+= hd->FreeCount*SIZEDATA_INDEX_TO_SIZE(i); + FreeBytes+= hd->FreeCount*cur_size; } } fprintf(f, "%12s %10lu%10lu\n", "Total:", nrUsed, nrFree); From eaf4979433912b8a9a8a306c1e118ecaa89768b8 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 21 Aug 2024 20:00:19 -0400 Subject: [PATCH 34/53] Macroizing more. --- DHT/fxf.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 129ddbcbad..3c83fbff3a 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -156,10 +156,15 @@ typedef struct { #define CLIP_TO_MAX_POINTER_DIFFERENCE(x) (((x) > MAX_POINTER_DIFFERENCE) ? MAX_POINTER_DIFFERENCE : (x)) +#define ROUND_UP_TO_ALIGNMENT(s, a) (((((size_t) (s)) - 1U) & ~(((size_t) (a)) - 1U)) + (size_t) (a)) +#define ROUND_DOWN_TO_ALIGNMENT(s, a) (((size_t) (s)) & ~(((size_t) (a)) - 1U)) +#define BOTTOM_BIT(s) (((size_t) (s)) & -((size_t) (s))) +#define CLEAR_BOTTOM_BIT(s) ((((size_t) (s)) - 1U) & (size_t) (s)) + #if defined(DOS) /* MSDOS 16 Bit support (maxmemory <= 1 MB) */ #define SEGMENTED -#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(32000) & ~(MAX_ALIGNMENT - 1U)) +#define ARENA_SEG_SIZE ROUND_DOWN_TO_ALIGNMENT(CLIP_TO_MAX_POINTER_DIFFERENCE(32000), MAX_ALIGNMENT) #define ARENA_SEG_COUNT ((1024*1024)/ARENA_SEG_SIZE) #define OSNAME "MSDOS" #define OSMAXMEM "1 MB" @@ -167,7 +172,7 @@ typedef struct { /* Win95/Win98/WinME can only allocate chunks up to 255 MB */ /* maxmemory <= 768 MB */ #define SEGMENTED -#define ARENA_SEG_SIZE (CLIP_TO_MAX_POINTER_DIFFERENCE(1000000) & ~(MAX_ALIGNMENT - 1U)) +#define ARENA_SEG_SIZE ROUND_DOWN_TO_ALIGNMENT(CLIP_TO_MAX_POINTER_DIFFERENCE(1000000), MAX_ALIGNMENT) #define ARENA_SEG_COUNT ((768*1024*1024)/ARENA_SEG_SIZE) #define OSNAME "Win95/Win98/WinME" #define OSMAXMEM "768 MB" @@ -177,7 +182,7 @@ typedef struct { #define DESIRED_MAX_ALLOC (256 * sizeof(void *)) /* TODO: Is the above sufficiently large for all of our needs without being excessive? */ #define DESIRED_MAX_ALLOC_ALIGNMENT ((DESIRED_MAX_ALLOC < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) -#define ROUNDED_DESIRED_MAXIMUM_ALLOCATION (((DESIRED_MAX_ALLOC - 1U) & ~(DESIRED_MAX_ALLOC_ALIGNMENT - 1U)) + DESIRED_MAX_ALLOC_ALIGNMENT) +#define ROUNDED_DESIRED_MAXIMUM_ALLOCATION ROUND_UP_TO_ALIGNMENT(DESIRED_MAX_ALLOC, DESIRED_MAX_ALLOC_ALIGNMENT) enum { fxfMINSIZE = sizeof(void *), /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ @@ -203,11 +208,11 @@ enum { }; #define MIN_ALIGNMENT_UNDERESTIMATE (((NOT_MULTIPLE_ALIGNMENT>>1) < fxfMINSIZE) ? NOT_MULTIPLE_ALIGNMENT : \ - ((fxfMINSIZE & (fxfMINSIZE - 1U)) ? ((fxfMINSIZE & -(largest_integer_type)fxfMINSIZE)<<2) : \ - fxfMINSIZE)) + (CLEAR_BOTTOM_BIT(fxfMINSIZE) ? (BOTTOM_BIT(fxfMINSIZE)<<2) : \ + fxfMINSIZE)) static size_t min_alignment= NOT_MULTIPLE_ALIGNMENT; /* for now */ -#define ROUNDED_MIN_SIZE_UNDERESTIMATE (((fxfMINSIZE - 1U) & ~(MIN_ALIGNMENT_UNDERESTIMATE - 1U)) + MIN_ALIGNMENT_UNDERESTIMATE) +#define ROUNDED_MIN_SIZE_UNDERESTIMATE ROUND_UP_TO_ALIGNMENT(fxfMINSIZE, MIN_ALIGNMENT_UNDERESTIMATE) #define SIZEDATA_SIZE_TO_INDEX(s) (((s) - ROUNDED_MIN_SIZE_UNDERESTIMATE)/MIN_ALIGNMENT_UNDERESTIMATE) #define SIZEDATA_INDEX_TO_SIZE(x) ((size_t)(((x) * MIN_ALIGNMENT_UNDERESTIMATE) + ROUNDED_MIN_SIZE_UNDERESTIMATE)) static SizeHead SizeData[1 + SIZEDATA_SIZE_TO_INDEX(fxfMAXSIZE)]; @@ -328,7 +333,7 @@ size_t fxfMaxAllocation(void) { return fxfMAXSIZE; } -#define ALIGN_TO_MINIMUM(s) ((((s) - 1U) & ~(min_alignment - 1U)) + min_alignment) +#define ALIGN_TO_MINIMUM(s) ROUND_UP_TO_ALIGNMENT(s, min_alignment) size_t fxfInit(size_t Size) { #if defined(LOG) @@ -371,12 +376,12 @@ size_t fxfInit(size_t Size) { if (Size > MAX_POINTER_DIFFERENCE) Size= MAX_POINTER_DIFFERENCE; if (Size < NOT_MULTIPLE_ALIGNMENT) - while (Size & (Size - 1U)) - Size&= (Size - 1U); + while (CLEAR_BOTTOM_BIT(Size)) + Size= CLEAR_BOTTOM_BIT(Size); else if (Size < MAX_ALIGNMENT) - Size&= ~(NOT_MULTIPLE_ALIGNMENT - 1U); + Size= ROUND_DOWN_TO_ALIGNMENT(Size, NOT_MULTIPLE_ALIGNMENT); else - Size&= ~(MAX_ALIGNMENT - 1U); + Size= ROUND_DOWN_TO_ALIGNMENT(Size, MAX_ALIGNMENT); Arena= nNewUntyped(Size, char); if (!Arena) { ERROR_LOG3("%s: Sorry, cannot allocate arena of %" SIZE_T_PRINTF_SPECIFIER " <= %" SIZE_T_PRINTF_SPECIFIER " bytes\n", @@ -412,8 +417,9 @@ size_t fxfInit(size_t Size) { SizeData[Size].FreeHead= Nil(void); } - while ((min_alignment>>1) >= fxfMINSIZE) - min_alignment>>= 1; + if ((NOT_MULTIPLE_ALIGNMENT>>1) >= fxfMINSIZE) /* compile-time check that's likely false */ + while ((min_alignment>>1) >= fxfMINSIZE) + min_alignment>>= 1; return GlobalSize; } @@ -645,7 +651,7 @@ void *fxfAlloc(size_t size) { if ((needed_alignment_mask - curBottomIndex) >= (sizeCurrentSeg - size)) goto NEXT_SEGMENT; do { - size_t const cur_alignment= (curBottomIndex & -curBottomIndex); + size_t const cur_alignment= BOTTOM_BIT(curBottomIndex); pushOntoFreeStore(BotFreePtr, cur_alignment); BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)cur_alignment); curBottomIndex+= cur_alignment; @@ -675,7 +681,7 @@ void *fxfAlloc(size_t size) { size_t curBottomIndex= (size_t)pointerDifference(BotFreePtr,Arena[CurrentSeg]); while (curBottomIndex & (NOT_MULTIPLE_ALIGNMENT-1U)) { - size_t const cur_alignment= (curBottomIndex & -curBottomIndex); + size_t const cur_alignment= BOTTOM_BIT(curBottomIndex); pushOntoFreeStore(BotFreePtr, cur_alignment); BotFreePtr= stepPointer(BotFreePtr, (ptrdiff_t)cur_alignment); curBottomIndex+= cur_alignment; From c5b57f0bdc673d033c8da37fad3b81f384452ad4 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 22 Aug 2024 20:15:18 -0400 Subject: [PATCH 35/53] Fixing a dumb typo. --- DHT/fxf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 3c83fbff3a..e840188f88 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -182,14 +182,14 @@ typedef struct { #define DESIRED_MAX_ALLOC (256 * sizeof(void *)) /* TODO: Is the above sufficiently large for all of our needs without being excessive? */ #define DESIRED_MAX_ALLOC_ALIGNMENT ((DESIRED_MAX_ALLOC < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) -#define ROUNDED_DESIRED_MAXIMUM_ALLOCATION ROUND_UP_TO_ALIGNMENT(DESIRED_MAX_ALLOC, DESIRED_MAX_ALLOC_ALIGNMENT) +#define ROUNDED_DESIRED_MAXIMUM_ALLOC ROUND_UP_TO_ALIGNMENT(DESIRED_MAX_ALLOC, DESIRED_MAX_ALLOC_ALIGNMENT) enum { fxfMINSIZE = sizeof(void *), /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ #if defined(SEGMENTED) - fxfMAXSIZE = ((ROUNDED_DESIRED_MAXIMUM_ALLOCATION > ARENA_SEG_SIZE) ? ARENA_SEG_SIZE : ROUNDED_DESIRED_MAXIMUM_ALLOCATION) + fxfMAXSIZE = ((ROUNDED_DESIRED_MAXIMUM_ALLOC > ARENA_SEG_SIZE) ? ARENA_SEG_SIZE : ROUNDED_DESIRED_MAXIMUM_ALLOC) #else - fxfMAXSIZE = ROUNDED_DESIRED_MAXIMUM_ALLOCATION + fxfMAXSIZE = ROUNDED_DESIRED_MAXIMUM_ALLOC #endif }; @@ -197,7 +197,7 @@ enum { ENSURE_FXFMINSIZE_GT_0 = 1/(fxfMINSIZE > 0), ENSURE_FXFMAXSIZE_GE_FXFMINSIZE = 1/(fxfMAXSIZE >= fxfMINSIZE), #if defined(SEGMENT) - ENSURE_SEGMENTS_ALIGNED = 1/!((ARENeA_SEG_SIZE & (((ARENA_SEG_SIZE < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) - 1U)) && + ENSURE_SEGMENTS_ALIGNED = 1/!((ARENA_SEG_SIZE & (((ARENA_SEG_SIZE < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) - 1U)) && (ARENA_SEG_SIZE & (ARENA_SEG_SIZE - 1U))), #endif ENSURE_FXFMAXSIZE_ALIGNED = 1/((!(fxfMAXSIZE & (fxfMAXSIZE - 1U))) || From 1dbcabf5affca8de31aa43326546376be2b8c1b9 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 23 Aug 2024 13:11:46 -0400 Subject: [PATCH 36/53] Simplifying the expression. If the sum overflow, then so does the intended answer anyway. --- DHT/fxf.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index e840188f88..237803d2af 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -155,8 +155,7 @@ typedef struct { } SizeHead; #define CLIP_TO_MAX_POINTER_DIFFERENCE(x) (((x) > MAX_POINTER_DIFFERENCE) ? MAX_POINTER_DIFFERENCE : (x)) - -#define ROUND_UP_TO_ALIGNMENT(s, a) (((((size_t) (s)) - 1U) & ~(((size_t) (a)) - 1U)) + (size_t) (a)) +#define ROUND_UP_TO_ALIGNMENT(s, a) (((((size_t) (a)) - 1U) + (size_t) (s)) & ~(((size_t) (a)) - 1U)) #define ROUND_DOWN_TO_ALIGNMENT(s, a) (((size_t) (s)) & ~(((size_t) (a)) - 1U)) #define BOTTOM_BIT(s) (((size_t) (s)) & -((size_t) (s))) #define CLEAR_BOTTOM_BIT(s) ((((size_t) (s)) - 1U) & (size_t) (s)) From a6b0b14c5da50f4902e93e456ceb9cfc92a49b25 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 23 Aug 2024 16:14:53 -0400 Subject: [PATCH 37/53] Reusing code. --- DHT/fxf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 237803d2af..1377e50541 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -155,9 +155,9 @@ typedef struct { } SizeHead; #define CLIP_TO_MAX_POINTER_DIFFERENCE(x) (((x) > MAX_POINTER_DIFFERENCE) ? MAX_POINTER_DIFFERENCE : (x)) -#define ROUND_UP_TO_ALIGNMENT(s, a) (((((size_t) (a)) - 1U) + (size_t) (s)) & ~(((size_t) (a)) - 1U)) #define ROUND_DOWN_TO_ALIGNMENT(s, a) (((size_t) (s)) & ~(((size_t) (a)) - 1U)) -#define BOTTOM_BIT(s) (((size_t) (s)) & -((size_t) (s))) +#define ROUND_UP_TO_ALIGNMENT(s, a) ROUND_DOWN_TO_ALIGNMENT((((size_t) (a)) - 1U) + (size_t) (s), a) +#define BOTTOM_BIT(s) (((size_t) (s)) & -(size_t) (s)) #define CLEAR_BOTTOM_BIT(s) ((((size_t) (s)) - 1U) & (size_t) (s)) #if defined(DOS) From fa4df877898899347981b3a4e1632bac80ad7400 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 23 Aug 2024 18:03:42 -0400 Subject: [PATCH 38/53] We have a macro for that. --- DHT/fxf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 1377e50541..ac1e164871 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -199,11 +199,11 @@ enum { ENSURE_SEGMENTS_ALIGNED = 1/!((ARENA_SEG_SIZE & (((ARENA_SEG_SIZE < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) - 1U)) && (ARENA_SEG_SIZE & (ARENA_SEG_SIZE - 1U))), #endif - ENSURE_FXFMAXSIZE_ALIGNED = 1/((!(fxfMAXSIZE & (fxfMAXSIZE - 1U))) || + ENSURE_FXFMAXSIZE_ALIGNED = 1/((!CLEAR_BOTTOM_BIT(fxfMAXSIZE)) || ((fxfMAXSIZE < MAX_ALIGNMENT) && !(fxfMAXSIZE & (NOT_MULTIPLE_ALIGNMENT - 1U))) || !(fxfMAXSIZE & (MAX_ALIGNMENT - 1U))), ENSURE_ALIGNMENT_ORDERED = 1/((NOT_MULTIPLE_ALIGNMENT > 0) && (NOT_MULTIPLE_ALIGNMENT <= MAX_ALIGNMENT)), - ENSURE_ALIGNMENTS_POWERS_OF_2 = 1/!((NOT_MULTIPLE_ALIGNMENT & (NOT_MULTIPLE_ALIGNMENT - 1U)) || (MAX_ALIGNMENT & (MAX_ALIGNMENT - 1U))) + ENSURE_ALIGNMENTS_POWERS_OF_2 = 1/!(CLEAR_BOTTOM_BIT(NOT_MULTIPLE_ALIGNMENT) || CLEAR_BOTTOM_BIT(MAX_ALIGNMENT)) }; #define MIN_ALIGNMENT_UNDERESTIMATE (((NOT_MULTIPLE_ALIGNMENT>>1) < fxfMINSIZE) ? NOT_MULTIPLE_ALIGNMENT : \ From e8226879ba04207a30b92d9e6370b5a6580a09f1 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 23 Aug 2024 19:50:02 -0400 Subject: [PATCH 39/53] That should really be configurable at compile-time. --- DHT/fxf.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index ac1e164871..604161e490 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -178,10 +178,13 @@ typedef struct { #endif /* The maximum size an fxfAlloc can handle */ -#define DESIRED_MAX_ALLOC (256 * sizeof(void *)) -/* TODO: Is the above sufficiently large for all of our needs without being excessive? */ -#define DESIRED_MAX_ALLOC_ALIGNMENT ((DESIRED_MAX_ALLOC < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) -#define ROUNDED_DESIRED_MAXIMUM_ALLOC ROUND_UP_TO_ALIGNMENT(DESIRED_MAX_ALLOC, DESIRED_MAX_ALLOC_ALIGNMENT) +#if !defined(FXF_DESIRED_MAX_ALLOC) +# define FXF_DESIRED_MAX_ALLOC (256U * sizeof(void *)) +/* TODO: Is the above a good default, sufficiently large for all of our needs without being excessive? */ +#endif + +#define DESIRED_MAX_ALLOC_ALIGNMENT ((FXF_DESIRED_MAX_ALLOC < MAX_ALIGNMENT) ? NOT_MULTIPLE_ALIGNMENT : MAX_ALIGNMENT) +#define ROUNDED_DESIRED_MAXIMUM_ALLOC ROUND_UP_TO_ALIGNMENT(FXF_DESIRED_MAX_ALLOC, DESIRED_MAX_ALLOC_ALIGNMENT) enum { fxfMINSIZE = sizeof(void *), /* Different size of fxfMINSIZE for 32-/64/Bit compilation */ From 6c046274a16ef15ebe82e20becd10dcd5bc26249 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 24 Aug 2024 14:14:09 -0400 Subject: [PATCH 40/53] If memory is leaking then treat it as still allocated rather than available. --- DHT/fxf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 604161e490..207256eae3 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -527,9 +527,6 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { !(size & (min_alignment - 1U))); cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; assert((!cur_sh->FreeHead) == (!cur_sh->FreeCount)); -#if defined(FREEMAP) && !defined(SEGMENTED) - SetRange(pointerDifference(ptr, Arena), size); -#endif if ((ROUNDED_MIN_SIZE_UNDERESTIMATE < sizeof cur_sh->FreeHead) /* compile-time check that's likely false */ && (size < sizeof cur_sh->FreeHead) && cur_sh->FreeHead) @@ -541,6 +538,9 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { memcpy(ptr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); cur_sh->FreeHead= ptr; cur_sh->FreeCount++; +#if defined(FREEMAP) && !defined(SEGMENTED) + SetRange(pointerDifference(ptr, Arena), size); +#endif TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); return 1; } From efaf8b050792291b5732f1708aee5c1433856e0f Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 24 Aug 2024 16:01:44 -0400 Subject: [PATCH 41/53] It looks a bit better this way. --- DHT/fxf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 207256eae3..75eeb09c2a 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -538,10 +538,10 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { memcpy(ptr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); cur_sh->FreeHead= ptr; cur_sh->FreeCount++; + TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); #if defined(FREEMAP) && !defined(SEGMENTED) SetRange(pointerDifference(ptr, Arena), size); #endif - TMDBG(printf(" FreeCount:%lu",cur_sh->FreeCount)); return 1; } return 0; From 81b58bc4bf347140d4fe632700f3f86ee57702c9 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 27 Aug 2024 22:21:14 -0400 Subject: [PATCH 42/53] Eliminating compiler warnings, because I can. --- DHT/fxf.c | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 75eeb09c2a..8c53481e00 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -338,19 +338,19 @@ size_t fxfMaxAllocation(void) { #define ALIGN_TO_MINIMUM(s) ROUND_UP_TO_ALIGNMENT(s, min_alignment) size_t fxfInit(size_t Size) { -#if defined(LOG) +#if defined(LOG) && !defined(SEGMENTED) static char const * const myname= "fxfInit"; #endif #if defined(SEGMENTED) size_t maxSegCnt= (Size / ARENA_SEG_SIZE); if (maxSegCnt > ARENA_SEG_COUNT) maxSegCnt= ARENA_SEG_COUNT; - while (ArenaSegCnt > maxSegCnt) { + while (maxSegCnt < (size_t)ArenaSegCnt) { --ArenaSegCnt; free(Arena[ArenaSegCnt]); Arena[ArenaSegCnt]= Nil(void); } - while (ArenaSegCnt < maxSegCnt) { + while (maxSegCnt > (size_t)ArenaSegCnt) { Arena[ArenaSegCnt]= nNewUntyped(ARENA_SEG_SIZE, char); if (!Arena[ArenaSegCnt]) break; @@ -361,7 +361,7 @@ size_t fxfInit(size_t Size) { TopFreePtr= Arena[CurrentSeg]; if (TopFreePtr) TopFreePtr= stepPointer(TopFreePtr, ARENA_SEG_SIZE); - GlobalSize= ArenaSegCnt*ARENA_SEG_SIZE; + GlobalSize= ARENA_SEG_SIZE*(size_t)ArenaSegCnt; #else #if defined(FREEMAP) if (FreeMap) @@ -530,7 +530,7 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { if ((ROUNDED_MIN_SIZE_UNDERESTIMATE < sizeof cur_sh->FreeHead) /* compile-time check that's likely false */ && (size < sizeof cur_sh->FreeHead) && cur_sh->FreeHead) - TMDBG(printf(" leaking %" SIZE_T_PRINTF_SPECIFIER " byte(s) instead of freeing them\n", (size_t_printf_type)curBottomIndex)); + TMDBG(printf(" leaking %" SIZE_T_PRINTF_SPECIFIER " byte(s) instead of freeing them\n", (size_t_printf_type)size)); else { if ((ROUNDED_MIN_SIZE_UNDERESTIMATE >= sizeof cur_sh->FreeHead) /* compile-time check that's likely true */ || @@ -717,6 +717,9 @@ void *fxfAlloc(size_t size) { void fxfFree(void *ptr, size_t size) { +#if defined(DEBUG) + static char const * const myname= "fxfFree"; +#endif SizeHead *sh; #if defined(FXF_ENABLE_TMDBG) || !defined(NDEBUG) @@ -736,15 +739,15 @@ void fxfFree(void *ptr, size_t size) do { convert_pointer_to_int_type segment_begin= (convert_pointer_to_int_type)Arena[ptrSegment]; if (tmp >= segment_begin) { - ptrIndex= (tmp - segment_begin); - if (ptrIndex < ARENA_SEG_SIZE) + ptrIndex= (ptrdiff_t)(tmp - segment_begin); + if (ARENA_SEG_SIZE > (size_t)ptrIndex) goto FOUND_PUTATIVE_SEGMENT; } } while (0 <= --ptrSegment); ptrIndex= -1; } else { ptrIndex= pointerDifference(ptr,Arena[0]); - assert((ptrIndex >= 0) && (ptrIndex < ARENA_SEG_SIZE)); + assert((ptrIndex >= 0) && (ARENA_SEG_SIZE > (size_t)ptrIndex)); } FOUND_PUTATIVE_SEGMENT: TMDBG(printf("fxfFree - ptr-Arena[%d]:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,ptrSegment,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); @@ -753,7 +756,7 @@ void fxfFree(void *ptr, size_t size) # if defined(FXF_ENABLE_TMDBG) || !defined(NDEBUG) ptrIndex= pointerDifference(ptr,Arena); # endif - assert((ptrIndex >= 0) && (ptrIndex < GlobalSize)); + assert((ptrIndex >= 0) && (GlobalSize > (size_t)ptrIndex)); TMDBG(printf("fxfFree - ptr-Arena:%" PTRDIFF_T_PRINTF_SPECIFIER " size:%" SIZE_T_PRINTF_SPECIFIER,(ptrdiff_t_printf_type)ptrIndex,(size_t_printf_type)size)); #endif /*!SEGMENTED*/ DBG((df, "%s(%p, %" SIZE_T_PRINTF_SPECIFIER ")\n", myname, (void *)ptr, (size_t_printf_type) size)); @@ -769,12 +772,12 @@ void fxfFree(void *ptr, size_t size) if (!CurrentSeg) /* Otherwise we'd be relying on converting to convert_pointer_to_int_type, and such calculations aren't guaranteed to provide exactly what we need. */ { - assert(size <= (ARENA_SEG_SIZE - ptrIndex)); - assert(((ptrIndex + size) <= pointerDifference(BotFreePtr,Arena[0])) || (ptr >= TopFreePtr)); + assert(size <= (ARENA_SEG_SIZE - (size_t)ptrIndex)); + assert(((size + (size_t)ptrIndex) <= (size_t)pointerDifference(BotFreePtr,Arena[0])) || (ptr >= TopFreePtr)); # else { - assert(size <= (GlobalSize - ptrIndex)); - assert(((ptrIndex + size) <= pointerDifference(BotFreePtr,Arena)) || (ptr >= TopFreePtr)); + assert(size <= (GlobalSize - (size_t)ptrIndex)); + assert(((size + (size_t)ptrIndex) <= (size_t)pointerDifference(BotFreePtr,Arena)) || (ptr >= TopFreePtr)); #endif if (ptrIndex > 0) { @@ -833,11 +836,11 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { and such calculations aren't guaranteed to provide exactly what we need. */ { ptrdiff_t const ptrIndex= pointerDifference(ptr,Arena[0]); - assert(ptrIndex < ARENA_SEG_SIZE); + assert(ARENA_SEG_SIZE > (size_t)ptrIndex); # else { ptrdiff_t const ptrIndex= pointerDifference(ptr,Arena); - assert(ptrIndex < GlobalSize); + assert(GlobalSize > (size_t)ptrIndex); # endif assert(ptrIndex >= 0); if (ptrIndex > 0) @@ -852,9 +855,9 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { } allocatedSize= ALIGN_TO_MINIMUM(allocatedSize); # if defined(SEGMENTED) - assert(allocatedSize <= (ARENA_SEG_SIZE - ptrIndex)); + assert(allocatedSize <= (ARENA_SEG_SIZE - (size_t)ptrIndex)); # else - assert(allocatedSize <= (GlobalSize - ptrIndex)); + assert(allocatedSize <= (GlobalSize - (size_t)ptrIndex)); # endif if (allocatedSize&PTRMASK) { @@ -917,7 +920,7 @@ void fxfInfo(FILE *f) { size_t const sizeArenaUsed = GlobalSize-sizeCurrentSeg #if defined(SEGMENTED) - - (ArenaSegCnt-CurrentSeg-1)*ARENA_SEG_SIZE + - ARENA_SEG_SIZE*(size_t)(ArenaSegCnt-CurrentSeg-1) #endif /*SEGMENTED*/ ; assert(GlobalSize/one_kilo<=ULONG_MAX); @@ -940,7 +943,7 @@ void fxfInfo(FILE *f) { for (i=0; i<((sizeof SizeData)/(sizeof *SizeData)); i++,hd++) { if (hd->MallocCount || hd->FreeCount) { size_t const cur_size= SIZEDATA_INDEX_TO_SIZE(i); - fprintf(f, "%12zu %10lu%10lu\n", cur_size, hd->MallocCount, hd->FreeCount); + fprintf(f, "%12" SIZE_T_PRINTF_SPECIFIER " %10lu%10lu\n", (size_t_printf_type)cur_size, hd->MallocCount, hd->FreeCount); nrUsed+= hd->MallocCount; UsedBytes+= hd->MallocCount*cur_size; nrFree+= hd->FreeCount; From 7d4c2333bf9efbb7b774dbfa19ddd5c94b356ec4 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 28 Aug 2024 17:30:35 -0400 Subject: [PATCH 43/53] Shift the operation to the compile-time constant now that the outer check ensures that there's no overflow. --- DHT/fxf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 8c53481e00..3d75213a35 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -420,7 +420,7 @@ size_t fxfInit(size_t Size) { } if ((NOT_MULTIPLE_ALIGNMENT>>1) >= fxfMINSIZE) /* compile-time check that's likely false */ - while ((min_alignment>>1) >= fxfMINSIZE) + while (min_alignment >= (((size_t)fxfMINSIZE)<<1)) min_alignment>>= 1; return GlobalSize; From 3de22b7a490d929bdf13fcc6cfd5b6457ecd2f50 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 31 Aug 2024 10:56:57 -0400 Subject: [PATCH 44/53] Documenting where one of our ideas came from. --- DHT/fxf.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DHT/fxf.c b/DHT/fxf.c index 3d75213a35..2d6fb6031e 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -55,6 +55,8 @@ # elif (defined(__cplusplus) && (__cplusplus >= 201103L)) # define MAX_ALIGNMENT alignof(FXF_MAX_ALIGNMENT_TYPE) # else + /* This technique for getting a type's alignment is taken from Steve Jessop's comment at + https://stackoverflow.com/a/228015/1019990. */ struct GET_MAX_ALIGNMENT_TYPE { unsigned char c; FXF_MAX_ALIGNMENT_TYPE max_alignment_member; @@ -70,6 +72,8 @@ # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # include # endif + /* This technique for getting the maximum alignment is taken from Steve Jessop's comment at + https://stackoverflow.com/a/228015/1019990. */ struct GET_MAX_ALIGNMENT_TYPE { unsigned char c; union { @@ -93,6 +97,8 @@ MAX_ALIGNMENT : \ alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE)) # else + /* This technique for getting a type's alignment is taken from Steve Jessop's comment at + https://stackoverflow.com/a/228015/1019990. */ struct GET_NOT_EVEN_ALIGNMENT_TYPE { unsigned char c; FXF_NOT_MULTIPLE_ALIGNMENT_TYPE not_even_alignment_member; From 84ec9dc87df760ff29ca73b4156bbb261646c9d6 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sun, 1 Sep 2024 22:36:29 -0400 Subject: [PATCH 45/53] The comments should be based on the FXF infrastructure, not on how Popeye specifically uses it. --- DHT/fxf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 2d6fb6031e..600446f455 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -425,7 +425,7 @@ size_t fxfInit(size_t Size) { SizeData[Size].FreeHead= Nil(void); } - if ((NOT_MULTIPLE_ALIGNMENT>>1) >= fxfMINSIZE) /* compile-time check that's likely false */ + if ((NOT_MULTIPLE_ALIGNMENT>>1) >= fxfMINSIZE) /* compile-time check */ while (min_alignment >= (((size_t)fxfMINSIZE)<<1)) min_alignment>>= 1; @@ -533,13 +533,13 @@ static int pushOntoFreeStore(void * const ptr, size_t const size) { !(size & (min_alignment - 1U))); cur_sh= &SizeData[SIZEDATA_SIZE_TO_INDEX(size)]; assert((!cur_sh->FreeHead) == (!cur_sh->FreeCount)); - if ((ROUNDED_MIN_SIZE_UNDERESTIMATE < sizeof cur_sh->FreeHead) /* compile-time check that's likely false */ && + if ((ROUNDED_MIN_SIZE_UNDERESTIMATE < sizeof cur_sh->FreeHead) /* compile-time check */ && (size < sizeof cur_sh->FreeHead) && cur_sh->FreeHead) TMDBG(printf(" leaking %" SIZE_T_PRINTF_SPECIFIER " byte(s) instead of freeing them\n", (size_t_printf_type)size)); else { - if ((ROUNDED_MIN_SIZE_UNDERESTIMATE >= sizeof cur_sh->FreeHead) /* compile-time check that's likely true */ || + if ((ROUNDED_MIN_SIZE_UNDERESTIMATE >= sizeof cur_sh->FreeHead) /* compile-time check */ || (size >= sizeof cur_sh->FreeHead)) memcpy(ptr, &cur_sh->FreeHead, sizeof cur_sh->FreeHead); cur_sh->FreeHead= ptr; @@ -565,7 +565,7 @@ static void * popOffFreeStore(size_t const size) assert((!ptr) == (!cur_sh->FreeCount)); if (ptr) { cur_sh->FreeCount--; - if ((ROUNDED_MIN_SIZE_UNDERESTIMATE < sizeof cur_sh->FreeHead) /* compile-time check that's likely false */ && + if ((ROUNDED_MIN_SIZE_UNDERESTIMATE < sizeof cur_sh->FreeHead) /* compile-time check */ && (size < sizeof cur_sh->FreeHead)) cur_sh->FreeHead= Nil(void); else From d5913314e3a8571304ed40388755a1727bed6d88 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 6 Sep 2024 21:19:57 -0400 Subject: [PATCH 46/53] We might as well make this macro signed regardless of how it's defined. --- DHT/fxf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index 600446f455..7a40483b98 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -31,7 +31,6 @@ # define PTRDIFF_T_PRINTF_SPECIFIER "td" # define SIZE_T_PRINTF_SPECIFIER "zu" #else -# define MAX_POINTER_DIFFERENCE (((size_t)-1)>>1) /* just a guess */ # if defined(LLONG_MAX) /* We have long long integer types. */ typedef long long int ptrdiff_t_printf_type; typedef unsigned long long int size_t_printf_type; @@ -47,6 +46,7 @@ # define PTRDIFF_T_PRINTF_SPECIFIER "ld" # define SIZE_T_PRINTF_SPECIFIER "lu" # endif +# define MAX_POINTER_DIFFERENCE ((ptrdiff_t_printf_type)(((size_t)-1)>>1)) /* just a guess */ #endif #if defined(FXF_MAX_ALIGNMENT_TYPE) From e00318f738649df92c0bc843b2d3eb433f664708 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Sat, 7 Sep 2024 20:46:19 -0400 Subject: [PATCH 47/53] One more sanity check. --- DHT/fxf.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DHT/fxf.c b/DHT/fxf.c index 7a40483b98..1ec21a9822 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -47,6 +47,11 @@ # define SIZE_T_PRINTF_SPECIFIER "lu" # endif # define MAX_POINTER_DIFFERENCE ((ptrdiff_t_printf_type)(((size_t)-1)>>1)) /* just a guess */ +enum +{ + ENSURE_MAX_POINTER_DIFFERENCE_POSITIVE = 1/(MAX_POINTER_DIFFERENCE > 0) /* I'm not sure what would happen if this fails, + but it likely wouldn't be pretty. */ +}; #endif #if defined(FXF_MAX_ALIGNMENT_TYPE) From f880972e8ad99a604341427b7efc4f9da5b50f2b Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 17 Sep 2024 21:17:02 -0400 Subject: [PATCH 48/53] Adding an alignment sanity check in debug mode. Maybe someday we'll do more with this alignment information, but for now let's just be sure our assumptions hold. --- DHT/dht.c | 21 ++++++++++++++------- DHT/dhtbcmem.c | 2 +- DHT/dhtcmem.c | 2 +- DHT/dhtcmem.h | 2 +- DHT/dhtmem.c | 2 +- DHT/dhtmem.h | 2 +- DHT/dhtstrng.c | 2 +- DHT/dhtvalue.h | 2 +- DHT/fxf.c | 30 ++++++++++++++++++++++++------ DHT/fxf.h | 14 ++++++++++++-- 10 files changed, 57 insertions(+), 22 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index 549ed94527..a8b0e82d4d 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -50,11 +50,18 @@ void set_dhtDebug(int const d) {dhtDebug = d;} #endif /*DEBUG_DHT*/ #if !defined(New) /* TODO: Is this the correct check for all of the below lines? */ -# define New(type) ((type *)fxfAlloc(sizeof(type))) -# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type))) +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) +# define DHT_ALIGNMENT_OF_TYPE(type) _Alignof(type) +#elif (defined(__cplusplus) && (__cplusplus >= 201103L)) +# define DHT_ALIGNMENT_OF_TYPE(type) alignof(type) +#else +# define DHT_ALIGNMENT_OF_TYPE(type) offsetof(struct {unsigned char c; type t;}, t) +#endif +# define New(type) ((type *)fxfAlloc(sizeof(type), type)) +# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),DHT_ALIGNMENT_OF_TYPE(type))) # define Nil(type) ((type *)0) -static inline void * nNewImpl(size_t const nmemb, size_t const size) { - return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : fxfAlloc(nmemb*size)); +static inline void * nNewImpl(size_t const nmemb, size_t const size, size_t alignment) { + return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : fxfAllocWithAlignment(nmemb*size, alignment)); } #endif /*New*/ @@ -197,7 +204,7 @@ static boolean appendDirTable_recursive(dirTable *dt, TraceFunctionEntry(__func__); TraceFunctionParam("%p",(void *)dt); TraceFunctionParam("%p",elmt_to_append); - TraceFunctionParam("%d",elmt_depth); + TraceFunctionParam("%u",elmt_depth); TraceFunctionParamListEnd(); if (elmt_depth>dt->level) @@ -263,7 +270,7 @@ static boolean appendDirTable_recursive(dirTable *dt, } TraceFunctionExit(__func__); - TraceFunctionResult("%d",result); + TraceFunctionResult("%d",(int)result); TraceFunctionResultEnd(); return result; } @@ -423,7 +430,7 @@ typedef struct dht { #if !defined(HashTable) #define HashTable struct dht #endif -#define NewHashTable ((HashTable *)fxfAlloc(sizeof(dht))) +#define NewHashTable ((HashTable *)fxfAlloc(sizeof(dht), HashTable)) #define FreeHashTable(h) fxfFree(h, sizeof(dht)) #define OVERFLOW_SAVE 1 #if defined(OVERFLOW_SAVE) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index dede2285ea..295a435c2b 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -101,7 +101,7 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) size += (length - num_bytes_in_Data); } - result = (BCMemValue *)fxfAlloc(size); + result = (BCMemValue *)fxfAlloc(size, BCMemValue); if (result) { result->Leng = length; diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index a6a486403a..6db18c2dd6 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -96,7 +96,7 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) size += (length - num_bytes_in_Data); } - result = (CompactMemVal *)fxfAlloc(size); + result = (CompactMemVal *)fxfAlloc(size, CompactMemVal); if (result) { result->Leng = length; diff --git a/DHT/dhtcmem.h b/DHT/dhtcmem.h index 85b24a1c4a..cdce794390 100644 --- a/DHT/dhtcmem.h +++ b/DHT/dhtcmem.h @@ -18,6 +18,6 @@ typedef struct CompactMemVal { unsigned char Data[1]; } CompactMemVal; #define NilCompactMemVal ((CompactMemVal *)0) -#define NewCompactMemVal(n) ((CompactMemVal *)fxfAlloc(sizeof(CompactMemVal)+(n)*sizeof(uChar))) +#define NewCompactMemVal(n) ((CompactMemVal *)fxfAlloc(sizeof(CompactMemVal)+(n)*sizeof(uChar), CompactMemVal)) #define FreeCompactMemVal(v) fxfFree(v, sizeof(CompactMemVal)+((CompactMemVal const *)(v))->Leng*sizeof(uChar)) #endif /*DHTCMEM_INCLUDED*/ diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 92ee473338..190f9ce5c5 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -96,7 +96,7 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) if (mv) { mv->Leng= length; if (length) { - void *newBuffer= fxfAlloc(length); + void *newBuffer= fxfAlloc(length, unsigned char); if (newBuffer) { memcpy(newBuffer, data, length); mv->Data = (uChar *)newBuffer; diff --git a/DHT/dhtmem.h b/DHT/dhtmem.h index c7640bfa77..9ddff19573 100644 --- a/DHT/dhtmem.h +++ b/DHT/dhtmem.h @@ -20,7 +20,7 @@ typedef struct MemVal { unsigned char *Data; } MemVal; #define NilMemVal ((MemVal *)0) -#define NewMemVal ((MemVal *)fxfAlloc(sizeof(MemVal))) +#define NewMemVal ((MemVal *)fxfAlloc(sizeof(MemVal), MemVal)) #define FreeMemVal(v) fxfFree(v, sizeof(MemVal)) #define DeleteMemVal(v) do {if (((MemVal const *)(v))!=NilMemVal) fxfFree(((MemVal const *)(v))->Data, ((MemVal const *)(v))->Leng), FreeMemVal(v);} while (0) diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 1f79b28c76..79a042e5fe 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -62,7 +62,7 @@ static int DupString(dhtValue v, dhtValue *output) len= strlen(original); if ((len < ((size_t)-1)) && !original[len]) { ++len; - nv= (char *)fxfAlloc(len); + nv= (char *)fxfAlloc(len, char); if (nv) { memcpy(nv, original, len); output->object_pointer= nv; diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index cc389da97d..6d12b330a8 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -31,7 +31,7 @@ #include "fxf.h" #else #include -#define fxfAlloc(x) malloc(x) /* TODO: Should we track allocations to ensure that we never allocate more than some chosen number (e.g., hashtable_kilos*1024) of total byte(s)? */ +#define fxfAlloc(x,t) malloc(x) /* TODO: Should we track allocations to ensure that we never allocate more than some chosen number (e.g., hashtable_kilos*1024) of total byte(s)? */ #define fxfFree(x,n) free(x) #endif /*FXF*/ diff --git a/DHT/fxf.c b/DHT/fxf.c index 1ec21a9822..fadac356d0 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -582,13 +582,13 @@ static void * popOffFreeStore(size_t const size) return ptr; } -void *fxfAlloc(size_t size) { +void *fxfAllocWithAlignment(size_t size, size_t alignment) { #if defined(LOG) || defined(DEBUG) - static char const * const myname= "fxfAlloc"; + static char const * const myname= "fxfAllocWithAlignment"; #endif void *ptr= Nil(void); - TMDBG(printf("fxfAlloc - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); + TMDBG(printf("fxfAllocWithAlignment - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); DBG((stderr, "%s(%" SIZE_T_PRINTF_SPECIFIER ") =", myname, (size_t_printf_type)size)); if (!size) @@ -608,6 +608,24 @@ void *fxfAlloc(size_t size) { // Round up to a multiple of min_alignment size= ALIGN_TO_MINIMUM(size); +#if !defined(NDEBUG) + /* Check our alignment assumptions, purely for informational purposes. */ + if (size&PTRMASK) { + if (alignment > NOT_MULTIPLE_ALIGNMENT) + fprintf(stderr, "WARNING: Not fully aligned allocation of size %" SIZE_T_PRINTF_SPECIFIER "needs %" SIZE_T_PRINTF_SPECIFIER "-byte allocation, > the %" SIZE_T_PRINTF_SPECIFIER " byte(s) that FXF will guarantee\n", + (size_t_printf_type) size, + (size_t_printf_type) alignment, + (size_t_printf_type) NOT_MULTIPLE_ALIGNMENT); + } + else { + if (alignment > MAX_ALIGNMENT) + fprintf(stderr, "WARNING: fully aligned allocation of size %" SIZE_T_PRINTF_SPECIFIER " needs %" SIZE_T_PRINTF_SPECIFIER "-byte allocation, > the %" SIZE_T_PRINTF_SPECIFIER " byte(s) that FXF will guarantee\n", + (size_t_printf_type) size, + (size_t_printf_type) alignment, + (size_t_printf_type) NOT_MULTIPLE_ALIGNMENT); + } +#endif + ptr= popOffFreeStore(size); if (ptr) { @@ -832,13 +850,13 @@ void fxfFree(void *ptr, size_t size) TMDBG(putchar('\n')); } -void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { +void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t alignment) { void *nptr; size_t original_allocation; if (!ptr) { assert(!OldSize); - return fxfAlloc(NewSize); + return fxfAllocWithAlignment(NewSize, alignment); } assert(OldSize && (OldSize <= fxfMAXSIZE)); #if !defined(NDEBUG) @@ -901,7 +919,7 @@ void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize) { return ptr; /* TODO: Should we try to break apart this chunk? */ } - nptr= fxfAlloc(NewSize); + nptr= fxfAllocWithAlignment(NewSize, alignment); if (nptr) { memcpy(nptr, ptr, ((NewSize < OldSize) ? NewSize : OldSize)); diff --git a/DHT/fxf.h b/DHT/fxf.h index d56cbcfb3e..b34bdff105 100644 --- a/DHT/fxf.h +++ b/DHT/fxf.h @@ -6,8 +6,18 @@ size_t fxfInit(size_t Size); /* returns the number of bytes actually allocated */ int fxfInitialised(void); -void *fxfAlloc(size_t size); -void *fxfReAlloc(void *ptr, size_t OldSize, size_t NewSize); +void *fxfAllocWithAlignment(size_t size, size_t alignment); +void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t alignment); +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) +# define fxfAlloc(size, type) fxfAllocWithAlignment(size, _Alignof(type)) +# define fxfReAlloc(ptr, OldSize, NewSize, type) fxfReAllocWithAlignment(ptr, OldSize, NewSize, _Alignof(type)) +#elif (defined(__cplusplus) && (__cplusplus >= 201103L)) +# define fxfAlloc(size, type) fxfAllocWithAlignment(size, alignof(type)) +# define fxfReAlloc(size, type) fxfReAllocWithAlignment(ptr, OldSize, NewSize, alignof(type)) +#else +# define fxfAlloc(size, type) fxfAllocWithAlignment(size, offsetof(struct {unsigned char c; type t;}, t)) +# define fxfReAlloc(size, type) fxfReAllocWithAlignment(ptr, OldSize, NewSize, offsetof(struct {unsigned char c; type t;}, t)) +#endif void fxfFree(void *ptr, size_t size); void fxfInfo(FILE *); size_t fxfTotal(void); From aac65c45276994280ea136c18d8c9078d105eeec Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Tue, 17 Sep 2024 22:24:44 -0400 Subject: [PATCH 49/53] Ensure we "use" alignment. --- DHT/fxf.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DHT/fxf.c b/DHT/fxf.c index fadac356d0..55f5abf8cf 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -608,7 +608,9 @@ void *fxfAllocWithAlignment(size_t size, size_t alignment) { // Round up to a multiple of min_alignment size= ALIGN_TO_MINIMUM(size); -#if !defined(NDEBUG) +#if defined(NDEBUG) + (void) alignment; /* Ensure we "use" alignment. */ +#else /*NDEBUG*/ /* Check our alignment assumptions, purely for informational purposes. */ if (size&PTRMASK) { if (alignment > NOT_MULTIPLE_ALIGNMENT) @@ -624,7 +626,7 @@ void *fxfAllocWithAlignment(size_t size, size_t alignment) { (size_t_printf_type) alignment, (size_t_printf_type) NOT_MULTIPLE_ALIGNMENT); } -#endif +#endif /*!NDEBUG*/ ptr= popOffFreeStore(size); From b5190e7a26f7a56b6c0f79bc3f621de1d0438a80 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Wed, 18 Sep 2024 22:25:06 -0400 Subject: [PATCH 50/53] Some more sanity checks. --- DHT/dht.c | 15 +++++++-------- DHT/fxf.c | 23 ++++++++++++++--------- DHT/fxf.h | 4 ++-- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index a8b0e82d4d..ad6728d609 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -50,15 +50,14 @@ void set_dhtDebug(int const d) {dhtDebug = d;} #endif /*DEBUG_DHT*/ #if !defined(New) /* TODO: Is this the correct check for all of the below lines? */ -#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) -# define DHT_ALIGNMENT_OF_TYPE(type) _Alignof(type) -#elif (defined(__cplusplus) && (__cplusplus >= 201103L)) -# define DHT_ALIGNMENT_OF_TYPE(type) alignof(type) -#else -# define DHT_ALIGNMENT_OF_TYPE(type) offsetof(struct {unsigned char c; type t;}, t) -#endif # define New(type) ((type *)fxfAlloc(sizeof(type), type)) -# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),DHT_ALIGNMENT_OF_TYPE(type))) +# if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) +# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),_Alignof(type))) +# elif (defined(__cplusplus) && (__cplusplus >= 201103L)) +# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),alignof(type))) +# else +# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),offsetof(struct {unsigned char c; type t;}, t))) +# endif # define Nil(type) ((type *)0) static inline void * nNewImpl(size_t const nmemb, size_t const size, size_t alignment) { return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : fxfAllocWithAlignment(nmemb*size, alignment)); diff --git a/DHT/fxf.c b/DHT/fxf.c index 55f5abf8cf..7754655d94 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -582,7 +582,7 @@ static void * popOffFreeStore(size_t const size) return ptr; } -void *fxfAllocWithAlignment(size_t size, size_t alignment) { +void *fxfAllocWithAlignment(size_t size, size_t desired_alignment) { #if defined(LOG) || defined(DEBUG) static char const * const myname= "fxfAllocWithAlignment"; #endif @@ -590,10 +590,12 @@ void *fxfAllocWithAlignment(size_t size, size_t alignment) { TMDBG(printf("fxfAllocWithAlignment - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); DBG((stderr, "%s(%" SIZE_T_PRINTF_SPECIFIER ") =", myname, (size_t_printf_type)size)); + assert(desired_alignment && !CLEAR_BOTTOM_BIT(desired_alignment)); if (!size) return Nil(void); + assert(desired_alignment <= size); if (sizefxfMAXSIZE) @@ -609,21 +611,21 @@ void *fxfAllocWithAlignment(size_t size, size_t alignment) { size= ALIGN_TO_MINIMUM(size); #if defined(NDEBUG) - (void) alignment; /* Ensure we "use" alignment. */ + (void) desired_alignment; /* Ensure we "use" alignment. */ #else /*NDEBUG*/ /* Check our alignment assumptions, purely for informational purposes. */ if (size&PTRMASK) { - if (alignment > NOT_MULTIPLE_ALIGNMENT) + if (desired_alignment > NOT_MULTIPLE_ALIGNMENT) fprintf(stderr, "WARNING: Not fully aligned allocation of size %" SIZE_T_PRINTF_SPECIFIER "needs %" SIZE_T_PRINTF_SPECIFIER "-byte allocation, > the %" SIZE_T_PRINTF_SPECIFIER " byte(s) that FXF will guarantee\n", (size_t_printf_type) size, - (size_t_printf_type) alignment, + (size_t_printf_type) desired_alignment, (size_t_printf_type) NOT_MULTIPLE_ALIGNMENT); } else { - if (alignment > MAX_ALIGNMENT) + if (desired_alignment > MAX_ALIGNMENT) fprintf(stderr, "WARNING: fully aligned allocation of size %" SIZE_T_PRINTF_SPECIFIER " needs %" SIZE_T_PRINTF_SPECIFIER "-byte allocation, > the %" SIZE_T_PRINTF_SPECIFIER " byte(s) that FXF will guarantee\n", (size_t_printf_type) size, - (size_t_printf_type) alignment, + (size_t_printf_type) desired_alignment, (size_t_printf_type) NOT_MULTIPLE_ALIGNMENT); } #endif /*!NDEBUG*/ @@ -852,13 +854,13 @@ void fxfFree(void *ptr, size_t size) TMDBG(putchar('\n')); } -void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t alignment) { +void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t desired_alignment) { void *nptr; size_t original_allocation; if (!ptr) { assert(!OldSize); - return fxfAllocWithAlignment(NewSize, alignment); + return fxfAllocWithAlignment(NewSize, desired_alignment); } assert(OldSize && (OldSize <= fxfMAXSIZE)); #if !defined(NDEBUG) @@ -902,11 +904,14 @@ void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t } } #endif + assert(desired_alignment && !CLEAR_BOTTOM_BIT(desired_alignment)); + assert(desired_alignment <= OldSize); if (!NewSize) { fxfFree(ptr, OldSize); return Nil(void); } + assert(desired_alignment <= NewSize); original_allocation= OldSize; if (original_allocation < fxfMINSIZE) original_allocation= fxfMINSIZE; @@ -921,7 +926,7 @@ void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t return ptr; /* TODO: Should we try to break apart this chunk? */ } - nptr= fxfAllocWithAlignment(NewSize, alignment); + nptr= fxfAllocWithAlignment(NewSize, desired_alignment); if (nptr) { memcpy(nptr, ptr, ((NewSize < OldSize) ? NewSize : OldSize)); diff --git a/DHT/fxf.h b/DHT/fxf.h index b34bdff105..6bfd43c415 100644 --- a/DHT/fxf.h +++ b/DHT/fxf.h @@ -6,8 +6,8 @@ size_t fxfInit(size_t Size); /* returns the number of bytes actually allocated */ int fxfInitialised(void); -void *fxfAllocWithAlignment(size_t size, size_t alignment); -void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t alignment); +void *fxfAllocWithAlignment(size_t size, size_t desired_alignment); +void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t desired_alignment); #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) # define fxfAlloc(size, type) fxfAllocWithAlignment(size, _Alignof(type)) # define fxfReAlloc(ptr, OldSize, NewSize, type) fxfReAllocWithAlignment(ptr, OldSize, NewSize, _Alignof(type)) From 9f6cefe4450e8c6d09b4ffe74d5d8bd16b13af01 Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Thu, 19 Sep 2024 22:39:43 -0400 Subject: [PATCH 51/53] In the off-chance the offset we compute is a nontrivial multiple of the alignment we want, taking the bottom bit should be safe. --- DHT/dht.c | 2 +- DHT/fxf.c | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index ad6728d609..70299505b2 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -56,7 +56,7 @@ void set_dhtDebug(int const d) {dhtDebug = d;} # elif (defined(__cplusplus) && (__cplusplus >= 201103L)) # define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),alignof(type))) # else -# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),offsetof(struct {unsigned char c; type t;}, t))) +# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),(offsetof(struct {unsigned char c; type t;}, t)&-offsetof(struct {unsigned char c; type t;}, t)))) # endif # define Nil(type) ((type *)0) static inline void * nNewImpl(size_t const nmemb, size_t const size, size_t alignment) { diff --git a/DHT/fxf.c b/DHT/fxf.c index 7754655d94..82d32335b3 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -54,6 +54,8 @@ enum }; #endif +#define BOTTOM_BIT(s) (((size_t) (s)) & -(size_t) (s)) + #if defined(FXF_MAX_ALIGNMENT_TYPE) # if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) # define MAX_ALIGNMENT _Alignof(FXF_MAX_ALIGNMENT_TYPE) @@ -61,12 +63,14 @@ enum # define MAX_ALIGNMENT alignof(FXF_MAX_ALIGNMENT_TYPE) # else /* This technique for getting a type's alignment is taken from Steve Jessop's comment at - https://stackoverflow.com/a/228015/1019990. */ + https://stackoverflow.com/a/228015/1019990. + We take the bottom bit in case the calculated value is some multiple of the correct alignment; + this should at least be closer to the correct value. */ struct GET_MAX_ALIGNMENT_TYPE { unsigned char c; FXF_MAX_ALIGNMENT_TYPE max_alignment_member; }; -# define MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_alignment_member) +# define MAX_ALIGNMENT BOTTOM_BIT(offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_alignment_member)) # endif #else /*FXF_MAX_ALIGNMENT_TYPE*/ # if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) @@ -78,7 +82,9 @@ enum # include # endif /* This technique for getting the maximum alignment is taken from Steve Jessop's comment at - https://stackoverflow.com/a/228015/1019990. */ + https://stackoverflow.com/a/228015/1019990. + We take the bottom bit in case the calculated value is some multiple of the correct alignment; + this should at least be closer to the correct value. */ struct GET_MAX_ALIGNMENT_TYPE { unsigned char c; union { @@ -88,7 +94,7 @@ enum long double floating_point; } max_aligned_union; }; -# define MAX_ALIGNMENT offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union) +# define MAX_ALIGNMENT BOTTOM_BIT(offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union)) # endif #endif /*!FXF_MAX_ALIGNMENT_TYPE*/ @@ -103,14 +109,16 @@ enum alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE)) # else /* This technique for getting a type's alignment is taken from Steve Jessop's comment at - https://stackoverflow.com/a/228015/1019990. */ + https://stackoverflow.com/a/228015/1019990. + We take the bottom bit in case the calculated value is some multiple of the correct alignment; + this should at least be closer to the correct value. */ struct GET_NOT_EVEN_ALIGNMENT_TYPE { unsigned char c; FXF_NOT_MULTIPLE_ALIGNMENT_TYPE not_even_alignment_member; }; -# define NOT_MULTIPLE_ALIGNMENT ((offsetof(struct GET_NOT_EVEN_ALIGNMENT_TYPE, not_even_alignment_member) > MAX_ALIGNMENT) ? \ +# define NOT_MULTIPLE_ALIGNMENT ((BOTTOM_BIT(offsetof(struct GET_NOT_EVEN_ALIGNMENT_TYPE, not_even_alignment_member)) > MAX_ALIGNMENT) ? \ MAX_ALIGNMENT : \ - offsetof(struct GET_NOT_EVEN_ALIGNMENT_TYPE, not_even_alignment_member)) + BOTTOM_BIT(offsetof(struct GET_NOT_EVEN_ALIGNMENT_TYPE, not_even_alignment_member))) # endif #else /*FXF_NOT_MULTIPLE_ALIGNMENT_TYPE*/ # define NOT_MULTIPLE_ALIGNMENT MAX_ALIGNMENT @@ -168,7 +176,6 @@ typedef struct { #define CLIP_TO_MAX_POINTER_DIFFERENCE(x) (((x) > MAX_POINTER_DIFFERENCE) ? MAX_POINTER_DIFFERENCE : (x)) #define ROUND_DOWN_TO_ALIGNMENT(s, a) (((size_t) (s)) & ~(((size_t) (a)) - 1U)) #define ROUND_UP_TO_ALIGNMENT(s, a) ROUND_DOWN_TO_ALIGNMENT((((size_t) (a)) - 1U) + (size_t) (s), a) -#define BOTTOM_BIT(s) (((size_t) (s)) & -(size_t) (s)) #define CLEAR_BOTTOM_BIT(s) ((((size_t) (s)) - 1U) & (size_t) (s)) #if defined(DOS) From a3d4d674bc88d83c4f805494204cca684ab2e23d Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Fri, 20 Sep 2024 20:04:24 -0400 Subject: [PATCH 52/53] Isolating the alignment hackery to one location. --- DHT/dht.c | 12 +++--------- DHT/fxf.c | 58 ++++++++----------------------------------------------- DHT/fxf.h | 24 +++++++++++++---------- 3 files changed, 25 insertions(+), 69 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index 70299505b2..b0e0b57e0d 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -51,16 +51,10 @@ void set_dhtDebug(int const d) {dhtDebug = d;} #if !defined(New) /* TODO: Is this the correct check for all of the below lines? */ # define New(type) ((type *)fxfAlloc(sizeof(type), type)) -# if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) -# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),_Alignof(type))) -# elif (defined(__cplusplus) && (__cplusplus >= 201103L)) -# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),alignof(type))) -# else -# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),(offsetof(struct {unsigned char c; type t;}, t)&-offsetof(struct {unsigned char c; type t;}, t)))) -# endif +# define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),ALIGNMENT_OF_TYPE(type))) # define Nil(type) ((type *)0) -static inline void * nNewImpl(size_t const nmemb, size_t const size, size_t alignment) { - return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : fxfAllocWithAlignment(nmemb*size, alignment)); +static inline void * nNewImpl(size_t const nmemb, size_t const size, size_t desired_alignment) { + return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : fxfAllocWithAlignment(nmemb*size, desired_alignment)); } #endif /*New*/ diff --git a/DHT/fxf.c b/DHT/fxf.c index 82d32335b3..32a45974c4 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -57,21 +57,7 @@ enum #define BOTTOM_BIT(s) (((size_t) (s)) & -(size_t) (s)) #if defined(FXF_MAX_ALIGNMENT_TYPE) -# if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) -# define MAX_ALIGNMENT _Alignof(FXF_MAX_ALIGNMENT_TYPE) -# elif (defined(__cplusplus) && (__cplusplus >= 201103L)) -# define MAX_ALIGNMENT alignof(FXF_MAX_ALIGNMENT_TYPE) -# else - /* This technique for getting a type's alignment is taken from Steve Jessop's comment at - https://stackoverflow.com/a/228015/1019990. - We take the bottom bit in case the calculated value is some multiple of the correct alignment; - this should at least be closer to the correct value. */ - struct GET_MAX_ALIGNMENT_TYPE { - unsigned char c; - FXF_MAX_ALIGNMENT_TYPE max_alignment_member; - }; -# define MAX_ALIGNMENT BOTTOM_BIT(offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_alignment_member)) -# endif +# define MAX_ALIGNMENT ALIGNMENT_OF_TYPE(FXF_MAX_ALIGNMENT_TYPE) #else /*FXF_MAX_ALIGNMENT_TYPE*/ # if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) # define MAX_ALIGNMENT _Alignof(max_align_t) @@ -81,45 +67,17 @@ enum # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # include # endif - /* This technique for getting the maximum alignment is taken from Steve Jessop's comment at - https://stackoverflow.com/a/228015/1019990. - We take the bottom bit in case the calculated value is some multiple of the correct alignment; - this should at least be closer to the correct value. */ - struct GET_MAX_ALIGNMENT_TYPE { - unsigned char c; - union { - largest_integer_type unsigned_integer; - const volatile void * object_pointer; - void (*function_pointer)(void); - long double floating_point; - } max_aligned_union; - }; -# define MAX_ALIGNMENT BOTTOM_BIT(offsetof(struct GET_MAX_ALIGNMENT_TYPE, max_aligned_union)) +# define MAX_ALIGNMENT ALIGNMENT_OF_TYPE(union {largest_integer_type unsigned_integer; \ + const volatile void * object_pointer; \ + void (*function_pointer)(void); \ + long double floating_point;}) # endif #endif /*!FXF_MAX_ALIGNMENT_TYPE*/ #if defined(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE) -# if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) -# define NOT_MULTIPLE_ALIGNMENT ((_Alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE) > MAX_ALIGNMENT) ? \ - MAX_ALIGNMENT : \ - _Alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE)) -# elif (defined(__cplusplus) && (__cplusplus >= 201103L)) -# define NOT_MULTIPLE_ALIGNMENT ((alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE) > MAX_ALIGNMENT) ? \ - MAX_ALIGNMENT : \ - alignof(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE)) -# else - /* This technique for getting a type's alignment is taken from Steve Jessop's comment at - https://stackoverflow.com/a/228015/1019990. - We take the bottom bit in case the calculated value is some multiple of the correct alignment; - this should at least be closer to the correct value. */ - struct GET_NOT_EVEN_ALIGNMENT_TYPE { - unsigned char c; - FXF_NOT_MULTIPLE_ALIGNMENT_TYPE not_even_alignment_member; - }; -# define NOT_MULTIPLE_ALIGNMENT ((BOTTOM_BIT(offsetof(struct GET_NOT_EVEN_ALIGNMENT_TYPE, not_even_alignment_member)) > MAX_ALIGNMENT) ? \ - MAX_ALIGNMENT : \ - BOTTOM_BIT(offsetof(struct GET_NOT_EVEN_ALIGNMENT_TYPE, not_even_alignment_member))) -# endif +# define NOT_MULTIPLE_ALIGNMENT ((ALIGNMENT_OF_TYPE(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE) > MAX_ALIGNMENT) ? \ + MAX_ALIGNMENT : \ + ALIGNMENT_OF_TYPE(FXF_NOT_MULTIPLE_ALIGNMENT_TYPE)) #else /*FXF_NOT_MULTIPLE_ALIGNMENT_TYPE*/ # define NOT_MULTIPLE_ALIGNMENT MAX_ALIGNMENT #endif /*!FXF_NOT_MULTIPLE_ALIGNMENT_TYPE*/ diff --git a/DHT/fxf.h b/DHT/fxf.h index 6bfd43c415..3f9624f992 100644 --- a/DHT/fxf.h +++ b/DHT/fxf.h @@ -4,20 +4,24 @@ #include #include -size_t fxfInit(size_t Size); /* returns the number of bytes actually allocated */ -int fxfInitialised(void); -void *fxfAllocWithAlignment(size_t size, size_t desired_alignment); -void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t desired_alignment); #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) -# define fxfAlloc(size, type) fxfAllocWithAlignment(size, _Alignof(type)) -# define fxfReAlloc(ptr, OldSize, NewSize, type) fxfReAllocWithAlignment(ptr, OldSize, NewSize, _Alignof(type)) +# define ALIGNMENT_OF_TYPE(type) _Alignof(type) #elif (defined(__cplusplus) && (__cplusplus >= 201103L)) -# define fxfAlloc(size, type) fxfAllocWithAlignment(size, alignof(type)) -# define fxfReAlloc(size, type) fxfReAllocWithAlignment(ptr, OldSize, NewSize, alignof(type)) +# define ALIGNMENT_OF_TYPE(type) alignof(type) #else -# define fxfAlloc(size, type) fxfAllocWithAlignment(size, offsetof(struct {unsigned char c; type t;}, t)) -# define fxfReAlloc(size, type) fxfReAllocWithAlignment(ptr, OldSize, NewSize, offsetof(struct {unsigned char c; type t;}, t)) + /* This technique for getting a type's alignment is taken from Steve Jessop's comment at + https://stackoverflow.com/a/228015/1019990. + We take the bottom bit in case the calculated value is some multiple of the correct alignment; + this should at least be closer to the correct value. */ +# define ALIGNMENT_OF_TYPE(type) (offsetof(struct {unsigned char c; type t;}, t) & -offsetof(struct {unsigned char c; type t;}, t)) #endif + +size_t fxfInit(size_t Size); /* returns the number of bytes actually allocated */ +int fxfInitialised(void); +void *fxfAllocWithAlignment(size_t size, size_t desired_alignment); +void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t desired_alignment); +#define fxfAlloc(size, type) fxfAllocWithAlignment(size, ALIGNMENT_OF_TYPE(type)) +#define fxfReAlloc(ptr, OldSize, NewSize, type) fxfReAllocWithAlignment(ptr, OldSize, NewSize, ALIGNMENT_OF_TYPE(type)) void fxfFree(void *ptr, size_t size); void fxfInfo(FILE *); size_t fxfTotal(void); From 06deb8845c64cdae4f62253a8c2285b0df2a341d Mon Sep 17 00:00:00 2001 From: Joshua Green Date: Mon, 23 Sep 2024 18:57:28 -0400 Subject: [PATCH 53/53] Making fxfAlloc type-safe. --- DHT/dht.c | 6 +++--- DHT/dhtbcmem.c | 2 +- DHT/dhtcmem.c | 2 +- DHT/dhtcmem.h | 2 +- DHT/dhtmem.c | 4 ++-- DHT/dhtmem.h | 2 +- DHT/dhtstrng.c | 2 +- DHT/dhtvalue.h | 2 +- DHT/fxf.c | 12 ++++++------ DHT/fxf.h | 8 ++++---- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/DHT/dht.c b/DHT/dht.c index b0e0b57e0d..eba59a2c87 100644 --- a/DHT/dht.c +++ b/DHT/dht.c @@ -50,11 +50,11 @@ void set_dhtDebug(int const d) {dhtDebug = d;} #endif /*DEBUG_DHT*/ #if !defined(New) /* TODO: Is this the correct check for all of the below lines? */ -# define New(type) ((type *)fxfAlloc(sizeof(type), type)) +# define New(type) fxfAlloc(sizeof(type), type) # define nNew(n,type) ((type *)nNewImpl(n,sizeof(type),ALIGNMENT_OF_TYPE(type))) # define Nil(type) ((type *)0) static inline void * nNewImpl(size_t const nmemb, size_t const size, size_t desired_alignment) { - return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : fxfAllocWithAlignment(nmemb*size, desired_alignment)); + return ((size && (nmemb > (((size_t)-1)/size))) ? Nil(void) : fxfAllocRaw(nmemb*size, desired_alignment)); } #endif /*New*/ @@ -423,7 +423,7 @@ typedef struct dht { #if !defined(HashTable) #define HashTable struct dht #endif -#define NewHashTable ((HashTable *)fxfAlloc(sizeof(dht), HashTable)) +#define NewHashTable fxfAlloc(sizeof(dht), HashTable) #define FreeHashTable(h) fxfFree(h, sizeof(dht)) #define OVERFLOW_SAVE 1 #if defined(OVERFLOW_SAVE) diff --git a/DHT/dhtbcmem.c b/DHT/dhtbcmem.c index 295a435c2b..2d0f7cb1ea 100644 --- a/DHT/dhtbcmem.c +++ b/DHT/dhtbcmem.c @@ -101,7 +101,7 @@ static int DupBCMemValue(dhtValue kv, dhtValue *output) size += (length - num_bytes_in_Data); } - result = (BCMemValue *)fxfAlloc(size, BCMemValue); + result = fxfAlloc(size, BCMemValue); if (result) { result->Leng = length; diff --git a/DHT/dhtcmem.c b/DHT/dhtcmem.c index 6db18c2dd6..2eaddfe1c5 100644 --- a/DHT/dhtcmem.c +++ b/DHT/dhtcmem.c @@ -96,7 +96,7 @@ static int DupCompactMemoryValue(dhtValue kv, dhtValue *output) size += (length - num_bytes_in_Data); } - result = (CompactMemVal *)fxfAlloc(size, CompactMemVal); + result = fxfAlloc(size, CompactMemVal); if (result) { result->Leng = length; diff --git a/DHT/dhtcmem.h b/DHT/dhtcmem.h index cdce794390..5b69994dd7 100644 --- a/DHT/dhtcmem.h +++ b/DHT/dhtcmem.h @@ -18,6 +18,6 @@ typedef struct CompactMemVal { unsigned char Data[1]; } CompactMemVal; #define NilCompactMemVal ((CompactMemVal *)0) -#define NewCompactMemVal(n) ((CompactMemVal *)fxfAlloc(sizeof(CompactMemVal)+(n)*sizeof(uChar), CompactMemVal)) +#define NewCompactMemVal(n) fxfAlloc(sizeof(CompactMemVal)+(n)*sizeof(uChar), CompactMemVal) #define FreeCompactMemVal(v) fxfFree(v, sizeof(CompactMemVal)+((CompactMemVal const *)(v))->Leng*sizeof(uChar)) #endif /*DHTCMEM_INCLUDED*/ diff --git a/DHT/dhtmem.c b/DHT/dhtmem.c index 190f9ce5c5..11390b4569 100644 --- a/DHT/dhtmem.c +++ b/DHT/dhtmem.c @@ -96,10 +96,10 @@ static int DupMemoryValue(dhtValue kv, dhtValue *output) if (mv) { mv->Leng= length; if (length) { - void *newBuffer= fxfAlloc(length, unsigned char); + uChar *newBuffer= fxfAlloc(length, uChar); if (newBuffer) { memcpy(newBuffer, data, length); - mv->Data = (uChar *)newBuffer; + mv->Data = newBuffer; output->object_pointer = mv; return 0; } else { diff --git a/DHT/dhtmem.h b/DHT/dhtmem.h index 9ddff19573..126ed8c724 100644 --- a/DHT/dhtmem.h +++ b/DHT/dhtmem.h @@ -20,7 +20,7 @@ typedef struct MemVal { unsigned char *Data; } MemVal; #define NilMemVal ((MemVal *)0) -#define NewMemVal ((MemVal *)fxfAlloc(sizeof(MemVal), MemVal)) +#define NewMemVal fxfAlloc(sizeof(MemVal), MemVal) #define FreeMemVal(v) fxfFree(v, sizeof(MemVal)) #define DeleteMemVal(v) do {if (((MemVal const *)(v))!=NilMemVal) fxfFree(((MemVal const *)(v))->Data, ((MemVal const *)(v))->Leng), FreeMemVal(v);} while (0) diff --git a/DHT/dhtstrng.c b/DHT/dhtstrng.c index 79a042e5fe..ab8011d5dd 100644 --- a/DHT/dhtstrng.c +++ b/DHT/dhtstrng.c @@ -62,7 +62,7 @@ static int DupString(dhtValue v, dhtValue *output) len= strlen(original); if ((len < ((size_t)-1)) && !original[len]) { ++len; - nv= (char *)fxfAlloc(len, char); + nv= fxfAlloc(len, char); if (nv) { memcpy(nv, original, len); output->object_pointer= nv; diff --git a/DHT/dhtvalue.h b/DHT/dhtvalue.h index 6d12b330a8..b5df6c1ce0 100644 --- a/DHT/dhtvalue.h +++ b/DHT/dhtvalue.h @@ -31,7 +31,7 @@ #include "fxf.h" #else #include -#define fxfAlloc(x,t) malloc(x) /* TODO: Should we track allocations to ensure that we never allocate more than some chosen number (e.g., hashtable_kilos*1024) of total byte(s)? */ +#define fxfAlloc(x,type) ((type *) malloc(x)) /* TODO: Should we track allocations to ensure that we never allocate more than some chosen number (e.g., hashtable_kilos*1024) of total byte(s)? */ #define fxfFree(x,n) free(x) #endif /*FXF*/ diff --git a/DHT/fxf.c b/DHT/fxf.c index 32a45974c4..460ed876ac 100644 --- a/DHT/fxf.c +++ b/DHT/fxf.c @@ -547,13 +547,13 @@ static void * popOffFreeStore(size_t const size) return ptr; } -void *fxfAllocWithAlignment(size_t size, size_t desired_alignment) { +void *fxfAllocRaw(size_t size, size_t desired_alignment) { #if defined(LOG) || defined(DEBUG) - static char const * const myname= "fxfAllocWithAlignment"; + static char const * const myname= "fxfAllocRaw"; #endif void *ptr= Nil(void); - TMDBG(printf("fxfAllocWithAlignment - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); + TMDBG(printf("fxfAllocRaw - size:%" SIZE_T_PRINTF_SPECIFIER,(size_t_printf_type)size)); DBG((stderr, "%s(%" SIZE_T_PRINTF_SPECIFIER ") =", myname, (size_t_printf_type)size)); assert(desired_alignment && !CLEAR_BOTTOM_BIT(desired_alignment)); @@ -819,13 +819,13 @@ void fxfFree(void *ptr, size_t size) TMDBG(putchar('\n')); } -void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t desired_alignment) { +void *fxfReAllocRaw(void *ptr, size_t OldSize, size_t NewSize, size_t desired_alignment) { void *nptr; size_t original_allocation; if (!ptr) { assert(!OldSize); - return fxfAllocWithAlignment(NewSize, desired_alignment); + return fxfAllocRaw(NewSize, desired_alignment); } assert(OldSize && (OldSize <= fxfMAXSIZE)); #if !defined(NDEBUG) @@ -891,7 +891,7 @@ void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t return ptr; /* TODO: Should we try to break apart this chunk? */ } - nptr= fxfAllocWithAlignment(NewSize, desired_alignment); + nptr= fxfAllocRaw(NewSize, desired_alignment); if (nptr) { memcpy(nptr, ptr, ((NewSize < OldSize) ? NewSize : OldSize)); diff --git a/DHT/fxf.h b/DHT/fxf.h index 3f9624f992..da27ceeda6 100644 --- a/DHT/fxf.h +++ b/DHT/fxf.h @@ -18,10 +18,10 @@ size_t fxfInit(size_t Size); /* returns the number of bytes actually allocated */ int fxfInitialised(void); -void *fxfAllocWithAlignment(size_t size, size_t desired_alignment); -void *fxfReAllocWithAlignment(void *ptr, size_t OldSize, size_t NewSize, size_t desired_alignment); -#define fxfAlloc(size, type) fxfAllocWithAlignment(size, ALIGNMENT_OF_TYPE(type)) -#define fxfReAlloc(ptr, OldSize, NewSize, type) fxfReAllocWithAlignment(ptr, OldSize, NewSize, ALIGNMENT_OF_TYPE(type)) +void *fxfAllocRaw(size_t size, size_t desired_alignment); +void *fxfReAllocRaw(void *ptr, size_t OldSize, size_t NewSize, size_t desired_alignment); +#define fxfAlloc(size, type) ((type *) fxfAllocRaw(size, ALIGNMENT_OF_TYPE(type))) +#define fxfReAlloc(ptr, OldSize, NewSize, type) ((type *) fxfReAllocRaw(ptr, OldSize, NewSize, ALIGNMENT_OF_TYPE(type))) void fxfFree(void *ptr, size_t size); void fxfInfo(FILE *); size_t fxfTotal(void);