From dedb73a8e5ba814234e9b7ad13f8ee848815332e Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Fri, 15 Mar 2024 20:26:57 +0100 Subject: [PATCH] Streamline ja3, ja4, ssl code --- src/include/nfdump.h | 21 ++----- src/inline/nffile_inline.c | 8 ++- src/libnfdump/filter/filter.c | 48 ++++++-------- src/libnfdump/filter/grammar.y | 29 ++++----- src/libnfdump/ja3/ja3.c | 36 ++++++----- src/libnfdump/ja3/ja3.h | 5 +- src/libnfdump/ja4/ja4.c | 25 ++++---- src/libnfdump/ja4/ja4.h | 26 +++++++- src/libnfdump/ja4/ja4s.c | 97 +++++++++++----------------- src/libnfdump/ja4/ja4s.h | 66 ------------------- src/nfdump/nflowcache.c | 27 +++----- src/nfdump/nfstat.c | 89 +++++++++++++++----------- src/output/output_fmt.c | 112 ++++++++++++++++++--------------- src/output/output_json.c | 50 +++++++++------ src/output/output_raw.c | 70 +++++++++++++-------- 15 files changed, 335 insertions(+), 374 deletions(-) mode change 100644 => 100755 src/include/nfdump.h mode change 100644 => 100755 src/inline/nffile_inline.c mode change 100644 => 100755 src/libnfdump/filter/filter.c mode change 100644 => 100755 src/libnfdump/filter/grammar.y mode change 100644 => 100755 src/libnfdump/ja3/ja3.c mode change 100644 => 100755 src/libnfdump/ja3/ja3.h mode change 100644 => 100755 src/libnfdump/ja4/ja4.c mode change 100644 => 100755 src/libnfdump/ja4/ja4.h mode change 100644 => 100755 src/libnfdump/ja4/ja4s.c delete mode 100644 src/libnfdump/ja4/ja4s.h mode change 100644 => 100755 src/nfdump/nfstat.c mode change 100644 => 100755 src/output/output_fmt.c mode change 100644 => 100755 src/output/output_json.c mode change 100644 => 100755 src/output/output_raw.c diff --git a/src/include/nfdump.h b/src/include/nfdump.h old mode 100644 new mode 100755 index 1be1ff01..f5da6fe3 --- a/src/include/nfdump.h +++ b/src/include/nfdump.h @@ -80,10 +80,11 @@ typedef struct ip_addr_s { typedef struct exporter_info_record_s exporter_info_record_t; typedef struct extension_map_s extension_map_t; -#define EXlocal MAXEXTENSIONS -#define SSLindex MAXEXTENSIONS + 1 -#define JA4index MAXEXTENSIONS + 2 -#define MAXLISTSIZE MAXEXTENSIONS + 3 +enum { EXlocal = MAXEXTENSIONS, + SSLindex, + JA3index, + JA4index, + MAXLISTSIZE }; typedef struct recordHandle_s { recordHeaderV3_t *recordHeaderV3; @@ -97,18 +98,6 @@ typedef struct recordHandle_s { #define OFFgeoSrcTunIP offsetof(recordHandle_t, geo) + 8 #define OFFgeoDstTunIP offsetof(recordHandle_t, geo) + 10 #define SizeGEOloc 2 - void *sslInfo; -#define OFFsslInfo offsetof(recordHandle_t, sslInfo) -#define SIZEsslInfo MemberSize(recordHandle_t, sslInfo) - uint8_t ja3[16]; -#define OFFja3 offsetof(recordHandle_t, ja3) -#define SIZEja3 MemberSize(recordHandle_t, ja3) - // for all ja4 hashes, ja4, ja4s, ja4h etc. there is one reference - // only, as a record can not have more thanone - void *ja4Info; -#define OFFja4Info offsetof(recordHandle_t, ja4Info) -#define SIZEja4Info MemberSize(recordHandle_t, ja4Info) - uint32_t ja4Type; uint32_t flowCount; #define OFFflowCount offsetof(recordHandle_t, flowCount) #define SIZEflowCount MemberSize(recordHandle_t, flowCount) diff --git a/src/inline/nffile_inline.c b/src/inline/nffile_inline.c old mode 100644 new mode 100755 index bbac72bd..ae607672 --- a/src/inline/nffile_inline.c +++ b/src/inline/nffile_inline.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2023, Peter Haag + * Copyright (c) 2009-2024, Peter Haag * Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung * All rights reserved. * @@ -63,8 +63,10 @@ static inline size_t CheckBufferSpace(nffile_t *nffile, size_t required) { } // End of CheckBufferSpace static inline int MapRecordHandle(recordHandle_t *handle, recordHeaderV3_t *recordHeaderV3, uint32_t flowCount) { - if (handle->sslInfo) free(handle->sslInfo); - if (handle->ja4Info) free(handle->ja4Info); + if (handle->extensionList[SSLindex]) free(handle->extensionList[SSLindex]); + if (handle->extensionList[JA3index]) free(handle->extensionList[JA3index]); + if (handle->extensionList[JA4index]) free(handle->extensionList[JA4index]); + memset((void *)handle, 0, sizeof(recordHandle_t)); handle->recordHeaderV3 = recordHeaderV3; diff --git a/src/libnfdump/filter/filter.c b/src/libnfdump/filter/filter.c old mode 100644 new mode 100755 index e09ad9d5..89352af6 --- a/src/libnfdump/filter/filter.c +++ b/src/libnfdump/filter/filter.c @@ -108,10 +108,10 @@ static uint64_t mpls_exp_function(void *dataPtr, uint32_t length, data_t data, r static uint64_t mpls_any_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); static uint64_t pblock_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); static uint64_t mmASLookup_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); -static uint64_t ja3_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); /* flow pre-processing functions */ static void *ssl_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); +static void *ja3_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); static void *ja4_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); /* @@ -133,12 +133,11 @@ static struct flow_procs_map_s { {FUNC_MPLS_ANY, "mpls any", mpls_any_function}, {FUNC_PBLOCK, "pblock", pblock_function}, {FUNC_MMAS_LOOKUP, "AS Lockup", mmASLookup_function}, - {FUNC_JA3, "ja3", ja3_function}, {0, NULL, NULL}}; static struct preprocess_s { preprocess_proc_t function; -} preprocess_map[] = {{ssl_preproc}, {ja4_preproc}, {NULL}}; +} preprocess_map[] = {{ssl_preproc}, {ja3_preproc}, {ja4_preproc}, {NULL}}; // 128bit compare for IPv6 static int IPNodeCMP(struct IPListNode *e1, struct IPListNode *e2) { @@ -292,29 +291,6 @@ static uint64_t mmASLookup_function(void *dataPtr, uint32_t length, data_t data, return as; } // End of mmASLookup_function -static uint64_t ja3_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *recordHandle) { - const uint8_t *payload = (const uint8_t *)recordHandle->extensionList[EXinPayloadID]; - - // check if ja3 already exists or no payload exists - if (recordHandle->ja3[0] != '\0' || payload == NULL) return 1; - - uint32_t len = ExtensionLength(payload); - - // check if ssl record already exists - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; - if (!ssl) { - ssl = sslProcess(payload, len); - recordHandle->sslInfo = (void *)ssl; - } - uint8_t *ja3 = ja3Process(ssl, recordHandle->ja3); - if (ja3 == NULL) { - return 0; - } - // else - found a valid ja3 hash from the ssl handshake record - return 1; - -} // End of ja3_function - static void *ssl_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle) { const uint8_t *payload = (uint8_t *)(handle->extensionList[EXinPayloadID]); if (payload == NULL) return NULL; @@ -330,10 +306,25 @@ static void *ssl_preproc(void *dataPtr, uint32_t length, data_t data, recordHand } // End of ssl_preproc +static void *ja3_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle) { + const uint8_t *payload = (const uint8_t *)handle->extensionList[EXinPayloadID]; + if (payload == NULL) return NULL; + + // return ja3 string if it already exists + if (handle->extensionList[JA3index]) return handle->extensionList[JA3index]; + + ssl_t *ssl = ssl_preproc(dataPtr, length, data, handle); + if (!ssl) return NULL; + + return ja3Process(ssl, NULL); + +} // End of ja3_preproc + static void *ja4_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle) { const uint8_t *payload = (uint8_t *)(handle->extensionList[EXinPayloadID]); if (payload == NULL) return NULL; + // return ja4 struct if it already exists if (handle->extensionList[JA4index]) return handle->extensionList[JA4index]; EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)(handle->extensionList[EXgenericFlowID]); @@ -345,12 +336,11 @@ static void *ja4_preproc(void *dataPtr, uint32_t length, data_t data, recordHand LogError("malloc() error in %s line %d: %s", __FILE__, __LINE__, strerror(errno)); return NULL; } - ja4->type = TYPE_JA4; - if (ja4Process(ssl, genericFlow->proto, ja4->string)) { + ja4 = ja4Process(ssl, genericFlow->proto); + if (ja4) { handle->extensionList[JA4index] = (void *)ja4; return (void *)ja4; } - free(ja4); return NULL; } // End of ja4_preproc diff --git a/src/libnfdump/filter/grammar.y b/src/libnfdump/filter/grammar.y old mode 100644 new mode 100755 index c7f05763..5377d7fe --- a/src/libnfdump/filter/grammar.y +++ b/src/libnfdump/filter/grammar.y @@ -48,6 +48,7 @@ #include "nfxV3.h" #include "ipconv.h" #include "sgregex.h" +#include "ja3/ja3.h" #include "ja4/ja4.h" #include "nfdump.h" @@ -1222,32 +1223,26 @@ static int AddPayload(char *type, char *arg, char *opt) { return NewElement(EXinPayloadID, 0, 0, 0, CMP_REGEX, FUNC_NONE, data); } else if (strcasecmp(type, "ssl") == 0) { if (strcasecmp(arg, "defined") == 0) { - return Invert(NewElement(EXlocal, OFFsslInfo, SIZEsslInfo, 0, CMP_EQ, FUNC_NONE, NULLPtr)); + return Invert(NewElement(SSLindex, 0, 0, 0, CMP_EQ, FUNC_NONE, NULLPtr)); } } else if (strcasecmp(type, "ja3") == 0) { - uint8_t *md5 = calloc(1,16); - if (!md5) { - yyerror("malloc() for md5 failed"); - return -1; - } - data_t data = {.dataPtr=md5}; if (strcasecmp(arg, "defined") == 0) { - return Invert(NewElement(EXlocal, OFFja3, SIZEja3, 0, CMP_BINARY, FUNC_JA3, data)); + return Invert(NewElement(JA3index, OFFja3String, SIZEja3String, 0, CMP_STRING, FUNC_NONE, NULLPtr)); } else { if (IsMD5(arg) == 0) { - yyerror("ja3 string %s is not an MD5 sum", arg); - free(md5); + yyerror("String %s is not a valid ja3 string", arg); return -1; } - for(int count = 0; count < 16; count++) { - sscanf(arg, "%2hhx", &md5[count]); - arg += 2; - } - return NewElement(EXlocal, OFFja3, SIZEja3, 0, CMP_BINARY, FUNC_JA3, data); + data_t data = {.dataPtr=strdup(arg)}; + return NewElement(JA3index, OFFja3String, SIZEja3String, 0, CMP_STRING, FUNC_NONE, data); } } else if (strcasecmp(type, "ja4") == 0) { - data_t data = {.dataPtr=strdup(arg)}; - return NewElement(JA4index, OFFja4String, SIZEja4String, 0, CMP_STRING, FUNC_NONE, data); + if ( ja4Check(arg) == 0 ){ + yyerror("String %s is not a valid ja4 string", arg); + return -1; + } + data_t data = {.dataPtr=strdup(arg)}; + return NewElement(JA4index, OFFja4String, SIZEja4String, 0, CMP_STRING, FUNC_NONE, data); } else { yyerror("Unknown PAYLOAD argument: %s\n", type); return -1; diff --git a/src/libnfdump/ja3/ja3.c b/src/libnfdump/ja3/ja3.c old mode 100644 new mode 100755 index b5d41216..bfeb8050 --- a/src/libnfdump/ja3/ja3.c +++ b/src/libnfdump/ja3/ja3.c @@ -53,22 +53,29 @@ } \ } -char *ja3String(uint8_t *ja3Hash) { - static char out[36]; +static char *ja3String(uint8_t *ja3Hash, char *buff) { + if (buff == NULL) { + buff = malloc(SIZEja3String + 1); + if (buff == NULL) { + LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); + return NULL; + } + } + buff[0] = '\0'; int i, j; for (i = 0, j = 0; i < 16; i++) { uint8_t ln = ja3Hash[i] & 0xF; uint8_t hn = (ja3Hash[i] >> 4) & 0xF; - out[j++] = hn <= 9 ? hn + '0' : hn + 'a' - 10; - out[j++] = ln <= 9 ? ln + '0' : ln + 'a' - 10; + buff[j++] = hn <= 9 ? hn + '0' : hn + 'a' - 10; + buff[j++] = ln <= 9 ? ln + '0' : ln + 'a' - 10; } - out[j] = '\0'; + buff[j] = '\0'; - return out; + return buff; } // End of ja3String -uint8_t *ja3Process(ssl_t *ssl, uint8_t *hash) { +char *ja3Process(ssl_t *ssl, char *buff) { if (!ssl) return NULL; size_t sLen = 6 * (1 + 1 + LenArray(ssl->cipherSuites) + 1 + LenArray(ssl->extensions) + 1 + LenArray(ssl->ellipticCurves) + 1 + @@ -80,6 +87,7 @@ uint8_t *ja3Process(ssl_t *ssl, uint8_t *hash) { LogError("calloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); return NULL; } + char *s = ja3_r; snprintf(s, sLen, "%u,", ssl->protocolVersion); size_t len = strlen(s); @@ -123,29 +131,25 @@ uint8_t *ja3Process(ssl_t *ssl, uint8_t *hash) { if (sLen == 0) { LogError("sLen error in %s line %d: %s\n", __FILE__, __LINE__, "Size == 0"); + free(ja3_r); return NULL; } *s++ = '\0'; - if (hash == NULL) { - hash = malloc(16); - if (!hash) { - LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); - return NULL; - } - } + uint8_t hash[16]; md5_hash((uint8_t *)ja3_r, strlen(ja3_r), (uint32_t *)hash); #ifdef MAIN printf("SSL/TLS info:\n"); sslPrint(ssl); printf("JA3_r : %s\n", ja3_r); - printf("JA3 : %s\n", ja3String(hash)); + printf("JA3 : %s\n", ja3String(hash, buff)); #endif free(ja3_r); - return hash; + return ja3String(hash, buff); + } // End of ja3Process #ifdef MAIN diff --git a/src/libnfdump/ja3/ja3.h b/src/libnfdump/ja3/ja3.h old mode 100644 new mode 100755 index a71b385d..d050b0cd --- a/src/libnfdump/ja3/ja3.h +++ b/src/libnfdump/ja3/ja3.h @@ -38,8 +38,9 @@ #define JA3DEFINED(ja3) (*((uint64_t *)ja3) != 0) #define JA3UNDEFINED(ja3) (*((uint64_t *)ja3) == 0) -uint8_t *ja3Process(ssl_t *ssl, uint8_t *hash); +#define OFFja3String 0 +#define SIZEja3String 32 -char *ja3String(uint8_t *ja3Hash); +char *ja3Process(ssl_t *ssl, char *buff); #endif diff --git a/src/libnfdump/ja4/ja4.c b/src/libnfdump/ja4/ja4.c old mode 100644 new mode 100755 index 0ed4e26c..776f6535 --- a/src/libnfdump/ja4/ja4.c +++ b/src/libnfdump/ja4/ja4.c @@ -86,16 +86,18 @@ int ja4Check(char *ja4String) { (s)[3] = hexChars[(u) & 0xF]; \ } -char *ja4Process(ssl_t *ssl, uint8_t proto, char *buff) { +ja4_t *ja4Process(ssl_t *ssl, uint8_t proto) { if (!ssl || ssl->type != CLIENTssl) return NULL; - if (buff == NULL) buff = malloc(SIZEja4String + 1); - if (buff == NULL) { + ja4_t *ja4 = malloc(sizeof(ja4_t) + SIZEja4String + 1); + if (ja4 == NULL) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); return NULL; } - buff[0] = '\0'; + ja4->type = TYPE_UNDEF; + ja4->string[0] = '\0'; + char *buff = ja4->string; // create ja4_a buff[0] = proto == IPPROTO_TCP ? 't' : 'q'; buff[1] = ssl->tlsCharVersion[0]; @@ -105,7 +107,7 @@ char *ja4Process(ssl_t *ssl, uint8_t proto, char *buff) { uint32_t num = LenArray(ssl->cipherSuites); if (num > 99) { - buff[0] = '\0'; + free(ja4); return NULL; } uint32_t ones = num % 10; @@ -115,7 +117,7 @@ char *ja4Process(ssl_t *ssl, uint8_t proto, char *buff) { num = LenArray(ssl->extensions); if (num > 99) { - buff[0] = '\0'; + free(ja4); return NULL; } ones = num % 10; @@ -198,9 +200,10 @@ char *ja4Process(ssl_t *ssl, uint8_t proto, char *buff) { HexString(sha256Digest, 6, buff + 24); buff[36] = '\0'; + ja4->type = TYPE_JA4; free(hashString); - return buff; + return ja4; } // End of DecodeJA4 @@ -276,7 +279,7 @@ int main(int argc, char **argv) { 0x00, 0x00, 0x00 // ... }; /* - [JA4: t13d1715h2_5b57614c22b0_3d5424432f57] + JA4: t13d1715h2_5b57614c22b0_3d5424432f57 JA4_r: t13d1715h2_002f,0035,009c,009d,1301,1302,1303,c009,c00a,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0015,0017,001c,0022,0023,002b,002d,0033,ff01_0403,0503,0603,0804,0805,0806,0401,0501,0601,0203,0201] @@ -293,9 +296,9 @@ int main(int argc, char **argv) { exit(255); } - char buff[SIZEja4]; - if (ja4Process(ssl, IPPROTO_TCP, buff)) - printf("ja4: %s\n", buff); + ja4_t *ja4 = ja4Process(ssl, IPPROTO_TCP); + if (ja4) + printf("ja4: %s\n", ja4->string); else printf("Failed to parse ja4\n"); diff --git a/src/libnfdump/ja4/ja4.h b/src/libnfdump/ja4/ja4.h old mode 100644 new mode 100755 index 5cb16518..f4143c48 --- a/src/libnfdump/ja4/ja4.h +++ b/src/libnfdump/ja4/ja4.h @@ -54,11 +54,13 @@ _ */ -enum { TYPE_JA4 = 1, TYPE_JA4S }; +typedef enum { TYPE_UNDEF = 0, + TYPE_JA4, + TYPE_JA4S } ja4Type_t; // ex. t13d1516h2_8daaf6152771_b186095e22bb typedef struct ja4_s { - uint8_t type; + ja4Type_t type; char string[]; } ja4_t; #define OFFja4String offsetof(ja4_t, string) @@ -66,6 +68,24 @@ typedef struct ja4_s { int ja4Check(char *ja4String); -char *ja4Process(ssl_t *ssl, uint8_t proto, char *buff); +ja4_t *ja4Process(ssl_t *ssl, uint8_t proto); + +/* + + ------ Protocol, TCP = "I" QUIC= "q" + | + ---- TLS version, 1.2 = "12", 1.3 = "13" + | | + -- Number of Extensions + | | | +- ALPN Chosen (00 if no ALPN) + | | | | +- Cipher Suite Chosen + | | | | | + | | | | | +- Truncated SHA256 hash of the Extensions, in the order they appear + | | | | | | + JA45=t120400_C030_4e8089608790 + ja4s_a ja4s_b ja4s_c + +*/ + +#define SIZEja4sString 25 + +ja4_t *ja4sProcess(ssl_t *ssl, uint8_t proto); #endif diff --git a/src/libnfdump/ja4/ja4s.c b/src/libnfdump/ja4/ja4s.c old mode 100644 new mode 100755 index 5fe0a4c2..1370e308 --- a/src/libnfdump/ja4/ja4s.c +++ b/src/libnfdump/ja4/ja4s.c @@ -28,8 +28,6 @@ * */ -#include "ja4s.h" - #include #include #include @@ -40,41 +38,45 @@ #include #include "digest/sha256.h" +#include "ja4.h" #include "ssl/ssl.h" #include "util.h" #define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) -ja4s_t *ja4sProcess(ssl_t *ssl, uint8_t proto) { +ja4_t *ja4sProcess(ssl_t *ssl, uint8_t proto) { if (!ssl || ssl->type != SERVERssl) return NULL; - ja4s_t *ja4s = calloc(1, sizeof(ja4s_t)); - if (!ja4s) { - LogError("calloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); + ja4_t *ja4 = malloc(sizeof(ja4_t) + SIZEja4sString + 1); + if (ja4 == NULL) { + LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); return NULL; } + ja4->type = TYPE_UNDEF; + ja4->string[0] = '\0'; + char *buff = ja4->string; // create ja4s_a - ja4s->a[0] = proto == IPPROTO_TCP ? 't' : 'q'; - ja4s->a[1] = ssl->tlsCharVersion[0]; - ja4s->a[2] = ssl->tlsCharVersion[1]; + buff[0] = proto == IPPROTO_TCP ? 't' : 'q'; + buff[1] = ssl->tlsCharVersion[0]; + buff[2] = ssl->tlsCharVersion[1]; uint32_t num = LenArray(ssl->extensions); if (num > 99) return 0; uint32_t ones = num % 10; uint32_t tens = num / 10; - ja4s->a[3] = tens + '0'; - ja4s->a[4] = ones + '0'; + buff[3] = tens + '0'; + buff[4] = ones + '0'; if (ssl->alpnName[0]) { // first and last char - ja4s->a[5] = ssl->alpnName[0]; - ja4s->a[6] = ssl->alpnName[strlen(ssl->alpnName) - 1]; + buff[5] = ssl->alpnName[0]; + buff[6] = ssl->alpnName[strlen(ssl->alpnName) - 1]; } else { - ja4s->a[5] = '0'; - ja4s->a[6] = '0'; + buff[5] = '0'; + buff[6] = '0'; } - ja4s->a[7] = '\0'; + buff[7] = '_'; // create ja4s_b char hexChars[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; @@ -83,15 +85,11 @@ ja4s_t *ja4sProcess(ssl_t *ssl, uint8_t proto) { if (numCipher == 1) { cipher = ArrayElement(ssl->cipherSuites, 0); } - uint8_t n1 = cipher >> 12; - uint8_t n2 = (cipher >> 8) & 0xF; - uint8_t n3 = (cipher >> 4) & 0xF; - uint8_t n4 = cipher & 0xF; - ja4s->b[0] = hexChars[n1]; - ja4s->b[1] = hexChars[n2]; - ja4s->b[2] = hexChars[n3]; - ja4s->b[3] = hexChars[n4]; - ja4s->b[4] = '\0'; + buff[8] = hexChars[cipher >> 12]; + buff[9] = hexChars[(cipher >> 8) & 0xF]; + buff[10] = hexChars[(cipher >> 4) & 0xF]; + buff[11] = hexChars[cipher & 0xF]; + buff[12] = '_'; // create ja4s_c // generate string to sha256 @@ -104,14 +102,10 @@ ja4s_t *ja4sProcess(ssl_t *ssl, uint8_t proto) { uint32_t index = 0; for (int i = 0; i < LenArray(ssl->extensions); i++) { uint16_t val = ArrayElement(ssl->extensions, i); - uint8_t n1 = val >> 12; - uint8_t n2 = (val >> 8) & 0xF; - uint8_t n3 = (val >> 4) & 0xF; - uint8_t n4 = val & 0xF; - hashString[index++] = hexChars[n1]; - hashString[index++] = hexChars[n2]; - hashString[index++] = hexChars[n3]; - hashString[index++] = hexChars[n4]; + hashString[index++] = hexChars[val >> 12]; + hashString[index++] = hexChars[(val >> 8) & 0xF]; + hashString[index++] = hexChars[(val >> 4) & 0xF]; + hashString[index++] = hexChars[val & 0xF]; hashString[index++] = ','; } // overwrite last ',' with end of string @@ -129,35 +123,14 @@ ja4s_t *ja4sProcess(ssl_t *ssl, uint8_t proto) { HexString(sha256Digest, 6, sha256String); #endif - memcpy((void *)ja4s->c, (void *)sha256String, 12); - ja4s->c[13] = '\0'; + memcpy((void *)(buff + 13), (void *)sha256String, 12); + buff[25] = '\0'; - return ja4s; + ja4->type = TYPE_JA4S; + return ja4; } // End of ja4Process -char *ja4sString(ja4s_t *ja4s, char *buff) { -#define JA4SLEN 32 - if (buff == NULL) buff = malloc(JA4SLEN); - if (buff == NULL) { - LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); - return NULL; - } - // - snprintf(buff, JA4SLEN - 1, "%s_%s_%s", ja4s->a, ja4s->b, ja4s->c); - buff[JA4SLEN - 1] = '\0'; - return buff; -} // End of ja4sString - -void ja4sPrint(ja4s_t *ja4s) { - char buff[JA4SLEN]; - printf("ja4s: %s\n", ja4sString(ja4s, buff)); -} // End of ja4Print - -void ja4sFree(ja4s_t *ja4s) { - if (ja4s) free(ja4s); -} // End of ja4Free - #ifdef MAIN int main(int argc, char **argv) { @@ -342,6 +315,8 @@ int main(int argc, char **argv) { 0xb5, 0x7d // .} }; + // ja4s: t130200_1301_234ea6891581 + size_t len = sizeof(srvHello); ssl_t *ssl = sslProcess(srvHello, len); @@ -349,9 +324,9 @@ int main(int argc, char **argv) { printf("Failed to parse ssl\n"); exit(255); } - ja4s_t *ja4s = ja4sProcess(ssl, IPPROTO_TCP); - if (ja4s) - ja4sPrint(ja4s); + ja4_t *ja4 = ja4sProcess(ssl, IPPROTO_TCP); + if (ja4) + printf("ja4s: %s\n", ja4->string); else printf("Failed to parse ja4s\n"); diff --git a/src/libnfdump/ja4/ja4s.h b/src/libnfdump/ja4/ja4s.h deleted file mode 100644 index 49758bd7..00000000 --- a/src/libnfdump/ja4/ja4s.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2024, Peter Haag - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * * Neither the name of SWITCH nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef _jA4S_H -#define _jA4S_H 1 - -#include -#include - -#include "ssl/ssl.h" - -/* - + ------ Protocol, TCP = "I" QUIC= "q" - | + ---- TLS version, 1.2 = "12", 1.3 = "13" - | | + -- Number of Extensions - | | | +- ALPN Chosen (00 if no ALPN) - | | | | +- Cipher Suite Chosen - | | | | | - | | | | | +- Truncated SHA256 hash of the Extensions, in the order they appear - | | | | | | - JA45=t120400_C030_4e8089608790 - ja4s_a ja4s_b ja4s_c - - -*/ -typedef struct ja4s_s { - char a[8]; // max 7 chars + '\0' - char b[8]; // max 4 chars + '\0' - char c[14]; // max 12 chars + '\0' -} ja4s_t; - -ja4s_t *ja4sProcess(ssl_t *ssl, uint8_t proto); - -char *ja4sString(ja4s_t *ja4s, char *buff); - -void ja4sPrint(ja4s_t *ja4s); - -void ja4sFree(ja4s_t *ja4s); -#endif diff --git a/src/nfdump/nflowcache.c b/src/nfdump/nflowcache.c index 7065dc83..173c45d9 100755 --- a/src/nfdump/nflowcache.c +++ b/src/nfdump/nflowcache.c @@ -49,7 +49,6 @@ #include "blocksort.h" #include "config.h" #include "exporter.h" -#include "ja3/ja3.h" #include "khash.h" #include "klist.h" #include "maxmind/maxmind.h" @@ -60,7 +59,12 @@ #include "output.h" #include "util.h" -typedef enum { NOPREPROCESS = 0, SRC_GEO, DST_GEO, SRC_AS, DST_AS, JA3 } preprocess_t; +typedef enum { NOPREPROCESS = 0, + SRC_GEO, + DST_GEO, + SRC_AS, + DST_AS +} preprocess_t; typedef struct aggregate_param_s { uint32_t extID; // extension ID @@ -177,7 +181,9 @@ typedef struct FlowHashRecord { } FlowHashRecord_t; // printing order definitions -typedef enum FlowDir { IN = 0, OUT, INOUT } flowDir_t; +typedef enum FlowDir { IN = 0, + OUT, + INOUT } flowDir_t; typedef uint64_t (*order_proc_record_t)(FlowHashRecord_t *, flowDir_t); @@ -379,21 +385,6 @@ static inline void PreProcess(void *inPtr, preprocess_t process, recordHandle_t if (HasGeoDB == 0 || *as) return; *as = ipv4Flow ? LookupV4AS(ipv4Flow->dstAddr) : (ipv6Flow ? LookupV6AS(ipv6Flow->dstAddr) : 0); } break; - case JA3: { - EXinPayload_t *payload = (EXinPayload_t *)recordHandle->extensionList[EXinPayloadID]; - if (payload == NULL) return; - - uint32_t payloadLength = ExtensionLength(payload); - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; - if (JA3DEFINED(recordHandle->ja3)) { - if (ssl == NULL) { - ssl = sslProcess((const uint8_t *)payload, payloadLength); - recordHandle->sslInfo = (void *)ssl; - } - ja3Process(ssl, recordHandle->ja3); - } - - } break; } } // End of PreProcess diff --git a/src/nfdump/nfstat.c b/src/nfdump/nfstat.c old mode 100644 new mode 100755 index bfe8ff7f..82f42915 --- a/src/nfdump/nfstat.c +++ b/src/nfdump/nfstat.c @@ -50,7 +50,6 @@ #include "config.h" #include "ja3/ja3.h" #include "ja4/ja4.h" -#include "ja4/ja4s.h" #include "khash.h" #include "maxmind/maxmind.h" #include "nfdump.h" @@ -76,9 +75,18 @@ typedef enum { IS_GEO } elementType_t; -typedef enum { NOPROC = 0, SRC_GEO, DST_GEO, SRC_AS, DST_AS, PREV_AS, NEXT_AS, JA3, JA4 } preprocess_t; +typedef enum { NOPROC = 0, + SRC_GEO, + DST_GEO, + SRC_AS, + DST_AS, + PREV_AS, + NEXT_AS, + JA3, + JA4 } preprocess_t; -typedef enum { DESCENDING = 0, ASCENDING } direction_t; +typedef enum { DESCENDING = 0, + ASCENDING } direction_t; typedef struct flow_element_s { uint32_t extID; // extension ID @@ -181,8 +189,8 @@ struct StatParameter_s { {"sl", "Server Latency", {EXlatencyID, OFFusecServerNwDelay, SIZEusecServerNwDelay, 0}, IS_LATENCY, NOPROC}, {"al", "Application Latency", {EXlatencyID, OFFusecApplLatency, SIZEusecApplLatency, 0}, IS_LATENCY, NOPROC}, {"nbar", "Nbar", {EXnbarAppID, OFFnbarAppID, SIZEnbarAppID, 0}, IS_NBAR, NOPROC}, - {"ja3", " ja3", {EXlocal, OFFja3, SIZEja3, 0}, IS_JA3, JA3}, - {"ja4", " ja4", {EXlocal, 0, SIZEja4String, 0}, IS_JA4, JA4}, + {"ja3", " ja3", {JA3index, OFFja3String, SIZEja3String + 1, 0}, IS_JA3, JA3}, + {"ja4", " ja4", {JA4index, OFFja4String, SIZEja4String + 1, 0}, IS_JA4, JA4}, {"odid", "Obs DomainID", {EXobservationID, OFFdomainID, SIZEdomainID, 0}, IS_HEXNUMBER, NOPROC}, {"opid", "Obs PointID", {EXobservationID, OFFpointID, SIZEpointID, 0}, IS_HEXNUMBER, NOPROC}, {"event", " Event", {EXnselCommonID, OFFfwEvent, SIZEfwEvent, 0}, IS_EVENT, NOPROC}, @@ -240,7 +248,9 @@ typedef struct StatRecord { * pps, bps and bpp are not directly available in the flow/stat record * therefore we need a function to calculate these values */ -typedef enum flowDir { IN = 0, OUT, INOUT } flowDir_t; +typedef enum flowDir { IN = 0, + OUT, + INOUT } flowDir_t; typedef uint64_t (*order_proc_element_t)(StatRecord_t *, flowDir_t); static inline uint64_t null_element(StatRecord_t *record, flowDir_t inout); @@ -527,37 +537,46 @@ static inline void *PreProcess(void *inPtr, preprocess_t process, recordHandle_t case NEXT_AS: { } break; case JA3: { - EXinPayload_t *payload = (EXinPayload_t *)recordHandle->extensionList[EXinPayloadID]; - if (payload == NULL || genericFlow->proto != IPPROTO_TCP || JA3DEFINED(recordHandle->ja3)) return inPtr; + const uint8_t *payload = (const uint8_t *)recordHandle->extensionList[EXinPayloadID]; + if (payload == NULL || genericFlow->proto != IPPROTO_TCP || inPtr) return inPtr; - uint32_t payloadLength = ExtensionLength(payload); - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; + ssl_t *ssl = recordHandle->extensionList[SSLindex]; if (ssl == NULL) { - ssl = sslProcess((const uint8_t *)payload, payloadLength); - recordHandle->sslInfo = (void *)ssl; + uint32_t payloadLength = ExtensionLength(payload); + ssl = sslProcess(payload, payloadLength); + recordHandle->extensionList[SSLindex] = ssl; + if (ssl == NULL) { + return NULL; + } } - ja3Process(ssl, recordHandle->ja3); - return inPtr; + // ssl is defined + char *ja3 = ja3Process(ssl, NULL); + recordHandle->extensionList[JA3index] = ja3; + return ja3; } break; case JA4: { EXinPayload_t *payload = (EXinPayload_t *)recordHandle->extensionList[EXinPayloadID]; if (payload == NULL || genericFlow->proto != IPPROTO_TCP) return NULL; - uint32_t payloadLength = ExtensionLength(payload); - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; + ssl_t *ssl = recordHandle->extensionList[SSLindex]; if (ssl == NULL) { - ssl = sslProcess((const uint8_t *)payload, payloadLength); - if (ssl == NULL || ssl->type != CLIENTssl) return NULL; - recordHandle->sslInfo = (void *)ssl; + uint32_t payloadLength = ExtensionLength(payload); + ssl = sslProcess(payload, payloadLength); + recordHandle->extensionList[SSLindex] = ssl; + if (ssl == NULL) { + return NULL; + } } // ssl is defined - static char ja4StrBuff[40]; - if (ja4Process(ssl, genericFlow->proto, ja4StrBuff)) { - inPtr = (void *)ja4StrBuff; - return inPtr; + ja4_t *ja4 = NULL; + if (ssl->type == CLIENTssl) { + ja4 = ja4Process(ssl, genericFlow->proto); } else { - return NULL; + ja4 = ja4sProcess(ssl, genericFlow->proto); } + recordHandle->extensionList[JA4index] = ja4; + return ja4; + } break; } @@ -580,12 +599,12 @@ void AddElementStat(recordHandle_t *recordHandle) { void *inPtr = recordHandle->extensionList[extID]; if (inPtr == NULL) { - index++; - continue; + if (extID <= MAXEXTENSIONS) { + index++; + continue; + } } - inPtr += offset; - uint32_t length = StatParameters[index].element.length; preprocess_t lookup = StatParameters[index].preprocess; inPtr = PreProcess(inPtr, lookup, recordHandle); if (inPtr == NULL) { @@ -593,6 +612,8 @@ void AddElementStat(recordHandle_t *recordHandle) { continue; } + inPtr += offset; + uint32_t length = StatParameters[index].element.length; switch (length) { case 0: break; @@ -757,16 +778,8 @@ static void PrintStatLine(stat_record_t *stat, outputParams_t *outputParams, Sta } break; case IS_JA3: { - uint8_t *u8 = (uint8_t *)&(StatData->hashkey.v0); - int i, j; - for (i = 0, j = 0; i < 16; i++, j += 2) { - uint8_t ln = u8[i] & 0xF; - uint8_t hn = (u8[i] >> 4) & 0xF; - valstr[j + 1] = ln <= 9 ? ln + '0' : ln + 'a' - 10; - valstr[j] = hn <= 9 ? hn + '0' : hn + 'a' - 10; - } - valstr[32] = '\0'; - + char *s = (char *)StatData->hashkey.ptr; + strcpy(valstr, s); } break; case IS_JA4: { char *s = (char *)StatData->hashkey.ptr; diff --git a/src/output/output_fmt.c b/src/output/output_fmt.c old mode 100644 new mode 100755 index da5f70ce..08aa69d6 --- a/src/output/output_fmt.c +++ b/src/output/output_fmt.c @@ -48,7 +48,6 @@ #include "ifvrf.h" #include "ja3/ja3.h" #include "ja4/ja4.h" -#include "ja4/ja4s.h" #include "maxmind/maxmind.h" #include "nbar.h" #include "nfdump.h" @@ -1056,85 +1055,89 @@ static void String_nbarName(FILE *stream, recordHandle_t *recordHandle) { } // End of String_nbarName static void String_ja3(FILE *stream, recordHandle_t *recordHandle) { - EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; - EXinPayload_t *payload = (EXinPayload_t *)recordHandle->extensionList[EXinPayloadID]; - + const uint8_t *payload = (uint8_t *)(recordHandle->extensionList[EXinPayloadID]); + EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)(recordHandle->extensionList[EXgenericFlowID]); if (payload == NULL || genericFlow->proto != IPPROTO_TCP) { fprintf(stream, "%38s", "no ja3"); return; } - uint32_t payloadLength = ExtensionLength(payload); - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; - if (JA3UNDEFINED(recordHandle->ja3)) { + char *ja3 = recordHandle->extensionList[JA3index]; + ssl_t *ssl = recordHandle->extensionList[SSLindex]; + if (ja3 == NULL) { if (ssl == NULL) { - ssl = sslProcess((const uint8_t *)payload, payloadLength); - recordHandle->sslInfo = (void *)ssl; + uint32_t payloadLength = ExtensionLength(payload); + ssl = sslProcess(payload, payloadLength); + ja3 = ja3Process(ssl, NULL); + recordHandle->extensionList[SSLindex] = ssl; + recordHandle->extensionList[JA3index] = ja3; + if (ssl == NULL || ja3 == NULL) { + fprintf(stream, "%38s", "no ja3"); + return; + } } - ja3Process(ssl, recordHandle->ja3); } - if (ssl) { - if (ssl->type == CLIENTssl) - fprintf(stream, "ja3 : %32s", ja3String(recordHandle->ja3)); - else - fprintf(stream, "ja3s: %32s", ja3String(recordHandle->ja3)); - } else { - fprintf(stream, "%32s", ""); - } + if (ssl->type == CLIENTssl) + fprintf(stream, "ja3 : %32s", ja3); + else + fprintf(stream, "ja3s: %32s", ja3); } // End of String_ja3 static void String_ja4(FILE *stream, recordHandle_t *recordHandle) { + const uint8_t *payload = (const uint8_t *)recordHandle->extensionList[EXinPayloadID]; EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; - EXinPayload_t *payload = (EXinPayload_t *)recordHandle->extensionList[EXinPayloadID]; - if (payload == NULL || genericFlow->proto != IPPROTO_TCP) { fprintf(stream, "%38s", "no ja4"); return; } - uint32_t payloadLength = ExtensionLength(payload); - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; - if (ssl == NULL) { - ssl = sslProcess((const uint8_t *)payload, payloadLength); - recordHandle->sslInfo = (void *)ssl; - } - - if (ssl == NULL) { - fprintf(stream, "%38s", "no ja4"); - return; + ja4_t *ja4 = recordHandle->extensionList[JA4index]; + ssl_t *ssl = recordHandle->extensionList[SSLindex]; + if (ja4 == NULL) { + if (ssl == NULL) { + uint32_t payloadLength = ExtensionLength(payload); + ssl = sslProcess(payload, payloadLength); + ja4 = ja4Process(ssl, genericFlow->proto); + recordHandle->extensionList[SSLindex] = ssl; + recordHandle->extensionList[JA4index] = ja4; + if (ssl == NULL || ja4 == NULL) { + fprintf(stream, "%38s", "no ja4"); + return; + } + } } - char buff[64]; - if (ssl->type == CLIENTssl) { - if (ja4Process(ssl, genericFlow->proto, buff)) fprintf(stream, "ja4 : %32s", buff); + // ja4 is defined + if (ja4->type == TYPE_JA4) { + fprintf(stream, "ja4 : %32s", ja4->string); } else { - ja4s_t *ja4s = ja4sProcess(ssl, genericFlow->proto); - fprintf(stream, "ja4s: %32s", ja4sString(ja4s, buff)); + fprintf(stream, "ja4s: %32s", ja4->string); } } // End of String_ja4 static void String_tlsVersion(FILE *stream, recordHandle_t *recordHandle) { EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; - EXinPayload_t *payload = (EXinPayload_t *)recordHandle->extensionList[EXinPayloadID]; + const uint8_t *payload = (const uint8_t *)recordHandle->extensionList[EXinPayloadID]; if (payload == NULL || genericFlow->proto != IPPROTO_TCP) { fprintf(stream, " 0"); return; } - uint32_t payloadLength = ExtensionLength(payload); - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; - if (ssl == NULL) { - ssl = sslProcess((uint8_t *)payload, payloadLength); - recordHandle->sslInfo = (void *)ssl; - } + ssl_t *ssl = recordHandle->extensionList[SSLindex]; if (ssl == NULL) { - fprintf(stream, " 0"); - return; + uint32_t payloadLength = ExtensionLength(payload); + ssl = sslProcess(payload, payloadLength); + recordHandle->extensionList[SSLindex] = ssl; + if (ssl == NULL) { + fprintf(stream, " 0"); + return; + } } + /* 0x0304 = TLS 1.3 = “13” 0x0303 = TLS 1.2 = “12” @@ -1145,6 +1148,7 @@ static void String_tlsVersion(FILE *stream, recordHandle_t *recordHandle) { 0x0100 = SSL 1.0 = “s1” */ + // ssl is defined switch (ssl->tlsCharVersion[0]) { case 0: fprintf(stream, " 0"); @@ -1163,20 +1167,26 @@ static void String_tlsVersion(FILE *stream, recordHandle_t *recordHandle) { } // End of String_tlsVersion static void String_sniName(FILE *stream, recordHandle_t *recordHandle) { - EXinPayload_t *payload = (EXinPayload_t *)recordHandle->extensionList[EXinPayloadID]; + EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; + const uint8_t *payload = (const uint8_t *)recordHandle->extensionList[EXinPayloadID]; - if (payload == NULL) { - fprintf(stream, "%6s", ""); + if (payload == NULL || genericFlow->proto != IPPROTO_TCP) { + fprintf(stream, " 0"); return; } - uint32_t payloadLength = ExtensionLength(payload); - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; + ssl_t *ssl = recordHandle->extensionList[SSLindex]; if (ssl == NULL) { - ssl = sslProcess((uint8_t *)payload, payloadLength); - recordHandle->sslInfo = (void *)ssl; + uint32_t payloadLength = ExtensionLength(payload); + ssl = sslProcess(payload, payloadLength); + recordHandle->extensionList[SSLindex] = ssl; + if (ssl == NULL) { + fprintf(stream, " 0"); + return; + } } + // ssl is defined fprintf(stream, "%6s", ssl != NULL ? ssl->sniName : ""); } // End of String_sniName diff --git a/src/output/output_json.c b/src/output/output_json.c old mode 100644 new mode 100755 index 28d07c17..2fa1703e --- a/src/output/output_json.c +++ b/src/output/output_json.c @@ -46,7 +46,6 @@ #include "ja3/ja3.h" #include "ja4/ja4.h" -#include "ja4/ja4s.h" #include "maxmind/maxmind.h" #include "nfdump.h" #include "nffile.h" @@ -403,26 +402,28 @@ static void stringEXlatency(FILE *stream, void *extensionRecord, const char *ind } // End of stringEXlatency static void String_payload(FILE *stream, recordHandle_t *recordHandle, void *extensionRecord, const char *indent, const char *fs) { + const uint8_t *payload = (const uint8_t *)extensionRecord; EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID]; - EXinPayload_t *payload = (EXinPayload_t *)extensionRecord; - uint32_t payloadLength = ExtensionLength(payload); // payload handled in output json: // ssl, ja3, ja4 - if (payloadLength == 0 || genericFlow->proto != IPPROTO_TCP) { + if (payload == NULL || genericFlow->proto != IPPROTO_TCP) { return; } - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; - if (JA3UNDEFINED(recordHandle->ja3)) { + ssl_t *ssl = recordHandle->extensionList[SSLindex]; + if (ssl == NULL) { + uint32_t payloadLength = ExtensionLength(payload); + ssl = sslProcess(payload, payloadLength); + recordHandle->extensionList[SSLindex] = ssl; if (ssl == NULL) { - ssl = sslProcess((const uint8_t *)payload, payloadLength); - recordHandle->sslInfo = (void *)ssl; + return; } - ja3Process(ssl, recordHandle->ja3); } + // ssl is defined + if (ssl) { switch (ssl->tlsCharVersion[0]) { case 's': @@ -439,20 +440,33 @@ static void String_payload(FILE *stream, recordHandle_t *recordHandle, void *ext if (ssl->sniName[0]) { fprintf(stream, "%s\"sni\" : %s%s\n", indent, ssl->sniName, fs); } - if (JA3DEFINED(recordHandle->ja3)) { - fprintf(stream, "%s\"ja3 hash\" : %s%s\n", indent, ja3String(recordHandle->ja3), fs); - } + } - char buff[64]; - if (ssl->type == CLIENTssl) { - if (ja4Process(ssl, genericFlow->proto, buff)) fprintf(stream, "%s\"ja4 hash\" : %s%s\n", indent, buff, fs); + char *ja3 = recordHandle->extensionList[JA3index]; + if (ja3 == NULL) { + ja3 = ja3Process(ssl, NULL); + recordHandle->extensionList[JA3index] = ja3; + } + if (ja3) { + fprintf(stream, "%s\"ja3 hash\" : %s%s\n", indent, ja3, fs); + } + ja4_t *ja4 = recordHandle->extensionList[JA4index]; + if (ja4 == NULL) { + if (ssl->type == CLIENTssl) { + ja4 = ja4Process(ssl, genericFlow->proto); } else { - if (JA3DEFINED(recordHandle->ja3)) fprintf(stream, " ja3s hash = %s\n", ja3String(recordHandle->ja3)); - ja4s_t *ja4s = ja4sProcess(ssl, genericFlow->proto); - if (ja4s) fprintf(stream, "%s\"ja4s hash\" : %s%s\n", indent, ja4sString(ja4s, buff), fs); + ja4 = ja4sProcess(ssl, genericFlow->proto); } + recordHandle->extensionList[JA4index] = ja4; } + if (ja4 == NULL) return; + + // ja4 is defined + if (ja4->type == TYPE_JA4) + fprintf(stream, "%s\"ja4 hash\" : %s%s\n", indent, ja4->string, fs); + else + fprintf(stream, "%s\"ja4s hash\" : %s%s\n", indent, ja4->string, fs); } // End of String_payload diff --git a/src/output/output_raw.c b/src/output/output_raw.c old mode 100644 new mode 100755 index 28ac33cf..0b54b6f2 --- a/src/output/output_raw.c +++ b/src/output/output_raw.c @@ -47,7 +47,6 @@ #include "ifvrf.h" #include "ja3/ja3.h" #include "ja4/ja4.h" -#include "ja4/ja4s.h" #include "maxmind/maxmind.h" #include "nbar.h" #include "nfdump.h" @@ -680,6 +679,7 @@ static void inoutPayload(FILE *stream, recordHandle_t *recordHandle, uint8_t *pa static void stringsEXinPayload(FILE *stream, recordHandle_t *recordHandle, void *extensionRecord) { EXinPayload_t *inPayload = (EXinPayload_t *)extensionRecord; + if (!inPayload) return; uint32_t payloadLength = ExtensionLength(inPayload); fprintf(stream, " in payload = %10u\n", payloadLength); @@ -688,6 +688,7 @@ static void stringsEXinPayload(FILE *stream, recordHandle_t *recordHandle, void static void stringsEXoutPayload(FILE *stream, recordHandle_t *recordHandle, void *extensionRecord) { EXoutPayload_t *outPayload = (EXoutPayload_t *)extensionRecord; + if (!outPayload) return; uint32_t payloadLength = ExtensionLength(outPayload); fprintf(stream, " out payload = %10u\n", payloadLength); @@ -712,40 +713,59 @@ static void inoutPayload(FILE *stream, recordHandle_t *recordHandle, uint8_t *pa if (ascii) { fprintf(stream, "%.*s\n", max, payload); } else if (genericFlow->proto == IPPROTO_TCP) { - ssl_t *ssl = (ssl_t *)recordHandle->sslInfo; - if (JA3UNDEFINED(recordHandle->ja3)) { + ssl_t *ssl = recordHandle->extensionList[SSLindex]; + if (ssl == NULL) { + ssl = sslProcess(payload, length); + recordHandle->extensionList[SSLindex] = ssl; if (ssl == NULL) { - ssl = sslProcess((const uint8_t *)payload, length); - recordHandle->sslInfo = (void *)ssl; + DumpHex(stream, payload, max); + return; } - ja3Process(ssl, recordHandle->ja3); } - if (ssl) { - switch (ssl->tlsCharVersion[0]) { - case 's': - fprintf(stream, " TLS version = SSL %c \n", ssl->tlsCharVersion[1]); - break; - case '1': - fprintf(stream, " TLS version = TLS 1.%c\n", ssl->tlsCharVersion[1]); - break; - default: - fprintf(stream, " TLS version = 0x%4x\n", ssl->tlsVersion); - break; - } + // ssl is defined + switch (ssl->tlsCharVersion[0]) { + case 's': + fprintf(stream, " TLS version = SSL %c \n", ssl->tlsCharVersion[1]); + break; + case '1': + fprintf(stream, " TLS version = TLS 1.%c\n", ssl->tlsCharVersion[1]); + break; + default: + fprintf(stream, " TLS version = 0x%4x\n", ssl->tlsVersion); + break; + } - if (ssl->sniName[0]) fprintf(stream, " sni name = %s\n", ssl->sniName); + if (ssl->sniName[0]) fprintf(stream, " sni name = %s\n", ssl->sniName); - char buff[64]; + char *ja3 = recordHandle->extensionList[JA3index]; + if (ja3 == NULL) { + ja3 = ja3Process(ssl, NULL); + recordHandle->extensionList[JA3index] = ja3; + } + if (ja3) { if (ssl->type == CLIENTssl) { - if (JA3DEFINED(recordHandle->ja3)) fprintf(stream, " ja3 hash = %s\n", ja3String(recordHandle->ja3)); - if (ja4Process(ssl, genericFlow->proto, buff)) fprintf(stream, " ja4 hash = %s\n", buff); + fprintf(stream, " ja3 hash = %s\n", ja3); + } else { + fprintf(stream, " ja3s hash = %s\n", ja3); + } + } + ja4_t *ja4 = recordHandle->extensionList[JA4index]; + if (ja4 == NULL) { + if (ssl->type == CLIENTssl) { + ja4 = ja4Process(ssl, genericFlow->proto); } else { - if (JA3DEFINED(recordHandle->ja3)) fprintf(stream, " ja3s hash = %s\n", ja3String(recordHandle->ja3)); - ja4s_t *ja4s = ja4sProcess(ssl, genericFlow->proto); - if (ja4s) fprintf(stream, " ja4s hash = %s\n", ja4sString(ja4s, buff)); + ja4 = ja4sProcess(ssl, genericFlow->proto); } + recordHandle->extensionList[JA4index] = ja4; + } + + if (ja4) { + if (ja4->type == TYPE_JA4) + fprintf(stream, " ja4 hash = %s\n", ja4->string); + else + fprintf(stream, " ja4s hash = %s\n", ja4->string); } }