Skip to content

Commit

Permalink
Streamline ja3, ja4, ssl code
Browse files Browse the repository at this point in the history
  • Loading branch information
@phaag
phaag committed Mar 15, 2024
1 parent d5220a9 commit dedb73a
Show file tree
Hide file tree
Showing 15 changed files with 335 additions and 374 deletions.
21 changes: 5 additions & 16 deletions src/include/nfdump.h
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,11 @@ typedef struct ip_addr_s {
typedef struct exporter_info_record_s exporter_info_record_t;
typedef struct extension_map_s extension_map_t;

#define EXlocal MAXEXTENSIONS
#define SSLindex MAXEXTENSIONS + 1
#define JA4index MAXEXTENSIONS + 2
#define MAXLISTSIZE MAXEXTENSIONS + 3
enum { EXlocal = MAXEXTENSIONS,
SSLindex,
JA3index,
JA4index,
MAXLISTSIZE };

typedef struct recordHandle_s {
recordHeaderV3_t *recordHeaderV3;
Expand All @@ -97,18 +98,6 @@ typedef struct recordHandle_s {
#define OFFgeoSrcTunIP offsetof(recordHandle_t, geo) + 8
#define OFFgeoDstTunIP offsetof(recordHandle_t, geo) + 10
#define SizeGEOloc 2
void *sslInfo;
#define OFFsslInfo offsetof(recordHandle_t, sslInfo)
#define SIZEsslInfo MemberSize(recordHandle_t, sslInfo)
uint8_t ja3[16];
#define OFFja3 offsetof(recordHandle_t, ja3)
#define SIZEja3 MemberSize(recordHandle_t, ja3)
// for all ja4 hashes, ja4, ja4s, ja4h etc. there is one reference
// only, as a record can not have more thanone
void *ja4Info;
#define OFFja4Info offsetof(recordHandle_t, ja4Info)
#define SIZEja4Info MemberSize(recordHandle_t, ja4Info)
uint32_t ja4Type;
uint32_t flowCount;
#define OFFflowCount offsetof(recordHandle_t, flowCount)
#define SIZEflowCount MemberSize(recordHandle_t, flowCount)
Expand Down
8 changes: 5 additions & 3 deletions src/inline/nffile_inline.c
100644 → 100755
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2009-2023, Peter Haag
* Copyright (c) 2009-2024, Peter Haag
* Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung
* All rights reserved.
*
Expand Down Expand Up @@ -63,8 +63,10 @@ static inline size_t CheckBufferSpace(nffile_t *nffile, size_t required) {
} // End of CheckBufferSpace

static inline int MapRecordHandle(recordHandle_t *handle, recordHeaderV3_t *recordHeaderV3, uint32_t flowCount) {
if (handle->sslInfo) free(handle->sslInfo);
if (handle->ja4Info) free(handle->ja4Info);
if (handle->extensionList[SSLindex]) free(handle->extensionList[SSLindex]);
if (handle->extensionList[JA3index]) free(handle->extensionList[JA3index]);
if (handle->extensionList[JA4index]) free(handle->extensionList[JA4index]);

memset((void *)handle, 0, sizeof(recordHandle_t));
handle->recordHeaderV3 = recordHeaderV3;

Expand Down
48 changes: 19 additions & 29 deletions src/libnfdump/filter/filter.c
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,10 +108,10 @@ static uint64_t mpls_exp_function(void *dataPtr, uint32_t length, data_t data, r
static uint64_t mpls_any_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle);
static uint64_t pblock_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle);
static uint64_t mmASLookup_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle);
static uint64_t ja3_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle);

/* flow pre-processing functions */
static void *ssl_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle);
static void *ja3_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle);
static void *ja4_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle);

/*
Expand All @@ -133,12 +133,11 @@ static struct flow_procs_map_s {
{FUNC_MPLS_ANY, "mpls any", mpls_any_function},
{FUNC_PBLOCK, "pblock", pblock_function},
{FUNC_MMAS_LOOKUP, "AS Lockup", mmASLookup_function},
{FUNC_JA3, "ja3", ja3_function},
{0, NULL, NULL}};

static struct preprocess_s {
preprocess_proc_t function;
} preprocess_map[] = {{ssl_preproc}, {ja4_preproc}, {NULL}};
} preprocess_map[] = {{ssl_preproc}, {ja3_preproc}, {ja4_preproc}, {NULL}};

// 128bit compare for IPv6
static int IPNodeCMP(struct IPListNode *e1, struct IPListNode *e2) {
Expand Down Expand Up @@ -292,29 +291,6 @@ static uint64_t mmASLookup_function(void *dataPtr, uint32_t length, data_t data,
return as;
} // End of mmASLookup_function

static uint64_t ja3_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *recordHandle) {
const uint8_t *payload = (const uint8_t *)recordHandle->extensionList[EXinPayloadID];

// check if ja3 already exists or no payload exists
if (recordHandle->ja3[0] != '\0' || payload == NULL) return 1;

uint32_t len = ExtensionLength(payload);

// check if ssl record already exists
ssl_t *ssl = (ssl_t *)recordHandle->sslInfo;
if (!ssl) {
ssl = sslProcess(payload, len);
recordHandle->sslInfo = (void *)ssl;
}
uint8_t *ja3 = ja3Process(ssl, recordHandle->ja3);
if (ja3 == NULL) {
return 0;
}
// else - found a valid ja3 hash from the ssl handshake record
return 1;

} // End of ja3_function

static void *ssl_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle) {
const uint8_t *payload = (uint8_t *)(handle->extensionList[EXinPayloadID]);
if (payload == NULL) return NULL;
Expand All @@ -330,10 +306,25 @@ static void *ssl_preproc(void *dataPtr, uint32_t length, data_t data, recordHand

} // End of ssl_preproc

static void *ja3_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle) {
const uint8_t *payload = (const uint8_t *)handle->extensionList[EXinPayloadID];
if (payload == NULL) return NULL;

// return ja3 string if it already exists
if (handle->extensionList[JA3index]) return handle->extensionList[JA3index];

ssl_t *ssl = ssl_preproc(dataPtr, length, data, handle);
if (!ssl) return NULL;

return ja3Process(ssl, NULL);

} // End of ja3_preproc

static void *ja4_preproc(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle) {
const uint8_t *payload = (uint8_t *)(handle->extensionList[EXinPayloadID]);
if (payload == NULL) return NULL;

// return ja4 struct if it already exists
if (handle->extensionList[JA4index]) return handle->extensionList[JA4index];

EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)(handle->extensionList[EXgenericFlowID]);
Expand All @@ -345,12 +336,11 @@ static void *ja4_preproc(void *dataPtr, uint32_t length, data_t data, recordHand
LogError("malloc() error in %s line %d: %s", __FILE__, __LINE__, strerror(errno));
return NULL;
}
ja4->type = TYPE_JA4;
if (ja4Process(ssl, genericFlow->proto, ja4->string)) {
ja4 = ja4Process(ssl, genericFlow->proto);
if (ja4) {
handle->extensionList[JA4index] = (void *)ja4;
return (void *)ja4;
}
free(ja4);
return NULL;
} // End of ja4_preproc

Expand Down
29 changes: 12 additions & 17 deletions src/libnfdump/filter/grammar.y
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
#include "nfxV3.h"
#include "ipconv.h"
#include "sgregex.h"
#include "ja3/ja3.h"
#include "ja4/ja4.h"
#include "nfdump.h"

Expand Down Expand Up @@ -1222,32 +1223,26 @@ static int AddPayload(char *type, char *arg, char *opt) {
return NewElement(EXinPayloadID, 0, 0, 0, CMP_REGEX, FUNC_NONE, data);
} else if (strcasecmp(type, "ssl") == 0) {
if (strcasecmp(arg, "defined") == 0) {
return Invert(NewElement(EXlocal, OFFsslInfo, SIZEsslInfo, 0, CMP_EQ, FUNC_NONE, NULLPtr));
return Invert(NewElement(SSLindex, 0, 0, 0, CMP_EQ, FUNC_NONE, NULLPtr));
}
} else if (strcasecmp(type, "ja3") == 0) {
uint8_t *md5 = calloc(1,16);
if (!md5) {
yyerror("malloc() for md5 failed");
return -1;
}
data_t data = {.dataPtr=md5};
if (strcasecmp(arg, "defined") == 0) {
return Invert(NewElement(EXlocal, OFFja3, SIZEja3, 0, CMP_BINARY, FUNC_JA3, data));
return Invert(NewElement(JA3index, OFFja3String, SIZEja3String, 0, CMP_STRING, FUNC_NONE, NULLPtr));
} else {
if (IsMD5(arg) == 0) {
yyerror("ja3 string %s is not an MD5 sum", arg);
free(md5);
yyerror("String %s is not a valid ja3 string", arg);
return -1;
}
for(int count = 0; count < 16; count++) {
sscanf(arg, "%2hhx", &md5[count]);
arg += 2;
}
return NewElement(EXlocal, OFFja3, SIZEja3, 0, CMP_BINARY, FUNC_JA3, data);
data_t data = {.dataPtr=strdup(arg)};
return NewElement(JA3index, OFFja3String, SIZEja3String, 0, CMP_STRING, FUNC_NONE, data);
}
} else if (strcasecmp(type, "ja4") == 0) {
data_t data = {.dataPtr=strdup(arg)};
return NewElement(JA4index, OFFja4String, SIZEja4String, 0, CMP_STRING, FUNC_NONE, data);
if ( ja4Check(arg) == 0 ){
yyerror("String %s is not a valid ja4 string", arg);
return -1;
}
data_t data = {.dataPtr=strdup(arg)};
return NewElement(JA4index, OFFja4String, SIZEja4String, 0, CMP_STRING, FUNC_NONE, data);
} else {
yyerror("Unknown PAYLOAD argument: %s\n", type);
return -1;
Expand Down
36 changes: 20 additions & 16 deletions src/libnfdump/ja3/ja3.c
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,22 +53,29 @@
} \
}

char *ja3String(uint8_t *ja3Hash) {
static char out[36];
static char *ja3String(uint8_t *ja3Hash, char *buff) {
if (buff == NULL) {
buff = malloc(SIZEja3String + 1);
if (buff == NULL) {
LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno));
return NULL;
}
}
buff[0] = '\0';

int i, j;
for (i = 0, j = 0; i < 16; i++) {
uint8_t ln = ja3Hash[i] & 0xF;
uint8_t hn = (ja3Hash[i] >> 4) & 0xF;
out[j++] = hn <= 9 ? hn + '0' : hn + 'a' - 10;
out[j++] = ln <= 9 ? ln + '0' : ln + 'a' - 10;
buff[j++] = hn <= 9 ? hn + '0' : hn + 'a' - 10;
buff[j++] = ln <= 9 ? ln + '0' : ln + 'a' - 10;
}
out[j] = '\0';
buff[j] = '\0';

return out;
return buff;
} // End of ja3String

uint8_t *ja3Process(ssl_t *ssl, uint8_t *hash) {
char *ja3Process(ssl_t *ssl, char *buff) {
if (!ssl) return NULL;

size_t sLen = 6 * (1 + 1 + LenArray(ssl->cipherSuites) + 1 + LenArray(ssl->extensions) + 1 + LenArray(ssl->ellipticCurves) + 1 +
Expand All @@ -80,6 +87,7 @@ uint8_t *ja3Process(ssl_t *ssl, uint8_t *hash) {
LogError("calloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno));
return NULL;
}

char *s = ja3_r;
snprintf(s, sLen, "%u,", ssl->protocolVersion);
size_t len = strlen(s);
Expand Down Expand Up @@ -123,29 +131,25 @@ uint8_t *ja3Process(ssl_t *ssl, uint8_t *hash) {

if (sLen == 0) {
LogError("sLen error in %s line %d: %s\n", __FILE__, __LINE__, "Size == 0");
free(ja3_r);
return NULL;
}

*s++ = '\0';
if (hash == NULL) {
hash = malloc(16);
if (!hash) {
LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno));
return NULL;
}
}
uint8_t hash[16];
md5_hash((uint8_t *)ja3_r, strlen(ja3_r), (uint32_t *)hash);

#ifdef MAIN
printf("SSL/TLS info:\n");
sslPrint(ssl);
printf("JA3_r : %s\n", ja3_r);
printf("JA3 : %s\n", ja3String(hash));
printf("JA3 : %s\n", ja3String(hash, buff));
#endif

free(ja3_r);

return hash;
return ja3String(hash, buff);

} // End of ja3Process

#ifdef MAIN
Expand Down
5 changes: 3 additions & 2 deletions src/libnfdump/ja3/ja3.h
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@
#define JA3DEFINED(ja3) (*((uint64_t *)ja3) != 0)
#define JA3UNDEFINED(ja3) (*((uint64_t *)ja3) == 0)

uint8_t *ja3Process(ssl_t *ssl, uint8_t *hash);
#define OFFja3String 0
#define SIZEja3String 32

char *ja3String(uint8_t *ja3Hash);
char *ja3Process(ssl_t *ssl, char *buff);

#endif
25 changes: 14 additions & 11 deletions src/libnfdump/ja4/ja4.c
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,16 +86,18 @@ int ja4Check(char *ja4String) {
(s)[3] = hexChars[(u) & 0xF]; \
}

char *ja4Process(ssl_t *ssl, uint8_t proto, char *buff) {
ja4_t *ja4Process(ssl_t *ssl, uint8_t proto) {
if (!ssl || ssl->type != CLIENTssl) return NULL;

if (buff == NULL) buff = malloc(SIZEja4String + 1);
if (buff == NULL) {
ja4_t *ja4 = malloc(sizeof(ja4_t) + SIZEja4String + 1);
if (ja4 == NULL) {
LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno));
return NULL;
}
buff[0] = '\0';
ja4->type = TYPE_UNDEF;
ja4->string[0] = '\0';

char *buff = ja4->string;
// create ja4_a
buff[0] = proto == IPPROTO_TCP ? 't' : 'q';
buff[1] = ssl->tlsCharVersion[0];
Expand All @@ -105,7 +107,7 @@ char *ja4Process(ssl_t *ssl, uint8_t proto, char *buff) {

uint32_t num = LenArray(ssl->cipherSuites);
if (num > 99) {
buff[0] = '\0';
free(ja4);
return NULL;
}
uint32_t ones = num % 10;
Expand All @@ -115,7 +117,7 @@ char *ja4Process(ssl_t *ssl, uint8_t proto, char *buff) {

num = LenArray(ssl->extensions);
if (num > 99) {
buff[0] = '\0';
free(ja4);
return NULL;
}
ones = num % 10;
Expand Down Expand Up @@ -198,9 +200,10 @@ char *ja4Process(ssl_t *ssl, uint8_t proto, char *buff) {

HexString(sha256Digest, 6, buff + 24);
buff[36] = '\0';
ja4->type = TYPE_JA4;

free(hashString);
return buff;
return ja4;

} // End of DecodeJA4

Expand Down Expand Up @@ -276,7 +279,7 @@ int main(int argc, char **argv) {
0x00, 0x00, 0x00 // ...
};
/*
[JA4: t13d1715h2_5b57614c22b0_3d5424432f57]
JA4: t13d1715h2_5b57614c22b0_3d5424432f57
JA4_r:
t13d1715h2_002f,0035,009c,009d,1301,1302,1303,c009,c00a,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0015,0017,001c,0022,0023,002b,002d,0033,ff01_0403,0503,0603,0804,0805,0806,0401,0501,0601,0203,0201]
Expand All @@ -293,9 +296,9 @@ int main(int argc, char **argv) {
exit(255);
}

char buff[SIZEja4];
if (ja4Process(ssl, IPPROTO_TCP, buff))
printf("ja4: %s\n", buff);
ja4_t *ja4 = ja4Process(ssl, IPPROTO_TCP);
if (ja4)
printf("ja4: %s\n", ja4->string);
else
printf("Failed to parse ja4\n");

Expand Down
Loading

0 comments on commit dedb73a

Please sign in to comment.