From 50d7a20699b0ded95d5a0decd7ae59d8a8ea6d02 Mon Sep 17 00:00:00 2001 From: Roman Malenko Date: Tue, 5 Nov 2024 11:41:46 +0200 Subject: [PATCH 1/5] Get logs method refactor --- application/Dockerfile | 3 - .../backend/app/containerdb/containerdb.go | 95 ++++++------------- application/backend/app/routes/routes.go | 23 +++-- application/build.sh | 6 +- .../frontend/src/Views/Logs/NewLogsV2.svelte | 8 +- 5 files changed, 49 insertions(+), 86 deletions(-) mode change 100644 => 100755 application/build.sh diff --git a/application/Dockerfile b/application/Dockerfile index 1f47cf2..15c0a63 100644 --- a/application/Dockerfile +++ b/application/Dockerfile @@ -28,6 +28,3 @@ FROM alpine COPY --from=frontbuilder /code/dist/ /dist/ COPY --from=backendbuilder /backend/main /backend/main CMD ["/backend/main"] - -# docker run -v /var/run/docker.sock:/var/run/docker.sock --rm -it $(docker build -f Dockerfile .) -# docker build . -t devforth/onlogs && docker push devforth/onlogs diff --git a/application/backend/app/containerdb/containerdb.go b/application/backend/app/containerdb/containerdb.go index 0fd573b..7385833 100644 --- a/application/backend/app/containerdb/containerdb.go +++ b/application/backend/app/containerdb/containerdb.go @@ -95,65 +95,6 @@ func getDateTimeFromKey(key string) string { return strings.Split(key, " +")[0] } -// # TODO: should be merged with GetLogs function -/* -Get logs line by line with filtering by logline status. - - getPrev - if true, will get logs from latest to oldest. - - include - if true, will include logs with startWith key. - -returns json obj same to GetLogs function. -*/ -func GetLogsByStatus(host string, container string, message string, status string, limit int, startWith string, getPrev bool, include bool, caseSensetivity bool) map[string]interface{} { - logs_db := util.GetDB(host, container, "logs") - db := util.GetDB(host, container, "statuses") - iter := db.NewIterator(nil, nil) - defer iter.Release() - to_return := map[string]interface{}{} - to_return["logs"] = [][]string{} - move_direction := getMoveDirection(getPrev, iter) - - if !searchInit(iter, startWith, getPrev, include, move_direction) { - to_return["is_end"] = true - return to_return - } - - counter := 0 - iteration := 0 - last_processed_key := []string{} - for counter < limit && iteration < 1000000 { - iteration += 1 - key := iter.Key() - if len(key) == 0 { - to_return["is_end"] = true - increaseAndMove(&counter, move_direction) - continue - } else { - to_return["is_end"] = false - } - - value := string(iter.Value()) - if value != status { - move_direction() - continue - } - - last_processed_key = []string{string(key), value} - res, _ := logs_db.Get(key, nil) - logLine := string(res) - - if !fitsForSearch(logLine, message, caseSensetivity) { - move_direction() - continue - } - - to_return["logs"] = append(to_return["logs"].([][]string), []string{getDateTimeFromKey(string(key)), logLine}) - increaseAndMove(&counter, move_direction) - } - - to_return["last_processed_key"] = last_processed_key - return to_return -} - /* Get logs line by line from container. - getPrev - if true, will get logs from latest to oldest. @@ -167,12 +108,19 @@ returns json obj like this: "is_end": false } */ -func GetLogs(getPrev bool, include bool, host string, container string, message string, limit int, startWith string, caseSensetivity bool) map[string]interface{} { +func GetLogs(getPrev bool, include bool, host string, container string, message string, limit int, startWith string, caseSensetivity bool, status *string) map[string]interface{} { logs_db := util.GetDB(host, container, "logs") + var statusDb *leveldb.DB + if status != nil { + statusDb = util.GetDB(host, container, "statuses") + } iter := logs_db.NewIterator(nil, nil) defer iter.Release() - to_return := map[string]interface{}{} - to_return["logs"] = [][]string{} + + to_return := map[string]interface{}{ + "logs": [][]string{}, + } + logs := [][]string{} move_direction := getMoveDirection(getPrev, iter) if !searchInit(iter, startWith, getPrev, include, move_direction) { @@ -185,7 +133,7 @@ func GetLogs(getPrev bool, include bool, host string, container string, message last_processed_key := "" for counter < limit && iteration < 1000000 { iteration += 1 - key := string(iter.Key()) + key := iter.Key() if len(key) == 0 { to_return["is_end"] = true increaseAndMove(&counter, move_direction) @@ -193,19 +141,36 @@ func GetLogs(getPrev bool, include bool, host string, container string, message } else { to_return["is_end"] = false } - last_processed_key = key + + keyStr := string(key) value := string(iter.Value()) + if status != nil { + statusValue, err := statusDb.Get(key, nil) + if err != nil || string(statusValue) != *status { + move_direction() + continue + } + } + if !fitsForSearch(value, message, caseSensetivity) { move_direction() continue } - to_return["logs"] = append(to_return["logs"].([][]string), []string{getDateTimeFromKey(key), value}) + logs = append(logs, []string{getDateTimeFromKey(keyStr), value}) increaseAndMove(&counter, move_direction) + last_processed_key = keyStr + } + + logs_len := len(logs) + for i, j := 0, logs_len-1; i < j; i, j = i+1, j-1 { + logs[i], logs[j] = logs[j], logs[i] } + to_return["logs"] = logs to_return["last_processed_key"] = last_processed_key + to_return["total_count"] = len(logs) return to_return } diff --git a/application/backend/app/routes/routes.go b/application/backend/app/routes/routes.go index dc4dad9..9308e75 100644 --- a/application/backend/app/routes/routes.go +++ b/application/backend/app/routes/routes.go @@ -415,7 +415,7 @@ func GetPrevLogs(w http.ResponseWriter, req *http.Request) { if params.Get("host") == "" { panic("Host is not mentioned!") } - json.NewEncoder(w).Encode(containerdb.GetLogs(true, false, params.Get("host"), params.Get("id"), params.Get("search"), limit, params.Get("startWith"), caseSensetive)) + json.NewEncoder(w).Encode(containerdb.GetLogs(true, false, params.Get("host"), params.Get("id"), params.Get("search"), limit, params.Get("startWith"), caseSensetive, nil)) } func GetLogs(w http.ResponseWriter, req *http.Request) { @@ -434,17 +434,16 @@ func GetLogs(w http.ResponseWriter, req *http.Request) { panic("Host is not mentioned!") } - if params.Get("status") != "" { - json.NewEncoder(w).Encode(containerdb.GetLogsByStatus( - params.Get("host"), params.Get("id"), params.Get("search"), params.Get("status"), - limit, params.Get("startWith"), false, true, caseSensetive, - )) - } else { - json.NewEncoder(w).Encode(containerdb.GetLogs( - false, false, params.Get("host"), params.Get("id"), params.Get("search"), - limit, params.Get("startWith"), caseSensetive, - )) + status := params.Get("status") + var statusPtr *string + if status != "" { + statusPtr = &status } + + json.NewEncoder(w).Encode(containerdb.GetLogs( + false, false, params.Get("host"), params.Get("id"), params.Get("search"), + limit, params.Get("startWith"), caseSensetive, statusPtr, + )) } func GetLogWithPrev(w http.ResponseWriter, req *http.Request) { @@ -458,7 +457,7 @@ func GetLogWithPrev(w http.ResponseWriter, req *http.Request) { if params.Get("host") == "" { panic("Host is not mentioned!") } - json.NewEncoder(w).Encode(containerdb.GetLogs(false, true, params.Get("host"), params.Get("id"), "", limit, params.Get("startWith"), false)) + json.NewEncoder(w).Encode(containerdb.GetLogs(false, true, params.Get("host"), params.Get("id"), "", limit, params.Get("startWith"), false, nil)) } // TODO return {"error": "Invalid host!"} when host is not exists diff --git a/application/build.sh b/application/build.sh old mode 100644 new mode 100755 index 0808271..1e7bc23 --- a/application/build.sh +++ b/application/build.sh @@ -1,2 +1,4 @@ -docker buildx create --use -# docker buildx build --platform=linux/amd64,linux/arm64 --tag "devforth/onlogs:latest" --tag "devforth/onlogs:1.1.1" --push . +# docker buildx create --use +docker buildx build --platform=linux/amd64,linux/arm64 --tag "devforth/onlogs:latest" --tag "devforth/onlogs:1.1.2" --push . +# docker run -v /var/run/docker.sock:/var/run/docker.sock --rm -it $(docker build -q -f Dockerfile .) +# docker build . -t devforth/onlogs && docker push devforth/onlogs diff --git a/application/frontend/src/Views/Logs/NewLogsV2.svelte b/application/frontend/src/Views/Logs/NewLogsV2.svelte index c258478..5ca2536 100644 --- a/application/frontend/src/Views/Logs/NewLogsV2.svelte +++ b/application/frontend/src/Views/Logs/NewLogsV2.svelte @@ -157,7 +157,7 @@ total_logs_amount += data.logs.length; if (initialService === $lastChosenService) { - setLastLogTime(data.logs.reverse()?.at(0)?.at(0)); + setLastLogTime(data.logs[data.total_count-1][0]); allLogs = [...allLogs, ...data.logs]; let allLogsCopy = [...allLogs]; @@ -252,7 +252,7 @@ } } if (initialService === $lastChosenService) { - allLogs = [...upperLogs.reverse(), ...viewLogs, ...downLogs]; + allLogs = [...upperLogs, ...viewLogs, ...downLogs]; let allLogsCopy = [...allLogs]; @@ -456,7 +456,7 @@ startWith: last_key, hostName: $lastChosenHost, signal, - })).logs.reverse(); + })).logs; total_logs = [...total_logs, ...data]; total_logs_amount += data.length; is_all_logs_processed = data.is_end; @@ -543,7 +543,7 @@ is_all_logs_processed = data.is_end; last_key = data.last_processed_key; total_received_logs_count += data.logs.length; - total_logs = [...total_logs, ...data.logs.reverse()]; + total_logs = [...total_logs, ...data.logs]; } isSearching.set(false); isFeatching.set(false); From 036695a8436e95c148d6e1e0ba90a0dc8c30520b Mon Sep 17 00:00:00 2001 From: Roman Malenko Date: Tue, 5 Nov 2024 11:51:13 +0200 Subject: [PATCH 2/5] Some fixes --- application/backend/app/containerdb/containerdb.go | 1 - application/frontend/src/Views/Logs/NewLogsV2.svelte | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/application/backend/app/containerdb/containerdb.go b/application/backend/app/containerdb/containerdb.go index 7385833..33ed3f5 100644 --- a/application/backend/app/containerdb/containerdb.go +++ b/application/backend/app/containerdb/containerdb.go @@ -170,7 +170,6 @@ func GetLogs(getPrev bool, include bool, host string, container string, message to_return["logs"] = logs to_return["last_processed_key"] = last_processed_key - to_return["total_count"] = len(logs) return to_return } diff --git a/application/frontend/src/Views/Logs/NewLogsV2.svelte b/application/frontend/src/Views/Logs/NewLogsV2.svelte index 5ca2536..e4bf737 100644 --- a/application/frontend/src/Views/Logs/NewLogsV2.svelte +++ b/application/frontend/src/Views/Logs/NewLogsV2.svelte @@ -157,7 +157,7 @@ total_logs_amount += data.logs.length; if (initialService === $lastChosenService) { - setLastLogTime(data.logs[data.total_count-1][0]); + setLastLogTime(data.logs[total_logs_amount-1][0]); allLogs = [...allLogs, ...data.logs]; let allLogsCopy = [...allLogs]; From ffbf57dde6672eba5b64176f681786e01e3771ff Mon Sep 17 00:00:00 2001 From: Roman Malenko Date: Tue, 5 Nov 2024 11:53:12 +0200 Subject: [PATCH 3/5] Test fix --- application/backend/app/containerdb/containerdb_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/backend/app/containerdb/containerdb_test.go b/application/backend/app/containerdb/containerdb_test.go index cf92fec..d5126ab 100644 --- a/application/backend/app/containerdb/containerdb_test.go +++ b/application/backend/app/containerdb/containerdb_test.go @@ -65,7 +65,7 @@ func TestGetLogs(t *testing.T) { db.Close() var logs [][]string - logs = GetLogs(false, true, "Test", "TestGetLogsCont", "", 30, vars.Year+"-02-10T12:57:09.230421754Z", false)["logs"].([][]string) + logs = GetLogs(false, true, "Test", "TestGetLogsCont", "", 30, vars.Year+"-02-10T12:57:09.230421754Z", false, nil)["logs"].([][]string) if len(logs) != 5 { t.Error("5 logItems must be returned!") } @@ -73,7 +73,7 @@ func TestGetLogs(t *testing.T) { t.Error("Invalid first logItem datetime: ", logs[0][0]) } - logs = GetLogs(true, false, "Test", "TestGetLogsCont", "", 30, vars.Year+"-02-10T12:51:09.230421754Z", false)["logs"].([][]string) + logs = GetLogs(true, false, "Test", "TestGetLogsCont", "", 30, vars.Year+"-02-10T12:51:09.230421754Z", false, nil)["logs"].([][]string) if len(logs) != 4 { t.Error("4 logItems must be returned!") } From f80b21710700d2e20032ad2a289124e7822da878 Mon Sep 17 00:00:00 2001 From: Roman Malenko Date: Tue, 5 Nov 2024 12:04:29 +0200 Subject: [PATCH 4/5] Test fix --- .../backend/app/containerdb/containerdb_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/application/backend/app/containerdb/containerdb_test.go b/application/backend/app/containerdb/containerdb_test.go index d5126ab..8e59e45 100644 --- a/application/backend/app/containerdb/containerdb_test.go +++ b/application/backend/app/containerdb/containerdb_test.go @@ -69,15 +69,21 @@ func TestGetLogs(t *testing.T) { if len(logs) != 5 { t.Error("5 logItems must be returned!") } - if logs[0][0] != vars.Year+"-02-10T12:57:09.230421754Z" { + if logs[0][0] != vars.Year+"-02-10T12:51:09.230421754Z" { t.Error("Invalid first logItem datetime: ", logs[0][0]) } + if logs[4][0] != vars.Year+"-02-10T12:57:09.230421754Z" { + t.Error("Invalid last logItem datetime: ", logs[4][0]) + } logs = GetLogs(true, false, "Test", "TestGetLogsCont", "", 30, vars.Year+"-02-10T12:51:09.230421754Z", false, nil)["logs"].([][]string) if len(logs) != 4 { t.Error("4 logItems must be returned!") } - if logs[0][0] != vars.Year+"-02-10T12:52:09.230421754Z" { + if logs[0][0] != vars.Year+"-02-10T12:57:09.230421754Z" { t.Error("Invalid first logItem datetime: ", logs[0][0]) } + if logs[3][0] != vars.Year+"-02-10T12:52:09.230421754Z" { + t.Error("Invalid last logItem datetime: ", logs[3][0]) + } } From a8f283d91918f0303ec548724d99af430be86d3b Mon Sep 17 00:00:00 2001 From: Roman Malenko Date: Tue, 5 Nov 2024 12:52:30 +0200 Subject: [PATCH 5/5] Some fixes --- application/backend/app/containerdb/containerdb.go | 5 ----- .../backend/app/containerdb/containerdb_test.go | 8 ++++---- application/frontend/src/Views/Logs/NewLogsV2.svelte | 10 +++++----- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/application/backend/app/containerdb/containerdb.go b/application/backend/app/containerdb/containerdb.go index 33ed3f5..86f7cce 100644 --- a/application/backend/app/containerdb/containerdb.go +++ b/application/backend/app/containerdb/containerdb.go @@ -163,11 +163,6 @@ func GetLogs(getPrev bool, include bool, host string, container string, message last_processed_key = keyStr } - logs_len := len(logs) - for i, j := 0, logs_len-1; i < j; i, j = i+1, j-1 { - logs[i], logs[j] = logs[j], logs[i] - } - to_return["logs"] = logs to_return["last_processed_key"] = last_processed_key return to_return diff --git a/application/backend/app/containerdb/containerdb_test.go b/application/backend/app/containerdb/containerdb_test.go index 8e59e45..4be7472 100644 --- a/application/backend/app/containerdb/containerdb_test.go +++ b/application/backend/app/containerdb/containerdb_test.go @@ -69,10 +69,10 @@ func TestGetLogs(t *testing.T) { if len(logs) != 5 { t.Error("5 logItems must be returned!") } - if logs[0][0] != vars.Year+"-02-10T12:51:09.230421754Z" { + if logs[0][0] != vars.Year+"-02-10T12:57:09.230421754Z" { t.Error("Invalid first logItem datetime: ", logs[0][0]) } - if logs[4][0] != vars.Year+"-02-10T12:57:09.230421754Z" { + if logs[4][0] != vars.Year+"-02-10T12:51:09.230421754Z" { t.Error("Invalid last logItem datetime: ", logs[4][0]) } @@ -80,10 +80,10 @@ func TestGetLogs(t *testing.T) { if len(logs) != 4 { t.Error("4 logItems must be returned!") } - if logs[0][0] != vars.Year+"-02-10T12:57:09.230421754Z" { + if logs[0][0] != vars.Year+"-02-10T12:52:09.230421754Z" { t.Error("Invalid first logItem datetime: ", logs[0][0]) } - if logs[3][0] != vars.Year+"-02-10T12:52:09.230421754Z" { + if logs[3][0] != vars.Year+"-02-10T12:57:09.230421754Z" { t.Error("Invalid last logItem datetime: ", logs[3][0]) } } diff --git a/application/frontend/src/Views/Logs/NewLogsV2.svelte b/application/frontend/src/Views/Logs/NewLogsV2.svelte index e4bf737..5fd59d8 100644 --- a/application/frontend/src/Views/Logs/NewLogsV2.svelte +++ b/application/frontend/src/Views/Logs/NewLogsV2.svelte @@ -157,8 +157,8 @@ total_logs_amount += data.logs.length; if (initialService === $lastChosenService) { - setLastLogTime(data.logs[total_logs_amount-1][0]); - allLogs = [...allLogs, ...data.logs]; + setLastLogTime(data.logs?.at(0)?.at(0)); + allLogs = [...allLogs, ...data.logs.reverse()]; let allLogsCopy = [...allLogs]; newLogs = allLogsCopy.splice(0, limit); @@ -252,7 +252,7 @@ } } if (initialService === $lastChosenService) { - allLogs = [...upperLogs, ...viewLogs, ...downLogs]; + allLogs = [...upperLogs.reverse(), ...viewLogs, ...downLogs]; let allLogsCopy = [...allLogs]; @@ -456,7 +456,7 @@ startWith: last_key, hostName: $lastChosenHost, signal, - })).logs; + })).logs.reverse(); total_logs = [...total_logs, ...data]; total_logs_amount += data.length; is_all_logs_processed = data.is_end; @@ -543,7 +543,7 @@ is_all_logs_processed = data.is_end; last_key = data.last_processed_key; total_received_logs_count += data.logs.length; - total_logs = [...total_logs, ...data.logs]; + total_logs = [...total_logs, ...data.logs.reverse()]; } isSearching.set(false); isFeatching.set(false);