Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework statistics logic #13

Merged
merged 4 commits into from
Nov 8, 2024
Merged

Rework statistics logic #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3a1f643
Rework statistics logic
Romsik788 Nov 7, 2024
5200ff3
Some improvements
Romsik788 Nov 8, 2024
0a8b9af
Some fixes
Romsik788 Nov 8, 2024
225eeab
Fix logs duplicate
Romsik788 Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions application/backend/app/agent/agent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package agent
import (
"bytes"
"encoding/json"
"io/ioutil"
"io"
"net/http"
"os"

Expand All @@ -25,7 +25,7 @@ func SendInitRequest(containers []string) {
}

if resp.StatusCode != 200 {
b, _ := ioutil.ReadAll(resp.Body)
b, _ := io.ReadAll(resp.Body)
panic("ERROR: Response status from host is " + resp.Status + "\nResponse body: " + string(b))
}
}
Expand Down Expand Up @@ -97,7 +97,7 @@ func AskForDelete() {

resp, _ := http.Post(os.Getenv("HOST")+"/api/v1/askForDelete", "application/json", responseBody)

body, _ := ioutil.ReadAll(resp.Body)
body, _ := io.ReadAll(resp.Body)
if string(body) != "" {
var toDelete map[string][]string
json.Unmarshal(body, &toDelete)
Expand Down
58 changes: 34 additions & 24 deletions application/backend/app/containerdb/containerdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,22 @@ import (
"github.com/syndtr/goleveldb/leveldb/iterator"
)

func GetLogStatusKey(message string) string {
if strings.Contains(message, "ERROR") || strings.Contains(message, "ERR") || // const statuses_errors = ["ERROR", "ERR", "Error", "Err"];
strings.Contains(message, "Error") || strings.Contains(message, "Err") {
return "error"
} else if strings.Contains(message, "WARN") || strings.Contains(message, "WARNING") { // const statuses_warnings = ["WARN", "WARNING"];
return "warn"
} else if strings.Contains(message, "DEBUG") { // const statuses_other = ["DEBUG", "INFO", "ONLOGS"];
return "debug"
} else if strings.Contains(message, "INFO") {
return "info"
} else if strings.Contains(message, "ONLOGS") {
return "meta"
}
return "other"
}

func PutLogMessage(db *leveldb.DB, host string, container string, message_item []string) error {
if len(message_item[0]) < 30 {
fmt.Println("WARNING: got broken timestamp: ", "timestamp: "+message_item[0], "message: "+message_item[1])
Expand All @@ -25,21 +41,10 @@ func PutLogMessage(db *leveldb.DB, host string, container string, message_item [
if vars.Statuses_DBs[location] == nil {
vars.Statuses_DBs[location] = util.GetDB(host, container, "statuses")
}

status_key := "other"
if strings.Contains(message_item[1], "ERROR") || strings.Contains(message_item[1], "ERR") || // const statuses_errors = ["ERROR", "ERR", "Error", "Err"];
strings.Contains(message_item[1], "Error") || strings.Contains(message_item[1], "Err") {
status_key = "error"
} else if strings.Contains(message_item[1], "WARN") || strings.Contains(message_item[1], "WARNING") { // const statuses_warnings = ["WARN", "WARNING"];
status_key = "warn"
} else if strings.Contains(message_item[1], "DEBUG") { // const statuses_other = ["DEBUG", "INFO", "ONLOGS"];
status_key = "debug"
} else if strings.Contains(message_item[1], "INFO") {
status_key = "info"
} else if strings.Contains(message_item[1], "ONLOGS") {
status_key = "meta"
}
vars.Counters_For_Containers_Last_30_Min[location][status_key]++
status_key := GetLogStatusKey(message_item[1])
vars.Mutex.Lock()
vars.Container_Stat_Counter[location][status_key]++
vars.Mutex.Unlock()
vars.Statuses_DBs[location].Put([]byte(message_item[0]), []byte(status_key), nil)

err := db.Put([]byte(message_item[0]), []byte(message_item[1]), nil)
Expand Down Expand Up @@ -77,15 +82,12 @@ func getMoveDirection(getPrev bool, iter iterator.Iterator) func() bool {
return func() bool { return iter.Next() }
}

func searchInit(iter iterator.Iterator, startWith string, getPrev bool, include bool, move_direction func() bool) bool {
func searchInit(iter iterator.Iterator, startWith string) bool {
iter.Last()

if startWith != "" {
if !iter.Seek([]byte(startWith)) {
return false
}
if !include {
move_direction()
return true
return startWith > getDateTimeFromKey(string(iter.Key()))
}
}
return true
Expand Down Expand Up @@ -123,7 +125,7 @@ func GetLogs(getPrev bool, include bool, host string, container string, message
logs := [][]string{}
move_direction := getMoveDirection(getPrev, iter)

if !searchInit(iter, startWith, getPrev, include, move_direction) {
if !searchInit(iter, startWith) {
to_return["is_end"] = true
return to_return
}
Expand All @@ -143,6 +145,11 @@ func GetLogs(getPrev bool, include bool, host string, container string, message
}

keyStr := string(key)
timeStr := getDateTimeFromKey(keyStr)
if !include && timeStr == startWith {
move_direction()
continue
}
value := string(iter.Value())

if status != nil {
Expand All @@ -158,7 +165,7 @@ func GetLogs(getPrev bool, include bool, host string, container string, message
continue
}

logs = append(logs, []string{getDateTimeFromKey(keyStr), value})
logs = append(logs, []string{timeStr, value})
increaseAndMove(&counter, move_direction)
last_processed_key = keyStr
}
Expand Down Expand Up @@ -191,5 +198,8 @@ func DeleteContainer(host string, container string, fullDelete bool) {
vars.Stat_Containers_DBs[host+"/"+container].Close()
vars.Statuses_DBs[host+"/"+container] = util.GetDB(host, container, "statistics")
}
vars.Counters_For_Containers_Last_30_Min[host+"/"+container] = map[string]uint64{"error": 0, "debug": 0, "info": 0, "warn": 0, "meta": 0, "other": 0}

vars.Mutex.Lock()
vars.Container_Stat_Counter[host+"/"+container] = map[string]uint64{"error": 0, "debug": 0, "info": 0, "warn": 0, "meta": 0, "other": 0}
vars.Mutex.Unlock()
}
15 changes: 13 additions & 2 deletions application/backend/app/containerdb/containerdb_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
func TestPutLogMessage(t *testing.T) {
cont := "testCont"
host := "testHost"
vars.Counters_For_Containers_Last_30_Min[host+"/"+cont] = map[string]uint64{"error": 0, "debug": 0, "info": 0, "warn": 0, "meta": 0, "other": 0}
vars.Container_Stat_Counter[host+"/"+cont] = map[string]uint64{"error": 0, "debug": 0, "info": 0, "warn": 0, "meta": 0, "other": 0}
db, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containers/"+cont+"/logs", nil)
statusDB, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containers/"+cont+"statuses", nil)
vars.Statuses_DBs[host+"/"+cont] = statusDB
Expand Down Expand Up @@ -52,7 +52,7 @@ func TestPutLogMessage(t *testing.T) {

func TestGetLogs(t *testing.T) {
db, _ := leveldb.OpenFile("leveldb/hosts/Test/containers/TestGetLogsCont/logs", nil)
vars.Counters_For_Containers_Last_30_Min["Test/TestGetLogsCont"] = map[string]uint64{"error": 0, "debug": 0, "info": 0, "warn": 0, "meta": 0, "other": 0}
vars.Container_Stat_Counter["Test/TestGetLogsCont"] = map[string]uint64{"error": 0, "debug": 0, "info": 0, "warn": 0, "meta": 0, "other": 0}
statusDB, _ := leveldb.OpenFile("leveldb/hosts/Test/containers/TestGetLogsCont/statuses", nil)
vars.Statuses_DBs["Test/TestGetLogsCont"] = statusDB
defer statusDB.Close()
Expand Down Expand Up @@ -86,4 +86,15 @@ func TestGetLogs(t *testing.T) {
if logs[3][0] != vars.Year+"-02-10T12:57:09.230421754Z" {
t.Error("Invalid last logItem datetime: ", logs[3][0])
}

logs = GetLogs(true, false, "Test", "TestGetLogsCont", "", 30, vars.Year+"-02-10T12:51:09.230421753Z", false, nil)["logs"].([][]string)
if len(logs) != 5 {
t.Error("4 logItems must be returned!")
}
if logs[0][0] != vars.Year+"-02-10T12:51:09.230421754Z" {
t.Error("Invalid first logItem datetime: ", logs[0][0])
}
if logs[4][0] != vars.Year+"-02-10T12:57:09.230421754Z" {
t.Error("Invalid last logItem datetime: ", logs[3][0])
}
}
4 changes: 2 additions & 2 deletions application/backend/app/daemon/daemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net"
"os"
"strconv"
Expand Down Expand Up @@ -158,7 +158,7 @@ func makeSocketRequest(path string) []byte {
}
fmt.Fprintf(connection, "GET /"+path+" HTTP/1.0\r\n\r\n")

body, _ := ioutil.ReadAll(connection)
body, _ := io.ReadAll(connection)

connection.Close()
return body
Expand Down
3 changes: 1 addition & 2 deletions application/backend/app/routes/routes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"os"
Expand Down Expand Up @@ -581,7 +580,7 @@ func UpdateUserSettings(w http.ResponseWriter, req *http.Request) {
}

var settings map[string]interface{}
body, _ := ioutil.ReadAll(req.Body)
body, _ := io.ReadAll(req.Body)
json.Unmarshal(body, &settings)
username, _ := util.GetUserFromJWT(*req)
userdb.UpdateUserSettings(username, settings)
Expand Down
28 changes: 14 additions & 14 deletions application/backend/app/routes/routes_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package routes
import (
"bytes"
"encoding/json"
"io/ioutil"
"io"
"net/http"
"net/http/httptest"
"os"
Expand All @@ -26,7 +26,7 @@ func TestFrontend(t *testing.T) {
rr1 := httptest.NewRecorder()
handler1 := http.HandlerFunc(Frontend)
handler1.ServeHTTP(rr1, req1)
body1, _ := ioutil.ReadAll(rr1.Result().Body)
body1, _ := io.ReadAll(rr1.Result().Body)
if string(body1) != "text" {
t.Error("Wrong file content!")
}
Expand All @@ -35,7 +35,7 @@ func TestFrontend(t *testing.T) {
rr2 := httptest.NewRecorder()
handler2 := http.HandlerFunc(Frontend)
handler2.ServeHTTP(rr2, req2)
body2, _ := ioutil.ReadAll(rr2.Result().Body)
body2, _ := io.ReadAll(rr2.Result().Body)
if string(body2) != "text" {
t.Error("Wrong file content!")
}
Expand Down Expand Up @@ -88,7 +88,7 @@ func TestGetHosts(t *testing.T) {
rr1 := httptest.NewRecorder()
handler1 := http.HandlerFunc(GetHosts)
handler1.ServeHTTP(rr1, req1)
b, _ := ioutil.ReadAll(rr1.Result().Body)
b, _ := io.ReadAll(rr1.Result().Body)
if string(b) != "[{\"host\":\"Test1\",\"services\":[{\"isDisabled\":true,\"isFavorite\":false,\"serviceName\":\"containerTest1\"},{\"isDisabled\":true,\"isFavorite\":false,\"serviceName\":\"containerTest2\"},{\"isDisabled\":true,\"isFavorite\":false,\"serviceName\":\"containerTest3\"}]},{\"host\":\"Test2\",\"services\":[{\"isDisabled\":true,\"isFavorite\":false,\"serviceName\":\"containerTest1\"},{\"isDisabled\":true,\"isFavorite\":false,\"serviceName\":\"containerTest2\"},{\"isDisabled\":true,\"isFavorite\":false,\"serviceName\":\"containerTest3\"}]},{\"host\":\""+util.GetHost()+"\",\"services\":[]}]" {
t.Error("Wrong containers or hosts list returned!\n" + string(b))
}
Expand All @@ -104,7 +104,7 @@ func TestSizeByAll(t *testing.T) {
rr1 := httptest.NewRecorder()
handler1 := http.HandlerFunc(GetSizeByAll)
handler1.ServeHTTP(rr1, req1)
b, _ := ioutil.ReadAll(rr1.Result().Body)
b, _ := io.ReadAll(rr1.Result().Body)
if !strings.Contains(string(b), "\"0.0\"") {
t.Error("Wrong size: ", string(b))
}
Expand All @@ -120,7 +120,7 @@ func TestSizeByService(t *testing.T) {
rr1 := httptest.NewRecorder()
handler1 := http.HandlerFunc(GetSizeByAll)
handler1.ServeHTTP(rr1, req1)
b, _ := ioutil.ReadAll(rr1.Result().Body)
b, _ := io.ReadAll(rr1.Result().Body)
if !strings.Contains(string(b), "\"0.0\"") {
t.Error("Wrong size: ", string(b))
}
Expand All @@ -137,7 +137,7 @@ func TestLogin(t *testing.T) {
rr1 := httptest.NewRecorder()
handler1 := http.HandlerFunc(Login)
handler1.ServeHTTP(rr1, req1)
b, _ := ioutil.ReadAll(rr1.Result().Body)
b, _ := io.ReadAll(rr1.Result().Body)
if !strings.Contains(string(b), "Wrong") {
t.Error("Password must be wrong!")
}
Expand All @@ -150,7 +150,7 @@ func TestLogin(t *testing.T) {
rr2 := httptest.NewRecorder()
handler2 := http.HandlerFunc(Login)
handler2.ServeHTTP(rr2, req2)
b2, _ := ioutil.ReadAll(rr2.Result().Body)
b2, _ := io.ReadAll(rr2.Result().Body)
if !strings.Contains(string(b2), "null") {
t.Error("Password must be wrong!")
}
Expand Down Expand Up @@ -190,10 +190,10 @@ func TestGetStats(t *testing.T) {
handler1.ServeHTTP(rr1, req1)
rr2 := httptest.NewRecorder()

vars.Counters_For_Containers_Last_30_Min["test/test"] = map[string]uint64{"error": 1, "debug": 2, "info": 3, "warn": 4, "meta": 0, "other": 5}
vars.Container_Stat_Counter["test/test"] = map[string]uint64{"error": 1, "debug": 2, "info": 3, "warn": 4, "meta": 0, "other": 5}
os.RemoveAll("leveldb/hosts/test/containers/test/statistics")
statDB, _ := leveldb.OpenFile("leveldb/hosts/test/containers/test/statistics", nil)
to_put, _ := json.Marshal(vars.Counters_For_Containers_Last_30_Min["test/test"])
to_put, _ := json.Marshal(vars.Container_Stat_Counter["test/test"])
datetime := strings.Replace(strings.Split(time.Now().UTC().String(), ".")[0], " ", "T", 1) + "Z"
statDB.Put([]byte(datetime), to_put, nil)
statDB.Close()
Expand All @@ -208,7 +208,7 @@ func TestGetStats(t *testing.T) {
handler2 := http.HandlerFunc(GetStats)
handler2.ServeHTTP(rr2, req2)

b, _ := ioutil.ReadAll(rr2.Result().Body)
b, _ := io.ReadAll(rr2.Result().Body)
res := map[string]int{}
json.Unmarshal(b, &res)
if res["debug"] != 4 || res["error"] != 2 ||
Expand All @@ -231,9 +231,9 @@ func TestGetChartData(t *testing.T) {

cur_db, _ := leveldb.OpenFile("leveldb/hosts/test/statistics", nil)
vars.Stat_Hosts_DBs["test"] = cur_db
vars.Counters_For_Containers_Last_30_Min["test/test"] = map[string]uint64{"error": 2, "debug": 1, "info": 3, "warn": 5, "meta": 0, "other": 4}
vars.Container_Stat_Counter["test/test"] = map[string]uint64{"error": 2, "debug": 1, "info": 3, "warn": 5, "meta": 0, "other": 4}
vars.Stat_Containers_DBs["test/test"] = cur_db
to_put, _ := json.Marshal(vars.Counters_For_Containers_Last_30_Min["test/test"])
to_put, _ := json.Marshal(vars.Container_Stat_Counter["test/test"])
datetime := strings.Replace(strings.Split(time.Now().UTC().String(), ".")[0], " ", "T", 1) + "Z"
cur_db.Put([]byte(datetime), to_put, nil)

Expand All @@ -250,7 +250,7 @@ func TestGetChartData(t *testing.T) {
handler2.ServeHTTP(rr2, req2)

res := map[string]map[string]int{}
b, _ := ioutil.ReadAll(rr2.Body)
b, _ := io.ReadAll(rr2.Body)
json.Unmarshal(b, &res)
datetime = datetime[:len(datetime)-6] + "00Z"
if res[datetime]["debug"] != 1 || res[datetime]["error"] != 2 ||
Expand Down
Loading
Loading