Merge branch 'main' of github.com:devforth/OnLogs

README.md

@@ -1,57 +1,61 @@
-# OnLogs
+# OnLogs - Lightweight docker logs web viewer
+
 <a href="https://devforth.io"><img src="./.assets/df_powered_by.svg" style="height:36px"/></a>
 
 ![Passing Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/LbP22/7a0933f8cba0bddbcc95c8b850e32663/raw/onlogs_passing__heads_main.json) ![Coverage Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/LbP22/7a0933f8cba0bddbcc95c8b850e32663/raw/onlogs_units_coverage__heads_main.json) ![License Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/LbP22/7a0933f8cba0bddbcc95c8b850e32663/raw/license_MIT.json) 
 
+![image](https://github.com/devforth/OnLogs/assets/1838656/38d0f184-3810-4389-a5af-2488b3a51276)
+
+
+
 ### Benefits
 
-- 🔑 Secure and simple way to access logs without server/ssh accession
-- 🏗️ Built with Golang & Svelte.js to extract maximum performance and tiny bundle. Logs storage implemented on top of LevelDB
-- 🧸 Setup is easy as docker run command/compose snippet
+- 🔑 Simple and secure way to access logs of any Docker containers without server/SSH connection
+- 🏗️ Built with Golang & Svelte.js to extract maximum performance and keep the image and RAM footprint as small as possible. Logs storage implemented on top of lightweight LevelDB
+- 🧸 Installation is easy as docker runs command/compose snippet. HTTP port exposed and could be routed from Nginx/Traefik/Directly
 - 🖱 Get every service realtime logs stream with 1 click <img src="./.assets/1.gif"/>
-- 📱 Check logs from smartphone (insane, but we know it happens that you need it)
-- 🧾 Open-Source commercial friendly MIT license
+- 📱 Check logs from your smartphone (insane, but we know it happens that you need it)
+- 🧾 Open-source, commercial-friendly MIT license
 - 💾 Small size of Docker image (~ 13 MB)
 - 👥 Share access to logs with team members, revoke any time
 
 ### Features
 
-- 💻 One host can be used to view logs from all other hosts in case if you are running Cluster
-- 🔗 Share log messages to collegues via link <img src="./.assets/2.gif"/>
+- 💻 One host can be used to view logs from all other hosts in case you are running Cluster
+- 🔗 Share log messages to colleagues via link <img src="./.assets/2.gif"/>
 - 💽 Clear original docker logs to keep your storage size.
 - 📊 Error/Info/Debug Statistics
-- 🔎 Search through logs (configurable case sensetivity)
-- 👁 View parameters (parsing JSON, show local/UTC time for every logline)
+- 🔎 Search through logs (configurable case sensitivity)
+- 👁 View parameters (parsing JSON, showing local/UTC time for every logline)
 - 🔴 Realtime logs updating
 
 ### Roadmap
 
 - 🗂 Grouping hosts
-- 🏷 Search & Filter by tags (log status, time)
+- 🏷 Search and filter by tags (log status, time)
 - 🔌Plugins and internal ability to notify about some event (e.g. notify when Error happens)
 - 📊 Improved statistics
 
-## Hello world & ussage
+## Hello world & usage
 ### Docker Compose example with traefik
 ```sh
   onlogs:
     image: devforth/onlogs
     restart: always
     environment:
-      - PASSWORD=<any password>
-      - PORT=<any port>
-    #  - ONLOGS_PATH_PREFIX=/<any path prefix> if using with path prefix
+      - ADMIN_USERNAME=admin
+      - ADMIN_PASSWORD=<any password>
+      - PORT=8798
+    #  - ONLOGS_PATH_PREFIX=/onlogs if want to use with path prefix
 
-    ports:
-      - <any port>:<any port>
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.onlogs.rule=Host(`<your host>`)"  # if using on subdomain
-    #  - traefik.http.routers.onlogs.rule=PathPrefix(`</any path prefix>`) # if using with path prefix
-      - "traefik.http.services.onlogs.loadbalancer.server.port=<any port>"
+      - "traefik.http.routers.onlogs.rule=Host(`<your host>`)"  # if using on subdomain, e.g. https://onlogs.yourdomain.com
+    #  - traefik.http.routers.onlogs.rule=PathPrefix(`/onlogs`) # if want to use with a path prefix, e.g. https://yourdomain.com/onlogs
+      - "traefik.http.services.onlogs.loadbalancer.server.port=8798"
     volumes:
      - /var/run/docker.sock:/var/run/docker.sock
-     - /var/lib/docker/containers:/var/lib/docker/containers # if you want to delete dublicating logs from docker
+     - /var/lib/docker/containers:/var/lib/docker/containers # if you want to delete duplicating logs from docker
      - /etc/hostname:/etc/hostname
      - onlogs-volume:/leveldb
 
@@ -61,25 +65,26 @@ volumes:
 
 ### Docker Run example with traefik
 ```sh
-docker run --restart always -e PASSWORD=<any password> -e PORT=<any port> \
+docker run --restart always -e ADMIN_USERNAME=admin -e PASSWORD=<any password> -e PORT=8798 \
     -v /var/run/docker.sock:/var/run/docker.sock:ro \
     -v /var/lib/docker/containers:/var/lib/docker/containers \
     -v /etc/hostname:/etc/hostname \
     -v onlogs-volume:/leveldb \ 
     --label traefik.enable=true \
     --label traefik.http.routers.onlogs.rule=Host\(\`<your host>\`\) \ 
-    --label traefik.http.services.onlogs.loadbalancer.server.port=2874 devforth/onlogs
+    --label traefik.http.services.onlogs.loadbalancer.server.port=8798 devforth/onlogs
 ```
 
 Once done, just go to <your host> and login as "admin" with <any password>.
 
 ## Available Environment Options:
 | Environment Variable       | Description   | Defaults | Required |
 |----------------------------|---------------------------------|--------|-----------------|
-| PASSWORD           | Password for default user                        |                    | if `AGENT=false`
+| ADMIN_USERNAME           | Username for initial user                        | `admin`                 | if `AGENT=false`
+| ADMIN_PASSWORD           | Password for initial user                        |                    | if `AGENT=false`
 | PORT               | Port to listen on                                | `2874`             | if `AGENT=false`
 | JWT_SECRET         | Secret for JWT tokens for users                  | Generates randomly | -
 | ONLOGS_PATH_PREFIX | Base path if you using OnLogs not on subdomain   |                    | only if using on path prefix
-| AGENT             | Toggles agent mode. If enabled, there will be no web interface available and all logs will be sent  and stored on HOST                                                      | `false` | -
+| AGENT             | Toggles agent mode. If enabled, there will be no web interface available, and all logs will be sent  and stored on HOST                                                      | `false` | -
 | HOST               | Url to OnLogs host from protocol to domain name. |                    | if `AGENT=true`
-| ONLOGS_TOKEN       | Token that will use agent to authorize and connect to HOST | Generates with OnLogs interface   | if `AGENT=true`
+| ONLOGS_TOKEN       | Token that will use an agent to authorize and connect to HOST | Generates with OnLogs interface   | if `AGENT=true`

application/Dockerfile

@@ -3,7 +3,7 @@ FROM node:16-alpine AS frontbuilder
 WORKDIR /code/
 ADD frontend/package-lock.json .
 ADD frontend/package.json .
-RUN npm install -g npm@latest && npm ci
+RUN npm install -g npm:16 && npm ci
 
 ADD frontend/. .
 
@@ -12,8 +12,6 @@ RUN npm run build
 COPY . /code/
 
 FROM alpine
-RUN apk add bash curl
-# tmp
 
 COPY --from=frontbuilder /code/dist/ /backend/dist/
 
@@ -26,10 +24,6 @@ RUN go mod download  \
   && go build -o main .
 
 FROM alpine
-RUN apk add bash curl
-# tmp
-
-EXPOSE 2874
 
 COPY --from=frontbuilder /code/dist/ /dist/
 COPY --from=backendbuilder /backend/main /backend/main

application/backend/app/agent/agent.go

@@ -46,7 +46,7 @@ func SendLogMessage(token string, container string, message_item []string) bool
 }
 
 func TryResend() {
-	token := os.Getenv("ONLOGS_TOKRN")
+	token := os.Getenv("ONLOGS_TOKEN")
 	containers, _ := os.ReadDir("leveldb/hosts/" + util.GetHost() + "/containers/")
 	for _, container := range containers {
 		tmpDB := vars.BrokenLogs_DBs[container.Name()]

application/backend/app/containerdb/containerdb.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/devforth/OnLogs/app/util"
 	"github.com/devforth/OnLogs/app/vars"
@@ -28,6 +29,9 @@ func PutLogMessage(db *leveldb.DB, host string, container string, message_item [
 		panic("Host is not mentioned!")
 	}
 	location := host + "/" + container
+	if vars.Statuses_DBs[location] == nil {
+		vars.Statuses_DBs[location] = util.GetDB(host, container, "statuses")
+	}
 
 	if strings.Contains(message_item[1], "ERROR") || strings.Contains(message_item[1], "ERR") || // const statuses_errors = ["ERROR", "ERR", "Error", "Err"];
 		strings.Contains(message_item[1], "Error") || strings.Contains(message_item[1], "Err") {
@@ -49,21 +53,28 @@ func PutLogMessage(db *leveldb.DB, host string, container string, message_item [
 	} else if strings.Contains(message_item[1], "ONLOGS") {
 		vars.Counters_For_Containers_Last_30_Min[location]["meta"]++
 		vars.Statuses_DBs[location].Put([]byte(message_item[0]), []byte("meta"), nil)
-
 	} else {
 		vars.Counters_For_Containers_Last_30_Min[location]["other"]++
 		vars.Statuses_DBs[location].Put([]byte(message_item[0]), []byte("other"), nil)
 	}
 
-	return db.Put([]byte(message_item[0]), []byte(message_item[1]), nil)
+	err := db.Put([]byte(message_item[0]), []byte(message_item[1]), nil)
+	tries := 0
+	for err != nil && tries < 10 {
+		db = util.GetDB(host, container, "logs")
+		err = db.Put([]byte(message_item[0]), []byte(message_item[1]), nil)
+		time.Sleep(10 * time.Millisecond)
+		tries++
+	}
+	if err != nil {
+		panic(err)
+	}
+	return err
 }
 
 func GetLogsByStatus(host string, container string, message string, status string, limit int, startWith string, getPrev bool, include bool, caseSensetivity bool) [][]string {
 	logs_db := util.GetDB(host, container, "logs")
 	db := util.GetDB(host, container, "statuses")
-	if host != util.GetHost() || vars.ActiveDBs[container] == nil {
-		defer logs_db.Close()
-	}
 
 	iter := db.NewIterator(nil, nil)
 	defer iter.Release()
@@ -136,9 +147,6 @@ func GetLogsByStatus(host string, container string, message string, status strin
 
 func GetLogs(getPrev bool, include bool, host string, container string, message string, limit int, startWith string, caseSensetivity bool) [][]string {
 	db := util.GetDB(host, container, "logs")
-	if host != util.GetHost() || vars.ActiveDBs[container] == nil {
-		defer db.Close()
-	}
 
 	iter := db.NewIterator(nil, nil)
 	defer iter.Release()
@@ -215,18 +223,15 @@ func DeleteContainer(host string, container string, fullDelete bool) {
 
 	if vars.ActiveDBs[container] != nil {
 		vars.ActiveDBs[container].Close()
-		newActiveDB, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containers/"+container+"/logs", nil)
-		vars.ActiveDBs[container] = newActiveDB
+		vars.ActiveDBs[container] = util.GetDB(host, container, "active")
 	}
 	if vars.Statuses_DBs[host+"/"+container] != nil {
 		vars.Statuses_DBs[host+"/"+container].Close()
-		newStatusesDB, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containers/"+container+"/statuses", nil)
-		vars.Statuses_DBs[host+"/"+container] = newStatusesDB
+		vars.Statuses_DBs[host+"/"+container] = util.GetDB(host, container, "statuses")
 	}
 	if vars.Stat_Containers_DBs[host+"/"+container] != nil {
 		vars.Stat_Containers_DBs[host+"/"+container].Close()
-		newStatDB, _ := leveldb.OpenFile("leveldb/hosts/"+host+"/containers/"+container+"/statistics", nil)
-		vars.Statuses_DBs[host+"/"+container] = newStatDB
+		vars.Statuses_DBs[host+"/"+container] = util.GetDB(host, container, "statistics")
 	}
 	vars.Counters_For_Containers_Last_30_Min[host+"/"+container] = map[string]uint64{"error": 0, "debug": 0, "info": 0, "warn": 0, "meta": 0, "other": 0}
 }

application/backend/app/daemon/daemon.go

@@ -41,7 +41,7 @@ func validateMessage(message string) (string, bool) {
 }
 
 func createConnection(containerName string) net.Conn {
-	connection, _ := net.Dial("unix", "/var/run/docker.sock")
+	connection, _ := net.Dial("unix", os.Getenv("DOCKER_SOCKET_PATH"))
 	fmt.Fprintf(
 		connection,
 		"GET /containers/"+containerName+"/logs?stdout=true&stderr=true&timestamps=true&follow=true&since="+strconv.FormatInt(time.Now().Add(-5*time.Second).Unix(), 10)+" HTTP/1.0\r\n\r\n",
@@ -66,7 +66,9 @@ func closeActiveStream(containerName string) {
 			newDaemonStreams = append(newDaemonStreams, stream)
 		}
 	}
-	vars.ActiveDBs[containerName].Close()
+	if vars.ActiveDBs[containerName] != nil {
+		vars.ActiveDBs[containerName].Close()
+	}
 	vars.ActiveDBs[containerName] = nil
 	vars.Active_Daemon_Streams = newDaemonStreams
 }
@@ -109,11 +111,10 @@ func CreateDaemonToDBStream(containerName string) {
 	reader := bufio.NewReader(connection)
 	readHeader(*reader)
 
-	current_db := vars.ActiveDBs[containerName]
 	host := util.GetHost()
+	current_db := util.GetDB(host, containerName, "logs")
 	createLogMessage(current_db, host, containerName, "ONLOGS: Container listening started!")
 
-	lastSleep := time.Now().Unix()
 	defer current_db.Close()
 	for { // reading body
 		logLine, get_string_error := reader.ReadString('\n')
@@ -145,16 +146,16 @@ func CreateDaemonToDBStream(containerName string) {
 			c.WriteMessage(1, to_send)
 		}
 
-		if time.Now().Unix()-lastSleep > 1 {
-			time.Sleep(5 * time.Millisecond)
-			lastSleep = time.Now().Unix()
-		}
+		time.Sleep(70 * time.Microsecond)
 	}
 }
 
 // make request to docker socket
 func makeSocketRequest(path string) []byte {
-	connection, _ := net.Dial("unix", "/var/run/docker.sock")
+	connection, err := net.Dial("unix", os.Getenv("DOCKER_SOCKET_PATH"))
+	if err != nil {
+		panic(err)
+	}
 	fmt.Fprintf(connection, "GET /"+path+" HTTP/1.0\r\n\r\n")
 
 	body, _ := ioutil.ReadAll(connection)
@@ -172,14 +173,23 @@ func GetContainersList() []string {
 	json.Unmarshal([]byte(body), &result)
 
 	var names []string
-	containersDB, _ := leveldb.OpenFile("leveldb/hosts/"+util.GetHost()+"/containersMeta", nil)
-	defer containersDB.Close()
+
+	containersMetaDB := vars.ContainersMeta_DBs[util.GetHost()]
+	if containersMetaDB == nil {
+		containersMetaDB, err := leveldb.OpenFile("leveldb/hosts/"+util.GetHost()+"/containersMeta", nil)
+		if err != nil {
+			panic(err)
+		}
+		vars.ContainersMeta_DBs[util.GetHost()] = containersMetaDB
+	}
+	containersMetaDB = vars.ContainersMeta_DBs[util.GetHost()]
+
 	for i := 0; i < len(result); i++ {
 		name := fmt.Sprintf("%v", result[i]["Names"].([]interface{})[0].(string))[1:]
 		id := result[i]["Id"].(string)
 
 		names = append(names, name)
-		containersDB.Put([]byte(name), []byte(id), nil)
+		containersMetaDB.Put([]byte(name), []byte(id), nil)
 	}
 
 	return names

application/backend/app/db/db.go

@@ -4,47 +4,46 @@ import (
 	"time"
 
 	"github.com/devforth/OnLogs/app/util"
+	"github.com/devforth/OnLogs/app/vars"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
 func CreateOnLogsToken() string {
-	tokensDB, _ := leveldb.OpenFile("leveldb/tokens", nil)
-	defer tokensDB.Close()
-
 	token := util.GenerateJWTSecret()
 	to_put := time.Now().UTC().Add(24 * time.Hour).String()
-	tokensDB.Put([]byte(token), []byte(to_put), nil)
+	err := vars.TokensDB.Put([]byte(token), []byte(to_put), nil)
+	if err != nil {
+		vars.TokensDB.Close()
+		vars.TokensDB, vars.TokensDBErr = leveldb.OpenFile("leveldb/tokens", nil)
+
+		err = vars.TokensDB.Put([]byte(token), []byte(to_put), nil)
+		if err != nil {
+			panic(err)
+		}
+	}
 	return token
 }
 
 func IsTokenExists(token string) bool {
-	tokensDB, _ := leveldb.OpenFile("leveldb/tokens", nil)
-	defer tokensDB.Close()
-
-	iter := tokensDB.NewIterator(nil, nil)
+	iter := vars.TokensDB.NewIterator(nil, nil)
 	defer iter.Release()
 	iter.First()
 	if string(iter.Key()) == token {
-		tokensDB.Put([]byte(token), []byte("was used"), nil)
+		vars.TokensDB.Put([]byte(token), []byte("was used"), nil)
 		return true
 	}
 	for iter.Next() {
 		if string(iter.Key()) == token {
-			tokensDB.Put([]byte(token), []byte("was used"), nil)
+			vars.TokensDB.Put([]byte(token), []byte("was used"), nil)
 			return true
 		}
 	}
-
 	return false
 }
 
 func DeleteUnusedTokens() {
 	for {
-		db, r := leveldb.OpenFile("leveldb/tokens", nil)
-		if r != nil {
-			panic(r)
-		}
-
+		db := vars.TokensDB
 		iter := db.NewIterator(nil, nil)
 		for iter.Next() {
 			wasUsed := string(iter.Value())