Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ETOS SSE v2alpha #91

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

ETOS SSE v2alpha #91

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5a5c7f5
Clean up configurations to utilize composition instead
t-persson Oct 23, 2024
28adb70
Add v2alpha version of the SSE server
t-persson Oct 31, 2024
4843dab
Create a streamer interface and add a file streamer
t-persson Nov 20, 2024
6227d24
Review comments
t-persson Nov 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions cmd/executionspace/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"runtime/debug"
"syscall"

config "github.com/eiffel-community/etos-api/internal/configs/executionspace"
"github.com/eiffel-community/etos-api/internal/config"
"github.com/eiffel-community/etos-api/internal/database/etcd"
"github.com/eiffel-community/etos-api/internal/executionspace/provider"
"github.com/eiffel-community/etos-api/internal/logging"
Expand All @@ -39,7 +39,7 @@ import (

// main sets up logging and starts up the webservice.
func main() {
cfg := config.Get()
cfg := config.NewExecutionSpaceConfig()
ctx := context.Background()

var hooks []logrus.Hook
Expand Down Expand Up @@ -120,7 +120,7 @@ func fileLogging(cfg config.Config) logrus.Hook {

// remoteLogging starts a new rabbitmq publisher if the rabbitmq parameters are set
// Warning: Must call publisher.Close() on the publisher returned from this function
func remoteLogging(cfg config.Config) *rabbitmq.Publisher {
func remoteLogging(cfg config.ExecutionSpaceConfig) *rabbitmq.Publisher {
if cfg.RabbitMQHookURL() != "" {
if cfg.RabbitMQHookExchangeName() == "" {
panic("-rabbitmq_hook_exchange (env:ETOS_RABBITMQ_EXCHANGE) must be set when using -rabbitmq_hook_url (env:ETOS_RABBITMQ_URL)")
Expand Down
4 changes: 2 additions & 2 deletions cmd/iut/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
"syscall"
"time"

config "github.com/eiffel-community/etos-api/internal/configs/iut"
"github.com/eiffel-community/etos-api/internal/config"
"github.com/eiffel-community/etos-api/internal/logging"
server "github.com/eiffel-community/etos-api/internal/server"
"github.com/eiffel-community/etos-api/pkg/application"
Expand All @@ -37,7 +37,7 @@ import (

// main sets up logging and starts up the webserver.
func main() {
cfg := config.Get()
cfg := config.NewIUTConfig()
ctx := context.Background()

var hooks []logrus.Hook
Expand Down
136 changes: 136 additions & 0 deletions cmd/keys/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
// Copyright Axis Communications AB.
//
// For a full list of individual contributors, please see the commit history.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
"context"
"net/http"
"os"
"os/signal"
"runtime/debug"
"syscall"
"time"

auth "github.com/eiffel-community/etos-api/internal/authorization"
"github.com/eiffel-community/etos-api/internal/config"
"github.com/eiffel-community/etos-api/internal/logging"
"github.com/eiffel-community/etos-api/internal/server"
"github.com/eiffel-community/etos-api/pkg/application"
v1alpha "github.com/eiffel-community/etos-api/pkg/keys/v1alpha"
"github.com/sirupsen/logrus"
"github.com/snowzach/rotatefilehook"
"go.elastic.co/ecslogrus"
)

// main sets up logging and starts up the key webserver.
func main() {
cfg := config.NewKeyConfig()
ctx := context.Background()

var hooks []logrus.Hook
if fileHook := fileLogging(cfg); fileHook != nil {
hooks = append(hooks, fileHook)
}
logger, err := logging.Setup(cfg.LogLevel(), hooks)
if err != nil {
logrus.Fatal(err.Error())
}

hostname, err := os.Hostname()
if err != nil {
logrus.Fatal(err.Error())
}
log := logger.WithFields(logrus.Fields{
"hostname": hostname,
"application": "ETOS API Key Server",
"version": vcsRevision(),
"name": "ETOS API",
})

log.Info("Loading Key routes")
t-persson marked this conversation as resolved.
Show resolved Hide resolved

pub, err := cfg.PublicKey()
if err != nil {
log.Fatal(err.Error())
}
priv, err := cfg.PrivateKey()
if err != nil {
log.Fatal(err.Error())
}
authorizer, err := auth.NewAuthorizer(pub, priv)
if err != nil {
log.Fatal(err.Error())
}
v1AlphaKeys := v1alpha.New(ctx, cfg, log, authorizer)
defer v1AlphaKeys.Close()
app := application.New(v1AlphaKeys)

srv := server.NewWebService(cfg, log, app)

done := make(chan os.Signal, 1)
signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)

go func() {
if err := srv.Start(); err != nil && err != http.ErrServerClosed {
log.Errorf("Webserver shutdown: %+v", err)
}
}()

sig := <-done
log.Infof("%s received", sig.String())

ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()

if err := srv.Close(ctx); err != nil {
log.Errorf("Webserver shutdown failed: %+v", err)
}
log.Info("Wait for shutdown to complete")
}

// fileLogging adds a hook into a slice of hooks, if the filepath configuration is set
func fileLogging(cfg config.Config) logrus.Hook {
if filePath := cfg.LogFilePath(); filePath != "" {
// TODO: Make these parameters configurable.
// NewRotateFileHook cannot return an error which is why it's set to '_'.
rotateFileHook, _ := rotatefilehook.NewRotateFileHook(rotatefilehook.RotateFileConfig{
Filename: filePath,
MaxSize: 10, // megabytes
MaxBackups: 3,
MaxAge: 0, // days
Level: logrus.DebugLevel,
Formatter: &ecslogrus.Formatter{
DataKey: "labels",
},
})
return rotateFileHook
}
return nil
}

// vcsRevision returns vcs revision from build info, if any. Otherwise '(unknown)'.
func vcsRevision() string {
buildInfo, ok := debug.ReadBuildInfo()
if !ok {
return "(unknown)"
}
for _, val := range buildInfo.Settings {
if val.Key == "vcs.revision" {
return val.Value
}
}
return "(unknown)"
}
4 changes: 2 additions & 2 deletions cmd/logarea/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
"syscall"
"time"

config "github.com/eiffel-community/etos-api/internal/configs/logarea"
"github.com/eiffel-community/etos-api/internal/config"
"github.com/eiffel-community/etos-api/internal/logging"
"github.com/eiffel-community/etos-api/internal/server"
"github.com/eiffel-community/etos-api/pkg/application"
Expand All @@ -36,7 +36,7 @@ import (

// main sets up logging and starts up the logarea webservice.
func main() {
cfg := config.Get()
cfg := config.NewLogAreaConfig()
ctx := context.Background()

var hooks []logrus.Hook
Expand Down
41 changes: 38 additions & 3 deletions cmd/sse/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,25 @@ import (
"syscall"
"time"

config "github.com/eiffel-community/etos-api/internal/configs/sse"
auth "github.com/eiffel-community/etos-api/internal/authorization"
"github.com/eiffel-community/etos-api/internal/config"
"github.com/eiffel-community/etos-api/internal/logging"
"github.com/eiffel-community/etos-api/internal/server"
"github.com/eiffel-community/etos-api/internal/stream"
"github.com/eiffel-community/etos-api/pkg/application"
v1 "github.com/eiffel-community/etos-api/pkg/sse/v1"
v1alpha "github.com/eiffel-community/etos-api/pkg/sse/v1alpha"
v2alpha "github.com/eiffel-community/etos-api/pkg/sse/v2alpha"
"github.com/julienschmidt/httprouter"
rabbitMQStream "github.com/rabbitmq/rabbitmq-stream-go-client/pkg/stream"
"github.com/sirupsen/logrus"
"github.com/snowzach/rotatefilehook"
"go.elastic.co/ecslogrus"
)

// main sets up logging and starts up the sse webserver.
func main() {
cfg := config.Get()
cfg := config.NewSSEConfig()
ctx := context.Background()

var hooks []logrus.Hook
Expand Down Expand Up @@ -66,7 +71,37 @@ func main() {
v1SSE := v1.New(cfg, log, ctx)
defer v1SSE.Close()

app := application.New(v1AlphaSSE, v1SSE)
pub, err := cfg.PublicKey()
if err != nil {
log.Fatal(err.Error())
}
var app *httprouter.Router
// Only load v2alpha if a public key exists.
if pub != nil {
authorizer, err := auth.NewAuthorizer(pub, nil)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume using nil works in this situation although it is kind of ambiguous what (as stated in the docs for New Authorizer) an empty []byte is, since var b []byte is not the same as b :=[]byte{}, although both could be considered an empty byte array.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any null value of byte works in this case. I.e. nil, var b []byte and b := []byte{} would all produce the same result
which is that the private key is ignored.

auth.go checks the len of the byte.

func main() {
  var b []byte
  fmt.Println(len(b))  // 0
  b = []byte{}
  fmt.Println(len(b))  // 0
  b = nil
  fmt.Println(len(b))  // 0
}

if err != nil {
log.Fatal(err.Error())
}

var streamer stream.Streamer
if cfg.RabbitMQURI() != "" {
log.Info("Starting up a RabbitMQStreamer")
streamer, err = stream.NewRabbitMQStreamer(*rabbitMQStream.NewEnvironmentOptions().SetUri(cfg.RabbitMQURI()), log)
} else {
log.Warning("RabbitMQURI is not set, defaulting to FileStreamer")
streamer, err = stream.NewFileStreamer(100*time.Millisecond, log)
}
if err != nil {
log.Fatal(err.Error())
}
v2AlphaSSE := v2alpha.New(ctx, cfg, log, streamer, authorizer)
defer v2AlphaSSE.Close()
app = application.New(v1AlphaSSE, v1SSE, v2AlphaSSE)
} else {
log.Warning("Public key does not exist, won't enable v2alpha endpoint")
app = application.New(v1AlphaSSE, v1SSE)
}

srv := server.NewWebService(cfg, log, app)

done := make(chan os.Signal, 1)
Expand Down
35 changes: 19 additions & 16 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,27 @@ toolchain go1.22.1
require (
github.com/eiffel-community/eiffelevents-sdk-go v0.0.0-20240807115026-5ca5c194b7dc
github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611
github.com/golang-jwt/jwt/v5 v5.2.1
github.com/google/uuid v1.6.0
github.com/jmespath/go-jmespath v0.4.0
github.com/julienschmidt/httprouter v1.3.0
github.com/machinebox/graphql v0.2.2
github.com/maxcnunes/httpfake v1.2.4
github.com/package-url/packageurl-go v0.1.3
github.com/rabbitmq/amqp091-go v1.10.0
github.com/rabbitmq/rabbitmq-stream-go-client v1.4.10
github.com/sethvargo/go-retry v0.3.0
github.com/sirupsen/logrus v1.9.3
github.com/snowzach/rotatefilehook v0.0.0-20220211133110-53752135082d
github.com/stretchr/testify v1.9.0
go.elastic.co/ecslogrus v1.0.0
go.etcd.io/etcd/api/v3 v3.5.15
go.etcd.io/etcd/client/v3 v3.5.15
go.etcd.io/etcd/server/v3 v3.5.14
go.opentelemetry.io/otel v1.20.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0
go.opentelemetry.io/otel/sdk v1.20.0
go.opentelemetry.io/otel/trace v1.20.0
k8s.io/api v0.31.1
k8s.io/apimachinery v0.31.1
k8s.io/client-go v0.31.1
)
Expand All @@ -46,6 +53,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
Expand All @@ -59,21 +67,22 @@ require (
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/magefile/mage v1.9.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/matryer/is v1.4.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.11.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
github.com/soheilhy/cmux v0.1.5 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
Expand All @@ -82,29 +91,24 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
go.etcd.io/bbolt v1.3.10 // indirect
go.etcd.io/etcd/api/v3 v3.5.15 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect
go.etcd.io/etcd/client/v2 v2.305.14 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.14 // indirect
go.etcd.io/etcd/raft/v3 v3.5.14 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect
go.opentelemetry.io/otel v1.20.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect
go.opentelemetry.io/otel/metric v1.20.0 // indirect
go.opentelemetry.io/otel/sdk v1.20.0 // indirect
go.opentelemetry.io/otel/trace v1.20.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.17.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/term v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/sys v0.23.0 // indirect
golang.org/x/term v0.23.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
Expand All @@ -114,7 +118,6 @@ require (
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.31.1 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
Expand Down
Loading
Loading