From 40e8d517e87966d86dde9e929619494955a71a03 Mon Sep 17 00:00:00 2001 From: Keith James Date: Thu, 11 Apr 2024 18:36:30 +0100 Subject: [PATCH] Add indexing of iRODS paths to homepage Paths tagged in iRODS with specific metadata are indexed automatically and presented on the homepage. --- README.md | 27 +++- cmd/sqyrrl.go | 22 ++- server/handlers.go | 113 ++++++++++++- server/index.go | 155 ++++++++++++++++++ server/irods.go | 45 ++++++ server/routes.go | 2 +- server/server.go | 271 +++++++++++++++++++------------- server/static/style.css | 105 +++++++++++++ server/templates/_footer.gothml | 1 - server/templates/_header.gohtml | 8 +- server/templates/home.gohtml | 83 +++++++++- 11 files changed, 694 insertions(+), 138 deletions(-) create mode 100644 server/index.go create mode 100644 server/static/style.css diff --git a/README.md b/README.md index 013af14..b9acf5a 100644 --- a/README.md +++ b/README.md @@ -37,15 +37,17 @@ Usage: sqyrrl start [flags] Flags: - --cert-file string Path to the SSL certificate file - -h, --help help for start - --host string Address on which to listen, host part (default "localhost") - --irods-env string Path to the iRODS environment file (default "~/.irods/irods_environment.json") - --key-file string Path to the SSL private key file - --port int Port on which to listen (default 3333) + --cert-file string Path to the SSL certificate file + -h, --help help for start + --host string Address on which to listen, host part (default "localhost") + --index-interval duration Interval at which update the index (default 1m0s) + --irods-env string Path to the iRODS environment file (default "/Users/kdj/.irods/irods_environment.json") + --key-file string Path to the SSL private key file + --port string Port on which to listen (default "3333") Global Flags: --log-level string Set the log level (trace, debug, info, warn, error) (default "info") + ``` To stop the server, send `SIGINT` or `SIGTERM` to the process. The server will wait for @@ -53,6 +55,17 @@ active connections to close before shutting down. For additional options, use the `--help` flag. +## Tagging iRODS data objects for display on the home page + +This is an experiment feature. It allows the user to tag iRODS data objects with metadata so that they will +be displayed in the Sqyrrl home page for convenience. To tag an iRODS data object, add a metadata attribute +`sqyrrl:index` with value `1`. Data objects may be grouped together on the page, under a title, known as a +"category". To specify a category for a data object, add a metadata attribute `sqyrrl:category` with the +value being the category name. + +The home page will be re-indexed at the interval specified by the `--index-interval` flag. The home page +auto-refreshes every 30 seconds. + ## Dependencies -Sqyrrl uses [go-irodsclient](https://github.com/cyverse/go-irodsclient) to connect to iRODS. \ No newline at end of file +Sqyrrl uses [go-irodsclient](https://github.com/cyverse/go-irodsclient) to connect to iRODS. \ No newline at end of file diff --git a/cmd/sqyrrl.go b/cmd/sqyrrl.go index 41af59f..ea51889 100644 --- a/cmd/sqyrrl.go +++ b/cmd/sqyrrl.go @@ -41,7 +41,9 @@ type cliFlags struct { host string // Address to listen on, host part level string // Logging level - port int // Port to listen on + port string // Port to listen on + + indexInterval time.Duration // Interval to index files } var cliFlagsSelected = cliFlags{ @@ -117,11 +119,12 @@ func startServer(cmd *cobra.Command, args []string) { logger := configureRootLogger(&cliFlagsSelected) server.ConfigureAndStart(logger, server.Config{ - Host: cliFlagsSelected.host, - Port: cliFlagsSelected.port, - CertFilePath: cliFlagsSelected.certFilePath, - KeyFilePath: cliFlagsSelected.keyFilePath, - EnvFilePath: cliFlagsSelected.envFilePath, + Host: cliFlagsSelected.host, + Port: cliFlagsSelected.port, + CertFilePath: cliFlagsSelected.certFilePath, + KeyFilePath: cliFlagsSelected.keyFilePath, + EnvFilePath: cliFlagsSelected.envFilePath, + IndexInterval: cliFlagsSelected.indexInterval, }) } @@ -147,8 +150,8 @@ func CLI() { startCmd.Flags().StringVar(&cliFlagsSelected.host, "host", "localhost", "Address on which to listen, host part") - startCmd.Flags().IntVar(&cliFlagsSelected.port, - "port", 3333, + startCmd.Flags().StringVar(&cliFlagsSelected.port, + "port", "3333", "Port on which to listen") startCmd.Flags().StringVar(&cliFlagsSelected.certFilePath, "cert-file", "", @@ -159,6 +162,9 @@ func CLI() { startCmd.Flags().StringVar(&cliFlagsSelected.envFilePath, "irods-env", server.IRODSEnvFilePath(), "Path to the iRODS environment file") + startCmd.Flags().DurationVar(&cliFlagsSelected.indexInterval, + "index-interval", server.DefaultIndexInterval, + "Interval at which update the index") rootCmd.AddCommand(startCmd) diff --git a/server/handlers.go b/server/handlers.go index 23daf9a..2c4b5bd 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -18,16 +18,29 @@ package server import ( + "context" + "github.com/rs/xid" + "github.com/rs/zerolog/hlog" "io/fs" "net/http" "path" + "time" "github.com/cyverse/go-irodsclient/irods/types" "github.com/rs/zerolog" ) +// HandlerChain is a function that takes an http.Handler and returns an new http.Handler +// wrapping the input handler. Each handler in the chain should process the request in +// some way, and then call the next handler. Ideally, the functionality of each handler +// should be orthogonal to the others. +// +// This is sometimes called "middleware" in Go. I haven't used that term here because it +// already has an established meaning in the context of operating systems and networking. +type HandlerChain func(http.Handler) http.Handler + // HandleHomePage is a handler for the static home page. -func HandleHomePage(logger zerolog.Logger) http.Handler { +func HandleHomePage(logger zerolog.Logger, index *ItemIndex) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { logger.Trace().Msg("HomeHandler called") @@ -42,12 +55,23 @@ func HandleHomePage(logger zerolog.Logger) http.Handler { http.Redirect(w, r, redirect, http.StatusPermanentRedirect) } - type customData struct { - URL string - Version string + type pageData struct { + Version string + Categories []string + CategorisedItems map[string][]Item } - data := customData{Version: Version, URL: r.URL.RequestURI()} + catItems := make(map[string][]Item) + cats := index.Categories() + for _, cat := range cats { + catItems[cat] = index.ItemsInCategory(cat) + } + + data := pageData{ + Version: Version, + Categories: cats, + CategorisedItems: catItems, + } tplName := "home.gohtml" if err := templates.ExecuteTemplate(w, tplName, data); err != nil { @@ -59,6 +83,8 @@ func HandleHomePage(logger zerolog.Logger) http.Handler { } func HandleStaticContent(logger zerolog.Logger) http.Handler { + logger.Trace().Msg("StaticContentHandler called") + sub := func(dir fs.FS, name string) fs.FS { f, err := fs.Sub(dir, name) if err != nil { @@ -100,3 +126,80 @@ func HandleIRODSGet(logger zerolog.Logger, account *types.IRODSAccount) http.Han getFileRange(rodsLogger, w, r, account, sanPath) }) } + +// AddRequestLogger adds an HTTP request suiteLogger to the handler chain. +// +// If a correlation ID is present in the request context, it is logged. +func AddRequestLogger(logger zerolog.Logger) HandlerChain { + return func(next http.Handler) http.Handler { + lh := hlog.NewHandler(logger) + + ah := hlog.AccessHandler(func(r *http.Request, status, size int, dur time.Duration) { + var corrID string + if val := r.Context().Value(correlationIDKey); val != nil { + corrID = val.(string) + } + + hlog.FromRequest(r).Info(). + Str("correlation_id", corrID). + Dur("duration", dur). + Int("size", size). + Int("status", status). + Str("method", r.Method). + Str("url", r.URL.RequestURI()). + Str("remote_addr", r.RemoteAddr). + Str("forwarded_for", r.Header.Get(HeaderForwardedFor)). + Str("user_agent", r.UserAgent()). + Msg("Request served") + }) + return lh(ah(next)) + } +} + +// AddCorrelationID adds a correlation ID to the request context and response headers. +func AddCorrelationID(logger zerolog.Logger) HandlerChain { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var corrID string + if corrID = r.Header.Get(HeaderCorrelationID); corrID == "" { + corrID = xid.New().String() + logger.Trace(). + Str("correlation_id", corrID). + Str("url", r.URL.RequestURI()). + Msg("Creating a new correlation ID") + w.Header().Add(HeaderCorrelationID, corrID) + } else { + logger.Trace(). + Str("correlation_id", corrID). + Str("url", r.URL.RequestURI()). + Msg("Using correlation ID from request") + } + + ctx := context.WithValue(r.Context(), correlationIDKey, corrID) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} + +func SanitiseRequestURL(logger zerolog.Logger) HandlerChain { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // URLs are already cleaned by the Go ServeMux. This is in addition + + dirtyPath := r.URL.Path + sanPath := userInputPolicy.Sanitize(dirtyPath) + if sanPath != dirtyPath { + logger.Warn(). + Str("sanitised_path", sanPath). + Str("dirty_path", dirtyPath). + Msg("Path was sanitised") + } + + url := r.URL + url.Path = sanPath + r.URL = url + + next.ServeHTTP(w, r) + }) + } +} diff --git a/server/index.go b/server/index.go new file mode 100644 index 0000000..deca5ea --- /dev/null +++ b/server/index.go @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2024. Genome Research Ltd. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package server + +import ( + "fmt" + "slices" + "sort" + "strings" + + "github.com/cyverse/go-irodsclient/irods/types" +) + +// ItemIndex is a collection of iRODS paths that have been tagged for presentation on +// the Sqyrrl web interface. +// +// The index is updated by the server as new items are added to the system. Each item +// has a category, which is used to group items together. The functions associated with +// the index allow the web interface to display the items in a structured way. +type ItemIndex struct { + items []Item +} + +// Item represents a item in the index. +type Item struct { + Path string // The iRODS path of the item. + Size int64 // The size of the item in bytes. + Metadata []*types.IRODSMeta + ACL []*types.IRODSAccess +} + +func NewItemIndex(items []Item) *ItemIndex { + return &ItemIndex{items: items} +} + +// Categories returns a sorted list of all the categories in the index. +func (index *ItemIndex) Categories() []string { + var categorySet = make(map[string]struct{}) + for _, item := range index.items { + categorySet[item.Category()] = struct{}{} + } + + categories := make([]string, len(categorySet)) + i := 0 + for cat := range categorySet { + categories[i] = cat + i++ + } + slices.Sort(categories) + + return categories +} + +// ItemsInCategory returns a sorted list of all the items in the index that are in the +// given category. +func (index *ItemIndex) ItemsInCategory(cat string) []Item { + var items []Item + for _, item := range index.items { + if item.Category() == cat { + items = append(items, item) + } + } + sort.SliceStable(items, func(i, j int) bool { + return items[i].Path < items[j].Path + }) + + return items +} + +func (index *ItemIndex) String() string { + var sb strings.Builder + sb.WriteString("") + + return sb.String() +} + +func (item *Item) Category() string { + var category string + for _, meta := range item.Metadata { + if meta.Name == CategoryAttr { + category = meta.Value + break + } + } + return category +} + +func (item *Item) SizeString() string { + if item.Size < 1024 { + return fmt.Sprintf("%d B", item.Size) + } + + return fmt.Sprintf("%d KiB", item.Size/1024) +} + +// MetadataStrings returns a sorted list of strings representing the metadata of the item. +func (item *Item) MetadataStrings() []string { + var meta []string + for _, m := range item.Metadata { + meta = append(meta, fmt.Sprintf("%s=%s", m.Name, m.Value)) + } + slices.Sort(meta) + + return meta +} + +// ACLStrings returns a sorted list of strings representing the ACL of the item. +func (item *Item) ACLStrings() []string { + var acl []string + for _, a := range item.ACL { + acl = append(acl, fmt.Sprintf("%s#%s:%s", a.UserName, a.UserZone, a.AccessLevel)) + } + slices.Sort(acl) + + return acl +} + +func (item *Item) String() string { + return fmt.Sprintf("", + item.Path, + item.Category(), + item.Size, + strings.Join(item.ACLStrings(), ", "), + strings.Join(item.MetadataStrings(), ", ")) +} diff --git a/server/irods.go b/server/irods.go index 7989de9..f205460 100644 --- a/server/irods.go +++ b/server/irods.go @@ -36,6 +36,12 @@ const iRODSEnvFileEnvVar = "IRODS_ENVIRONMENT_FILE" const PublicUser = "public" +const Namespace = "sqyrrl" +const NamespaceSeparator = ":" +const IndexAttr = Namespace + NamespaceSeparator + "index" +const IndexValue = "1" +const CategoryAttr = Namespace + NamespaceSeparator + "category" + // IRODSEnvFilePath returns the path to the iRODS environment file. If the path // is not set in the environment, the default path is returned. func IRODSEnvFilePath() string { @@ -181,6 +187,8 @@ func isPublicReadable(logger zerolog.Logger, filesystem *fs.FileSystem, return false, nil } +// getFileRange serves a file from iRODS to the client. It delegates to http.ServeContent +// which sets the appropriate headers, including Content-Type. func getFileRange(logger zerolog.Logger, w http.ResponseWriter, r *http.Request, account *types.IRODSAccount, rodsPath string) { @@ -257,3 +265,40 @@ func getFileRange(logger zerolog.Logger, w http.ResponseWriter, r *http.Request, logger.Info().Str("path", rodsPath).Msg("Serving file") http.ServeContent(w, r, rodsPath, time.Now(), fh) } + +// The index attribute and value should be configurable so that individual users can +// customise the metadata used to index their data. This will allow them to focus on +// specific data objects or collections interesting to them. + +// findItems runs a metadata query against iRODS to find any items that have metadata +// with the key sqyrrl::index and value 1. The items are grouped by the value of the +// metadata. +func findItems(filesystem *fs.FileSystem) ([]Item, error) { + entries, err := filesystem.SearchByMeta(IndexAttr, IndexValue) + if err != nil { + return nil, err + } + + filesystem.ClearCache() + + var items []Item + for _, entry := range entries { + acl, err := filesystem.ListACLs(entry.Path) + if err != nil { + return nil, err + } + + metadata, err := filesystem.ListMetadata(entry.Path) + if err != nil { + return nil, err + } + + items = append(items, Item{ + Path: entry.Path, + Size: entry.Size, + ACL: acl, + Metadata: metadata, + }) + } + return items, nil +} diff --git a/server/routes.go b/server/routes.go index 8cefe46..d255539 100644 --- a/server/routes.go +++ b/server/routes.go @@ -43,7 +43,7 @@ func (server *SqyrrlServer) addRoutes(mux *http.ServeMux) { // // Any requests relative to the root are redirected to the API endpoint mux.Handle("GET "+EndpointRoot, - correlate(logRequest(HandleHomePage(server.logger)))) + correlate(logRequest(HandleHomePage(server.logger, server.index)))) mux.Handle("GET "+EndPointStatic, correlate(logRequest(getStatic))) diff --git a/server/server.go b/server/server.go index 10022e3..39987cc 100644 --- a/server/server.go +++ b/server/server.go @@ -22,56 +22,52 @@ import ( "embed" "errors" "html/template" + "math" "net" "net/http" "os" "os/signal" - "strconv" "strings" "syscall" "time" _ "crypto/tls" // Leave this to ensure that the TLS package is linked + "github.com/cyverse/go-irodsclient/fs" "github.com/cyverse/go-irodsclient/icommands" "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/go-irodsclient/irods/util" "github.com/microcosm-cc/bluemonday" - "github.com/rs/xid" "github.com/rs/zerolog" - "github.com/rs/zerolog/hlog" ) type ContextKey string -// HandlerChain is a function that takes an http.Handler and returns an new http.Handler -// wrapping the input handler. Each handler in the chain should process the request in -// some way, and then call the next handler. Ideally, the functionality of each handler -// should be orthogonal to the others. -// -// This is sometimes called "middleware" in Go. I haven't used that term here because it -// already has an established meaning in the context of operating systems and networking. -type HandlerChain func(http.Handler) http.Handler - // SqyrrlServer is an HTTP server which contains an embedded iRODS client. type SqyrrlServer struct { http.Server - context context.Context // Context for clean shutdown - logger zerolog.Logger // Base logger from which the server creates its own sub-loggers - manager *icommands.ICommandsEnvironmentManager // iRODS manager for the embedded client - account *types.IRODSAccount // iRODS account for the embedded client + context context.Context // Context for clean shutdown + logger zerolog.Logger // Base logger from which the server creates its own sub-loggers + manager *icommands.ICommandsEnvironmentManager // iRODS manager for the embedded client + account *types.IRODSAccount // iRODS account for the embedded client + indexInterval time.Duration // Interval for indexing items + index *ItemIndex // ItemIndex of items in the iRODS server } type Config struct { - Host string - Port int - EnvFilePath string // Path to the iRODS environment file - CertFilePath string - KeyFilePath string + Host string + Port string + EnvFilePath string // Path to the iRODS environment file + CertFilePath string + KeyFilePath string + IndexInterval time.Duration } const AppName = "sqyrrl" +const MinIndexInterval = 10 * time.Second +const DefaultIndexInterval = 60 * time.Second + const correlationIDKey = ContextKey("correlation_id") const staticContentDir = "static" @@ -106,7 +102,7 @@ func NewSqyrrlServer(logger zerolog.Logger, config Config) (*SqyrrlServer, error if config.Host == "" { return nil, errors.New("missing host") } - if config.Port == 0 { + if config.Port == "" { return nil, errors.New("missing port") } if config.CertFilePath == "" { @@ -114,6 +110,15 @@ func NewSqyrrlServer(logger zerolog.Logger, config Config) (*SqyrrlServer, error errors.New("missing certificate file path") } + indexInterval := config.IndexInterval + if indexInterval < MinIndexInterval { + logger.Warn(). + Dur("interval", indexInterval). + Dur("min_interval", MinIndexInterval). + Msg("Index interval too short, using the default interval") + indexInterval = DefaultIndexInterval + } + // The sub-suiteLogger adds "hostname and" "component" field to the log entries. Further // fields are added by other components e.g. in the HTTP handlers. hostname, err := os.Hostname() @@ -144,7 +149,7 @@ func NewSqyrrlServer(logger zerolog.Logger, config Config) (*SqyrrlServer, error return nil, err } - addr := net.JoinHostPort(config.Host, strconv.Itoa(config.Port)) + addr := net.JoinHostPort(config.Host, config.Port) mux := http.NewServeMux() serverCtx, cancelServer := context.WithCancel(context.Background()) @@ -159,8 +164,11 @@ func NewSqyrrlServer(logger zerolog.Logger, config Config) (*SqyrrlServer, error subLogger, manager, account, + indexInterval, + NewItemIndex([]Item{}), } + server.setUpIndexing(serverCtx) server.setUpSignalHandler(cancelServer) server.addRoutes(mux) @@ -196,45 +204,6 @@ func (server *SqyrrlServer) Start(certFile string, keyFile string) { server.waitAndShutdown() } -func ConfigureAndStart(logger zerolog.Logger, config Config) { - if config.Host == "" { - logger.Error().Msg("Missing host component of address to listen on") - return - } - if config.Port == 0 { - logger.Error().Msg("Missing port component of address to listen on") - return - } - if config.CertFilePath == "" { - logger.Error().Msg("Missing certificate file path") - return - } - if config.KeyFilePath == "" { - logger.Error().Msg("Missing key file path") - return - } - if config.EnvFilePath == "" { - logger.Error().Msg("Missing iRODS environment file path") - return - } - - envFilePath, err := util.ExpandHomeDir(config.EnvFilePath) - if err != nil { - logger.Err(err).Str("path", config.EnvFilePath). - Msg("Failed to expand the iRODS environment file path") - return - } - config.EnvFilePath = envFilePath - - server, err := NewSqyrrlServer(logger, config) - if err != nil { - logger.Err(err).Msg("Failed to create a server") - return - } - - server.Start(config.CertFilePath, config.KeyFilePath) -} - func (server *SqyrrlServer) setUpSignalHandler(cancel context.CancelFunc) { logger := server.logger @@ -258,6 +227,39 @@ func (server *SqyrrlServer) setUpSignalHandler(cancel context.CancelFunc) { }() } +func (server *SqyrrlServer) setUpIndexing(ctx context.Context) { + logger := server.logger + + filesystem, err := fs.NewFileSystemWithDefault(server.account, AppName) + if err != nil { + panic(err) + } + + // Query the iRODS server for items at regular intervals + items := queryAtIntervalsWithBackoff(logger, ctx, server.indexInterval, + func() ([]Item, error) { + return findItems(filesystem) + }) + + logger.Info().Dur("interval", server.indexInterval).Msg("Indexing started") + + // Create an index of items + go func() { + for { + select { + case <-ctx.Done(): + logger.Info().Msg("Indexing cancelled") + return + case items := <-items: + server.index.items = items + logger.Info(). + Str("index", server.index.String()). + Msg("Updated index") + } + } + }() +} + func (server *SqyrrlServer) waitAndShutdown() { logger := server.logger @@ -270,58 +272,47 @@ func (server *SqyrrlServer) waitAndShutdown() { logger.Info().Msg("Server shutdown cleanly") } -// AddRequestLogger adds an HTTP request suiteLogger to the handler chain. -// -// If a correlation ID is present in the request context, it is logged. -func AddRequestLogger(logger zerolog.Logger) HandlerChain { - return func(next http.Handler) http.Handler { - lh := hlog.NewHandler(logger) - - ah := hlog.AccessHandler(func(r *http.Request, status, size int, dur time.Duration) { - var corrID string - if val := r.Context().Value(correlationIDKey); val != nil { - corrID = val.(string) - } - - hlog.FromRequest(r).Info(). - Str("correlation_id", corrID). - Dur("duration", dur). - Int("size", size). - Int("status", status). - Str("method", r.Method). - Str("url", r.URL.RequestURI()). - Str("remote_addr", r.RemoteAddr). - Str("forwarded_for", r.Header.Get(HeaderForwardedFor)). - Str("user_agent", r.UserAgent()). - Msg("Request served") - }) - return lh(ah(next)) +func ConfigureAndStart(logger zerolog.Logger, config Config) { + if config.Host == "" { + logger.Error().Msg("Missing host component of address to listen on") + return + } + if config.Port == "" { + logger.Error().Msg("Missing port component of address to listen on") + return + } + if config.CertFilePath == "" { + logger.Error().Msg("Missing certificate file path") + return + } + if config.KeyFilePath == "" { + logger.Error().Msg("Missing key file path") + return + } + if config.EnvFilePath == "" { + logger.Error().Msg("Missing iRODS environment file path") + return + } + if !(config.IndexInterval > 0) { + logger.Error().Msg("Missing index interval") + return } -} -// AddCorrelationID adds a correlation ID to the request context and response headers. -func AddCorrelationID(logger zerolog.Logger) HandlerChain { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var corrID string - if corrID = r.Header.Get(HeaderCorrelationID); corrID == "" { - corrID = xid.New().String() - logger.Trace(). - Str("correlation_id", corrID). - Str("url", r.URL.RequestURI()). - Msg("Creating a new correlation ID") - w.Header().Add(HeaderCorrelationID, corrID) - } else { - logger.Trace(). - Str("correlation_id", corrID). - Str("url", r.URL.RequestURI()). - Msg("Using correlation ID from request") - } + envFilePath, err := util.ExpandHomeDir(config.EnvFilePath) + if err != nil { + logger.Err(err).Str("path", config.EnvFilePath). + Msg("Failed to expand the iRODS environment file path") + return + } + config.EnvFilePath = envFilePath - ctx := context.WithValue(r.Context(), correlationIDKey, corrID) - next.ServeHTTP(w, r.WithContext(ctx)) - }) + server, err := NewSqyrrlServer(logger, config) + if err != nil { + logger.Err(err).Msg("Failed to create a server") + return } + + server.Start(config.CertFilePath, config.KeyFilePath) } func writeErrorResponse(logger zerolog.Logger, w http.ResponseWriter, code int, message ...string) { @@ -340,3 +331,63 @@ func writeErrorResponse(logger zerolog.Logger, w http.ResponseWriter, code int, http.Error(w, msg, code) } + +// queryAtIntervalsWithBackoff runs a query at regular intervals until the context is +// cancelled. +// +// If the query takes longer than the interval, the next query is delayed by +// the time taken by the previous query. If the query takes less time than the interval +// by a certain factor, the interval is shrunk to that value, but not below the original. +func queryAtIntervalsWithBackoff(logger zerolog.Logger, ctx context.Context, + interval time.Duration, queryFn func() ([]Item, error)) chan []Item { + items := make(chan []Item) + + origInterval := interval + shrinkFactor := 0.7 + findTick := time.NewTicker(interval) + + go func() { + defer close(items) + defer findTick.Stop() + + for { + select { + case <-findTick.C: + start := time.Now() + x, err := queryFn() + if err != nil { + logger.Err(err).Msg("Query failed") + } else { + items <- x + } + + elapsed := time.Since(start) + + // If the query took longer than the interval, back off by making the next + // query wait for the extra amount of time in excess of the internal that + // the last query took. + if elapsed > interval { + backoff := time.NewTimer(elapsed - interval) + select { + case <-backoff.C: + // Continue to the next iteration + case <-ctx.Done(): + logger.Info().Msg("Query cancelled") + return + } + } + // If the query took less time than the interval by the shrink factor, + // shrink the interval to that value, but not below the original value. + threshold := interval.Seconds() * shrinkFactor + if elapsed.Seconds() < threshold { + interval = time.Duration(math.Max(threshold, origInterval.Seconds())) + } + case <-ctx.Done(): + logger.Info().Msg("Query cancelled") + return + } + } + }() + + return items +} diff --git a/server/static/style.css b/server/static/style.css new file mode 100644 index 0000000..fbeccff --- /dev/null +++ b/server/static/style.css @@ -0,0 +1,105 @@ + +html { + background-color: lightgray; + font-family: Arial, sans-serif; +} +.container { + display: grid; + grid-template-columns: 1fr 10fr 1fr; + grid-template-rows: auto; +} + +.top-left-cell { + grid-column: 1; + margin: auto; +} + +.top-center { + grid-column: 2; + margin: auto; +} + +.top-right-cell { + grid-column: 3; + margin: auto; +} + +.left-cell { + grid-column: 1; +} + +.main-cell { + grid-column: 2; + background-color: lightgray; +} + +.right-cell { + grid-column: 3; + background-color: lightgray; +} + +.url-grid { + display: grid; + gap: 0.1rem; + grid-template-columns: repeat(20, [col] 1fr); + grid-template-rows: repeat(auto-fit, [row] auto); +} + +.url-cell { + text-align: left; + grid-column: col 1 / span 9; +} + +.info-cell { + font-size: smaller; + grid-column: col 10 / span 2; +} + +.acl-cell { + font-size: smaller; + grid-column: col 12 / span 4; +} + +.metadata-cell { + font-size: smaller; + grid-column: col 16 / span 4; +} + +.info-item { + padding: 0.4rem; + border-radius: calc(0.5rem); + text-align: center; +} + +.acl-bag { + padding: 0.2rem; + border-radius: calc(0.5rem); + gap: 0.2rem; + background: gray; + display: flex; + flex-wrap: wrap; +} + +.acl-item { + padding: 0.2rem; + border-radius: calc(0.5rem); + text-align: center; + background-color: orange; +} + +.metadata-bag { + padding: 0.2rem; + border-radius: calc(0.5rem); + gap: 0.2rem; + background: dimgray; + display: flex; + flex-wrap: wrap; +} + +.metadata-item { + padding: 0.2rem; + border-radius: calc(0.5rem); + text-align: center; + font-size-adjust: 0.5; + background-color: cornflowerblue; +} diff --git a/server/templates/_footer.gothml b/server/templates/_footer.gothml index 14552c3..ed50bff 100644 --- a/server/templates/_footer.gothml +++ b/server/templates/_footer.gothml @@ -1,4 +1,3 @@ {{define "footer"}} - {{end}} diff --git a/server/templates/_header.gohtml b/server/templates/_header.gohtml index bd56806..4258638 100644 --- a/server/templates/_header.gohtml +++ b/server/templates/_header.gohtml @@ -2,9 +2,11 @@ Sqyrrl - - - + + + + + {{end}} diff --git a/server/templates/home.gohtml b/server/templates/home.gohtml index de7e704..a004ff2 100644 --- a/server/templates/home.gohtml +++ b/server/templates/home.gohtml @@ -1,8 +1,85 @@ {{template "header" .}} -
-

Sqyrrl Homepage

+{{ $categories := .Categories }} +{{ $items := .CategorisedItems }} -

Application version: {{.Version}}

+
+
Sqyrrl
+ +

Sqyrrl

+ +
Version: {{ .Version}}
+ +
+
+

Categorised, tagged items

+ + {{ range $category := $categories }} + {{ if $category }} +

{{$category}}

+ +
+ {{ $citems := index $items $category }} + {{ range $citem := $citems }} +
{{ $citem.Path }}
+
+
{{ $citem.SizeString }}
+
+
+
+ {{ with $citem }} + {{ range $av := .ACLStrings }} +
{{ $av }}
+ {{ end }} + {{ end }} +
+
+
+
+ {{ with $citem }} + {{ range $av := .MetadataStrings }} +
{{ $av }}
+ {{ end }} + {{ end }} +
+
+ + {{ end }} +
+ {{ end }} + {{ end }} + +
+

Uncategorised, tagged items

+ +
+ {{ $citems := index $items "" }} + {{ range $citem := $citems }} +
{{ $citem.Path }}
+
+
{{ $citem.SizeString }}
+
+
+
+ {{ with $citem }} + {{ range $av := .ACLStrings }} +
{{ $av }}
+ {{ end }} + {{ end }} +
+
+
+
+ {{ with $citem }} + {{ range $av := .MetadataStrings }} +
{{ $av }}
+ {{ end }} + {{ end }} +
+
+ + {{ end }} +
+
{{template "footer"}}