Merge pull request #138 from czcorpus/conf-improv

Allow for using separate conf files for each corpus

cmd/service/fcs.go

@@ -35,13 +35,15 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/czcorpus/cnc-gokit/collections"
 	"github.com/czcorpus/cnc-gokit/logging"
 	"github.com/czcorpus/cnc-gokit/uniresp"
 	"github.com/czcorpus/mquery-common/concordance"
 	"github.com/gin-gonic/gin"
 	"github.com/rs/zerolog/log"
 
 	"github.com/czcorpus/mquery-sru/cnf"
+	"github.com/czcorpus/mquery-sru/corpus"
 	"github.com/czcorpus/mquery-sru/general"
 	"github.com/czcorpus/mquery-sru/handler"
 	"github.com/czcorpus/mquery-sru/handler/form"
@@ -215,6 +217,16 @@ func main() {
 	log.Info().Msg("MQuery-SRU initialization...")
 	cnf.ValidateAndDefaults(conf)
 
+	log.Info().
+		Strs(
+			"corpora",
+			collections.SliceMap(
+				conf.CorporaSetup.Resources,
+				func(item *corpus.CorpusSetup, i int) string { return item.ID },
+			),
+		).
+		Msgf("providing %d resources/corpora", len(conf.CorporaSetup.Resources))
+
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 

cnf/conf.go

@@ -21,6 +21,7 @@ package cnf
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"time"
@@ -162,6 +163,28 @@ func (conf *Conf) GetSourcePath() string {
 	return filepath.Join(cwd, conf.srcPath)
 }
 
+func loadResources(path string) ([]*corpus.CorpusSetup, error) {
+	ans := make([]*corpus.CorpusSetup, 0, 20)
+	items, err := os.ReadDir(path)
+	if err != nil {
+		return ans, fmt.Errorf("failed to list resource conf directory: %w", err)
+	}
+	for _, item := range items {
+		fmt.Println("item: ", item.Name())
+		rawConf, err := os.ReadFile(filepath.Join(path, item.Name()))
+		if err != nil {
+			return ans, fmt.Errorf("failed to list resource conf file %s: %w", item.Name(), err)
+		}
+		var cs corpus.CorpusSetup
+		err = json.Unmarshal(rawConf, &cs)
+		if err != nil {
+			return ans, fmt.Errorf("failed to parse resource conf file %s: %w", item.Name(), err)
+		}
+		ans = append(ans, &cs)
+	}
+	return ans, nil
+}
+
 func LoadConfig(path string) *Conf {
 	if path == "" {
 		log.Fatal().Msg("Cannot load config - path not specified")
@@ -176,6 +199,13 @@ func LoadConfig(path string) *Conf {
 	if err != nil {
 		log.Fatal().Err(err).Msg("Cannot load config")
 	}
+	if conf.CorporaSetup.ResourcesConfDir != "" {
+		rsrcs, err := loadResources(conf.CorporaSetup.ResourcesConfDir)
+		if err != nil {
+			log.Fatal().Err(err).Msg("Cannot load individual resource configs")
+		}
+		conf.CorporaSetup.Resources = append(conf.CorporaSetup.Resources, rsrcs...)
+	}
 	return &conf
 }
 

corpus/conf.go

@@ -423,6 +423,13 @@ type CorporaSetup struct {
 
 	// Resources is a description of configured corpora/resources
 	Resources SrchResources `json:"resources"`
+
+	// ResourcesConfDir is an alternative to Resources allowing to use
+	// a separate configuration file for each corpus. Both values can
+	// be used simultaneously but to maintain readability, it is better
+	// to stick with one of the two (inline solution for one or two corpora
+	// and this one for more)
+	ResourcesConfDir string `json:"resourcesConfDir"`
 }
 
 func (cs *CorporaSetup) GetRegistryPath(corpusID string) string {

go.mod

@@ -5,7 +5,7 @@ go 1.21
 toolchain go1.23.0
 
 require (
-	github.com/czcorpus/cnc-gokit v0.11.0
+	github.com/czcorpus/cnc-gokit v0.11.2
 	github.com/czcorpus/manabuild v0.1.0
 	github.com/czcorpus/mquery-common v0.4.3
 	github.com/gin-gonic/gin v1.10.0

go.sum

@@ -17,8 +17,8 @@ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJ
 github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
 github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/czcorpus/cnc-gokit v0.11.0 h1:0DSWVAMu6TyBLxeBfTRB/yezoFKQPy1zW8yqUJmcBzg=
-github.com/czcorpus/cnc-gokit v0.11.0/go.mod h1:BZSRrYUFIHXVIiuqnSoZbfXfL2X/gHWG3w35aIVW36U=
+github.com/czcorpus/cnc-gokit v0.11.2 h1:XlU6oZENE1BEOwJZMZ/C7hv3aK1EFP/1NDqukcVaCJI=
+github.com/czcorpus/cnc-gokit v0.11.2/go.mod h1:BZSRrYUFIHXVIiuqnSoZbfXfL2X/gHWG3w35aIVW36U=
 github.com/czcorpus/manabuild v0.1.0 h1:60sgRj4oM+XqbCKtn/HL+6URXzsfQQCy9TyBWY2iaZE=
 github.com/czcorpus/manabuild v0.1.0/go.mod h1:dj2iAsZObs4yJhF6KkQs5oH2AAyZlrmaNwMGV44hLbk=
 github.com/czcorpus/mquery-common v0.4.3 h1:Zv7UqIM3kyyWGosFhgkXSxsJmwh+gsnpFlo4q649v+c=