Merge branch 'schema-dev-df'

.gitignore

@@ -24,3 +24,5 @@ _build/**
 /graphOps/notebooks/analysis/converted/
 /graphOps/notebooks/analysis/store/
 /graphOps/users/odisClone/data/
+/graphOps/sqlOps/output
+/graphOps/graph2solr/stores/

book/architecture/images/component.d2

@@ -0,0 +1,44 @@
+direction: right
+
+classes: {
+coreclass: {
+    width: 200
+               style: {
+                     stroke-width: 0
+                     font-size: 18
+                    fill: lightgreen
+                    shadow: true
+                    border-radius: 5
+               }
+      }
+      actionclass: {
+    width: 250
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: lightblue
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+propclass: {
+    width: 250
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: lightyellow
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+prodclass: {
+    width: 250
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: honeydew
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+}

book/architecture/images/dataflow.d2

@@ -0,0 +1,154 @@
+direction: right
+
+classes: {
+coreclass: {
+    width: 200
+               style: {
+                     stroke-width: 0
+                     font-size: 18
+                    fill: lightgreen
+                    shadow: true
+                    border-radius: 5
+               }
+      }
+      actionclass: {
+                width: 200
+                height: 100
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: lightblue
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+propclass: {
+    width: 200
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: lightyellow
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+prodclass: {
+    width: 210
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: honeydew
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+}
+
+### resources section
+
+partners: JSON-LD files \n from partners
+partners.shape: Circle
+config: config files from \n git & odiscat
+config.shape: Circle
+gleaner: Gleaner
+etl_odis: ODIS ETL \n ODIS Configs
+etl: ODIS ETL \n OIH Configs
+
+oss: ODIS Object Store {
+    jsonld: JSON-LD files\n from partners
+    nq: RDF (nq)\n one per partner
+     jsonld.shape: Circle
+    nq.shape: Circle
+    jsonld.class: propclass
+    nq.class: propclass
+
+    jsonld -> nq: gleaner
+}
+
+sparql_odis: ODIS SPARQL \nendpoint
+sparql: OIH SPARQL \nendpoint
+odisprod: ODIS Products
+oihprod: OIH Products
+solr: OIH SOLR
+validator: SHACL Validator
+
+partners.class: propclass
+config.class: propclass
+gleaner.class: actionclass
+validator.class: actionclass
+sparql_odis.class: actionclass
+sparql.class: actionclass
+etl_odis.class: actionclass
+etl.class: actionclass
+solr.class: actionclass
+
+odisprod: {
+    csv: ODIS csv files \n one per partner
+    parquet: ODIS parquet files \n one per partner
+    csv.shape: Circle
+    parquet.shape: Circle
+    csv.class: prodclass
+    parquet.class: prodclass
+}
+
+oihprod: {
+   csv: OIH csv files \n one per partner
+    parquet: OIH parquet files \n one per partner
+    valreport: Validation Report \n SHACL RDF
+    csv.shape: Circle
+    parquet.shape: Circle
+    valreport.shape: Circle
+    csv.class: prodclass
+    parquet.class: prodclass
+    valreport.class: prodclass
+}
+
+sys: System System Interop {
+    wis2: WIS2 Products
+    nfra: AquaNFRA
+    west: WESR
+    *.class: actionclass
+}
+
+dash: Dashboard {
+  dashpy: Dashboard.py
+  streamlit: Streamlit UI
+
+  dashpy <-> streamlit
+
+      *.class: actionclass
+
+}
+
+oihprod.valreport -> dash.streamlit: used by (SPARQL)
+sparql_odis -> dash.streamlit: used by (SPARQL)
+odisprod.parquet -> dash.streamlit: via DuckDB (SQL)
+
+wmowis2: WIS2
+wmowis2.shape: cloud
+
+
+### connection section
+
+partners -> gleaner
+config -> gleaner
+
+gleaner -> oss.jsonld
+
+oss.nq -> sparql: gleaner + schema alignment
+oss.nq -> sparql_odis: gleaner
+
+oss.nq -> validator -> oihprod.valreport
+
+sparql -> etl
+sparql_odis -> etl_odis
+
+etl_odis -> odisprod.csv
+etl_odis -> odisprod.parquet
+
+etl -> oihprod.csv
+etl -> oihprod.parquet
+etl -> solr
+
+oihprod.parquet -> sys.wis2
+sys.wis2 -> wmowis2

book/architecture/images/deployment.d2

@@ -0,0 +1,80 @@
+direction: right
+
+classes: {
+coreclass: {
+    width: 200
+               style: {
+                     stroke-width: 0
+                     font-size: 18
+                    fill: lightgreen
+                    shadow: true
+                    border-radius: 5
+               }
+      }
+      actionclass: {
+    width: 250
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: lightblue
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+propclass: {
+    width: 100
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: lightyellow
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+prodclass: {
+    width: 150
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: honeydew
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+}
+
+*.class: actionclass
+
+dagster: {
+    gleaner
+    nabu
+    etl
+    *.class: prodclass
+
+}
+triplestore: qlever \n triplestore
+solr
+webui
+oss: object store
+dashboard
+router
+
+dagster.gleaner -> oss: http
+
+dagster.etl <- triplestore: http
+dagster.etl -> oss: http
+dagster.etl -> solr: http
+
+solr <- webui: http
+triplestore <- dashboard: http
+oss <- dashboard: http
+
+webui <-> router : http
+oss <-> router: http
+triplestore <-> router: http
+dashboard <-> router: http
+
+#router -> net: http
+#net: Internet
+#net.shape: cloud
+# net.class: coreclass

book/architecture/images/highlevel.d2

@@ -0,0 +1,80 @@
+direction: right
+
+classes: {
+coreclass: {
+    width: 200
+               style: {
+                     stroke-width: 0
+                     font-size: 18
+                    fill: lightgreen
+                    shadow: true
+                    border-radius: 5
+               }
+      }
+      actionclass: {
+    width: 150
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: lightblue
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+propclass: {
+    width: 250
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: lightyellow
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+prodclass: {
+    width: 125
+               style: {
+                     stroke-width: 1
+                     font-size: 18
+                     fill: honeydew
+                    shadow: false
+                    border-radius: 0
+               }
+      }
+}
+
+graph.class: prodclass
+prod.class: prodclass
+ui.class: prodclass
+sys.class: prodclass
+
+partners: Partners
+partners.shape: Person
+
+prod: Products
+graph: Graph
+ui: OIH UI \n Dahsboard
+sys: Systems
+
+community: Community
+community.shape: Person
+
+
+cloud: Internet\n Standards
+cloud.shape: cloud
+
+odis: ODIS Systems
+odis.shape: stored_data
+
+partners -> cloud -> odis
+
+odis -> partners: feedback and validation
+
+
+odis -> graph -> prod
+odis.class: actionclass
+
+graph <-> community
+prod -> ui
+prod <-> community
+prod -> sys