feat: enable refback column data to be included in csv/excel downloads (will be ignored during import)

backend/molgenis-emx2-io/src/test/java/org/molgenis/emx2/io/TestLegacyImport.java

@@ -77,10 +77,7 @@ private void executeTest(TableStore store, Schema schema) {
         schema
             .getTable("biobanks")
             .select(
-                s("name"),
-                s("contact_person", s("full_name")),
-                s("principal_investigators", s("full_name")),
-                s("juristic_person", s("name")))
+                s("name"), s("contact_person"), s("principal_investigators"), s("juristic_person"))
             .search("GrONingen")
             .retrieveRows();
     assertEquals(1, rows.size());

backend/molgenis-emx2-rdf/src/main/java/org/molgenis/emx2/rdf/ColumnTypeRdfMapper.java

@@ -3,8 +3,6 @@
 import static java.util.Map.entry;
 import static org.eclipse.rdf4j.model.util.Values.literal;
 import static org.molgenis.emx2.Constants.API_FILE;
-import static org.molgenis.emx2.Constants.COMPOSITE_REF_SEPARATOR;
-import static org.molgenis.emx2.Constants.SUBSELECT_SEPARATOR;
 import static org.molgenis.emx2.rdf.RdfUtils.getSchemaNamespace;
 
 import com.google.common.net.UrlEscapers;
@@ -68,7 +66,7 @@ public class ColumnTypeRdfMapper {
           // RELATIONSHIP
           entry(ColumnType.REF, RdfColumnType.REFERENCE),
           entry(ColumnType.REF_ARRAY, RdfColumnType.REFERENCE),
-          entry(ColumnType.REFBACK, RdfColumnType.REFBACK),
+          entry(ColumnType.REFBACK, RdfColumnType.REFERENCE),
 
           // LAYOUT and other constants
           entry(ColumnType.HEADING, RdfColumnType.SKIP), // Should not be in RDF output.
@@ -227,54 +225,6 @@ boolean isEmpty(Row row, Column column) {
         return column.getReferences().stream().anyMatch(i -> row.getString(i.getName()) == null);
       }
     },
-    REFBACK(CoreDatatype.XSD.ANYURI) {
-      @Override
-      Set<Value> retrieveValues(String baseURL, Row row, Column column) {
-        Map<String, String> colNameToRefTableColName = new HashMap<>();
-        if (row.getString(column.getName()) != null) {
-          colNameToRefTableColName.put(
-              column.getName(), column.getRefTable().getPrimaryKeyColumns().get(0).getName());
-        } else {
-          refBackSubColumns(
-              colNameToRefTableColName, column, column.getName() + SUBSELECT_SEPARATOR, "");
-        }
-
-        return RdfColumnType.retrieveReferenceValues(
-            baseURL, row, column, colNameToRefTableColName);
-      }
-
-      private void refBackSubColumns(
-          Map<String, String> colNameToRefTableColName,
-          Column column,
-          String colPrefix,
-          String refPrefix) {
-        for (Column refPrimaryKey : column.getRefTable().getPrimaryKeyColumns()) {
-          if (refPrimaryKey.isRef() || refPrimaryKey.isRefArray()) {
-            refBackSubColumns(
-                colNameToRefTableColName,
-                refPrimaryKey,
-                colPrefix + refPrimaryKey.getName() + SUBSELECT_SEPARATOR,
-                refPrefix + refPrimaryKey.getName() + COMPOSITE_REF_SEPARATOR);
-          } else {
-            colNameToRefTableColName.put(
-                colPrefix + refPrimaryKey.getName(), refPrefix + refPrimaryKey.getName());
-          }
-        }
-      }
-
-      @Override
-      boolean isEmpty(Row row, Column column) {
-        if (row.getString(column.getName()) != null) return false;
-
-        // Composite key requires all fields to be filled. If one is null, all should be null.
-        Optional<String> firstMatch =
-            row.getColumnNames().stream()
-                .filter(i -> i.startsWith(column.getName() + SUBSELECT_SEPARATOR))
-                .findFirst();
-
-        return firstMatch.isEmpty() || row.getString(firstMatch.get()) == null;
-      }
-    },
     ONTOLOGY(CoreDatatype.XSD.ANYURI) {
       // TODO: Implement Ontology behavior where it also returns ontologyTermURI as Value.
       @Override
@@ -331,7 +281,13 @@ private static Set<Value> basicRetrievalString(
     abstract Set<Value> retrieveValues(final String baseURL, final Row row, final Column column);
 
     boolean isEmpty(final Row row, final Column column) {
-      return row.getString(column.getName()) == null;
+      if (column.isReference() && column.getReferences().size() > 1) {
+        // check composite keys to be empty
+        return column.getReferences().stream()
+            .anyMatch(ref -> row.getString(ref.getName()) == null);
+      } else {
+        return row.getString(column.getName()) == null;
+      }
     }
 
     private static Set<Value> retrieveReferenceValues(

backend/molgenis-emx2-rdf/src/test/java/org/molgenis/emx2/rdf/ColumnTypeRdfMapperTest.java

@@ -249,8 +249,9 @@ private Set<Value> retrieveValues(String columnName) {
   /** Only primary key & AUTO_ID is filled. */
   private Set<Value> retrieveEmptyValues(String columnName) {
     // REFBACK causes duplicate row (with only REFBACK values being different).
-    // Therefore, 3rd row is empty one.
-    return retrieveValues(columnName, 2);
+    // That was a bug fixed in #4705
+    // Therefore, 2nd row is empty one.
+    return retrieveValues(columnName, 1);
   }
 
   private Set<Value> retrieveValues(String columnName, int row) {

backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlQuery.java

@@ -90,21 +90,18 @@ public List<Row> retrieveRows() {
     checkHasViewPermission(table);
     String tableAlias = "root-" + table.getTableName();
 
-    // if empty selection, we will add the default selection here, excl File and Refback
+    // if empty selection, we will add the default selection here, incl File and Refback
+    // will generally be all you need
     if (select == null || select.getColumNames().isEmpty()) {
       for (Column c : table.getColumns()) {
         // currently we don't download refBack (good) and files (that is bad)
         if (c.isFile()) {
           select.select(c.getName());
-        } else if (!c.isRefback()) {
-          if (c.isReference()) {
-            for (Reference ref : c.getReferences()) {
-              select.select(ref.getName());
-            }
-          } else {
-            // don't include refBack or files or mg_ columns
-            select.select(c.getName());
-          }
+        } else if (c.isReference()) {
+          // subselect primary keys, nested
+          select.subselect(getRefPrimaryKeySubselect(c));
+        } else if (!c.isHeading()) {
+          select.select(c.getName());
         }
       }
     }
@@ -113,7 +110,7 @@ public List<Row> retrieveRows() {
     SelectJoinStep<org.jooq.Record> from =
         table
             .getJooq()
-            .select(rowSelectFields(table, tableAlias, "", select))
+            .select(rowSelectFields(table, tableAlias, select))
             .from(tableWithInheritanceJoin(table).as(alias(tableAlias)));
 
     // joins, only filtered tables
@@ -141,6 +138,21 @@ public List<Row> retrieveRows() {
     }
   }
 
+  private SelectColumn getRefPrimaryKeySubselect(Column c) {
+    List<SelectColumn> result = new ArrayList<>();
+    c.getRefTable()
+        .getPrimaryKeyColumns()
+        .forEach(
+            key -> {
+              if (key.isReference()) {
+                result.add(s(key.getName(), getRefPrimaryKeySubselect(key)));
+              } else {
+                result.add(s(key.getName()));
+              }
+            });
+    return s(c.getName(), result.toArray(SelectColumn[]::new));
+  }
+
   private void checkHasViewPermission(SqlTableMetadata table) {
     if (!table.getTableType().equals(TableType.ONTOLOGIES)
         && !schema.getInheritedRolesForActiveUser().contains(VIEWER.toString())) {
@@ -149,12 +161,11 @@ private void checkHasViewPermission(SqlTableMetadata table) {
   }
 
   private List<Field<?>> rowSelectFields(
-      TableMetadata table, String tableAlias, String prefix, SelectColumn selection) {
+      TableMetadata table, String tableAlias, SelectColumn selection) {
 
     List<Field<?>> fields = new ArrayList<>();
     for (SelectColumn select : selection.getSubselect()) {
       Column column = getColumnByName(table, select.getColumn());
-      String columnAlias = prefix.equals("") ? column.getName() : prefix + "-" + column.getName();
       if (column.isFile()) {
         // check what they want to get, contents, mimetype, size, filename and/or extension
         if (select.getSubselect().isEmpty() || select.has("id")) {
@@ -175,77 +186,50 @@ private List<Field<?>> rowSelectFields(
         if (select.has("extension")) {
           fields.add(field(name(alias(tableAlias), column.getName() + "_extension")));
         }
-      } else if (column.isReference()
-          // if subselection, then we will add it as subselect
-          && !select.getSubselect().isEmpty()) {
+      } else if (column.isRef() || column.isRefArray()) {
+        shouldNotExpandBeyondPkey(select, column);
         fields.addAll(
-            rowSelectFields(
-                column.getRefTable(),
-                tableAlias + "-" + column.getName(),
-                columnAlias,
-                selection.getSubselect(column.getName())));
+            column.getReferences().stream()
+                .map(ref -> field(name(alias(tableAlias), ref.getName())))
+                .toList());
       } else if (column.isRefback()) {
-        fields.add(
-            field("array({0})", rowBackrefSubselect(column, tableAlias)).as(column.getName()));
-      } else if (column.isReference()) { // REF and REF_ARRAY
-        // might be composite column with same name
-        Reference ref = null;
-        for (Reference r : column.getReferences()) {
-          if (r.getName().equals(column.getName())) {
-            ref = r;
-          }
-        }
-        if (ref == null) {
-          throw new MolgenisException(
-              "Select of column '"
-                  + column.getName()
-                  + "' failed: composite foreign key requires subselection or explicit naming of underlying fields");
-        } else {
-          fields.add(
-              field(name(alias(tableAlias), column.getName()), ref.getJooqType()).as(columnAlias));
-        }
+        shouldNotExpandBeyondPkey(select, column);
+        // will come from refJoin table
+        fields.addAll(
+            column.getReferences().stream()
+                .map(
+                    ref ->
+                        field(
+                            name(
+                                alias(tableAlias + "-refbackjoin-" + column.getName()),
+                                ref.getName())))
+                .toList());
       } else if (!column.isHeading()) {
-        fields.add(
-            field(name(alias(tableAlias), column.getName()), column.getJooqType()).as(columnAlias));
+        fields.add(field(name(alias(tableAlias), column.getName()), column.getJooqType()));
       }
     }
     return fields;
   }
 
+  private static void shouldNotExpandBeyondPkey(SelectColumn select, Column column) {
+    select
+        .getSubselect()
+        .forEach(
+            subselect -> {
+              if (!column.getRefTable().getPrimaryKeys().contains(subselect.getColumn())) {
+                throw new MolgenisException(
+                    "Row subselect can only contain primary keys. Found: " + subselect.getColumn());
+              }
+            });
+  }
+
   private Field<String> intervalField(String tableAlias, Column column) {
     Field<?> intervalField = field(name(alias(tableAlias), column.getName()));
     Field<String> functionCallField =
         function("\"MOLGENIS\".interval_to_iso8601", String.class, intervalField);
     return functionCallField.as(name(column.getIdentifier()));
   }
 
-  private SelectConditionStep<org.jooq.Record> rowBackrefSubselect(
-      Column column, String tableAlias) {
-    Column refBack = column.getRefBackColumn();
-    List<Condition> where = new ArrayList<>();
-
-    // might be composite
-    for (Reference ref : refBack.getReferences()) {
-      if (refBack.isRef()) {
-        where.add(
-            field(name(refBack.getTable().getTableName(), ref.getName()))
-                .eq(field(name(alias(tableAlias), ref.getRefTo()))));
-      } else if (refBack.isRefArray()) {
-        where.add(
-            condition(
-                ANY_SQL,
-                field(name(alias(tableAlias), ref.getRefTo())),
-                field(name(refBack.getTable().getTableName(), ref.getName()))));
-      } else {
-        throw new MolgenisException(
-            "Internal error: Refback for type not matched for column " + column.getName());
-      }
-    }
-    return DSL.select(column.getRefTable().getPrimaryKeyFields())
-        .from(name(refBack.getSchemaName(), refBack.getTableName()))
-        .where(where);
-  }
-
   @Override
   public String retrieveJSON() {
     SelectColumn select = getSelect();
@@ -954,22 +938,53 @@ private SelectJoinStep<org.jooq.Record> refJoins(
       for (SelectColumn select : selection.getSubselect()) {
         // then do same as above
         Column column = getColumnByName(table, select.getColumn());
-        if (column.isReference()) {
-          String subAlias = tableAlias + "-" + column.getName();
-          // only join if subselection extists
-          if (!aliasList.contains(subAlias) && !select.getSubselect().isEmpty()) {
+        if (column.isRefback()) {
+          String subAlias = alias(tableAlias + "-refbackjoin-" + column.getName());
+          if (!aliasList.contains(subAlias)) {
+            // to ensure only join once
             aliasList.add(subAlias);
-            join.leftJoin(tableWithInheritanceJoin(column.getRefTable()).as(alias(subAlias)))
-                .on(refJoinCondition(column, tableAlias, subAlias));
-            // recurse
+            Column refBack = column.getRefBackColumn();
+            List<Field> refbackSelection = new ArrayList<>();
+            refbackSelection.addAll(
+                column.getReferences().stream()
+                    .map(
+                        ref ->
+                            field("array_agg({0})", name(ref.getRefTo())).as(name(ref.getName())))
+                    .toList());
+            if (refBack.isRefArray()) {
+              refbackSelection.addAll(
+                  refBack.getReferences().stream()
+                      .map(
+                          reference ->
+                              field("unnest({0})", name(reference.getName()))
+                                  .as(name("_refback_" + reference.getRefTo())))
+                      .toList());
+            } else {
+              refbackSelection.addAll(
+                  refBack.getReferences().stream()
+                      .map(
+                          reference ->
+                              field(name(reference.getName()))
+                                  .as(name("_refback_" + reference.getRefTo())))
+                      .toList());
+            }
+            // we create a natural joinable representation of refback that looks same as ref_array
             join =
-                refJoins(
-                    column.getRefTable(),
-                    subAlias,
-                    join,
-                    filters != null ? filters.getSubfilter(column.getName()) : null,
-                    select,
-                    aliasList);
+                join.leftJoin(
+                        DSL.select(refbackSelection)
+                            .from(column.getRefTable().getJooqTable())
+                            .groupBy(
+                                refBack.getReferences().stream()
+                                    .map(ref -> field(name("_refback_" + ref.getRefTo())))
+                                    .toList())
+                            .asTable(name(subAlias)))
+                    .on(
+                        refBack.getReferences().stream()
+                            .map(
+                                ref ->
+                                    field(name(subAlias, "_refback_" + ref.getRefTo()))
+                                        .eq(field(name(alias(tableAlias), ref.getRefTo()))))
+                            .toArray(Condition[]::new));
           }
         }
       }

backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/TestCompositeForeignKeys.java

@@ -292,7 +292,16 @@ public void testCompositeRefArray() throws JsonProcessingException {
                     .where(f("firstName", EQUALS, "Donald"))
                     .retrieveRows()
                     .get(0) //
-                    .getStringArray("cousins-firstName")) // TODO should be array?
+                    .getStringArray("cousins.firstName")) // TODO should be array?
+            .contains("Kwik"));
+
+    assertTrue(
+        List.of(
+                p.query()
+                    .where(f("firstName", EQUALS, "Donald"))
+                    .retrieveRows()
+                    .get(0) //
+                    .getStringArray("cousins.firstName")) // TODO should be array?
             .contains("Kwik"));
 
     // check we can sort on ref_array