Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: enable refback column data to be included in csv/excel downloads (will be ignored during import) #4705

Merged
merged 24 commits into from
Feb 20, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -38,24 +38,21 @@ private static void outputTable(TableStore store, Table table, boolean includeSy
.map(Column::getName)
.filter(n -> !n.startsWith("mg_") || includeSystemColumns)
.toList();
SelectColumn[] select =
downloadColumnNames.stream().map(SelectColumn::s).toArray(SelectColumn[]::new);

if (table.getMetadata().getColumnNames().contains(MG_TABLECLASS)) {
store.writeTable(
table.getName(),
downloadColumnNames,
table
.query()
.select(select)
.where(
f(
MG_TABLECLASS,
Operator.EQUALS,
table.getSchema().getName() + "." + table.getName()))
.retrieveRows());
} else {
store.writeTable(table.getName(), downloadColumnNames, table.select(select).retrieveRows());
store.writeTable(table.getName(), downloadColumnNames, table.retrieveRows());
}

// in case of zip file we include the attached files
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,7 @@ private void executeTest(TableStore store, Schema schema) {
schema
.getTable("biobanks")
.select(
s("name"),
s("contact_person", s("full_name")),
s("principal_investigators", s("full_name")),
s("juristic_person", s("name")))
s("name"), s("contact_person"), s("principal_investigators"), s("juristic_person"))
.search("GrONingen")
.retrieveRows();
assertEquals(1, rows.size());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
import static java.util.Map.entry;
import static org.eclipse.rdf4j.model.util.Values.literal;
import static org.molgenis.emx2.Constants.API_FILE;
import static org.molgenis.emx2.Constants.COMPOSITE_REF_SEPARATOR;
import static org.molgenis.emx2.Constants.SUBSELECT_SEPARATOR;
import static org.molgenis.emx2.rdf.RdfUtils.getSchemaNamespace;

import com.google.common.net.UrlEscapers;
Expand Down Expand Up @@ -68,7 +66,7 @@ public class ColumnTypeRdfMapper {
// RELATIONSHIP
entry(ColumnType.REF, RdfColumnType.REFERENCE),
entry(ColumnType.REF_ARRAY, RdfColumnType.REFERENCE),
entry(ColumnType.REFBACK, RdfColumnType.REFBACK),
entry(ColumnType.REFBACK, RdfColumnType.REFERENCE),

// LAYOUT and other constants
entry(ColumnType.HEADING, RdfColumnType.SKIP), // Should not be in RDF output.
Expand Down Expand Up @@ -227,54 +225,6 @@ boolean isEmpty(Row row, Column column) {
return column.getReferences().stream().anyMatch(i -> row.getString(i.getName()) == null);
}
},
REFBACK(CoreDatatype.XSD.ANYURI) {
@Override
Set<Value> retrieveValues(String baseURL, Row row, Column column) {
Map<String, String> colNameToRefTableColName = new HashMap<>();
if (row.getString(column.getName()) != null) {
colNameToRefTableColName.put(
column.getName(), column.getRefTable().getPrimaryKeyColumns().get(0).getName());
} else {
refBackSubColumns(
colNameToRefTableColName, column, column.getName() + SUBSELECT_SEPARATOR, "");
}

return RdfColumnType.retrieveReferenceValues(
baseURL, row, column, colNameToRefTableColName);
}

private void refBackSubColumns(
Map<String, String> colNameToRefTableColName,
Column column,
String colPrefix,
String refPrefix) {
for (Column refPrimaryKey : column.getRefTable().getPrimaryKeyColumns()) {
if (refPrimaryKey.isRef() || refPrimaryKey.isRefArray()) {
refBackSubColumns(
colNameToRefTableColName,
refPrimaryKey,
colPrefix + refPrimaryKey.getName() + SUBSELECT_SEPARATOR,
refPrefix + refPrimaryKey.getName() + COMPOSITE_REF_SEPARATOR);
} else {
colNameToRefTableColName.put(
colPrefix + refPrimaryKey.getName(), refPrefix + refPrimaryKey.getName());
}
}
}

@Override
boolean isEmpty(Row row, Column column) {
if (row.getString(column.getName()) != null) return false;

// Composite key requires all fields to be filled. If one is null, all should be null.
Optional<String> firstMatch =
row.getColumnNames().stream()
.filter(i -> i.startsWith(column.getName() + SUBSELECT_SEPARATOR))
.findFirst();

return firstMatch.isEmpty() || row.getString(firstMatch.get()) == null;
}
},
ONTOLOGY(CoreDatatype.XSD.ANYURI) {
// TODO: Implement Ontology behavior where it also returns ontologyTermURI as Value.
@Override
Expand Down Expand Up @@ -331,7 +281,13 @@ private static Set<Value> basicRetrievalString(
abstract Set<Value> retrieveValues(final String baseURL, final Row row, final Column column);

boolean isEmpty(final Row row, final Column column) {
return row.getString(column.getName()) == null;
if (column.isReference() && column.getReferences().size() > 1) {
// check composite keys to be empty
return column.getReferences().stream()
.anyMatch(ref -> row.getString(ref.getName()) == null);
} else {
return row.getString(column.getName()) == null;
}
}

private static Set<Value> retrieveReferenceValues(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import static org.molgenis.emx2.Constants.MG_TABLECLASS;
import static org.molgenis.emx2.FilterBean.f;
import static org.molgenis.emx2.Operator.EQUALS;
import static org.molgenis.emx2.SelectColumn.s;
import static org.molgenis.emx2.rdf.RdfUtils.getSchemaNamespace;

import com.google.common.net.UrlEscapers;
Expand Down Expand Up @@ -579,50 +578,24 @@ private String getLabelForRow(final Row row, final TableMetadata metadata) {
private List<Row> getRows(Table table, final String rowId) {
Query query = table.query();

List<SelectColumn> selectColumns = new ArrayList<>();
for (Column c : table.getMetadata().getColumns()) {
if (c.isFile()) {
selectColumns.add(s(c.getName(), s("id"), s("filename"), s("mimetype")));
} else if (c.isRef() || c.isRefArray()) {
c.getReferences().forEach(i -> selectColumns.add(s(i.getName())));
} else if (c.isRefback()) {
selectColumns.add(refBackSelect(c));
} else {
selectColumns.add(s(c.getName()));
}
}
SelectColumn[] selectArray = selectColumns.toArray(SelectColumn[]::new);

if (rowId != null) {
// first find from root table
PrimaryKey key = PrimaryKey.makePrimaryKeyFromEncodedKey(rowId);
List<Row> oneRow = query.select(selectArray).where(key.getFilter()).retrieveRows();
List<Row> oneRow = query.where(key.getFilter()).retrieveRows();
// if subclass
if (oneRow.size() == 1 && oneRow.get(0).getString(MG_TABLECLASS) != null) {
Row row = oneRow.get(0);
table = getSubclassTableForRowBasedOnMgTableclass(table, row);
return table.query().select(selectArray).where(key.getFilter()).retrieveRows();
return table.query().where(key.getFilter()).retrieveRows();
}
return oneRow;
} else {
if (table.getMetadata().getColumnNames().contains(MG_TABLECLASS)) {
var tableName = table.getSchema().getName() + "." + table.getName();
query.where(f("mg_tableclass", EQUALS, tableName));
}
return query.select(selectArray).retrieveRows();
}
}

private SelectColumn refBackSelect(Column column) {
List<SelectColumn> subSelects = new ArrayList<>();
for (Column subColumn : column.getRefTable().getPrimaryKeyColumns()) {
if (subColumn.isRef() || subColumn.isRefArray()) {
subSelects.add(refBackSelect(subColumn));
} else {
subSelects.add(s(subColumn.getName()));
}
return query.retrieveRows();
}
return s(column.getName(), subSelects.toArray(SelectColumn[]::new));
}

private IRI getIriForRow(final Row row, final Table table) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.molgenis.emx2.Column.column;
import static org.molgenis.emx2.Row.row;
import static org.molgenis.emx2.SelectColumn.s;
import static org.molgenis.emx2.TableMetadata.table;

import java.io.File;
Expand Down Expand Up @@ -218,22 +217,9 @@ public static void setup() {
row("id1", "a", "id2", "b", "ref", "lonelyString"),
row("id1", "c", "id2", "d", "ref", "lonelyString"));

// Use query to explicitly retrieve all rows as the following would exclude REFBACK values:
// allColumnTypes.getTable(TEST_TABLE).retrieveRows()
List<SelectColumn> selectColumnList =
Arrays.stream(ColumnType.values())
.map(i -> new SelectColumn(i.name()))
.collect(Collectors.toList());
// Add Composite columns manually.
selectColumnList.add(s(COLUMN_COMPOSITE_REF + ".ids"));
selectColumnList.add(s(COLUMN_COMPOSITE_REF + ".idi"));
selectColumnList.add(s(COLUMN_COMPOSITE_REF_ARRAY + ".ids"));
selectColumnList.add(s(COLUMN_COMPOSITE_REF_ARRAY + ".idi"));
selectColumnList.add(s(COLUMN_COMPOSITE_REFBACK, s("id1"), s("id2")));
SelectColumn[] selectColumns = selectColumnList.toArray(SelectColumn[]::new);

// Describes rows for easy access.
testRows = allColumnTypes.getTable(TEST_TABLE).query().select(selectColumns).retrieveRows();
// exclude mg columns because they might be not empty for the empty test
testRows = allColumnTypes.getTable(TEST_TABLE).retrieveRows(Query.Option.EXCLUDE_MG_COLUMNS);
}

@AfterAll
Expand All @@ -249,8 +235,9 @@ private Set<Value> retrieveValues(String columnName) {
/** Only primary key & AUTO_ID is filled. */
private Set<Value> retrieveEmptyValues(String columnName) {
// REFBACK causes duplicate row (with only REFBACK values being different).
// Therefore, 3rd row is empty one.
return retrieveValues(columnName, 2);
// That was a bug fixed in #4705
// Therefore, 2nd row is empty one.
return retrieveValues(columnName, 1);
}

private Set<Value> retrieveValues(String columnName, int row) {
Expand Down
Loading