Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: enable refback column data to be included in csv/excel downloads (will be ignored during import) #4705

Merged
merged 24 commits into from
Feb 20, 2025
Merged

feat: enable refback column data to be included in csv/excel downloads (will be ignored during import) #4705

Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
66293c1
feat: enable refbacks to be included in csv/excel downloads (will be …
mswertz Feb 11, 2025
e5cb49c
adapt test to new situation
mswertz Feb 12, 2025
14ae5a3
Merge branch 'master' into feat/4223
hslh Feb 12, 2025
4c81a79
Merge branch 'master' into feat/4223
connoratrug Feb 12, 2025
c1baaa0
Merge branch 'master' of github.com:molgenis/molgenis-emx2 into feat/…
mswertz Feb 12, 2025
5fb0624
wip
mswertz Feb 14, 2025
92e2b03
Merge branch 'master' of github.com:molgenis/molgenis-emx2 into feat/…
mswertz Feb 17, 2025
ff4b28f
wip
mswertz Feb 17, 2025
22fc8a5
disable subselection on retrieveRows
mswertz Feb 17, 2025
46c3288
fix test
mswertz Feb 17, 2025
857d543
all tests fixed except rdf?
mswertz Feb 17, 2025
31c851a
few more fixes, might need help with last one
mswertz Feb 18, 2025
5767c0d
Merge branch 'master' into feat/4223
mswertz Feb 18, 2025
721f1fb
fix bug in getReferences
mswertz Feb 18, 2025
e14609d
Merge branch 'feat/4223' of github.com:molgenis/molgenis-emx2 into fe…
mswertz Feb 18, 2025
a51c8ef
remove unneeded change
mswertz Feb 19, 2025
0a61467
fix review comments
mswertz Feb 19, 2025
c586cce
fix download
mswertz Feb 19, 2025
b6964e8
wip
mswertz Feb 19, 2025
b9054f2
fix file contents export
mswertz Feb 19, 2025
ddd2596
add some options to retrieveRows in cases people don't want to use se…
mswertz Feb 20, 2025
14e931d
refactor
mswertz Feb 20, 2025
22c67b8
forgot one exclude mg columns
mswertz Feb 20, 2025
2fe631d
fix test
mswertz Feb 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions backend/molgenis-emx2-io/src/test/java/org/molgenis/emx2/io/TestLegacyImport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,7 @@ private void executeTest(TableStore store, Schema schema) {
schema
.getTable("biobanks")
.select(
s("name"),
s("contact_person", s("full_name")),
s("principal_investigators", s("full_name")),
s("juristic_person", s("name")))
s("name"), s("contact_person"), s("principal_investigators"), s("juristic_person"))
.search("GrONingen")
.retrieveRows();
assertEquals(1, rows.size());
Expand Down
60 changes: 8 additions & 52 deletions backend/molgenis-emx2-rdf/src/main/java/org/molgenis/emx2/rdf/ColumnTypeRdfMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
import static java.util.Map.entry;
import static org.eclipse.rdf4j.model.util.Values.literal;
import static org.molgenis.emx2.Constants.API_FILE;
import static org.molgenis.emx2.Constants.COMPOSITE_REF_SEPARATOR;
import static org.molgenis.emx2.Constants.SUBSELECT_SEPARATOR;
import static org.molgenis.emx2.rdf.RdfUtils.getSchemaNamespace;

import com.google.common.net.UrlEscapers;
Expand Down Expand Up @@ -68,7 +66,7 @@ public class ColumnTypeRdfMapper {
// RELATIONSHIP
entry(ColumnType.REF, RdfColumnType.REFERENCE),
entry(ColumnType.REF_ARRAY, RdfColumnType.REFERENCE),
entry(ColumnType.REFBACK, RdfColumnType.REFBACK),
entry(ColumnType.REFBACK, RdfColumnType.REFERENCE),

// LAYOUT and other constants
entry(ColumnType.HEADING, RdfColumnType.SKIP), // Should not be in RDF output.
Expand Down Expand Up @@ -227,54 +225,6 @@ boolean isEmpty(Row row, Column column) {
return column.getReferences().stream().anyMatch(i -> row.getString(i.getName()) == null);
}
},
REFBACK(CoreDatatype.XSD.ANYURI) {
@Override
Set<Value> retrieveValues(String baseURL, Row row, Column column) {
Map<String, String> colNameToRefTableColName = new HashMap<>();
if (row.getString(column.getName()) != null) {
colNameToRefTableColName.put(
column.getName(), column.getRefTable().getPrimaryKeyColumns().get(0).getName());
} else {
refBackSubColumns(
colNameToRefTableColName, column, column.getName() + SUBSELECT_SEPARATOR, "");
}

return RdfColumnType.retrieveReferenceValues(
baseURL, row, column, colNameToRefTableColName);
}

private void refBackSubColumns(
Map<String, String> colNameToRefTableColName,
Column column,
String colPrefix,
String refPrefix) {
for (Column refPrimaryKey : column.getRefTable().getPrimaryKeyColumns()) {
if (refPrimaryKey.isRef() || refPrimaryKey.isRefArray()) {
refBackSubColumns(
colNameToRefTableColName,
refPrimaryKey,
colPrefix + refPrimaryKey.getName() + SUBSELECT_SEPARATOR,
refPrefix + refPrimaryKey.getName() + COMPOSITE_REF_SEPARATOR);
} else {
colNameToRefTableColName.put(
colPrefix + refPrimaryKey.getName(), refPrefix + refPrimaryKey.getName());
}
}
}

@Override
boolean isEmpty(Row row, Column column) {
if (row.getString(column.getName()) != null) return false;

// Composite key requires all fields to be filled. If one is null, all should be null.
Optional<String> firstMatch =
row.getColumnNames().stream()
.filter(i -> i.startsWith(column.getName() + SUBSELECT_SEPARATOR))
.findFirst();

return firstMatch.isEmpty() || row.getString(firstMatch.get()) == null;
}
},
ONTOLOGY(CoreDatatype.XSD.ANYURI) {
// TODO: Implement Ontology behavior where it also returns ontologyTermURI as Value.
@Override
Expand Down Expand Up @@ -331,7 +281,13 @@ private static Set<Value> basicRetrievalString(
abstract Set<Value> retrieveValues(final String baseURL, final Row row, final Column column);

boolean isEmpty(final Row row, final Column column) {
return row.getString(column.getName()) == null;
if (column.isReference() && column.getReferences().size() > 1) {
// check composite keys to be empty
return column.getReferences().stream()
.anyMatch(ref -> row.getString(ref.getName()) == null);
} else {
return row.getString(column.getName()) == null;
}
}

private static Set<Value> retrieveReferenceValues(
Expand Down
72 changes: 36 additions & 36 deletions backend/molgenis-emx2-rdf/src/test/java/org/molgenis/emx2/rdf/ColumnTypeRdfMapperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,8 +249,9 @@ private Set<Value> retrieveValues(String columnName) {
/** Only primary key & AUTO_ID is filled. */
private Set<Value> retrieveEmptyValues(String columnName) {
// REFBACK causes duplicate row (with only REFBACK values being different).
// Therefore, 3rd row is empty one.
return retrieveValues(columnName, 2);
// That was a bug fixed in #4705
// Therefore, 2nd row is empty one.
return retrieveValues(columnName, 1);
}

private Set<Value> retrieveValues(String columnName, int row) {
Expand Down Expand Up @@ -282,40 +283,38 @@ void validateAllColumnTypesCovered() {
void validateValueTypes() {
Row row = allColumnTypes.getTable(TEST_TABLE).retrieveRows().get(0);

assertAll(
// SIMPLE
() -> assertTrue(retrieveFirstValue(ColumnType.BOOL.name()).isLiteral()),
() -> assertTrue(retrieveFirstValue(ColumnType.UUID.name()).isIRI()),
() -> assertTrue(retrieveFirstValue(ColumnType.FILE.name()).isIRI()),
// STRING
() -> assertTrue(retrieveFirstValue(ColumnType.STRING.name()).isLiteral()),
() -> assertTrue(retrieveFirstValue(ColumnType.TEXT.name()).isLiteral()),
() -> assertTrue(retrieveFirstValue(ColumnType.JSON.name()).isLiteral()),

// NUMERIC
() -> assertTrue(retrieveFirstValue(ColumnType.INT.name()).isLiteral()),
() -> assertTrue(retrieveFirstValue(ColumnType.LONG.name()).isLiteral()),
() -> assertTrue(retrieveFirstValue(ColumnType.DECIMAL.name()).isLiteral()),
() -> assertTrue(retrieveFirstValue(ColumnType.DATE.name()).isLiteral()),
() -> assertTrue(retrieveFirstValue(ColumnType.DATETIME.name()).isLiteral()),
() -> assertTrue(retrieveFirstValue(ColumnType.PERIOD.name()).isLiteral()),

// RELATIONSHIP
() -> assertTrue(retrieveFirstValue(ColumnType.REF.name()).isIRI()),
() -> assertTrue(retrieveFirstValue(ColumnType.REFBACK.name()).isIRI()),

// LAYOUT and other constants
// ColumnType.HEADING.name() -> no Value should be present to validate on

// format flavors that extend a baseType
() -> assertTrue(retrieveFirstValue(ColumnType.AUTO_ID.name()).isLiteral()),
() -> assertTrue(retrieveFirstValue(ColumnType.ONTOLOGY.name()).isIRI()),
() -> assertTrue(retrieveFirstValue(ColumnType.EMAIL.name()).isIRI()),
() -> assertTrue(retrieveFirstValue(ColumnType.HYPERLINK.name()).isIRI()),

// Composite keys
() -> assertTrue(retrieveFirstValue(COLUMN_COMPOSITE_REF).isIRI()),
() -> assertTrue(retrieveFirstValue(COLUMN_COMPOSITE_REFBACK).isIRI()));
assertTrue(retrieveFirstValue(ColumnType.BOOL.name()).isLiteral());
assertTrue(retrieveFirstValue(ColumnType.UUID.name()).isIRI());
assertTrue(retrieveFirstValue(ColumnType.FILE.name()).isIRI());
// STRING
assertTrue(retrieveFirstValue(ColumnType.STRING.name()).isLiteral());
assertTrue(retrieveFirstValue(ColumnType.TEXT.name()).isLiteral());
assertTrue(retrieveFirstValue(ColumnType.JSON.name()).isLiteral());

// NUMERIC
assertTrue(retrieveFirstValue(ColumnType.INT.name()).isLiteral());
assertTrue(retrieveFirstValue(ColumnType.LONG.name()).isLiteral());
assertTrue(retrieveFirstValue(ColumnType.DECIMAL.name()).isLiteral());
assertTrue(retrieveFirstValue(ColumnType.DATE.name()).isLiteral());
assertTrue(retrieveFirstValue(ColumnType.DATETIME.name()).isLiteral());
assertTrue(retrieveFirstValue(ColumnType.PERIOD.name()).isLiteral());

// RELATIONSHIP
assertTrue(retrieveFirstValue(ColumnType.REF.name()).isIRI());
assertTrue(retrieveFirstValue(ColumnType.REFBACK.name()).isIRI());

// LAYOUT and other constants
// ColumnType.HEADING.name() -> no Value should be present to validate on

// format flavors that extend a baseType
assertTrue(retrieveFirstValue(ColumnType.AUTO_ID.name()).isLiteral());
assertTrue(retrieveFirstValue(ColumnType.ONTOLOGY.name()).isIRI());
assertTrue(retrieveFirstValue(ColumnType.EMAIL.name()).isIRI());
assertTrue(retrieveFirstValue(ColumnType.HYPERLINK.name()).isIRI());

// Composite keys
assertTrue(retrieveFirstValue(COLUMN_COMPOSITE_REF).isIRI());
assertTrue(retrieveFirstValue(COLUMN_COMPOSITE_REFBACK).isIRI());
}

@Test
Expand All @@ -329,6 +328,7 @@ void validateValuesRetrieval() {

// Validation
assertAll(
// todo: please don't use assertAll, this makes it hard to debug individual fails
// SIMPLE
() -> assertEquals(Set.of(Values.literal(true)), retrieveValues(ColumnType.BOOL.name())),
() ->
Expand Down
Loading