From f649e259d23d08e6b37caeb216609cfa5c24708e Mon Sep 17 00:00:00 2001 From: Sebastian Hofmann <7668803+sebhofmann@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:49:07 +0200 Subject: [PATCH 1/5] MCR-3104 missing database indexes (#2154) * MCR-3104 add indexes for MCRJob * MCR-3104 add indexes for MCRObjectInfoEntity --- .../backend/jpa/objectinfo/MCRObjectInfoEntity.java | 12 +++++++++++- .../java/org/mycore/services/queuedjob/MCRJob.java | 11 ++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/mycore-base/src/main/java/org/mycore/backend/jpa/objectinfo/MCRObjectInfoEntity.java b/mycore-base/src/main/java/org/mycore/backend/jpa/objectinfo/MCRObjectInfoEntity.java index d5d91a7aa5..4d07dc5cf9 100644 --- a/mycore-base/src/main/java/org/mycore/backend/jpa/objectinfo/MCRObjectInfoEntity.java +++ b/mycore-base/src/main/java/org/mycore/backend/jpa/objectinfo/MCRObjectInfoEntity.java @@ -28,11 +28,21 @@ import jakarta.persistence.Entity; import jakarta.persistence.Id; import jakarta.persistence.IdClass; +import jakarta.persistence.Index; import jakarta.persistence.Table; @Entity @IdClass(MCRObjectIDPK.class) -@Table(name = "MCRObject") +@Table(name = "MCRObject", + indexes = { + @Index(name = "MCRObject_state", columnList = "state"), + @Index(name = "MCRObject_createdate", columnList = "createdate"), + @Index(name = "MCRObject_modifydate", columnList = "modifydate"), + @Index(name = "MCRObject_createdby", columnList = "createdby"), + @Index(name = "MCRObject_modifiedby", columnList = "modifiedby"), + @Index(name = "MCRObject_deletedby", columnList = "deletedby"), + @Index(name = "MCRObject_deletedate", columnList = "deletedate") + }) public class MCRObjectInfoEntity implements MCRObjectInfo { private MCRObjectID id; diff --git a/mycore-jobqueue/src/main/java/org/mycore/services/queuedjob/MCRJob.java b/mycore-jobqueue/src/main/java/org/mycore/services/queuedjob/MCRJob.java index 1b27584db4..546192a993 100644 --- a/mycore-jobqueue/src/main/java/org/mycore/services/queuedjob/MCRJob.java +++ b/mycore-jobqueue/src/main/java/org/mycore/services/queuedjob/MCRJob.java @@ -36,6 +36,7 @@ import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; +import jakarta.persistence.Index; import jakarta.persistence.JoinColumn; import jakarta.persistence.MapKeyColumn; import jakarta.persistence.NamedQueries; @@ -52,7 +53,15 @@ @NamedQueries({ @NamedQuery(name = "mcrjob.classes", query = "select DISTINCT(o.action) from MCRJob o") }) -@Table(name = "MCRJob") +@Table(name = "MCRJob", + indexes = { + @Index(name = "MCRJob_status", columnList = "status"), + @Index(name = "MCRJob_added", columnList = "added"), + @Index(name = "MCRJob_start", columnList = "start"), + @Index(name = "MCRJob_finished", columnList = "finished"), + @Index(name = "MCRJob_action", columnList = "action"), + @Index(name = "MCRJob_tries", columnList = "tries") +}) public class MCRJob implements Cloneable { private Long id; From 11d1c954eeed38837dce07353e2dec4b7baba711 Mon Sep 17 00:00:00 2001 From: Torsten Krause Date: Wed, 19 Jun 2024 19:29:08 +0200 Subject: [PATCH 2/5] MCR-3131 align behaviour of command with normal repository operations --- mycore-pi/src/main/java/org/mycore/pi/cli/MCRPICommands.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mycore-pi/src/main/java/org/mycore/pi/cli/MCRPICommands.java b/mycore-pi/src/main/java/org/mycore/pi/cli/MCRPICommands.java index 081e3995f1..82d7b4b4d0 100644 --- a/mycore-pi/src/main/java/org/mycore/pi/cli/MCRPICommands.java +++ b/mycore-pi/src/main/java/org/mycore/pi/cli/MCRPICommands.java @@ -176,9 +176,8 @@ public static void controlObjectWithServiceAndAdditional(String objectIDString, LOGGER.info("Already present in Database: {}", serviceID); return; } - Date now = new Date(); MCRPI mcrpi = service.insertIdentifierToDatabase(mcrBase, trimAdditional, persistentIdentifier, - new MCRPIServiceDates(now, now)); + new MCRPIServiceDates(null, null)); MCRPIService.addFlagToObject(mcrBase, mcrpi); MCRMetadataManager.update(mcrBase); LOGGER.info("{}:{} is now under control of {}", objectID, trimAdditional, serviceID); From 836b39792a1af7a45498f5b5f960f33966132496 Mon Sep 17 00:00:00 2001 From: erodde <155449327+erodde@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:05:52 +0200 Subject: [PATCH 3/5] MCR-3133 Add fallback value to xpath based classification mapping (#2190) * MCR-3133 fallback mechanism for mods-specific x-path-mapping + fix: XPaths are now evaluated per classification file * MCR-3133 checkstyle * MCR-3133 fallback mechanism for oai-specific x-path-mapping * MCR-3133 test multiple fallbacks in one classification * MCR-3133 checkstyle * MCR-3133 retrigger checks --- .../MCRClassificationMappingEventHandler.java | 85 +++++++++++++------ ...ClassificationMappingEventHandlerTest.java | 81 +++++++++++++++++- .../dummyClassification.xml | 21 +++++ .../orcidWorkType.xml | 18 ++++ .../testMods3.xml | 12 +++ .../src/test/resources/mycore.properties | 2 +- .../MCRClassificationMappingEventHandler.java | 75 +++++++++++----- ...ClassificationMappingEventHandlerTest.java | 57 ++++++++++++- .../dummyClassification.xml | 15 ++++ .../orcidWorkType.xml | 2 + .../testMcrObject2.xml | 12 +++ 11 files changed, 332 insertions(+), 48 deletions(-) create mode 100644 mycore-mods/src/test/resources/MCRClassificationMappingEventHandlerTest/dummyClassification.xml create mode 100644 mycore-mods/src/test/resources/MCRClassificationMappingEventHandlerTest/testMods3.xml create mode 100644 mycore-oai/src/test/resources/MCRClassificationMappingEventHandlerTest/dummyClassification.xml create mode 100644 mycore-oai/src/test/resources/MCRClassificationMappingEventHandlerTest/testMcrObject2.xml diff --git a/mycore-mods/src/main/java/org/mycore/mods/classification/MCRClassificationMappingEventHandler.java b/mycore-mods/src/main/java/org/mycore/mods/classification/MCRClassificationMappingEventHandler.java index 1cc0985610..2f6b7f28ab 100644 --- a/mycore-mods/src/main/java/org/mycore/mods/classification/MCRClassificationMappingEventHandler.java +++ b/mycore-mods/src/main/java/org/mycore/mods/classification/MCRClassificationMappingEventHandler.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -49,14 +50,16 @@ * Maps classifications in Mods-Documents. *

You can define a label x-mapping in a classification with space seperated categoryIds * to which the classification will be mapped. You can further define a label x-mapping-xpath in a - * classification which creates a classification mapping when the xPath inside the x-mapping-xpath attribute - * is matched.

+ * classification which creates a classification mapping when the XPath inside the x-mapping-xpath attribute + * is matched. You can also define a label x-mapping-xpathfb which contains a fallback value + * in case that no other XPaths inside a classification match.

* * <category ID="article" counter="1">
*  <label xml:lang="en" text="Article / Chapter" />
*  <label xml:lang="de" text="Artikel / Aufsatz" />
*  <label xml:lang="x-mapping" text="diniPublType:article" />
*  <label xml:lang="x-mapping-xpath" text="mods:genre[text()='article']" />
+ *  <label xml:lang="x-mapping-xpathfb" text="mods:genre[text()='other']" />
* </category> * * @@ -68,6 +71,8 @@ public class MCRClassificationMappingEventHandler extends MCREventHandlerBase { public static final String LABEL_LANG_XPATH_MAPPING = "x-mapping-xpath"; + public static final String LABEL_LANG_XPATH_MAPPING_FALLBACK = "x-mapping-xpathfb"; + public static final String LABEL_LANG_X_MAPPING = "x-mapping"; public static final String XPATH_GENERATOR_NAME = "xpathmapping"; @@ -104,16 +109,22 @@ private static List> getXMappings(MCRCat /** * Searches all configured classifications * (see {@link MCRClassificationMappingEventHandler#X_PATH_MAPPING_CLASSIFICATIONS}) for categories - * with language label {@link MCRClassificationMappingEventHandler#LABEL_LANG_XPATH_MAPPING}. - * All categories with said label and present in database are returned in a list. - * @return a list of {@link MCRCategory categories} with the XPath-Mapping label + * with language labels {@link MCRClassificationMappingEventHandler#LABEL_LANG_XPATH_MAPPING} or + * {@link MCRClassificationMappingEventHandler#LABEL_LANG_XPATH_MAPPING_FALLBACK}. + * All categories with said label and present in database are returned in a Map of Sets, + * separated by their classification. + * @return a Map with classification-IDs as key and Sets of {@link MCRCategory categories} + * with the XPath-Mapping label as value */ - private static List loadAllXPathMappings() { + private static Map> loadAllXPathMappings() { final MCRCategoryDAO dao = MCRCategoryDAOFactory.getInstance(); return Arrays.stream(X_PATH_MAPPING_CLASSIFICATIONS.trim().split(",")) - .flatMap( - relevantClass -> dao.getCategoriesByClassAndLang(relevantClass, LABEL_LANG_XPATH_MAPPING).stream()) - .collect(Collectors.toList()); + .collect(Collectors.toMap( + relevantClass -> relevantClass, + relevantClass -> Stream.concat( + dao.getCategoriesByClassAndLang(relevantClass, LABEL_LANG_XPATH_MAPPING).stream(), + dao.getCategoriesByClassAndLang(relevantClass, LABEL_LANG_XPATH_MAPPING_FALLBACK).stream()) + .collect(Collectors.toSet()))); } @Override @@ -173,21 +184,47 @@ private void createMapping(MCRObject obj) { mappedClassification.setAttribute("generator", generator); MCRClassMapper.assignCategory(mappedClassification, mapping.getValue()); }); - // check x-mapping-xpath-mappings - for (MCRCategory category : loadAllXPathMappings()) { - if (category.getLabel(LABEL_LANG_XPATH_MAPPING).isPresent()) { - - String xPath = category.getLabel(LABEL_LANG_XPATH_MAPPING).get().getText(); - MCRXPathEvaluator evaluator = new MCRXPathEvaluator(new HashMap<>(), mcrmodsWrapper.getMODS()); - - if (evaluator.test(xPath)) { - String taskMessage = String.format(Locale.ROOT, "add x-path-mapping from '%s'", - category.getId().toString()); - LOGGER.info(taskMessage); - Element mappedClassification = mcrmodsWrapper.addElement("classification"); - String generator = getXPathMappingGenerator(category.getId()); - mappedClassification.setAttribute("generator", generator); - MCRClassMapper.assignCategory(mappedClassification, category.getId()); + + final Map> xPathMappings = loadAllXPathMappings(); + for (Set categoriesPerClass : xPathMappings.values()) { + boolean isXPathMatched = false; + // check x-mapping-xpath-mappings + for (MCRCategory category : categoriesPerClass) { + if (category.getLabel(LABEL_LANG_XPATH_MAPPING).isPresent()) { + + String xPath = category.getLabel(LABEL_LANG_XPATH_MAPPING).get().getText(); + MCRXPathEvaluator evaluator = new MCRXPathEvaluator(new HashMap<>(), mcrmodsWrapper.getMODS()); + + if (evaluator.test(xPath)) { + String taskMessage = String.format(Locale.ROOT, "add x-path-mapping from '%s'", + category.getId().toString()); + LOGGER.info(taskMessage); + Element mappedClassification = mcrmodsWrapper.addElement("classification"); + String generator = getXPathMappingGenerator(category.getId()); + mappedClassification.setAttribute("generator", generator); + MCRClassMapper.assignCategory(mappedClassification, category.getId()); + isXPathMatched = true; + } + } + } + //check x-mapping-xpath-fallback-mappings + if (!isXPathMatched) { + for (MCRCategory category : categoriesPerClass) { + if (category.getLabel(LABEL_LANG_XPATH_MAPPING_FALLBACK).isPresent()) { + + String xPath = category.getLabel(LABEL_LANG_XPATH_MAPPING_FALLBACK).get().getText(); + MCRXPathEvaluator evaluator = new MCRXPathEvaluator(new HashMap<>(), mcrmodsWrapper.getMODS()); + + if (evaluator.test(xPath)) { + String taskMessage = String.format(Locale.ROOT, "add x-path-mapping-fallback from '%s'", + category.getId().toString()); + LOGGER.info(taskMessage); + Element mappedClassification = mcrmodsWrapper.addElement("classification"); + String generator = getXPathMappingGenerator(category.getId()); + mappedClassification.setAttribute("generator", generator); + MCRClassMapper.assignCategory(mappedClassification, category.getId()); + } + } } } } diff --git a/mycore-mods/src/test/java/org/mycore/mods/classification/MCRClassificationMappingEventHandlerTest.java b/mycore-mods/src/test/java/org/mycore/mods/classification/MCRClassificationMappingEventHandlerTest.java index 8b3eb2eacf..c382f9b4aa 100644 --- a/mycore-mods/src/test/java/org/mycore/mods/classification/MCRClassificationMappingEventHandlerTest.java +++ b/mycore-mods/src/test/java/org/mycore/mods/classification/MCRClassificationMappingEventHandlerTest.java @@ -95,7 +95,8 @@ public void testMapping() throws IOException, JDOMException, URISyntaxException } /** - * Tests if an XPath-mapping is properly added into a Mods-Document. + * Tests if an XPath-mapping is properly added into a Mods-Document. Also tests, if multiple fallbacks per + * classification are considered * @throws IOException in case of error * @throws JDOMException in case of error * @throws URISyntaxException in case of error @@ -109,29 +110,103 @@ public void testXPathMapping() throws IOException, JDOMException, URISyntaxExcep loadCategory("genre.xml"); loadCategory("orcidWorkType.xml"); + loadCategory("dummyClassification.xml"); Document document = saxBuilder.build(classLoader.getResourceAsStream(TEST_DIRECTORY + "testMods2.xml")); MCRObject mcro = new MCRObject(); MCRMODSWrapper mw = new MCRMODSWrapper(mcro); mw.setMODS(document.getRootElement().detach()); - mw.setID("junit", 1); + mw.setID("junit", 2); MCRClassificationMappingEventHandler mapper = new MCRClassificationMappingEventHandler(); mapper.handleObjectUpdated(null, mcro); Document xml = mcro.createXML(); + String expression = "//mods:classification[contains(@generator,'-mycore') and " + + "contains(@generator,'xpathmapping2orcidWorkType') and contains(@valueURI, 'journal-article')]"; + XPathExpression expressionObject = XPathFactory.instance() + .compile(expression, Filters.element(), null, MCRConstants.MODS_NAMESPACE, MCRConstants.XLINK_NAMESPACE); + + Assert.assertNotNull("The mapped classification should be in the MyCoReObject now!", + expressionObject.evaluateFirst( + xml)); + + expression = "//mods:classification[contains(@generator,'-mycore') and " + + "contains(@generator,'xpathmapping2dummyClassification') and contains(@valueURI, 'dummy-text')]"; + expressionObject = XPathFactory.instance() + .compile(expression, Filters.element(), null, MCRConstants.MODS_NAMESPACE, MCRConstants.XLINK_NAMESPACE); + + Assert.assertNotNull("The mapped dummy classification should be in the MyCoReObject now!", + expressionObject.evaluateFirst(xml)); + + expression = "//mods:classification[contains(@generator,'-mycore') and " + + "contains(@generator,'xpathmapping2dummyClassification') and contains(@valueURI, 'dummy-fbonly')]"; + expressionObject = XPathFactory.instance() + .compile(expression, Filters.element(), null, MCRConstants.MODS_NAMESPACE, MCRConstants.XLINK_NAMESPACE); + + Assert.assertNotNull("The mapped dummy classification should be in the MyCoReObject now!", + expressionObject.evaluateFirst(xml)); + LOGGER.info(new XMLOutputter(Format.getPrettyFormat()).outputString(xml)); + } + + /** + * Tests if a fallback XPath-mapping is properly added into a Mods-Document if the original mapping doesn't match. + * @throws IOException in case of error + * @throws JDOMException in case of error + * @throws URISyntaxException in case of error + */ + @Test + public void testXPathMappingFallback() throws IOException, JDOMException, URISyntaxException { + MCRSessionMgr.getCurrentSession(); + MCRTransactionHelper.isTransactionActive(); + ClassLoader classLoader = getClass().getClassLoader(); + SAXBuilder saxBuilder = new SAXBuilder(); + + loadCategory("genre.xml"); + loadCategory("orcidWorkType.xml"); + loadCategory("dummyClassification.xml"); + + Document document = saxBuilder.build(classLoader.getResourceAsStream(TEST_DIRECTORY + "testMods3.xml")); + MCRObject mcro = new MCRObject(); + + MCRMODSWrapper mw = new MCRMODSWrapper(mcro); + mw.setMODS(document.getRootElement().detach()); + mw.setID("junit", 3); + + MCRClassificationMappingEventHandler mapper = new MCRClassificationMappingEventHandler(); + mapper.handleObjectUpdated(null, mcro); + + Document xml = mcro.createXML(); String expression = "//mods:classification[contains(@generator,'-mycore') and " - + "contains(@generator,'xpathmapping2orcidWorkType') and contains(@valueURI, 'journal-article')]"; + + "contains(@generator,'xpathmapping2orcidWorkType') and contains(@valueURI, 'online-resource')]"; XPathExpression expressionObject = XPathFactory.instance() .compile(expression, Filters.element(), null, MCRConstants.MODS_NAMESPACE, MCRConstants.XLINK_NAMESPACE); Assert.assertNotNull("The mapped classification should be in the MyCoReObject now!", expressionObject.evaluateFirst( xml)); + + expression = "//mods:classification[contains(@generator,'-mycore') and " + + "contains(@generator,'xpathmapping2dummyClassification') and contains(@valueURI, 'dummy-text')]"; + expressionObject = XPathFactory.instance() + .compile(expression, Filters.element(), null, MCRConstants.MODS_NAMESPACE, MCRConstants.XLINK_NAMESPACE); + + Assert.assertNotNull("The mapped dummy classification should be in the MyCoReObject now!", + expressionObject.evaluateFirst(xml)); + + expression = "//mods:classification[contains(@generator,'-mycore') and " + + "contains(@generator,'xpathmapping2dummyClassification') and contains(@valueURI, 'dummy-fbonly')]"; + expressionObject = XPathFactory.instance() + .compile(expression, Filters.element(), null, MCRConstants.MODS_NAMESPACE, MCRConstants.XLINK_NAMESPACE); + + Assert.assertNotNull("The mapped dummy classification should be in the MyCoReObject now!", + expressionObject.evaluateFirst(xml)); + + LOGGER.info(new XMLOutputter(Format.getPrettyFormat()).outputString(xml)); } private void loadCategory(String categoryFileName) throws URISyntaxException, JDOMException, IOException { diff --git a/mycore-mods/src/test/resources/MCRClassificationMappingEventHandlerTest/dummyClassification.xml b/mycore-mods/src/test/resources/MCRClassificationMappingEventHandlerTest/dummyClassification.xml new file mode 100644 index 0000000000..debba5222b --- /dev/null +++ b/mycore-mods/src/test/resources/MCRClassificationMappingEventHandlerTest/dummyClassification.xml @@ -0,0 +1,21 @@ + + + diff --git a/mycore-mods/src/test/resources/MCRClassificationMappingEventHandlerTest/orcidWorkType.xml b/mycore-mods/src/test/resources/MCRClassificationMappingEventHandlerTest/orcidWorkType.xml index 110d774239..39f007063d 100644 --- a/mycore-mods/src/test/resources/MCRClassificationMappingEventHandlerTest/orcidWorkType.xml +++ b/mycore-mods/src/test/resources/MCRClassificationMappingEventHandlerTest/orcidWorkType.xml @@ -10,5 +10,23 @@ description="Articles in peer-reviewed publications that disseminate the results of original research and scholarship."/>