Version 2.2.0

# Conflicts:
#	proline-cortex/src/main/scala/fr/proline/cortex/service/dps/msi/ImportValidateGenerateSM.scala

msparser-native-libraries/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <artifactId>proline-server</artifactId>
     <groupId>fr.proline</groupId>
-    <version>2.1.2</version>
+    <version>2.2.0</version>
   </parent>
 
   <groupId>matrixscience</groupId>

pm-dataset-exporter/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>proline-server</artifactId>
     <groupId>fr.proline</groupId>
-    <version>2.1.2</version>
+    <version>2.2.0</version>  <!-- Snapshot Trunk version -->
   </parent>
 
   <artifactId>pm-dataset-exporter</artifactId>

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/commons/config/CustomFieldConfigFactory.scala

@@ -135,6 +135,7 @@ object CustomFieldConfigFactory {
     val fieldsBuffer = ArrayBuffer(
       CustomFieldConfig(FIELD_PSM_PEPTIDE_ID, "peptide_id"),
       CustomFieldConfig(FIELD_PSM_SEQUENCE, "sequence"),
+      CustomFieldConfig(FIELD_PSM_SEQUENCE_NONE_AMBIGUOUS, "none_ambiguous_psm_sequence"),
       CustomFieldConfig(FIELD_PSM_MODIFICATIONS, "modifications"),
       CustomFieldConfig(FIELD_PSM_PTM_PROTEIN_POSITIONS, "ptm_protein_positions"),
       CustomFieldConfig(FIELD_PSM_ID, "psm_id"),
@@ -188,12 +189,39 @@ object CustomFieldConfigFactory {
     getPeptideMatchesSheetFields(fromXIC, fromSC)
   }
 
+  // get all fields for masterQuantPeptideIon
+  def getMasterQuantPeptideSheetFields(): Array[CustomFieldConfig] = {
+
+    val fieldsBuffer = ArrayBuffer(
+      CustomFieldConfig(FIELD_PSM_PEPTIDE_ID, "peptide_id"),
+      CustomFieldConfig(FIELD_PSM_SEQUENCE, "sequence"),
+      CustomFieldConfig(FIELD_PSM_SEQUENCE_NONE_AMBIGUOUS, "none_ambiguous_psm_sequence"),
+      CustomFieldConfig(FIELD_PSM_MODIFICATIONS, "modifications"),
+      CustomFieldConfig(FIELD_PSM_QUANT_MASTER_QUANT_PEPTIDE_ID, "master_quant_peptide_id"),
+      CustomFieldConfig(FIELD_PSM_QUANT_ELUTION_TIME, "master_elution_time"),
+      CustomFieldConfig(FIELD_PSM_QUANT_SELECTION_LEVEL, "is_validated_for_quanti"),
+      CustomFieldConfig(FIELD_PSM_NB_PROTEIN_SETS, "#psm_prot_sets"),
+      CustomFieldConfig(FIELD_MASTER_QUANT_PEPTIDE_PROTEIN_SETS, "protein_sets_list"),
+      CustomFieldConfig(FIELD_MASTER_QUANT_PEPTIDE_ION_MOZ, "master_quant_peptide_moz"),
+      CustomFieldConfig(FIELD_MASTER_QUANT_PEPTIDE_ION_ELUTION_TIME, "master_quant_peptide_elution_time")
+    )
+
+    fieldsBuffer ++= Array(
+      CustomFieldConfig(FIELD_PROTEIN_SETS_QUANT_PSM_COUNT, "psm_count"), // incremental
+      CustomFieldConfig(FIELD_QUANT_PEPTIDE_ION_ELUTION_TIME, "elution_time"), // incremental
+      CustomFieldConfig(FIELD_PROTEIN_SETS_QUANT_RAW_ABUNDANCE, "raw_abundance"), // incremental
+      CustomFieldConfig(FIELD_PROTEIN_SETS_QUANT_ABUNDANCE, "abundance") // incremental
+    )
+
+    fieldsBuffer.toArray
+  }
   // get all fields for masterQuantPeptideIon
   def getMasterQuantPepIonSheetFields(): Array[CustomFieldConfig] = {
 
     val fieldsBuffer = ArrayBuffer(
       CustomFieldConfig(FIELD_PSM_PEPTIDE_ID, "peptide_id"),
       CustomFieldConfig(FIELD_PSM_SEQUENCE, "sequence"),
+      CustomFieldConfig(FIELD_PSM_SEQUENCE_NONE_AMBIGUOUS, "none_ambiguous_psm_sequence"),
       CustomFieldConfig(FIELD_PSM_MODIFICATIONS, "modifications"),
       CustomFieldConfig(FIELD_PSM_PTM_PROTEIN_POSITIONS, "ptm_protein_positions"),
       CustomFieldConfig(FIELD_PSM_QUANT_MASTER_QUANT_PEPTIDE_ID, "master_quant_peptide_id"),
@@ -257,6 +285,9 @@ object CustomFieldConfigFactory {
       CustomFieldConfig(FIELD_PTM_CLUSTER_NB_PEPTIDES, "ptm_cluster_nb_peptides"),
       CustomFieldConfig(FIELD_PTM_CLUSTER_NB_SITES, "ptm_cluster_nb_sites"),
       CustomFieldConfig(FIELD_PTM_CLUSTER_SITES_LOCALISATION,"ptm_cluster_sites_localisations"),
+      CustomFieldConfig(FIELD_PTM_CLUSTER_IS_VALIDATED, "ptm_cluster_is_validated"),
+      CustomFieldConfig(FIELD_PTM_CLUSTER_VALIDATION_CONFIDENCE, "ptm_cluster_validation_confidence"),
+      CustomFieldConfig(FIELD_PTM_CLUSTER_VALIDATION_INFO, "ptm_cluster_validation_information"),
       CustomFieldConfig(FIELD_PTM_CLUSTER_LOCALISATION_CONFIDENCE, "ptm_cluster_localisation_confidence"),
       CustomFieldConfig(FIELD_PSM_PEPTIDE_ID, "peptide_id"),
       CustomFieldConfig(FIELD_PSM_SEQUENCE, "sequence"),

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/commons/config/ExportConfig.scala

@@ -81,7 +81,7 @@ object ExportConfig {
         ExportConfigSheet.getProteinSetsSheetExportConfig(fromXIC, fromSC),
         ExportConfigSheet.getProteinMatchesSheetExportConfig(defaultDisplayed = false),
         ExportConfigSheet.getBestPeptideMatchesSheetExportConfig(fromXIC, fromSC, defaultDisplayed = false),
-        //ExportConfigSheet.getMasterQuantPeptidesSheetExportConfig(),
+        ExportConfigSheet.getMasterQuantPeptidesSheetExportConfig(),
         ExportConfigSheet.getMasterQuantPepIonsSheetSheetExportConfig(),
         ExportConfigSheet.getPTMClusterSheetExportConfig(fromXIC, defaultDisplayed = false),
         ExportConfigSheet.getStatisticsSheetExportConfig(defaultDisplayed = false)

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/commons/config/ExportConfigConstant.scala

@@ -133,6 +133,7 @@ object ExportConfigConstant {
   // see  also FIELD_PROTEIN_SETS_*
   val FIELD_PSM_PEPTIDE_ID = "psm_peptide_id"
   val FIELD_PSM_SEQUENCE = "psm_sequence"
+  val FIELD_PSM_SEQUENCE_NONE_AMBIGUOUS = "none_ambiguous_psm_sequence"
   val FIELD_PSM_MODIFICATIONS = "psm_modifications"
   val FIELD_PSM_PTM_PROTEIN_POSITIONS = "psm_ptm_protein_positions"
   val FIELD_PSM_ID = "psm_id"
@@ -178,7 +179,10 @@ object ExportConfigConstant {
 
   // MasterQuantReporterIon sheet
   val SHEET_MASTER_QUANT_REPORTER_ION = "master_quant_reporter_ion"
-
+  val FIELD_MASTER_QUANT_PEPTIDE_PROTEIN_SETS = "master_quant_peptide_protein_sets"
+
+  val SHEET_MASTER_QUANT_PEPTIDE = "master_quant_peptide"
+
   // Some properties shared between quantitation sheets
   val FIELD_PROFILIZER_RATIO = "ratio" // incremental
   val FIELD_PROFILIZER_TTEST_PVALUE = "t-test_pvalue" // incremental
@@ -214,6 +218,9 @@ object ExportConfigConstant {
   val FIELD_PTM_CLUSTER_NB_PEPTIDES = "ptm_cluster_nb_peptides"
   val FIELD_PTM_CLUSTER_NB_SITES = "ptm_cluster_nb_sites"
   val FIELD_PTM_CLUSTER_SITES_LOCALISATION = "ptm_cluster_sites_localisation"
+  val FIELD_PTM_CLUSTER_IS_VALIDATED = "ptm_cluster_is_validated"
+  val FIELD_PTM_CLUSTER_VALIDATION_CONFIDENCE= "ptm_cluster_validation_confidence"
+  val FIELD_PTM_CLUSTER_VALIDATION_INFO= "ptm_cluster_validation_information"
   val FIELD_PTM_CLUSTER_LOCALISATION_CONFIDENCE= "ptm_cluster_localisation_confidence"
 
 }

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/commons/config/ExportConfigSheet.scala

@@ -98,7 +98,14 @@ object ExportConfigSheet {
   }
 
   def getMasterQuantPeptidesSheetExportConfig(): ExportConfigSheet = {
-    this.getPeptideMatchesSheetExportConfig(fromXIC = true, fromSC = false, title = "Peptides from protein sets")
+    ExportConfigSheet(
+      id = ExportConfigConstant.SHEET_MASTER_QUANT_PEPTIDE,
+      title = "Quantified Peptides",
+      presentation = ExportConfigConstant.PRESENTATION_SHEET_COLUMNS,
+      fields = CustomFieldConfigFactory.getMasterQuantPeptideSheetFields(),
+      defaultDisplayed = false
+    )
+
   }
 
   // get full config for masterQuantPeptideIon

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/commons/config/view/DatasetViewType.scala

@@ -14,6 +14,7 @@ object DatasetViewType extends IViewTypeEnumeration {
   val PROT_SET_TO_ALL_PEPTIDE_MATCHES = Value
   val TYPICAL_PROT_MATCH_TO_ALL_PEP_MATCHES = Value
   val MASTER_QUANT_PEPTIDE_ION = Value
+  val MASTER_QUANT_PEPTIDE = Value
   val MASTER_QUANT_REPORTER_ION = Value
   val PTM_CLUSTER = Value
 }

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/dataset/IdentDataset.scala

@@ -24,11 +24,12 @@ class IdentDataset(
                     protected val loadBioSequences: () => Array[BioSequence],
                     protected val loadSpectraDescriptors: (Array[Long]) => Array[Spectrum],
                     val loadPepMatches: (Array[Long]) => Array[PeptideMatch],
+                    val loadPeptides: (Array[Long]) => Array[Peptide],
                     val loadPtmDataset: () => Option[PtmDataSet]
 ) extends LazyLogging {
 
   // Count the number of protein sets and proteins matches related to a given peptide match
-  val validProtSetIdSetByPeptideId = new LongMap[HashSet[Long]]()
+  val validProtSetsByPeptideId = new LongMap[HashSet[ProteinSet]]()
   val validSamesetProtMatchIdSetByPepMatchId = new LongMap[HashSet[Long]]()
   val validProtMatchIdSetByPepMatchId = new LongMap[HashSet[Long]]()
 
@@ -43,7 +44,7 @@ class IdentDataset(
     }
     //#19643 use peptide Ids instead if psmId (may need information from leaf psm (in case of quant peptide ion for example)
     protSet.peptideSet.getPeptideIds.foreach {pepId =>
-      validProtSetIdSetByPeptideId.getOrElseUpdate(pepId, new HashSet[Long]) += protSet.id
+      validProtSetsByPeptideId.getOrElseUpdate(pepId, new HashSet[ProteinSet]) += protSet
     }
 
     val protMatchIdByPepSet = protSet.getAllProteinMatchesIdByPeptideSet

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/dataset/QuantDataset.scala

@@ -19,14 +19,15 @@ class QuantDataset(
                     override protected val loadBioSequences: () => Array[BioSequence],
                     override protected val loadSpectraDescriptors: (Array[Long]) => Array[Spectrum],
                     override val loadPepMatches: (Array[Long]) => Array[PeptideMatch],
+                    override val loadPeptides :  (Array[Long]) => Array[Peptide],
                     val loadPepMatchesByMsQIs: (Array[Long]) => Array[PeptideMatch],
                     // val qcNameById: LongMap[String], //  update and use the QuantChannel name in the UDSdb
                     //val protMatchStatusByIdPepMatchByQCId: Map[Long, Map[Long, String]],
                     val peptideCountByProtMatchIdByQCId: Option[LongMap[LongMap[Int]]],
                     val peptideCountByMqProtSetByQCId: Option[LongMap[LongMap[Int]]],
                     override val loadPtmDataset: () => Option[PtmDataSet]
 
-) extends IdentDataset(projectName,quantRSM.lazyResultSummary,loadLeaveResultSummaries,loadLeaveResultSets, loadBioSequences,loadSpectraDescriptors, loadPepMatches, loadPtmDataset) {
+) extends IdentDataset(projectName,quantRSM.lazyResultSummary,loadLeaveResultSummaries,loadLeaveResultSets, loadBioSequences,loadSpectraDescriptors, loadPepMatches, loadPeptides, loadPtmDataset) {
 
 
   lazy val identRsmById = childResultSummaries.mapByLong(_.id)

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/dataset/template/IDatasetTemplate.scala

@@ -33,6 +33,7 @@ trait IDatasetTemplate extends IViewSetTemplate with StrictLogging {
       case SHEET_MASTER_QUANT_REPORTER_ION => MASTER_QUANT_REPORTER_ION
       case SHEET_STAT => STATISTICS
       case SHEET_PTM_CLUSTER => PTM_CLUSTER
+      case SHEET_MASTER_QUANT_PEPTIDE => MASTER_QUANT_PEPTIDE
       case _ => throw new Exception(s"Invalid sheet id: ${sheet.id}")
     }
 

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/dataset/view/AbstractProtSetToPepMatchView.scala

@@ -12,6 +12,7 @@ abstract class AbstractProtSetToPepMatchView extends AbstractProtSetToTypicalPro
   protected val pepMatchFieldSet = Set(
     FIELD_PSM_PEPTIDE_ID,
     FIELD_PSM_SEQUENCE,
+    FIELD_PSM_SEQUENCE_NONE_AMBIGUOUS,
     FIELD_PSM_MODIFICATIONS,
     FIELD_PSM_PTM_PROTEIN_POSITIONS,
     FIELD_PSM_ID,
@@ -107,6 +108,7 @@ abstract class AbstractProtSetToPepMatchView extends AbstractProtSetToTypicalPro
       val fieldValue: Any = fieldConfig.id match {
         case FIELD_PSM_PEPTIDE_ID => peptide.id
         case FIELD_PSM_SEQUENCE => peptide.sequence
+        case FIELD_PSM_SEQUENCE_NONE_AMBIGUOUS => pepMatch.getDisambiguatedSeq()
         case FIELD_PSM_MODIFICATIONS => peptide.readablePtmString
         case FIELD_PSM_PTM_PROTEIN_POSITIONS => ptmProtPositions
         case FIELD_PSM_ID => pepMatch.id
@@ -123,7 +125,7 @@ abstract class AbstractProtSetToPepMatchView extends AbstractProtSetToTypicalPro
         case FIELD_PSM_CD_PRETTY_RANK => pepMatch.cdPrettyRank
         case FIELD_PSM_FRAGMENT_MATCHES_COUNT => pepMatch.fragmentMatchesCount
         case FIELD_PSM_SPECTRUM_TITLE => msQueryOpt.map(_.spectrumTitle).orNull
-        case FIELD_PSM_NB_PROTEIN_SETS => identDS.validProtSetIdSetByPeptideId.get(peptide.id).map(_.size).getOrElse(0)
+        case FIELD_PSM_NB_PROTEIN_SETS => identDS.validProtSetsByPeptideId.get(peptide.id).map(_.size).getOrElse(0)
         case FIELD_PSM_NB_SAMESET_PROTEIN_MATCHES => identDS.validSamesetProtMatchIdSetByPepMatchId.get(pepMatch.id).map(_.size).getOrElse(0)
         case FIELD_PSM_NB_PROTEIN_MATCHES => identDS.validProtMatchIdSetByPepMatchId.get(pepMatch.id).map(_.size).getOrElse(0)
         case FIELD_PSM_NB_DATABANK_PROTEIN_MATCHES => identDS.allProtMatchSetByPepId.get(peptide.id).map(_.size).getOrElse(0)

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/dataset/view/BuildDatasetView.scala

@@ -55,6 +55,9 @@ object BuildDatasetView {
         case ExportConfigConstant.SHEET_ALL_PSM => {
           mapBuilder += (PROT_SET_TO_ALL_PEPTIDE_MATCHES -> { ds: IdentDataset => new ProtSetToAllPepMatchesView(ds, sheet, dateFormat, smartDecimalFormat, titleSep, exportAllProteinSet, exportBestProfile) })
         }
+        case ExportConfigConstant.SHEET_MASTER_QUANT_PEPTIDE => {
+          mapBuilder += (MASTER_QUANT_PEPTIDE -> { ds: IdentDataset => new MasterQuantPeptideView(ds, sheet, dateFormat, smartDecimalFormat, titleSep, exportAllProteinSet, exportBestProfile) })
+        }
         case ExportConfigConstant.SHEET_MASTER_QUANT_PEPTIDE_ION => {
           mapBuilder += (MASTER_QUANT_PEPTIDE_ION -> { ds: IdentDataset => new MasterQuantPeptideIonView(ds, sheet, dateFormat, smartDecimalFormat, titleSep, exportAllProteinSet, exportBestProfile) })
         }

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/dataset/view/BuildDatasetViewSet.scala

@@ -187,6 +187,7 @@ object BuildDatasetViewSet extends LazyLogging {
         this._buildBioSequenceLoader(msiDbCtx, lazyRsm),
         this._buildSpectraLoader(msiDbCtx),
         this._buildPepMatchesLoader(executionContext),
+        this._buildPeptidesLoader(executionContext),
         this._buildPTMDatasetLoader(executionContext, rsmId)
       )
 
@@ -398,6 +399,7 @@ object BuildDatasetViewSet extends LazyLogging {
         this._buildBioSequenceLoader(msiDbCtx, lazyQuantRSM.lazyResultSummary),
         this._buildSpectraLoader(msiDbCtx),
         this._buildPepMatchesLoader(executionContext),
+        this._buildPeptidesLoader(executionContext),
         this._buildPepMatchesByMsQIdLoader(executionContext),
         Option(peptideCountByProtMatchIdByQCId),
         Option(peptideCountByMqProtSetIdByQCId),
@@ -452,6 +454,13 @@ object BuildDatasetViewSet extends LazyLogging {
 
   }
 
+  private def _buildPeptidesLoader(executionContext: IExecutionContext) = { pepsId: Array[Long] =>
+    logger.debug(s"Loading peptides ... ${pepsId.length}")
+    val peptidesProvider = new SQLPeptideProvider(PeptideCacheExecutionContext(executionContext))
+    peptidesProvider.getPeptides(pepsId)
+
+  }
+
   private def _buildPepMatchesByMsQIdLoader(executionContext: IExecutionContext) = { msQueriesIds: Array[Long] =>
     logger.debug(s"Loading peptide Matches from msQuery Ids ... ${msQueriesIds.length}")
     val peptideMatchProvider = new SQLPeptideMatchProvider(PeptideCacheExecutionContext(executionContext))
@@ -464,14 +473,26 @@ object BuildDatasetViewSet extends LazyLogging {
 
     DoJDBCReturningWork.withEzDBC(executionContext.getMSIDbConnectionContext) { msiEzDBC =>
 
-      val ptmObjectTreeQuery = new SelectQueryBuilder1(MsiDbResultSummaryObjectTreeMapTable).mkSelectQuery((t, c) =>
-        List(t.*) -> "WHERE " ~ t.RESULT_SUMMARY_ID ~ s" = '${rsmId}'" ~ " AND " ~ t.SCHEMA_NAME ~ s" = '${SchemaName.PTM_DATASET.toString}'"
+      val annotatedPtmObjectTreeQuery = new SelectQueryBuilder1(MsiDbResultSummaryObjectTreeMapTable).mkSelectQuery((t, c) =>
+        List(t.*) -> "WHERE " ~ t.RESULT_SUMMARY_ID ~ s" = '${rsmId}'" ~ " AND " ~ t.SCHEMA_NAME ~ s" = '${SchemaName.PTM_DATASET_ANNOTATED.toString}'"
       )
 
-      val objectTreeIdOpt = msiEzDBC.select(ptmObjectTreeQuery) { r =>
+      var objectTreeIdOpt = msiEzDBC.select(annotatedPtmObjectTreeQuery) { r =>
         toLong(r.getAny(MsiDbResultSummaryObjectTreeMapTable.columns.OBJECT_TREE_ID))
       }.headOption
 
+      if(!objectTreeIdOpt.isDefined) {
+        val ptmObjectTreeQuery = new SelectQueryBuilder1(MsiDbResultSummaryObjectTreeMapTable).mkSelectQuery((t, c) =>
+          List(t.*) -> "WHERE " ~ t.RESULT_SUMMARY_ID ~ s" = '${rsmId}'" ~ " AND " ~ t.SCHEMA_NAME ~ s" = '${SchemaName.PTM_DATASET.toString}'"
+        )
+
+        objectTreeIdOpt = msiEzDBC.select(ptmObjectTreeQuery) { r =>
+          toLong(r.getAny(MsiDbResultSummaryObjectTreeMapTable.columns.OBJECT_TREE_ID))
+        }.headOption
+
+      }
+
+
       if (objectTreeIdOpt.isDefined) {
         val ptmSiteQuery = new SelectQueryBuilder1(MsiDbObjectTreeTable).mkSelectQuery((t, c) =>
           List(t.*) -> "WHERE " ~ t.ID ~ s" = '${objectTreeIdOpt.get}'"

pm-dataset-exporter/src/main/scala/fr/proline/module/exporter/dataset/view/MasterQuantPeptideIonView.scala

@@ -182,7 +182,6 @@ class MasterQuantPeptideIonView(
 
               val quantRecordBuildingCtx = new MasterQuantPeptideIonBuildingContext(
                 pepMatch = parentOrChildBestPepMatchOpt.get,
-                protMatch = reprProtMatch,
                 seqMatch = seqMatchByPepId(peptideId),
                 protMatchBuildingCtx = Some(protMatchBuildingCtx),
                 mqPep,