Ticket 48872: Unable to customize columns in PTM reports or use locat…

…ion and amino acid separately

resources/queries/targetedms/PTMPercents.query.xml

@@ -2,6 +2,7 @@
     <metadata>
         <tables xmlns="http://labkey.org/data/xml">
             <table tableName="PTMPercents" tableDbType="TABLE">
+                <javaCustomizer class="org.labkey.targetedms.query.PTMPercentsCustomizer" />
                 <columns>
                     <column columnName="PeptideModifiedSequence">
                         <displayColumnFactory>

resources/queries/targetedms/PTMPercents.sql

@@ -10,6 +10,8 @@ SELECT
   SampleName,
   -- Explicitly cast for SQLServer to avoid trying to add as numeric types
   SUBSTRING(Sequence, IndexAA + 1, 1) || CAST(StartIndex + IndexAA + 1 AS VARCHAR) AS SiteLocation,
+  SUBSTRING(Sequence, IndexAA + 1, 1) AS AminoAcid,
+  StartIndex + IndexAA + 1 AS Location,
   PeptideGroupId
 FROM PTMPercentsPrepivot
 GROUP BY

resources/queries/targetedms/PTMPercentsGrouped.sql

@@ -1,6 +1,9 @@
 SELECT
     PeptideGroupId,
-    SiteLocation,
+    -- Explicitly cast for SQLServer to avoid trying to add as numeric types
+    AminoAcid || CAST(Location AS VARCHAR) AS SiteLocation,
+    AminoAcid,
+    Location,
     PeptideModifiedSequence,
     Sequence @hidden,
     -- We have a special rule for this C-term modification - we're actually interested in the _unmodified_ percentage
@@ -26,7 +29,8 @@ GROUP BY
     NextAA,
     PeptideModifiedSequence,
     PeptideGroupId,
-    SiteLocation,
+    AminoAcid,
+    Location,
     Modification.Name,
     ModificationCount
 PIVOT PercentModified, TotalPercentModified BY SampleName

resources/queries/targetedms/PTMPercentsGroupedPrepivot.sql

@@ -11,30 +11,30 @@ SELECT p1.Modification,
        PreviousAA @hidden,
        NextAA @hidden,
        p1.SampleName,
-       p1.SiteLocation,
+       p1.AminoAcid,
+       p1.Location,
        p1.PeptideGroupId
 
 FROM
     -- First, calculate the percentage for each individual modification
     (SELECT StructuralModId                                                                  AS Modification,
             SUM(ModifiedAreaProportion)                                                      AS PercentModified,
             MIN(Id)                                                                          AS Id,
-            PeptideModifiedSequence,
+            MAX(PeptideModifiedSequence) AS PeptideModifiedSequence,
             Sequence @hidden,
             PreviousAA @hidden,
             NextAA @hidden,
             SampleName,
             StartIndex,
             IndexAA,
-            -- Explicitly cast for SQLServer to avoid trying to add as numeric types
-            SUBSTRING(Sequence, IndexAA + 1, 1) || CAST(StartIndex + IndexAA + 1 AS VARCHAR) AS SiteLocation,
+            SUBSTRING(Sequence, IndexAA + 1, 1) AS AminoAcid,
+            StartIndex + IndexAA + 1 AS Location,
             PeptideGroupId
      FROM PTMPercentsPrepivot
      GROUP BY SampleName,
               Sequence,
               PreviousAA,
               NextAA,
-              PeptideModifiedSequence,
               StartIndex,
               PeptideGroupId,
               IndexAA,
@@ -45,11 +45,12 @@ FROM
     -- Second, calculate total percent across all modifications for each amino acid
     (SELECT
          SUM(ModifiedAreaProportion) AS TotalPercentModified,
-         COUNT(*) AS ModificationCount,
+         COUNT(DISTINCT StructuralModId) AS ModificationCount,
          SampleName,
          Sequence,
          PeptideGroupId,
-         SUBSTRING(Sequence, IndexAA + 1, 1) || CAST(StartIndex + IndexAA + 1 AS VARCHAR) AS SiteLocation,
+         SUBSTRING(Sequence, IndexAA + 1, 1) AS AminoAcid,
+         StartIndex + IndexAA + 1 AS Location,
      FROM PTMPercentsPrepivot
      GROUP BY SampleName,
               Sequence,
@@ -60,7 +61,8 @@ FROM
     ON p1.SampleName = p2.SampleName AND
        p1.Sequence = p2.Sequence AND
        p1.PeptideGroupId = p2.PeptideGroupId AND
-       p1.SiteLocation = p2.SiteLocation
+       p1.AminoAcid = p2.AminoAcid AND
+       p1.Location = p2.Location
 
     INNER JOIN
 
@@ -69,22 +71,24 @@ FROM
                     MIN(TotalPercentModified) AS MinPercentModified,
                     Sequence,
                     PeptideGroupId,
-                    SiteLocation
+                    AminoAcid,
+                    Location
              FROM (SELECT SUM(ModifiedAreaProportion)                                                      AS TotalPercentModified,
                           SampleName                                                                       AS SampleName2,
                           Sequence,
                           PeptideGroupId,
-                          SUBSTRING(Sequence, IndexAA + 1, 1) ||
-                          CAST(StartIndex + IndexAA + 1 AS VARCHAR)                                        AS SiteLocation,
+                          SUBSTRING(Sequence, IndexAA + 1, 1) AS AminoAcid,
+                          StartIndex + IndexAA + 1 AS Location,
                    FROM PTMPercentsPrepivot
                    GROUP BY SampleName,
                             Sequence,
                             PeptideGroupId,
                             IndexAA,
                             StartIndex) x
-             GROUP BY Sequence, PeptideGroupId, SiteLocation
+             GROUP BY Sequence, PeptideGroupId, AminoAcid, Location
              ) p3
 
     ON p1.Sequence = p3.Sequence AND
        p1.PeptideGroupId = p3.PeptideGroupId AND
-       p1.SiteLocation = p3.SiteLocation
+       p1.AminoAcid = p3.AminoAcid AND
+       p1.Location = p3.Location

src/org/labkey/targetedms/TargetedMSController.java

@@ -268,6 +268,8 @@
 import static org.labkey.targetedms.TargetedMSModule.PROTEIN_TAB_NAME;
 import static org.labkey.targetedms.TargetedMSModule.PROTEIN_TAB_WEB_PARTS;
 import static org.labkey.targetedms.TargetedMSModule.QC_FOLDER_WEB_PARTS;
+import static org.labkey.targetedms.TargetedMSSchema.QUERY_PTM_PERCENTS_GROUPED_PREFIX;
+import static org.labkey.targetedms.TargetedMSSchema.QUERY_PTM_PERCENTS_PREFIX;
 
 public class TargetedMSController extends SpringActionController
 {
@@ -4470,23 +4472,9 @@ public ShowPTMReportAction()
         @Override
         protected QueryView createQueryView(RunDetailsForm form, BindException errors, boolean forExport, String dataRegion)
         {
-            QuerySettings settings = new QuerySettings(getViewContext(), _dataRegionName, "PTMPercents")
-            {
-                @Override
-                protected QueryDefinition createQueryDef(UserSchema schema)
-                {
-                    QueryDefinition queryDef = super.createQueryDef(schema);
-
-                    String queryName = "PTMPercents" + form.getId();
-                    QueryDefinition tempDef = QueryService.get().createQueryDef(getUser(), getContainer(), schema.getSchemaPath(), queryName);
-                    tempDef.setIsHidden(true);
-                    tempDef.setIsTemporary(true);
-                    tempDef.setSql(queryDef.getSql() + " IN (SELECT sf.SampleName FROM targetedms.SampleFile sf WHERE sf.ReplicateId.RunId = " + form.getId() + ")");
-                    tempDef.setMetadataXml(queryDef.getMetadataXml());
 
-                    return tempDef;
-                }
-            };
+            String queryName = QUERY_PTM_PERCENTS_PREFIX + form.getId();
+            QuerySettings settings = new QuerySettings(getViewContext(), _dataRegionName, queryName);
             // Issue 40731- prevent expensive cross-folder queries when the results will always be scoped to the current
             // run anyway
             settings.setContainerFilterName(null);
@@ -4508,30 +4496,21 @@ public ShowEarlyStagePTMReportAction()
         @Override
         protected QueryView createQueryView(RunDetailsForm form, BindException errors, boolean forExport, String dataRegion)
         {
-            QuerySettings settings = new QuerySettings(getViewContext(), _dataRegionName, "PTMPercentsGrouped")
-            {
-                @Override
-                protected QueryDefinition createQueryDef(UserSchema schema)
-                {
-                    QueryDefinition queryDef = super.createQueryDef(schema);
-
-                    String queryName = "PTMPercentsGrouped" + form.getId();
-                    QueryDefinition tempDef = QueryService.get().createQueryDef(getUser(), getContainer(), schema.getSchemaPath(), queryName);
-                    tempDef.setIsHidden(true);
-                    tempDef.setIsTemporary(true);
-                    tempDef.setSql(queryDef.getSql() + " IN (SELECT sf.SampleName FROM targetedms.SampleFile sf WHERE sf.ReplicateId.RunId = " + form.getId() + ")");
-                    tempDef.setMetadataXml(queryDef.getMetadataXml());
-
-                    return tempDef;
-                }
-            };
+            String queryName = QUERY_PTM_PERCENTS_GROUPED_PREFIX + form.getId();
+            QuerySettings settings = new QuerySettings(getViewContext(), _dataRegionName, queryName);
             // Issue 40731- prevent expensive cross-folder queries when the results will always be scoped to the current
             // run anyway
             settings.setContainerFilterName(null);
             settings.setBaseFilter(new SimpleFilter(FieldKey.fromParts("PeptideGroupId", "RunId"), form.getId()));
             // Issue 47668 - need to sort on PeptideGroupId, Sequence, and SiteLocation to make sure peptides
             // with multiple modification sites have them reported in the right order
-            settings.setBaseSort(new Sort("PeptideGroupId, Sequence, SiteLocation"));
+            Sort sort = new Sort();
+            for (FieldKey sortField : PTMPercentsGroupedCustomizer.EXPECTED_SORTS)
+            {
+                sort.appendSortColumn(sortField, Sort.SortDirection.ASC, false);
+            }
+            sort.appendSortColumn(FieldKey.fromParts("Modification"), Sort.SortDirection.ASC, false);
+            settings.setBaseSort(sort);
             TargetedMSSchema schema = new TargetedMSSchema(getUser(), getContainer());
             QueryView result = schema.createView(getViewContext(), settings, errors);
             result.setShadeAlternatingRows(false);

src/org/labkey/targetedms/TargetedMSSchema.java

@@ -35,10 +35,10 @@
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.FilteredTable;
 import org.labkey.api.query.LookupForeignKey;
-import org.labkey.api.query.QueryAction;
 import org.labkey.api.query.QueryDefinition;
 import org.labkey.api.query.QueryForeignKey;
 import org.labkey.api.query.QuerySchema;
+import org.labkey.api.query.QueryService;
 import org.labkey.api.query.QuerySettings;
 import org.labkey.api.query.QueryView;
 import org.labkey.api.query.SchemaKey;
@@ -51,7 +51,6 @@
 import org.labkey.api.util.ContainerContext;
 import org.labkey.api.util.HtmlString;
 import org.labkey.api.util.Pair;
-import org.labkey.api.util.StringExpression;
 import org.labkey.api.util.UnexpectedException;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.PopupMenu;
@@ -60,7 +59,6 @@
 import org.labkey.panoramapremium.query.QCEmailNotificationsTable;
 import org.labkey.targetedms.parser.Chromatogram;
 import org.labkey.targetedms.parser.ChromatogramBinaryFormat;
-import org.labkey.targetedms.parser.ReplicateAnnotation;
 import org.labkey.targetedms.parser.SkylineBinaryParser;
 import org.labkey.targetedms.query.*;
 import org.labkey.targetedms.view.FontAwesomeLinkColumn;
@@ -78,6 +76,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.Spliterator;
 import java.util.function.Consumer;
@@ -190,6 +189,8 @@ public class TargetedMSSchema extends UserSchema
     public static final String TABLE_SKYLINE_AUDITLOG_MESSAGE = "AuditLogMessage";
 
     public static final String TABLE_SKYLINE_AUDITLOG_PREFIX = "AuditLog_Run";
+    public static final String QUERY_PTM_PERCENTS_PREFIX = "PTMPercents_";
+    public static final String QUERY_PTM_PERCENTS_GROUPED_PREFIX = "PTMPercentsGrouped_";
 
     public static final String TABLE_LIST_DEFINITION = "ListDefinition";
     public static final String TABLE_LIST_COLUMN_DEFINITION = "ListColumnDefinition";
@@ -1734,6 +1735,43 @@ public Set<String> getTableNames()
         return getAllTableNames(true);
     }
 
+    @Override
+    public QueryDefinition getQueryDef(@NotNull String queryName)
+    {
+        QueryDefinition result = super.getQueryDef(queryName);
+
+        if (result == null && StringUtils.startsWithIgnoreCase(queryName, QUERY_PTM_PERCENTS_PREFIX))
+        {
+            String runIdString = queryName.substring(QUERY_PTM_PERCENTS_PREFIX.length());
+            try
+            {
+                int runId = Integer.parseInt(runIdString);
+                QueryDefinition queryDef = Objects.requireNonNull(getQueryDef("PTMPercents"));
+                result = QueryService.get().createQueryDef(getUser(), getContainer(), getSchemaPath(), queryName);
+                result.setSql(queryDef.getSql() + " IN (SELECT sf.SampleName FROM targetedms.SampleFile sf WHERE sf.ReplicateId.RunId = " + runId + ")");
+                result.setMetadataXml(queryDef.getMetadataXml());
+            }
+            catch (NumberFormatException ignored) {}
+        }
+
+        if (result == null && StringUtils.startsWithIgnoreCase(queryName, QUERY_PTM_PERCENTS_GROUPED_PREFIX))
+        {
+            String runIdString = queryName.substring(QUERY_PTM_PERCENTS_GROUPED_PREFIX.length());
+            try
+            {
+                int runId = Integer.parseInt(runIdString);
+                QueryDefinition queryDef = Objects.requireNonNull(getQueryDef("PTMPercentsGrouped"));
+                result = QueryService.get().createQueryDef(getUser(), getContainer(), getSchemaPath(), queryName);
+                result.setSql(queryDef.getSql() + " IN (SELECT sf.SampleName FROM targetedms.SampleFile sf WHERE sf.ReplicateId.RunId = " + runId + ")");
+                result.setMetadataXml(queryDef.getMetadataXml());
+            }
+            catch (NumberFormatException ignored) {}
+        }
+        return result;
+    }
+
+
+
     public static class ChromatogramDisplayColumn extends DataColumn
     {
         private final boolean _times;

src/org/labkey/targetedms/query/PTMPercentsCustomizer.java

@@ -0,0 +1,55 @@
+package org.labkey.targetedms.query;
+
+import org.apache.commons.collections4.MultiValuedMap;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.collections.CaseInsensitiveHashMap;
+import org.labkey.api.data.ColumnInfo;
+import org.labkey.api.data.CompareType;
+import org.labkey.api.data.ConditionalFormat;
+import org.labkey.api.data.DataColumn;
+import org.labkey.api.data.DisplayColumn;
+import org.labkey.api.data.DisplayColumnFactory;
+import org.labkey.api.data.MutableColumnInfo;
+import org.labkey.api.data.RenderContext;
+import org.labkey.api.data.SQLFragment;
+import org.labkey.api.data.SimpleFilter;
+import org.labkey.api.data.SqlSelector;
+import org.labkey.api.data.TableCustomizer;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.query.FieldKey;
+import org.labkey.api.util.Pair;
+import org.labkey.targetedms.TargetedMSManager;
+import org.labkey.targetedms.parser.Protein;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Customizes the set of columns available on a pivot query that operates on samples, hiding all of the pivot values
+ * that aren't part of the run that's being filtered on. This lets you view a single document's worth of data
+ * without seeing empty columns for all of the other samples in the same container.
+ */
+public class PTMPercentsCustomizer implements TableCustomizer
+{
+
+    /** Referenced from query XML metadata */
+    @SuppressWarnings("unused")
+    public PTMPercentsCustomizer(MultiValuedMap<String, String> props)
+    {
+
+    }
+
+    @Override
+    public void customize(TableInfo tableInfo)
+    {
+        List<FieldKey> defaultCols = new ArrayList<>(tableInfo.getDefaultVisibleColumns());
+        defaultCols.remove(FieldKey.fromParts("AminoAcid"));
+        defaultCols.remove(FieldKey.fromParts("Location"));
+        tableInfo.setDefaultVisibleColumns(defaultCols);
+    }
+}