Merge branch 'master' into validator-fixes

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,10 +5,10 @@ Describe changes proposed in this pull request:
 - b
 
 # Checks
-- [ ] Runs on heroku
+- [ ] The commit log is comprehensible. It follows [7 rules of great commit messages](http://chris.beams.io/posts/git-commit/). We can fix this during merge by using a squash+merge if necessary
 - [ ] Has tests or has a separate issue that describes the types of test that should be created. If no test is included it should explicitly be mentioned in the PR why there is no test.
-- [ ] The commit log is comprehensible. It follows [7 rules of great commit messages](http://chris.beams.io/posts/git-commit/). For most PRs a single commit should suffice, in some cases multiple topical commits can be useful. During review it is ok to see tiny commits (e.g. Fix reviewer comments), but right before the code gets merged to master or rc branch, any such commits should be squashed since they are useless to the other developers. Definitely avoid [merge commits, use rebase instead.](http://nathanleclaire.com/blog/2014/09/14/dont-be-scared-of-git-rebase/)
 - [ ] Is this PR adding logic based on one or more **clinical** attributes? If yes, please make sure validation for this attribute is also present in the data validation / data loading layers (in backend repo) and documented in [File-Formats Clinical data section](https://github.com/cBioPortal/cbioportal/blob/master/docs/File-Formats.md#clinical-data)!
+- [ ] Make sure your PR has one of the labels defined in https://github.com/cBioPortal/cbioportal/blob/master/.github/release-drafter.yml
 
 # Any screenshots or GIFs?
 If this is a new visual feature please add a before/after screenshot or gif

.github/workflows/dockerimage.yml

@@ -5,6 +5,7 @@ on:
       - master
       - release-*
       - rc
+      - rfc*
       - demo-*
       - fusion-sv-migration
       - redis-branch-up-to-date

.github/workflows/integration-test.yml

@@ -42,10 +42,12 @@ jobs:
         working-directory: ./cbioportal-docker-compose
         run: |
           cd ./data && ./init.sh && rm -rf ./studies/* && cd ../config && \
-          cat $PORTAL_SOURCE_DIR/portal/target/portal/WEB-INF/classes/portal.properties | \
-              sed 's/db.host=.*/db.host=cbioportal-database:3306/g' | \
-              sed 's|db.connection_string=.*|db.connection_string=jdbc:mysql://cbioportal-database:3306/|g' \
-              > portal.properties
+          cat $PORTAL_SOURCE_DIR/portal/target/classes/portal.properties | \
+              sed 's|db.host=.*||' | \
+              sed 's|db.portal_db_name=.*||' | \
+              sed 's|db.use_ssl=.*||' | \
+              sed 's|db.connection_string=.*|db.connection_string=jdbc:mysql://cbioportal-database:3306/cbioportal?useSSL=false\&allowPublicKeyRetrieval=true|' \
+              > portal.properties && more portal.properties
       - name: 'Start cbioportal-docker-compose'
         working-directory: ./cbioportal-docker-compose
         run: |

.github/workflows/label-check.yml

@@ -0,0 +1,53 @@
+# This GitHub Actions workflow is designed to automatically check pull requests in the cBioPortal repository for valid labels before they can be merged.
+# The workflow ensures that pull requests have labels that are defined in the .github/release-drafter.yml file's "categories" section.
+# If a pull request lacks a valid label, the workflow will fail, preventing the merge until valid labels are applied.
+name: Label Check
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+      - unlabeled
+
+jobs:
+  label-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check PR Labels
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: |
+          wget https://github.com/mikefarah/yq/releases/download/v4.34.2/yq_linux_amd64 -O /usr/local/bin/yq
+          chmod +x /usr/local/bin/yq
+      - name: Get Labels from release-drafter.yml
+        id: get_labels
+        run: |
+          curl -s "https://raw.githubusercontent.com/cBioPortal/cbioportal/master/.github/release-drafter.yml" | \
+            yq -r '.categories[].labels[]' > labels.txt
+      - name: Check Labels
+        id: check_labels
+        run: |
+          PR_NUMBER=$(jq -r ".number" $GITHUB_EVENT_PATH)
+          PR_LABELS=$(curl -s "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER" | \
+                       jq -r '.labels[].name')
+          mapfile -t AVAILABLE_LABELS < labels.txt
+          for LABEL in ${PR_LABELS[@]}; do
+            if [[ "$LABEL" == "skip-changelog" ]]; then
+              echo "PR contains a valid label: skip-changelog"
+              exit 0  # Valid label found, exit successfully
+            fi
+            for AVAILABLE_LABEL in "${AVAILABLE_LABELS[@]}"; do
+              if [[ "$AVAILABLE_LABEL" == "$LABEL" ]]; then
+                echo "PR contains a valid label: $LABEL"
+                exit 0  # Valid label found, exit successfully
+              fi
+            done
+          done
+          echo "No valid label found on PR."
+          echo "Available label options from release-drafter.yml:"
+          cat labels.txt
+          exit 1  # No valid label found, exit with an error

CONTRIBUTING.md

@@ -26,10 +26,16 @@ From the [GitHub Help Page of Using Pull Requests](https://help.github.com/artic
 
 ## Getting Started
 
+### Programming
  * Make sure you have a [GitHub account](https://github.com/signup/free).
- * Create an issue in our issues tracker, assuming one does not already exist.
- * Fork the cbioportal project on GitHub.  For general instructions on forking a GitHub project, see [Forking a Repo](https://help.github.com/articles/fork-a-repo/) and [Syncing a fork](https://help.github.com/articles/syncing-a-fork/).
  * Familiarize yourself with the [project documentation](https://docs.cbioportal.org), including the [Feature Development Guide](https://docs.cbioportal.org/4.-development/feature-development-guide), the [Architecture docs](https://docs.cbioportal.org/2.1-deployment/architecture-overview), the [backend code organization](docs/Backend-Code-Organization.md) and [backend development guidelines](docs/Backend-Development-Guidelines.md).
+ * Find a [good first issue](https://github.com/cBioPortal/cbioportal/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) to start with
+ * Check if the issue will require frontend or backend changes. If it is for the frontend look at how to set up the [frontend repo](https://github.com/cbioPortal/cbioportal-frontend/) instead
+ * Fork the cbioportal or cbioportal-frontend project on GitHub depending on what your working on.  For general instructions on forking a GitHub project, see [Forking a Repo](https://help.github.com/articles/fork-a-repo/) and [Syncing a fork](https://help.github.com/articles/syncing-a-fork/).
+ * Reach out on slack or our [Google user group](https://groups.google.com/g/cbioportal) if you run into any issues
+
+### Documentation
+If you'd like to improve our documentation, have a look at https://docs.cbioportal.org. At the bottom of each page is an "edit page" button to make changes. You can use the GitHub UI to edit the pages and submit them.
 
 ## Contributing Code Changes via a Pull Request
 

core/src/main/java/org/mskcc/cbio/portal/dao/JdbcDataSource.java

@@ -3,28 +3,48 @@
 import org.apache.commons.dbcp2.BasicDataSource;
 import org.mskcc.cbio.portal.util.DatabaseProperties;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.util.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Data source that self-initializes based on cBioPortal configuration.
  */
 public class JdbcDataSource extends BasicDataSource {
+
+    private static final Logger LOG = LoggerFactory.getLogger(JdbcDataSource.class);
+
     public JdbcDataSource () {
         DatabaseProperties dbProperties = DatabaseProperties.getInstance();
+
         String host = dbProperties.getDbHost();
         String userName = dbProperties.getDbUser();
         String password = dbProperties.getDbPassword();
         String mysqlDriverClassName = dbProperties.getDbDriverClassName();
         String database = dbProperties.getDbName();
-        String useSSL = (!StringUtils.isBlank(dbProperties.getDbUseSSL())) ? dbProperties.getDbUseSSL() : "false";
         String enablePooling = (!StringUtils.isBlank(dbProperties.getDbEnablePooling())) ? dbProperties.getDbEnablePooling(): "false";
-        String url ="jdbc:mysql://" + host + "/" + database +
-                        "?user=" + userName + "&password=" + password +
-                        "&zeroDateTimeBehavior=convertToNull&useSSL=" + useSSL;
+        String connectionURL = dbProperties.getConnectionURL();
+
+        if (defined(host) || defined(database) || defined(dbProperties.getDbUseSSL())) {
+		LOG.warn("\n----------------------------------------------------------------------------------------------------------------" +
+			"-- Connection error:\n" +
+			"-- You try to connect to the database using the deprecated 'db.host', 'db.portal_db_name' and 'db.use_ssl' properties.\n" +
+			"-- Please remove these properties and use the 'db.connection_string' property instead. See https://docs.cbioportal.org/deployment/customization/portal.properties-reference/\n" +
+			"-- for assistance on building a valid connection string.\n" +
+			"----------------------------------------------------------------------------------------------------------------\n"
+		);
+	}
+
+        Assert.hasText(userName, errorMessage("username", "db.user"));
+        Assert.hasText(password, errorMessage("password", "db.password"));
+        Assert.hasText(mysqlDriverClassName, errorMessage("driver class name", "db.driver"));
+
+        this.setUrl(connectionURL);
+
         //  Set up poolable data source
         this.setDriverClassName(mysqlDriverClassName);
         this.setUsername(userName);
         this.setPassword(password);
-        this.setUrl(url);
         // Disable this to avoid caching statements
         this.setPoolPreparedStatements(Boolean.valueOf(enablePooling));
         // these are the values cbioportal has been using in their production
@@ -37,4 +57,12 @@ public JdbcDataSource () {
         this.setValidationQuery("SELECT 1");
         this.setJmxName("org.cbioportal:DataSource=" + database);
     }
+
+    private String errorMessage(String displayName, String propertyName) {
+        return String.format("No %s provided for database connection. Please set '%s' in portal.properties.", displayName, propertyName);
+    }
+
+    private boolean defined(String property) {
+        return property != null && !property.isEmpty();
+    }
 }

core/src/main/java/org/mskcc/cbio/portal/servlet/CalcFisherExactTest.java

@@ -44,13 +44,13 @@
 import org.mskcc.cbio.portal.stats.FisherExact;
 
 /**
- * Calculate the cumulative (one-tail) p-value out of fisher exact test
+ * Calculate the cumulative (two-tail) p-value out of fisher exact test
  * 
  * @param a a, b, c, d are the four cells in a 2x2 matrix
  * @param b
  * @param c
  * @param d
- * @return one-tailed P-value (right or left, whichever is smallest)
+ * @return two-tailed P-value (right or left, whichever is smallest)
  * 
  */
 public class CalcFisherExactTest extends HttpServlet  {
@@ -86,7 +86,7 @@ protected void doPost(HttpServletRequest httpServletRequest,
                 int c = Integer.parseInt(dataSet.split(" ")[2]);
                 int d = Integer.parseInt(dataSet.split(" ")[3]);    
                 FisherExact fisher = new FisherExact(a + b + c + d);
-                double pValue = fisher.getCumlativeP(a, b, c, d);
+                double pValue = fisher.getTwoTailedP(a, b, c, d);
                 result = result.concat(String.valueOf(pValue) + " ");                
             }
         }

core/src/main/java/org/mskcc/cbio/portal/stats/OddsRatio.java

@@ -87,7 +87,7 @@ public OddsRatio(ProfileDataSummary pDataSummary, String geneA, String geneB) {
 
         oddsRatio = ((double) (a * d)) / ((double) (b * c));
         FisherExact fisher = new FisherExact(a + b + c + d);
-        pValue = fisher.getCumlativeP(a, b, c, d);
+        pValue = fisher.getTwoTailedP(a, b, c, d);
         lowerConfidenceInterval = Math.exp(Math.log(oddsRatio) - 1.96 * (Math.sqrt(1 / (double) a
                 + 1 / (double) b + 1 / (double) c + 1 / (double) d)));
         upperConfidenceInterval = Math.exp(Math.log(oddsRatio) + 1.96 * (Math.sqrt(1 / (double) a
@@ -103,7 +103,7 @@ public double getOddsRatio() {
         return oddsRatio;
     }
 
-    public double getCumulativeP() {
+    public double getTwoTailedP() {
         return pValue;
     }
 

core/src/main/java/org/mskcc/cbio/portal/util/DatabaseProperties.java

@@ -47,6 +47,7 @@ public class DatabaseProperties {
     private String dbDriverClassName;
     private String dbUseSSL;
     private String dbEnablePooling;
+    private String connectionURL;
 
     // No production keys stored in filesystem or code: digest the key; put it in properties; load it into dbms on startup
     private static DatabaseProperties dbProperties;
@@ -63,6 +64,7 @@ public static DatabaseProperties getInstance() {
             dbProperties.setDbDriverClassName(GlobalProperties.getProperty("db.driver"));
             dbProperties.setDbUseSSL(GlobalProperties.getProperty("db.use_ssl"));
             dbProperties.setDbEnablePooling(GlobalProperties.getProperty("db.enable_pooling"));
+            dbProperties.setConnectionURL(GlobalProperties.getProperty("db.connection_string"));
         }
         return dbProperties;
     }
@@ -134,4 +136,12 @@ public void setDbEnablePooling(String dbEnablePooling) {
         this.dbEnablePooling = dbEnablePooling;
     }
 
+    public String getConnectionURL() {
+        return connectionURL;
+    }
+
+    public void setConnectionURL(String connectionURL) {
+        this.connectionURL = connectionURL;
+    }
+
 }

core/src/main/java/org/mskcc/cbio/portal/util/EnrichmentsAnalysisUtil.java

@@ -652,7 +652,7 @@ private double runFisherExactTest(HashMap<Integer, String> singleGeneCaseValueMa
             }
         }
         FisherExact fisher = new FisherExact(a + b + c + d);
-        return fisher.getCumlativeP(a, b, c, d);
+        return fisher.getTwoTailedP(a, b, c, d);
     }
 
     private String getCytoband(int geneticEntityId, String geneticProfileStableId) {

core/src/main/java/org/mskcc/cbio/portal/util/GlobalProperties.java

@@ -50,11 +50,13 @@
 import java.util.List;
 import java.util.Properties;
 
+import org.apache.commons.lang.StringUtils;
 import org.cbioportal.security.spring.authentication.PortalUserDetails;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.security.core.Authentication;
+import org.springframework.security.core.authority.SimpleGrantedAuthority;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Component;
 import org.springframework.util.ResourceUtils;
@@ -120,6 +122,7 @@ public class GlobalProperties {
     public static final String SKIN_RIGHT_NAV_SHOW_EXAMPLES = "skin.right_nav.show_examples";
     public static final String SKIN_RIGHT_NAV_SHOW_TESTIMONIALS = "skin.right_nav.show_testimonials";
     public static final String SKIN_RIGHT_NAV_SHOW_WHATS_NEW = "skin.right_nav.show_whats_new";
+    public static final String SKIN_RIGHT_NAV_SHOW_WEB_TOURS = "skin.right_nav.show_web_tours";
     private static String skinAuthorizationMessage;
     @Value("${skin.authorization_message:Access to this portal is only available to authorized users.}")
     public void setSkinAuthorizationMessage(String property) { skinAuthorizationMessage = property; }
@@ -180,6 +183,12 @@ public void setOncoprintClinicalTracksConfigJson(String property) {
         oncoprintClinicalTracksConfigJson = property; 
     }
 
+    private static String skinPatientViewCustomSampleTypeColorsJson;
+    @Value("${skin.patient_view.custom_sample_type_colors_json:}") // default is empty string
+    public void setSkinPatientViewCustomSampleTypeColorsJson(String property) {
+        skinPatientViewCustomSampleTypeColorsJson = property; 
+    }
+
     // properties for showing the right logo in the header_bar and default logo
     public static final String SKIN_RIGHT_LOGO = "skin.right_logo";
 
@@ -362,6 +371,20 @@ public static String parseUrl(String url)
     @Value("${frontend.url.runtime:}") 
     public void setFrontendUrlRuntime(String property) { frontendUrlRuntime = property; }
 
+    private static String downloadGroup;
+    @Value("${download_group:}") // default is empty string
+    public void setDownloadGroup(String property) { downloadGroup = property; }
+
+    public static final String DEFAULT_DAT_METHOD = "none";
+
+    private static String dataAccessTokenMethod;
+    @Value("${dat.method:none}") // default is empty string
+    public void setDataAccessTokenMethod(String property) { dataAccessTokenMethod = property; }
+
+    private static String tokenAccessUserRole;
+    @Value("${dat.filter_user_role:}") // default is empty string
+    public void setTokenAccessUserRole(String property) { tokenAccessUserRole = property; }
+
     private static Logger LOG = LoggerFactory.getLogger(GlobalProperties.class);
     private static ConfigPropertyResolver portalProperties = new ConfigPropertyResolver();
     private static Properties mavenProperties = initializeProperties(MAVEN_PROPERTIES_FILE_NAME);
@@ -789,6 +812,12 @@ public static boolean showRightNavWhatsNew()
         return showFlag == null || Boolean.parseBoolean(showFlag);
     }
 
+    public static boolean showRightNavWebTours()
+    {
+        String showFlag = portalProperties.getProperty(SKIN_RIGHT_NAV_SHOW_WEB_TOURS);
+        return showFlag == null || Boolean.parseBoolean(showFlag);
+    }
+
     public static String getAuthorizationMessage()
     {
         return skinAuthorizationMessage;
@@ -1234,6 +1263,14 @@ public static String getOncoprintClinicalTracksConfigJson() {
         }
     }
 
+    public static String getSkinPatientViewCustomSampleTypeColorsJson() {
+        if (skinPatientViewCustomSampleTypeColorsJson.length() > 0) {
+            return readFile(skinPatientViewCustomSampleTypeColorsJson);
+        } else {
+            return null;
+        }
+    }
+
     public static String getQuerySetsOfGenes() {
         String fileName = portalProperties.getProperty(SETSOFGENES_LOCATION, null);
         return readFile(fileName);
@@ -1262,4 +1299,43 @@ public static String getReferenceGenomeName() {
     public static String getOncoKbToken()  {
         return portalProperties.getProperty(ONCOKB_TOKEN, null);
     }
+
+    public static String getDownloadControl() {
+        Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
+
+        if (authentication != null &&
+            StringUtils.isNotEmpty(downloadGroup) &&
+            authentication.getAuthorities().contains(new SimpleGrantedAuthority(downloadGroup))) {
+            return "show";
+        } else {
+            String downloadControlOption = getProperty("skin.hide_download_controls");
+            /*
+                skin.hide_download_controls   return_value
+                        true                    hide
+                        false                   show
+                        data                    data
+                        null/empty              show
+             */
+            switch ((downloadControlOption != null) ? downloadControlOption.trim().toLowerCase() : "false") {
+                case "true":
+                    return "hide";
+                case "data":
+                    return "data";
+                case "false":
+                default:
+                    return "show";
+            }
+        }
+    }
+
+    public static String getDataAccessTokenMethod() {
+        Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
+
+        if (authentication != null &&
+            StringUtils.isNotEmpty(tokenAccessUserRole)) {
+            return authentication.getAuthorities().contains(new SimpleGrantedAuthority(tokenAccessUserRole)) ? dataAccessTokenMethod : DEFAULT_DAT_METHOD;
+        } else {
+            return dataAccessTokenMethod;
+        }
+    }
 }