Merge remote-tracking branch 'upstream/master' into cbiomaster-latest

.github/PULL_REQUEST_TEMPLATE.md

@@ -9,6 +9,7 @@ Describe changes proposed in this pull request:
 - [ ] Has tests or has a separate issue that describes the types of test that should be created. If no test is included it should explicitly be mentioned in the PR why there is no test.
 - [ ] The commit log is comprehensible. It follows [7 rules of great commit messages](http://chris.beams.io/posts/git-commit/). For most PRs a single commit should suffice, in some cases multiple topical commits can be useful. During review it is ok to see tiny commits (e.g. Fix reviewer comments), but right before the code gets merged to master or rc branch, any such commits should be squashed since they are useless to the other developers. Definitely avoid [merge commits, use rebase instead.](http://nathanleclaire.com/blog/2014/09/14/dont-be-scared-of-git-rebase/)
 - [ ] Is this PR adding logic based on one or more **clinical** attributes? If yes, please make sure validation for this attribute is also present in the data validation / data loading layers (in backend repo) and documented in [File-Formats Clinical data section](https://github.com/cBioPortal/cbioportal/blob/master/docs/File-Formats.md#clinical-data)!
+- [ ] Make sure your PR has one of the labels defined in https://github.com/cBioPortal/cbioportal/blob/master/.github/release-drafter.yml
 
 # Any screenshots or GIFs?
 If this is a new visual feature please add a before/after screenshot or gif

.github/release-drafter.yml

@@ -43,6 +43,10 @@ categories:
     labels:
       - 'documentation'
       - 'cl-documentation'
+  - title: '🖊️ Data Importer & Validator'
+    labels:
+      - 'importer'
+      - 'validator'
   - title: '🧹 Cleanup & Refactoring 🔧'
     labels:
       - 'cleanup'
@@ -79,4 +83,4 @@ template: |
   - Frontend: https://github.com/cBioPortal/cbioportal-frontend/compare/$PREVIOUS_TAG...v$NEXT_PATCH_VERSION
   
   ## 🏷Notes on versioning and release procedure
-  https://docs.cbioportal.org/4.-development/release-procedure#a-note-on-versioning
+  https://docs.cbioportal.org/development/release-procedure/#a-note-on-versioning

.github/workflows/dockerimage.yml

@@ -5,6 +5,7 @@ on:
       - master
       - release-*
       - rc
+      - rfc*
       - demo-*
       - fusion-sv-migration
       - redis-branch-up-to-date

.github/workflows/integration-test.yml

@@ -42,10 +42,12 @@ jobs:
         working-directory: ./cbioportal-docker-compose
         run: |
           cd ./data && ./init.sh && rm -rf ./studies/* && cd ../config && \
-          cat $PORTAL_SOURCE_DIR/portal/target/portal/WEB-INF/classes/portal.properties | \
-              sed 's/db.host=.*/db.host=cbioportal-database:3306/g' | \
-              sed 's|db.connection_string=.*|db.connection_string=jdbc:mysql://cbioportal-database:3306/|g' \
-              > portal.properties
+          cat $PORTAL_SOURCE_DIR/portal/target/classes/portal.properties | \
+              sed 's|db.host=.*||' | \
+              sed 's|db.portal_db_name=.*||' | \
+              sed 's|db.use_ssl=.*||' | \
+              sed 's|db.connection_string=.*|db.connection_string=jdbc:mysql://cbioportal-database:3306/cbioportal?useSSL=false\&allowPublicKeyRetrieval=true|' \
+              > portal.properties && more portal.properties
       - name: 'Start cbioportal-docker-compose'
         working-directory: ./cbioportal-docker-compose
         run: |

.github/workflows/label-check.yml

@@ -0,0 +1,50 @@
+# This GitHub Actions workflow is designed to automatically check pull requests in the cBioPortal repository for valid labels before they can be merged.
+# The workflow ensures that pull requests have labels that are defined in the .github/release-drafter.yml file's "categories" section.
+# If a pull request lacks a valid label, the workflow will fail, preventing the merge until valid labels are applied.
+name: Label Check
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+
+jobs:
+  label-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check PR Labels
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: |
+          wget https://github.com/mikefarah/yq/releases/download/v4.34.2/yq_linux_amd64 -O /usr/local/bin/yq
+          chmod +x /usr/local/bin/yq
+      - name: Get Labels from release-drafter.yml
+        id: get_labels
+        run: |
+          curl -s "https://raw.githubusercontent.com/cBioPortal/cbioportal/master/.github/release-drafter.yml" | \
+            yq -r '.categories[].labels[]' > labels.txt
+      - name: Check Labels
+        id: check_labels
+        run: |
+          PR_NUMBER=$(jq -r ".number" $GITHUB_EVENT_PATH)
+          PR_LABELS=$(curl -s "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER" | \
+                       jq -r '.labels[].name')
+          mapfile -t AVAILABLE_LABELS < labels.txt
+          for LABEL in ${PR_LABELS[@]}; do
+            if [[ "$LABEL" == "skip-changelog" ]]; then
+              echo "PR contains a valid label: skip-changelog"
+              exit 0  # Valid label found, exit successfully
+            fi
+            for AVAILABLE_LABEL in "${AVAILABLE_LABELS[@]}"; do
+              if [[ "$AVAILABLE_LABEL" == "$LABEL" ]]; then
+                echo "PR contains a valid label: $LABEL"
+                exit 0  # Valid label found, exit successfully
+              fi
+            done
+          done
+          echo "No valid label found on PR."
+          echo "Available label options from release-drafter.yml:"
+          cat labels.txt
+          exit 1  # No valid label found, exit with an error

core/src/main/java/org/mskcc/cbio/portal/dao/DaoCosmicData.java

@@ -39,7 +39,6 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -116,7 +115,7 @@ private static Set<CosmicMutationFrequency> filterTruncatingCosmicByPosition(
 
         Set<CosmicMutationFrequency> ret = new HashSet<CosmicMutationFrequency>();
         Pattern p = Pattern.compile("[0-9]+");
-        int mutPos = mut.getOncotatorProteinPosStart();
+        int mutPos = mut.getProteinPosStart();
         for (CosmicMutationFrequency cmf : cmfs) {
             String aa = cmf.getAminoAcidChange();
             Matcher m = p.matcher(aa);

core/src/main/java/org/mskcc/cbio/portal/dao/DaoMutation.java

@@ -145,23 +145,16 @@ public static int addMutationEvent(ExtendedMutation.MutationEvent event) throws
                 event.getTumorSeqAllele(),
                 event.getProteinChange(),
                 event.getMutationType(),
-                event.getFunctionalImpactScore(),
-                Float.toString(event.getFisValue()),
-                event.getLinkXVar(),
-                event.getLinkPdb(),
-                event.getLinkMsa(),
                 event.getNcbiBuild(),
                 event.getStrand(),
                 event.getVariantType(),
                 event.getDbSnpRs(),
                 event.getDbSnpValStatus(),
-                event.getOncotatorDbSnpRs(),
-                event.getOncotatorRefseqMrnaId(),
-                event.getOncotatorCodonChange(),
-                event.getOncotatorUniprotName(),
-                event.getOncotatorUniprotAccession(),
-                Integer.toString(event.getOncotatorProteinPosStart()),
-                Integer.toString(event.getOncotatorProteinPosEnd()),
+                event.getRefseqMrnaId(),
+                event.getCodonChange(),
+                event.getUniprotAccession(),
+                Integer.toString(event.getProteinPosStart()),
+                Integer.toString(event.getProteinPosEnd()),
                 boolToStr(event.isCanonicalTranscript()),
                 keyword==null ? "\\N":(event.getGene().getHugoGeneSymbolAllCaps()+" "+keyword));
         return 1;
@@ -800,24 +793,17 @@ private static ExtendedMutation.MutationEvent extractMutationEvent(ResultSet rs)
         event.setEndPosition(rs.getLong("END_POSITION"));
         event.setProteinChange(rs.getString("PROTEIN_CHANGE"));
         event.setMutationType(rs.getString("MUTATION_TYPE"));
-        event.setFunctionalImpactScore(rs.getString("FUNCTIONAL_IMPACT_SCORE"));
-        event.setFisValue(rs.getFloat("FIS_VALUE"));
-        event.setLinkXVar(rs.getString("LINK_XVAR"));
-        event.setLinkPdb(rs.getString("LINK_PDB"));
-        event.setLinkMsa(rs.getString("LINK_MSA"));
         event.setNcbiBuild(rs.getString("NCBI_BUILD"));
         event.setStrand(rs.getString("STRAND"));
         event.setVariantType(rs.getString("VARIANT_TYPE"));
         event.setDbSnpRs(rs.getString("DB_SNP_RS"));
         event.setDbSnpValStatus(rs.getString("DB_SNP_VAL_STATUS"));
         event.setReferenceAllele(rs.getString("REFERENCE_ALLELE"));
-        event.setOncotatorDbSnpRs(rs.getString("ONCOTATOR_DBSNP_RS"));
-        event.setOncotatorRefseqMrnaId(rs.getString("ONCOTATOR_REFSEQ_MRNA_ID"));
-        event.setOncotatorCodonChange(rs.getString("ONCOTATOR_CODON_CHANGE"));
-        event.setOncotatorUniprotName(rs.getString("ONCOTATOR_UNIPROT_ENTRY_NAME"));
-        event.setOncotatorUniprotAccession(rs.getString("ONCOTATOR_UNIPROT_ACCESSION"));
-        event.setOncotatorProteinPosStart(rs.getInt("ONCOTATOR_PROTEIN_POS_START"));
-        event.setOncotatorProteinPosEnd(rs.getInt("ONCOTATOR_PROTEIN_POS_END"));
+        event.setRefseqMrnaId(rs.getString("REFSEQ_MRNA_ID"));
+        event.setCodonChange(rs.getString("CODON_CHANGE"));
+        event.setUniprotAccession(rs.getString("UNIPROT_ACCESSION"));
+        event.setProteinPosStart(rs.getInt("PROTEIN_POS_START"));
+        event.setProteinPosEnd(rs.getInt("PROTEIN_POS_END"));
         event.setCanonicalTranscript(rs.getBoolean("CANONICAL_TRANSCRIPT"));
         event.setTumorSeqAllele(rs.getString("TUMOR_SEQ_ALLELE"));
         event.setKeyword(rs.getString("KEYWORD"));
@@ -1391,12 +1377,12 @@ public static Collection<Map<String, Object>> countSamplesWithProteinPosStarts(
         if (geneIdSet.size() == 0 || internalProfileIds.size() == 0) return new ArrayList<Map<String, Object>>(); //empty IN() clause would be a SQL error below
         try {
             con = JdbcUtil.getDbConnection(DaoMutation.class);
-            String sql = "SELECT ONCOTATOR_PROTEIN_POS_START, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID, count(DISTINCT SAMPLE_ID) " +
+            String sql = "SELECT PROTEIN_POS_START, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID, count(DISTINCT SAMPLE_ID) " +
                     "FROM mutation INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " +
                     "WHERE mutation.ENTREZ_GENE_ID IN (" + StringUtils.join(geneIdSet, ",") + ") " +
                     "AND GENETIC_PROFILE_ID IN (" + StringUtils.join(internalProfileIds, ",") + ") " +
-                    "AND (mutation.ENTREZ_GENE_ID, ONCOTATOR_PROTEIN_POS_START) IN (" + StringUtils.join(proteinPosStarts, ",") + ") " +
-                    "GROUP BY ONCOTATOR_PROTEIN_POS_START, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID";
+                    "AND (mutation.ENTREZ_GENE_ID, PROTEIN_POS_START) IN (" + StringUtils.join(proteinPosStarts, ",") + ") " +
+                    "GROUP BY PROTEIN_POS_START, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID";
             pstmt = con.prepareStatement(sql);
             rs = pstmt.executeQuery();
             Collection<Map<String, Object>> data = new ArrayList<Map<String, Object>>();

core/src/main/java/org/mskcc/cbio/portal/dao/JdbcDataSource.java

@@ -3,28 +3,44 @@
 import org.apache.commons.dbcp2.BasicDataSource;
 import org.mskcc.cbio.portal.util.DatabaseProperties;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.util.Assert;
 
 /**
  * Data source that self-initializes based on cBioPortal configuration.
  */
 public class JdbcDataSource extends BasicDataSource {
+
     public JdbcDataSource () {
         DatabaseProperties dbProperties = DatabaseProperties.getInstance();
+
         String host = dbProperties.getDbHost();
         String userName = dbProperties.getDbUser();
         String password = dbProperties.getDbPassword();
         String mysqlDriverClassName = dbProperties.getDbDriverClassName();
         String database = dbProperties.getDbName();
-        String useSSL = (!StringUtils.isBlank(dbProperties.getDbUseSSL())) ? dbProperties.getDbUseSSL() : "false";
         String enablePooling = (!StringUtils.isBlank(dbProperties.getDbEnablePooling())) ? dbProperties.getDbEnablePooling(): "false";
-        String url ="jdbc:mysql://" + host + "/" + database +
-                        "?user=" + userName + "&password=" + password +
-                        "&zeroDateTimeBehavior=convertToNull&useSSL=" + useSSL;
+        String connectionURL = dbProperties.getConnectionURL();
+
+        Assert.isTrue(
+            !defined(host) && !defined(database) && !defined(dbProperties.getDbUseSSL()),
+            "\n----------------------------------------------------------------------------------------------------------------" +
+                "-- Connection error:\n" +
+                "-- You try to connect to the database using the deprecated 'db.host', 'db.portal_db_name' and 'db.use_ssl' properties.\n" +
+                "-- Please remove these properties and use the 'db.connection_string' property instead. See https://docs.cbioportal.org/deployment/customization/portal.properties-reference/\n" +
+                "-- for assistance on building a valid connection string.\n" +
+                "----------------------------------------------------------------------------------------------------------------\n"
+        );
+
+        Assert.hasText(userName, errorMessage("username", "db.user"));
+        Assert.hasText(password, errorMessage("password", "db.password"));
+        Assert.hasText(mysqlDriverClassName, errorMessage("driver class name", "db.driver"));
+
+        this.setUrl(connectionURL);
+
         //  Set up poolable data source
         this.setDriverClassName(mysqlDriverClassName);
         this.setUsername(userName);
         this.setPassword(password);
-        this.setUrl(url);
         // Disable this to avoid caching statements
         this.setPoolPreparedStatements(Boolean.valueOf(enablePooling));
         // these are the values cbioportal has been using in their production
@@ -37,4 +53,12 @@ public JdbcDataSource () {
         this.setValidationQuery("SELECT 1");
         this.setJmxName("org.cbioportal:DataSource=" + database);
     }
+
+    private String errorMessage(String displayName, String propertyName) {
+        return String.format("No %s provided for database connection. Please set '%s' in portal.properties.", displayName, propertyName);
+    }
+
+    private boolean defined(String property) {
+        return property != null && !property.isEmpty();
+    }
 }