Release

.github/workflows/dev-hail-search-release.yaml

@@ -47,11 +47,11 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: update image tag in the broad seqr chart
+      - name: update image tag in the dev broad seqr chart
         uses: mikefarah/yq@v4.22.1
         with:
           cmd: >
-            yq -i '.hail-search.image.tag = "${{ github.event.workflow_run.head_sha }}"' charts/broad-seqr/values-dev.yaml
+            yq -i '.hail-search.image.tag = "${{ github.event.workflow_run.head_sha }}"' charts/dev-broad-seqr/values.yaml
 
       - name: Commit and Push changes
         uses: Andro999b/push@v1.3

.github/workflows/docker-lint.yaml

@@ -11,6 +11,7 @@ on:
       - deploy/docker/seqr/Dockerfile
       - hail_search/deploy/Dockerfile
       - .hadolint.yaml
+      - .docker-compose.yaml
       - .github/workflows/docker-lint.yaml
   pull_request:
     types: [opened, synchronize, reopened]
@@ -21,13 +22,16 @@ on:
       - deploy/docker/seqr/Dockerfile
       - hail_search/deploy/Dockerfile
       - .hadolint.yaml
+      - .docker-compose.yaml
       - .github/workflows/docker-lint.yaml
 
 jobs:
   hadolint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2 
+      - uses: actions/checkout@v2
+      - name: Validate docker compose
+        run: docker compose -f docker-compose.yml config
       - uses: hadolint/hadolint-action@v1.5.0
         with:
           dockerfile: deploy/docker/seqr/Dockerfile

.github/workflows/hail-search-persistent-volume-snapshot-release.yaml

@@ -4,30 +4,33 @@ on:
     inputs:
       environment:
         type: choice
-        options: 
+        options:
         - dev
         - prod
       reference_genome:
          type: choice
          description: Reference Genome
-         options: 
+         options:
          - GRCh37
          - GRCh38
          required: true
       dataset_type:
         type: choice
         description: Dataset Type
-        options: 
+        options:
         - SNV_INDEL
         - MITO
         - GCNV
         - SV
         required: true
       version:
-        required: true
+        required: false
       volume_handle:
         required: true
 
+env:
+  CHART_NAME: "${{ inputs.environment == 'dev' && 'dev-' || '' }}broad-seqr"
+
 jobs:
   helm_update:
     runs-on: ubuntu-latest
@@ -42,16 +45,17 @@ jobs:
           fetch-depth: 0
 
       - name: update dataset version in the broad-seqr chart
+        if: "${{ inputs.version != '' }}"
         uses: mikefarah/yq@v4.22.1
         with:
           cmd: >
-            yq -i '.global.hail_search.datasetVersions.${{ inputs.reference_genome }}/${{ inputs.dataset_type }} = "${{ inputs.version }}"' charts/broad-seqr/values-${{ inputs.environment }}.yaml
+            yq -i '.global.hail_search.datasetVersions.${{ inputs.reference_genome }}/${{ inputs.dataset_type }} = "${{ inputs.version }}"' charts/${{ env.CHART_NAME }}/values.yaml
 
       - name: update volume handle in the broad-seqr chart
         uses: mikefarah/yq@v4.22.1
         with:
           cmd: >
-            yq -i '.hail-search.persistentVolume.volumeHandle = "${{ inputs.volume_handle }}"' charts/broad-seqr/values-${{ inputs.environment }}.yaml
+            yq -i '.hail-search.persistentVolume.volumeHandle = "${{ inputs.volume_handle }}"' charts/${{ env.CHART_NAME }}/values.yaml
 
       - name: Commit and Push changes
         uses: Andro999b/push@v1.3
@@ -61,4 +65,4 @@ jobs:
           github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }}
           author_email: ${{ github.actor }}@users.noreply.github.com
           author_name: tgg-automation
-          message: "Updating ${{ inputs.environment }} ${{ inputs.reference_genome }}/${{ inputs.dataset_type }} dataset version to ${{ inputs.version }} and volume handle to ${{ inputs.volume_handle }} "
+          message: "Updating ${{ inputs.environment }} ${{ inputs.reference_genome }}/${{ inputs.dataset_type }} ${{ inputs.version != '' && format('{0} {1} {2}', 'dataset version to', inputs.version, 'and') || ''}} volume handle to ${{ inputs.volume_handle }} "

.github/workflows/hail-search-unit-tests.yaml

@@ -28,7 +28,7 @@ jobs:
       - name: Run coverage tests
         run: |
           export DATASETS_DIR=./hail_search/fixtures
-          export ONT_ENABLED=true
+          export MAX_GENE_INTERVALS=3
           export MACHINE_MEM=24
           export JAVA_OPTS_XSS=16M
           coverage run --source="./hail_search" --omit="./hail_search/__main__.py","./hail_search/test_utils.py" -m pytest hail_search/

.github/workflows/trivy.yml

@@ -1,14 +1,15 @@
-
 name: trivy
 on:
   # runs on default branch
   workflow_dispatch:
   schedule:
-    - cron:  '0 22 * * 0' # each Monday at 9am AEST+10 / 10am AEDT+11
+    - cron: '0 22 * * 0' # each Monday at 9am AEST+10 / 10am AEDT+11
 
 permissions:
   id-token: write
+  security-events: write
   contents: read
+  actions: write
 
 jobs:
   trivy-prod:
@@ -21,13 +22,14 @@ jobs:
       DOCKER_IMAGE: australia-southeast1-docker.pkg.dev/seqr-308602/seqr-project/seqr:gcloud-prod
 
     steps:
-      # - uses: actions/checkout@v2
-      - id: "google-cloud-auth"
-        name: "Authenticate to Google Cloud"
-        uses: "google-github-actions/auth@v2"
+      - uses: actions/checkout@v4
+
+      - id: 'google-cloud-auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v2'
         with:
-          workload_identity_provider: "projects/1021400127367/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
-          service_account: "github-trivy-workflow@seqr-308602.iam.gserviceaccount.com"
+          workload_identity_provider: 'projects/1021400127367/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+          service_account: 'github-trivy-workflow@seqr-308602.iam.gserviceaccount.com'
 
       - name: gcloud docker auth
         run: |
@@ -48,9 +50,9 @@ jobs:
           format: 'template'
           template: '@/contrib/sarif.tpl'
           output: 'trivy-results-prod.sarif'
-          
+
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v1
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: 'trivy-results-prod.sarif'
 
@@ -65,12 +67,12 @@ jobs:
 
     steps:
       # - uses: actions/checkout@v2
-      - id: "google-cloud-auth"
-        name: "Authenticate to Google Cloud"
-        uses: "google-github-actions/auth@v2"
+      - id: 'google-cloud-auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v2'
         with:
-          workload_identity_provider: "projects/1021400127367/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
-          service_account: "github-trivy-workflow@seqr-308602.iam.gserviceaccount.com"
+          workload_identity_provider: 'projects/1021400127367/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+          service_account: 'github-trivy-workflow@seqr-308602.iam.gserviceaccount.com'
 
       - name: gcloud docker auth
         run: |
@@ -91,8 +93,8 @@ jobs:
           format: 'template'
           template: '@/contrib/sarif.tpl'
           output: 'trivy-results-dev.sarif'
-          
+
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v1
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: 'trivy-results-dev.sarif'

.github/workflows/unit-tests.yml

@@ -74,6 +74,7 @@ jobs:
         pushd ui
         npm install
         npm run build
+        mkdir ../static
         cp dist/* ../static/
         popd
     - name: Run coverage tests

CHANGELOG.md

@@ -2,6 +2,34 @@
 
 ## dev
 
+## 8/14/24
+* Remove ONT support (REQUIRES DB MIGRATION)
+* Add "Validated Name" functional tag (REQUIRES DB MIGRATION)
+
+## 8/9/24
+* Update directory structure for search backend
+
+## 8/2/24
+* Adds index_file_path to IGV Sample model (REQUIRES DB MIGRATION)
+
+## 7/24/24
+* Split RNA Sample models (REQUIRES DB MIGRATION)
+
+## 7/8/24
+* Add VLM contact for Projects (REQUIRES DB MIGRATION)
+
+## 6/11/24
+* Add "Partial Phenotype Contribution" functional tag (REQUIRES DB MIGRATION)
+
+## 5/24/24
+* Adds external_data to Family model (REQUIRES DB MIGRATION)
+* Adds post_discovery_mondo_id to Family model (REQUIRES DB MIGRATION)
+* Adds guid and created fields to PhenotypePrioritization model (REQUIRES DB MIGRATION)
+* Enable "Reports" tab by default for local installations
+
+## 5/8/24
+* Adds dynamic analysis groups (REQUIRES DB MIGRATION)
+
 ## 4/4/24
 * Add ability to import project metadata from gregor metadata
   * Only enabled for a project if tag is first created via 

deploy/LOCAL_DEVELOPMENT_INSTALL.md

@@ -116,7 +116,7 @@ Before running seqr, make sure the following are currently running/ started:
   - If you want ES running but do not need production data/ are working with a standalone seqr instance, 
   use docker-compose
     ```bash
-    docker-compose up elasticsearch
+    docker compose up elasticsearch
     ```
 
 ### Run ui asset server

deploy/LOCAL_INSTALL.md

@@ -31,10 +31,10 @@ SEQR_DIR=$(pwd)
 
 wget https://raw.githubusercontent.com/populationgenomics/seqr/master/docker-compose.yml
 
-docker-compose up -d seqr   # start up the seqr docker image in the background after also starting other components it depends on (postgres, redis, elasticsearch). This may take 10+ minutes.
-docker-compose logs -f seqr  # (optional) continuously print seqr logs to see when it is done starting up or if there are any errors. Type Ctrl-C to exit from the logs. 
+docker compose up -d seqr   # start up the seqr docker image in the background after also starting other components it depends on (postgres, redis, elasticsearch). This may take 10+ minutes.
+docker compose logs -f seqr  # (optional) continuously print seqr logs to see when it is done starting up or if there are any errors. Type Ctrl-C to exit from the logs. 
 
-docker-compose exec seqr python manage.py createsuperuser  # create a seqr Admin user 
+docker compose exec seqr python manage.py createsuperuser  # create a seqr Admin user 
 
 open http://localhost     # open the seqr landing page in your browser. Log in to seqr using the email and password from the previous step
 ```
@@ -45,15 +45,15 @@ Updating your local installation of seqr involves pulling the latest version of
 
 ```bash
 # run this from the directory containing your docker-compose.yml file
-docker-compose pull
-docker-compose up -d seqr
+docker compose pull
+docker compose up -d seqr
 
-docker-compose logs -f seqr  # (optional) continuously print seqr logs to see when it is done starting up or if there are any errors. Type Ctrl-C to exit from the logs. 
+docker compose logs -f seqr  # (optional) continuously print seqr logs to see when it is done starting up or if there are any errors. Type Ctrl-C to exit from the logs. 
 ```
 
 To update reference data in seqr, such as OMIM, HPO, etc., run the following
 ```bash
-docker-compose exec seqr ./manage.py update_all_reference_data --use-cached-omim --skip-gencode
+docker compose exec seqr ./manage.py update_all_reference_data --use-cached-omim --skip-gencode
 ```
 
 ### Annotating and loading VCF callsets 
@@ -79,7 +79,7 @@ The steps below describe how to annotate a callset and then load it into your on
 
 1. start a pipeline-runner container which has the necessary tools and environment for starting and submitting jobs to a Dataproc cluster.
    ```bash
-   docker-compose up -d pipeline-runner            # start the pipeline-runner container 
+   docker compose up -d pipeline-runner            # start the pipeline-runner container 
    ```
 
 1. if you haven't already, upload reference data to your own google bucket. 
@@ -88,7 +88,7 @@ This is expected to take a while
    ```bash
    BUILD_VERSION=38                 # can be 37 or 38
 
-   docker-compose exec pipeline-runner copy_reference_data_to_gs.sh $BUILD_VERSION $GS_BUCKET
+   docker compose exec pipeline-runner copy_reference_data_to_gs.sh $BUILD_VERSION $GS_BUCKET
 
    ```
    Periodically, you may want to update the reference data in order to get the latest versions of these annotations. 
@@ -115,7 +115,7 @@ annotations, but you will need to re-load previously loaded projects to get the
 
    INPUT_FILE_PATH=/${GS_FILE_PATH}/${FILENAME}  
 
-   docker-compose exec pipeline-runner load_data_dataproc.sh $BUILD_VERSION $SAMPLE_TYPE $INDEX_NAME $GS_BUCKET $INPUT_FILE_PATH
+   docker compose exec pipeline-runner load_data_dataproc.sh $BUILD_VERSION $SAMPLE_TYPE $INDEX_NAME $GS_BUCKET $INPUT_FILE_PATH
 
    ``` 
 
@@ -138,13 +138,13 @@ The steps below describe how to annotate a callset and then load it into your on
 
 1. start a pipeline-runner container
    ```bash
-   docker-compose up -d pipeline-runner            # start the pipeline-runner container 
+   docker compose up -d pipeline-runner            # start the pipeline-runner container 
    ```
 
 1. authenticate into your google cloud account.
 This is required for hail to access buckets hosted on gcloud.
    ```bash
-   docker-compose exec pipeline-runner  gcloud auth application-default login
+   docker compose exec pipeline-runner  gcloud auth application-default login
    ```
 
 1. if you haven't already, download VEP and other reference data to the docker image's mounted directories. 
@@ -153,7 +153,7 @@ This is expected to take a while
    ```bash
    BUILD_VERSION=38                 # can be 37 or 38
 
-   docker-compose exec pipeline-runner download_reference_data.sh $BUILD_VERSION
+   docker compose exec pipeline-runner download_reference_data.sh $BUILD_VERSION
 
    ``` 
    Periodically, you may want to update the reference data in order to get the latest versions of these annotations. 
@@ -163,12 +163,12 @@ annotations, but you will need to re-load previously loaded projects to get the
    BUILD_VERSION=38                 # can be 37 or 38
 
    # Update clinvar 
-   docker-compose exec pipeline-runner rm -rf "/seqr-reference-data/GRCh${BUILD_VERSION}/clinvar.GRCh${BUILD_VERSION}.ht"
-   docker-compose exec pipeline-runner gsutil rsync -r "gs://seqr-reference-data/GRCh${BUILD_VERSION}/clinvar/clinvar.GRCh${BUILD_VERSION}.ht" "/seqr-reference-data/GRCh${BUILD_VERSION}/clinvar.GRCh${BUILD_VERSION}.ht"
+   docker compose exec pipeline-runner rm -rf "/seqr-reference-data/GRCh${BUILD_VERSION}/clinvar.GRCh${BUILD_VERSION}.ht"
+   docker compose exec pipeline-runner gsutil rsync -r "gs://seqr-reference-data/GRCh${BUILD_VERSION}/clinvar/clinvar.GRCh${BUILD_VERSION}.ht" "/seqr-reference-data/GRCh${BUILD_VERSION}/clinvar.GRCh${BUILD_VERSION}.ht"
 
    # Update all other reference data
-   docker-compose exec pipeline-runner rm -rf "/seqr-reference-data/GRCh${BUILD_VERSION}/combined_reference_data_grch${BUILD_VERSION}.ht"
-   docker-compose exec pipeline-runner gsutil rsync -r "gs://seqr-reference-data/GRCh${BUILD_VERSION}/all_reference_data/combined_reference_data_grch${BUILD_VERSION}.ht" "/seqr-reference-data/GRCh${BUILD_VERSION}/combined_reference_data_grch${BUILD_VERSION}.ht"
+   docker compose exec pipeline-runner rm -rf "/seqr-reference-data/GRCh${BUILD_VERSION}/combined_reference_data_grch${BUILD_VERSION}.ht"
+   docker compose exec pipeline-runner gsutil rsync -r "gs://seqr-reference-data/GRCh${BUILD_VERSION}/all_reference_data/combined_reference_data_grch${BUILD_VERSION}.ht" "/seqr-reference-data/GRCh${BUILD_VERSION}/combined_reference_data_grch${BUILD_VERSION}.ht"
     ```
 
 1. run the loading command in the pipeline-runner container. Adjust the arguments as needed
@@ -179,7 +179,7 @@ annotations, but you will need to re-load previously loaded projects to get the
 
    INPUT_FILE_PATH=${FILE_PATH}/${FILENAME}  
 
-   docker-compose exec pipeline-runner load_data.sh $BUILD_VERSION $SAMPLE_TYPE $INDEX_NAME $INPUT_FILE_PATH
+   docker compose exec pipeline-runner load_data.sh $BUILD_VERSION $SAMPLE_TYPE $INDEX_NAME $INPUT_FILE_PATH
 
    ``` 
 

deploy/docker/seqr/Dockerfile

@@ -7,7 +7,7 @@
 WORKDIR /app/seqr
 COPY requirements.txt /app/seqr/
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-transport-https ca-certificates curl gnupg \
     # install dependencies of the HaploPainter.pl script used to generate static pedigree images
     libgtk2-perl libdbi-perl libtk-perl libsort-naturally-perl && \
@@ -22,7 +22,6 @@
 COPY matchmaker /app/seqr/matchmaker
 COPY reference_data /app/seqr/reference_data
 COPY seqr /app/seqr/seqr
-COPY static ui/dist /app/seqr/static/
 COPY ui/dist /app/seqr/ui/dist
 COPY panelapp /app/seqr/panelapp
 COPY wsgi.py settings.py manage.py deploy/docker/seqr/entrypoint.sh deploy/docker/seqr/init_db.sh deploy/docker/seqr/config/ /app/seqr/

hail_search/__main__.py

@@ -1,5 +1,4 @@
 from aiohttp import web
-import hail as hl
 import logging
 
 from hail_search.web_app import init_web_app

hail_search/constants.py

@@ -15,6 +15,10 @@
 SPLICE_AI_FIELD = 'splice_ai'
 NEW_SV_FIELD = 'new_structural_variants'
 SCREEN_KEY = 'SCREEN'  # uses all caps to match filter provided by the seqr UI
+UTR_ANNOTATOR_KEY = 'UTRAnnotator'
+EXTENDED_SPLICE_KEY = 'extended_splice_site'
+MOTIF_FEATURES_KEY = 'motif_feature'
+REGULATORY_FEATURES_KEY = 'regulatory_feature'
 CLINVAR_KEY = 'clinvar'
 CLINVAR_MITO_KEY = 'clinvar_mito'
 HGMD_KEY = 'hgmd'
@@ -23,7 +27,7 @@
 GENOTYPES_FIELD = 'genotypes'
 
 ANNOTATION_OVERRIDE_FIELDS = [
-    SCREEN_KEY, SPLICE_AI_FIELD, NEW_SV_FIELD, STRUCTURAL_ANNOTATION_FIELD,
+    SCREEN_KEY, SPLICE_AI_FIELD, NEW_SV_FIELD, STRUCTURAL_ANNOTATION_FIELD, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY,
 ]
 ALLOWED_TRANSCRIPTS = 'allowed_transcripts'
 ALLOWED_SECONDARY_TRANSCRIPTS = 'allowed_transcripts_secondary'
@@ -35,6 +39,7 @@
 PATHOGENICTY_HGMD_SORT_KEY = 'pathogenicity_hgmd'
 ABSENT_PATH_SORT_OFFSET = 12.5
 CONSEQUENCE_SORT = 'protein_consequence'
+ALPHAMISSENSE_SORT = 'alphamissense'
 OMIM_SORT = 'in_omim'
 
 ALT_ALT = 'alt_alt'