diff --git a/.dockerignore b/.dockerignore index bf881194fb..52d1cb5546 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,3 @@ -ui/node_modules/ \ No newline at end of file +ui/node_modules/ +.idea/ +.vscode/ \ No newline at end of file diff --git a/.envrc.template b/.envrc.template new file mode 100644 index 0000000000..1e8d0585a8 --- /dev/null +++ b/.envrc.template @@ -0,0 +1,26 @@ +# Database settings for Django +export POSTGRES_SERVICE_HOSTNAME=localhost +export POSTGRES_SERVICE_PORT=5433 +export POSTGRES_USERNAME=postgres +export POSTGRES_PASSWORD=a-very-secure-password +export POSTGRES_DATABASE=seqrdb + +# Leave blank for local env +export DEPLOYMENT_TYPE= + +# Convenience commands for creating database +export CREATE_REF_DB_CMD="psql -W -U ${POSTGRES_USERNAME} -h ${POSTGRES_SERVICE_HOSTNAME} -p ${POSTGRES_SERVICE_PORT} -c 'create database reference_data_db;'" +export LIST_DBS_CMD="psql -W -U ${POSTGRES_USERNAME} -h ${POSTGRES_SERVICE_HOSTNAME} -p ${POSTGRES_SERVICE_PORT} -c '\list'" + +# ---------------------------------------------------------------------------- +# Apple Developers +# ---------------------------------------------------------------------------- +# Install libpg for the Postgres client library without the server (brew install libpg) +# Export these variables so that pip packages know where to find the appropriate binaries, libs and headers. +# export PATH="/opt/homebrew/opt/libpq/bin:$PATH" +# export LDFLAGS="-L/opt/homebrew/opt/libpq/lib $LDFLAGS" +# export CPPFLAGS="-I/opt/homebrew/opt/libpq/include $CPPFLAGS" + +# Apple silicon (M1 chip etc) fix for grpcio +# export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 +# export GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 diff --git a/.gitignore b/.gitignore index ffe9555219..e7dd445a2c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea* +.vscode* *.iml *.sqlite3 *.log @@ -22,3 +23,7 @@ seqr_settings django_key static/ .coverage + +.envrc + +data/ \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000000..8bd604a5ed --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,19 @@ +version: '2.4' + +services: + postgres: + image: postgres:12.6 + command: postgres -c listen_addresses='*' + environment: + - PGPORT=5433 + - POSTGRES_DB=seqrdb + - POSTGRES_PASSWORD=a-very-secure-password + volumes: + - ./data/postgres:/var/lib/postgresql/data + ports: + - "5433:5433" + healthcheck: + test: pg_isready -h postgres -U postgres + interval: 5s + timeout: 10s + retries: 100 diff --git a/environment-dev.yaml b/environment-dev.yaml new file mode 100644 index 0000000000..88875070ad --- /dev/null +++ b/environment-dev.yaml @@ -0,0 +1,13 @@ +name: seqr + +channels: + - cpg + - bioconda + - conda-forge +dependencies: + - python=3.8 + - pip + - nodejs==16.10.0 + - pip: + - '-r requirements-dev.txt' + - '-r requirements.txt' diff --git a/ui/config/webpack.config.dev.js b/ui/config/webpack.config.dev.js index 96f7d9c7f2..04f7d744aa 100644 --- a/ui/config/webpack.config.dev.js +++ b/ui/config/webpack.config.dev.js @@ -10,6 +10,10 @@ const eslintFormatter = require('react-dev-utils/eslintFormatter'); const glob = require('glob'); const paths = require('./paths'); +// HACK: OpenSSL 3 does not support md4 any more, but webpack hardcodes it all over the place: https://github.com/webpack/webpack/issues/13572 +const crypto = require('crypto'); +const cryptoOriginalCreateHash = crypto.createHash; +crypto.createHash = algorithm => cryptoOriginalCreateHash(algorithm === 'md4' ? 'sha256' : algorithm); // This is the development configuration. diff --git a/ui/config/webpack.config.prod.js b/ui/config/webpack.config.prod.js index bd72fc805a..44c6c6f455 100644 --- a/ui/config/webpack.config.prod.js +++ b/ui/config/webpack.config.prod.js @@ -16,6 +16,11 @@ const eslintFormatter = require('react-dev-utils/eslintFormatter'); const paths = require('./paths'); const getClientEnvironment = require('./env'); +// HACK: OpenSSL 3 does not support md4 any more, but webpack hardcodes it all over the place: https://github.com/webpack/webpack/issues/13572 +const crypto = require('crypto'); +const cryptoOriginalCreateHash = crypto.createHash; +crypto.createHash = algorithm => cryptoOriginalCreateHash(algorithm === 'md4' ? 'sha256' : algorithm); + // Webpack uses `publicPath` to determine where the app is being served from. // It requires a trailing slash, or the file assets will get an incorrect path. const publicPath = paths.servedPath; diff --git a/ui/package-lock.json b/ui/package-lock.json index 36bd9317ca..31785d6935 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -24,6 +24,7 @@ "lodash": "^4.17.21", "markdown-draft-js": "^2.4.0", "object-hash": "^1.3.0", + "papaparse": "^5.3.1", "pedigreejs": "github:CCGE-BOADICEA/pedigreejs#2e17296", "prop-types": "^15.7.2", "query-string": "^6.1.0", @@ -15205,6 +15206,11 @@ "node": ">=4" } }, + "node_modules/papaparse": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.1.tgz", + "integrity": "sha512-Dbt2yjLJrCwH2sRqKFFJaN5XgIASO9YOFeFP8rIBRG2Ain8mqk5r1M6DkfvqEVozVcz3r3HaUGw253hA1nLIcA==" + }, "node_modules/param-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", @@ -31938,6 +31944,11 @@ "version": "1.0.0", "dev": true }, + "papaparse": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.1.tgz", + "integrity": "sha512-Dbt2yjLJrCwH2sRqKFFJaN5XgIASO9YOFeFP8rIBRG2Ain8mqk5r1M6DkfvqEVozVcz3r3HaUGw253hA1nLIcA==" + }, "param-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", diff --git a/ui/package.json b/ui/package.json index 14b4380393..a53b471bdd 100644 --- a/ui/package.json +++ b/ui/package.json @@ -83,6 +83,7 @@ "lodash": "^4.17.21", "markdown-draft-js": "^2.4.0", "object-hash": "^1.3.0", + "papaparse": "^5.3.1", "pedigreejs": "github:CCGE-BOADICEA/pedigreejs#2e17296", "prop-types": "^15.7.2", "query-string": "^6.1.0", diff --git a/ui/pages/Project/Project.jsx b/ui/pages/Project/Project.jsx index f9e29ebac2..843037f6e2 100644 --- a/ui/pages/Project/Project.jsx +++ b/ui/pages/Project/Project.jsx @@ -12,6 +12,7 @@ import CaseReview from './components/CaseReview' import FamilyPage from './components/FamilyPage' import Matchmaker from './components/Matchmaker' import SavedVariants from './components/SavedVariants' +import DataLoadingWizard from './components/DataLoadingWizard/DataLoadingWizard' class Project extends React.PureComponent { @@ -45,6 +46,7 @@ class Project extends React.PureComponent { + ) diff --git a/ui/pages/Project/components/DataLoadingWizard/DataLoadingWizard.jsx b/ui/pages/Project/components/DataLoadingWizard/DataLoadingWizard.jsx new file mode 100644 index 0000000000..1f4bc46990 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/DataLoadingWizard.jsx @@ -0,0 +1,196 @@ +/* eslint-disable react-perf/jsx-no-new-function-as-prop */ +/* eslint-disable no-console */ +/* eslint-disable no-alert */ + +import React, { useState, useEffect, useCallback } from 'react' +import PropTypes from 'prop-types' +import { connect } from 'react-redux' +import { Breadcrumb } from 'semantic-ui-react' + +import { Error404 } from '../../../../shared/components/page/Errors' +import { getCurrentProject } from '../../selectors' + +import { Centered, FormSection, FormStepButtons } from './ui' +import { Welcome, TemplateUpload } from './steps' +import PedigreeTemplateColumns from './templates/Pedigree' +import IndividualTemplateColumns from './templates/Individual' +import FamilyTemplateColumns from './templates/Family' +import TemplateFile from './templates/File/TemplateFile' +import TemplateHelp from './TemplateHelp' + +const BREADCRUMBS = [ + { key: 0, content: 'Welcome', link: true, active: true }, + { key: 1, content: 'Pedigree', link: true, active: false }, + { key: 2, content: 'Individual metadata', link: true, active: false }, + { key: 3, content: 'Family metadata', link: true, active: false }, + { key: 4, content: 'Sample mapping', link: true, active: false }, + { key: 5, content: 'Review', link: true, active: false }, +] + +const BaseMultistepForm = ({ project }) => { + // ---- Debug ---- // + console.group('Project Object') + console.log(project) + console.groupEnd() + + // ---- State ----- // + const [breadCrumbs, setBreadcrumbs] = useState(BREADCRUMBS.map(b => ({ ...b }))) + const [activeFormStepIndex, setActiveFormStepIndex] = useState(0) + const [formSteps, setFormSteps] = useState(BREADCRUMBS.map( + (_, index) => ({ formData: {}, isComplete: (index === 0) }), + )) + + // ---- Callbacks ----- // + const updateFormStep = useCallback((stepNumber, { formData, isComplete }) => { + setFormSteps(formSteps.map( + (step, index) => { + if (stepNumber === index) { + return { formData, isComplete } + } + return { ...step } + }, + )) + }, [formSteps]) + + const enableReview = useCallback(() => formSteps + .slice(0, formSteps.length - 1) + .reduce((acc, step) => acc && step.isComplete, true), [formSteps]) + + const enableNext = useCallback(() => { + if (activeFormStepIndex === (formSteps.length - 2)) { + return enableReview() + } + + return activeFormStepIndex < formSteps.length + }, [formSteps, activeFormStepIndex, enableReview]) + + const getFormStepComponent = useCallback(() => { + const onFormChange = ({ formData, isComplete }) => updateFormStep(activeFormStepIndex, { formData, isComplete }) + + switch (activeFormStepIndex) { + case 0: + return + case 1: + return ( + } + project={project} + /> + ) + case 2: + return ( + } + project={project} + /> + ) + case 3: + return ( + } + project={project} + /> + ) + case 4: + return
{ breadCrumbs[activeFormStepIndex].content }
+ case 5: + return
{ breadCrumbs[activeFormStepIndex].content }
+ default: + return + } + }, [project, activeFormStepIndex, updateFormStep]) + + // ---- Effects ---- // + useEffect(() => { + setBreadcrumbs(breadCrumbs.map((b, index) => ({ ...b, active: (index === activeFormStepIndex) }))) + }, [activeFormStepIndex]) + + useEffect(() => { + setBreadcrumbs( + breadCrumbs.map((b, index) => { + if (index === (breadCrumbs.length - 1)) { + return { + ...b, + link: enableReview(), + onClick: enableReview() ? () => setActiveFormStepIndex(index) : null, + } + } + + return { + ...b, + link: true, + onClick: () => setActiveFormStepIndex(index), + } + }), + ) + }, [enableReview]) + + // ---- Render ----- // + return ( +
+ + + + + + + + {getFormStepComponent()} + + + + setActiveFormStepIndex(Math.min(activeFormStepIndex + 1, formSteps.length - 1))} + onBack={() => setActiveFormStepIndex(Math.max(0, activeFormStepIndex - 1))} + enableNext={enableNext()} + enableSubmit={enableReview()} + onSubmit={() => alert('Yay')} + /> + + + + + + + +
+ ) +} + +BaseMultistepForm.propTypes = { + project: PropTypes.object.isRequired, +} + +const mapStateToProps = state => ({ + project: getCurrentProject(state), +}) + +export const MultistepForm = connect(mapStateToProps)(BaseMultistepForm) + +const DataLoadingWizard = () => + +export default DataLoadingWizard diff --git a/ui/pages/Project/components/DataLoadingWizard/TemplateDirectoryLink.jsx b/ui/pages/Project/components/DataLoadingWizard/TemplateDirectoryLink.jsx new file mode 100644 index 0000000000..533b102b83 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/TemplateDirectoryLink.jsx @@ -0,0 +1,13 @@ +import React from 'react' + +const TemplateDirectoryLink = () => ( + + here + +) + +export default TemplateDirectoryLink diff --git a/ui/pages/Project/components/DataLoadingWizard/TemplateHelp.jsx b/ui/pages/Project/components/DataLoadingWizard/TemplateHelp.jsx new file mode 100644 index 0000000000..4233af88f5 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/TemplateHelp.jsx @@ -0,0 +1,17 @@ +import React from 'react' + +import { ReadableText } from './ui' +import TemplateDirectoryLink from './TemplateDirectoryLink' + +const TemplateHelp = () => ( + + In this section, you will provide all information relating to the family pedigree and associated metadata in your + project. Please download the families template from + {' '} + + { '. ' } + Once you have filled in this template, upload it via this step and correct any validation errors before proceeding. + +) + +export default TemplateHelp diff --git a/ui/pages/Project/components/DataLoadingWizard/steps/IndividualMetadataUpload.jsx b/ui/pages/Project/components/DataLoadingWizard/steps/IndividualMetadataUpload.jsx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ui/pages/Project/components/DataLoadingWizard/steps/Review.jsx b/ui/pages/Project/components/DataLoadingWizard/steps/Review.jsx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ui/pages/Project/components/DataLoadingWizard/steps/SampleMapping.jsx b/ui/pages/Project/components/DataLoadingWizard/steps/SampleMapping.jsx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ui/pages/Project/components/DataLoadingWizard/steps/TemplateUpload.jsx b/ui/pages/Project/components/DataLoadingWizard/steps/TemplateUpload.jsx new file mode 100644 index 0000000000..258165c97c --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/steps/TemplateUpload.jsx @@ -0,0 +1,210 @@ +import React, { useCallback, useState } from 'react' +import PropTypes from 'prop-types' +import { capitalize, isBoolean, isEqual } from 'lodash' + +import { Form, Table, Icon, List, Message } from 'semantic-ui-react' + +import TemplateFile from '../templates/File/TemplateFile' +import { Scrollable, WithSpace } from '../ui' + +const renderCellContent = (value) => { + if (Array.isArray(value)) { + return value.length ? {value.map(v => {v})} : '-' + } + + if (isBoolean(value)) { + return capitalize(value.toString()) + } + + return value == null ? '-' : value.toString() +} + +const ERROR_LIST_STYLE = { width: 250 } + +/** + * @param {TemplateRow} row + * @returns {JSX.Element} + */ +const renderRowErrorList = (row) => { + if (row.valid) return null + + return ( + + { + row.columns.map(({ data, definition }) => { + if (data.valid) return null + + const errorList = data.errors.map(e => {e}) + return ( + +
{definition.key}
+ {errorList} + + ) + }) + } + + ) +} + +const TemplateUpload = ({ id, label, information, template, project, onFormChange }) => { + const [isLoading, setIsLoading] = useState(false) + const [errors, setErrors] = useState(/** @type {string[]|JSXElement[]} */ []) + const [valid, setValid] = useState(true) + const [rows, setRows] = useState(/** @type {TemplateRow[]} */ []) + + const parseFile = useCallback((event) => { + if (event.target.files[0]) { + if (!event.target.files[0].name.match(/\.(tsv|csv)$/)) { + setErrors(['Please upload a TSV or CSV file.']) + setValid(false) + setRows([]) + return + } + + setIsLoading(true) + template.parse( + { + file: event.target.files[0], + onComplete: ((result) => { + console.debug(result) + + setIsLoading(false) + + // Copy array before sorting since sort function has side effects and modifies the original array + const columnMismatch = !isEqual([...result.header].sort(), template.columns.map(c => c.key).sort()) + + if (columnMismatch) { + setErrors([ + ( +
+

Template file should have the following header columns:

+ + {template.columns.map(c => {c.key})} + +

However, your file contains the header columns:

+ + {result.header.map(f => {f})} + +
+ ), + ]) + setValid(false) + setRows([]) + + return + } + + setErrors(result.errors) + setValid(result.valid) + setRows(result.rows) + }), + onError: ((e) => { + setErrors([e.message]) + setIsLoading(false) + setValid(false) + setRows([]) + }), + }, + ) + } + }, [template, setIsLoading, setErrors, setRows, setValid]) + + const renderTableBody = useCallback(() => { + if (!valid && !rows.length) { + return ( + + Data could not be loaded. See errors for help. + + ) + } + + // Render rows if we have them even if the parsing result is not valid, so we can display the errors + // to the user. + if (rows.length) { + return rows.map(row => ( + + + {row.valid ? : } + + + {renderRowErrorList(row)} + + { + row.columns.map(({ data, definition }) => ( + + {renderCellContent(data.value)} + + )) + } + + )) + } + + return ( + + No data to display + + ) + }, [rows, valid, errors]) + + return ( +
+ {information} + +
+ + + + { + !valid ? ( + {e})} + /> + ) : null + } + + + + + + + Status + Comments + { + template + .columns + .map(c => ( + + {`${c.key}${c.required ? ' *' : ''}`} + + )) + } + + + + {renderTableBody()} + +
+ + +
+ ) +} + +TemplateUpload.propTypes = { + id: PropTypes.string.isRequired, + label: PropTypes.string.isRequired, + information: PropTypes.element, + template: PropTypes.instanceOf(TemplateFile).isRequired, + project: PropTypes.object.isRequired, + onFormChange: PropTypes.func.isRequired, +} + +TemplateUpload.defaultProps = { + information: null, +} + +export default TemplateUpload diff --git a/ui/pages/Project/components/DataLoadingWizard/steps/Welcome.jsx b/ui/pages/Project/components/DataLoadingWizard/steps/Welcome.jsx new file mode 100644 index 0000000000..fd916c7a4f --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/steps/Welcome.jsx @@ -0,0 +1,36 @@ +import React from 'react' +import styled from 'styled-components' +import { ReadableText } from '../ui' + +const WelcomeSection = styled.section` + margin: 1em 0; +` + +const Welcome = () => ( +
+ + + Welcome to the seqr data loading wizard! This wizard will guide you through the process of uploading + the relevant individual template, family template and associated metadata files. Each step will validate + these files to ensure that the information relating individuals, families and samples is correctly formatted. + + + + +

Introduction

+ Overview of what the collaborator needs to do goes here + + + +

Resources

+ Overview of required template and mapping formats + + + +

FAQ

+ Frequently asked questions and answers + +
+) + +export default Welcome diff --git a/ui/pages/Project/components/DataLoadingWizard/steps/index.js b/ui/pages/Project/components/DataLoadingWizard/steps/index.js new file mode 100644 index 0000000000..c2fb5f7051 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/steps/index.js @@ -0,0 +1,7 @@ +import Welcome from './Welcome' +import TemplateUpload from './TemplateUpload' + +export { + Welcome, + TemplateUpload, +} diff --git a/ui/pages/Project/components/DataLoadingWizard/templates/Family.js b/ui/pages/Project/components/DataLoadingWizard/templates/Family.js new file mode 100644 index 0000000000..e76be572a7 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/templates/Family.js @@ -0,0 +1,41 @@ +import TemplateColumn from './File/TemplateColumn' +import { parseString, parseStringArray } from './utils/parsers' + +const FamilyTemplateColumns = () => ([ + new TemplateColumn({ + id: 'familyId', + key: 'Family ID', + index: 0, + required: true, + parser: parseString, + validators: [ + v => ((!v) ? 'A family ID must be present.' : null), + ], + }), + new TemplateColumn({ + id: 'displayName', + key: 'Display Name', + index: 1, + required: false, + parser: parseString, + validators: [], + }), + new TemplateColumn({ + id: 'description', + key: 'Description', + index: 2, + required: false, + parser: parseString, + validators: [], + }), + new TemplateColumn({ + id: 'codedPhenotype', + key: 'Coded Phenotype', + index: 3, + required: false, + parser: parseStringArray, + validators: [], + }), +]) + +export default FamilyTemplateColumns diff --git a/ui/pages/Project/components/DataLoadingWizard/templates/File/TemplateColumn.js b/ui/pages/Project/components/DataLoadingWizard/templates/File/TemplateColumn.js new file mode 100644 index 0000000000..b10701d723 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/templates/File/TemplateColumn.js @@ -0,0 +1,41 @@ +class TemplateColumn { + + constructor({ id, key, index, required = false, parser = x => x, validators = [] }) { + this.id = id + this.key = key + this.index = index + this.required = required || false + this.parser = parser || (x => x) + this.validators = validators || [] + } + + /** + * @param {*[]|{}} row + * @returns {{value: *, valid: boolean, errors: string[]}} + */ + parse(row) { + let result = null + if (Array.isArray(row)) { + result = this.parser(row[this.index]) + } else if (row.constructor.name === 'Object') { + result = this.parser(row[this.key]) + } else { + throw new Error( + `TemplateColumn 'parse' function expects an Array or an Object, but received ${row.constructor.name}`, + ) + } + + // Some validators may return a list of errors if they validate a field which is parsed into an array, for example + // a comma-delimited list of string values. Use flatMap to flatten all validation errors into a single array. + const validationErrors = this.validators.flatMap(validator => validator(result)).filter(e => e != null) + + return { + value: result, + errors: validationErrors, + valid: validationErrors.length === 0, + } + } + +} + +export default TemplateColumn diff --git a/ui/pages/Project/components/DataLoadingWizard/templates/File/TemplateFile.js b/ui/pages/Project/components/DataLoadingWizard/templates/File/TemplateFile.js new file mode 100644 index 0000000000..6fb18e52bf --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/templates/File/TemplateFile.js @@ -0,0 +1,103 @@ +import * as Papa from 'papaparse' +import { isEqual } from 'lodash' +import TemplateRow from './TemplateRow' + +/** + * Papa-parse error array + * @typedef {{type: string, code: string, message: string, row: number}[]} ParseErrorList + */ + +/** + * Papa-parse metadata + * @typedef { + * {delimiter: string, linebreak: string, aborted: boolean, fields: ?string[], truncated: boolean} +* } ParseMetadata + */ + +/** + * TemplateFile `parse` onComplete callback signature + * @callback onComplete + * @param {{ rows: TemplateRow[], valid: boolean, errors: string[], header: string[] }} result + */ + +/** + * TemplateFile `onError` onError callback signature + * @callback onError + * @param {Error} error + */ + +class TemplateFile { + + /** + * @param {TemplateColumn[]} columns + */ + constructor(columns) { + if (!Array.isArray(columns)) { + throw new Error("Parameter 'columns' must be an array of TemplateColumn objects") + } + + /** + * @type {TemplateColumn[]} + */ + this.columns = Array.from(columns) + } + + /** + * @param {File} file + * @param {onComplete} onComplete + * @param {onError} onError + * + * @returns {TemplateFile} + */ + parse({ file, onComplete, onError }) { + Papa.parse( + file, + { + header: true, + skipEmptyLines: true, + /** + * @param {{data: *[], errors: ParseErrorList, meta: ParseMetadata}} results + */ + complete: (results) => { + const errors = results.errors.map(e => `(Row ${e.row}) ${e.message}`) + + if (!isEqual(this.columns.map(c => c.key).sort(), [...(results.meta.fields || [])].sort())) { + onComplete({ + rows: [], + valid: false, + errors: ['File contains invalid columns in header'], + header: results.meta.fields || [], + }) + return + } + + if (errors.length > 0) { + onComplete({ rows: [], valid: false, errors, header: results.meta.fields || [] }) + } else { + const rows = results.data.map((row, index) => { + const columns = this.columns.map(c => ({ data: c.parse(row), definition: c })) + return new TemplateRow({ index, columns }) + }) + + const valid = rows.reduce((isValid, row) => isValid && row.valid, true) + const validationErrors = valid ? [] : ['Some rows contain invalid information'] + + onComplete({ + rows, + valid, + errors: validationErrors, + header: results.meta.fields || [], + }) + } + }, + /** + * @param {Error} error + */ + error: error => onError(error), + }, + ) + } + +} + +export default TemplateFile diff --git a/ui/pages/Project/components/DataLoadingWizard/templates/File/TemplateRow.js b/ui/pages/Project/components/DataLoadingWizard/templates/File/TemplateRow.js new file mode 100644 index 0000000000..449834ddcf --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/templates/File/TemplateRow.js @@ -0,0 +1,34 @@ +class TemplateRow { + + /** @type {number} #index */ + #index + + /** @type {{data: {value: *, valid: boolean, errors: string[]}, definition: TemplateColumn }[]} #columns */ + #columns + + /** @type {boolean} #valid */ + #valid + + /** + * @param {number} index Row number + * @param {{data: {value: *, valid: boolean, errors: string[]}, definition: TemplateColumn }[]} columns + */ + constructor({ index, columns }) { + if (!Array.isArray(columns)) { + throw new Error("Parameter 'columns' must be an array") + } + + this.#index = index + this.#columns = Array.from(columns) + this.#valid = this.columns.map(column => column.data.valid).reduce((a, b) => a && b, true) + } + + get index() { return this.#index } + + get columns() { return this.#columns } + + get valid() { return this.#valid } + +} + +export default TemplateRow diff --git a/ui/pages/Project/components/DataLoadingWizard/templates/Individual.js b/ui/pages/Project/components/DataLoadingWizard/templates/Individual.js new file mode 100644 index 0000000000..dd7a6b18da --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/templates/Individual.js @@ -0,0 +1,196 @@ +import TemplateColumn from './File/TemplateColumn' +import { parseString, parseStringArray, parseBoolean, parseYear } from './utils/parsers' +import { validateHpoTerms, validateModeOfInheritanceTerms, validateOmimTerms, validateOnsetCategory } from './utils/validators' + +const IndividualTemplateColumns = () => ([ + new TemplateColumn({ + id: 'familyId', + key: 'Family ID', + index: 0, + required: true, + parser: parseString, + validators: [ + v => ((!v) ? 'A family ID must be present.' : null), + ], + }), + new TemplateColumn({ + id: 'individualId', + key: 'Individual ID', + index: 1, + required: true, + parser: parseString, + validators: [ + v => ((!v) ? 'An individual ID must be present.' : null), + ], + }), + new TemplateColumn({ + id: 'hpoTermsPresent', + key: 'HPO Terms (present)', + index: 2, + required: false, + parser: parseStringArray, + validators: [validateHpoTerms], + }), + new TemplateColumn({ + id: 'hpoTermsAbsent', + key: 'HPO Terms (absent)', + index: 3, + required: false, + parser: parseStringArray, + validators: [validateHpoTerms], + }), + new TemplateColumn({ + id: 'birthYear', + key: 'Birth Year', + index: 4, + required: false, + parser: parseYear, + validators: [], + }), + new TemplateColumn({ + id: 'deathYear', + key: 'Death Year', + index: 5, + required: false, + parser: parseYear, + validators: [], + }), + new TemplateColumn({ + id: 'ageOfOnset', + key: 'Age of Onset', + index: 6, + required: false, + parser: parseString, + validators: [validateOnsetCategory], + }), + new TemplateColumn({ + id: 'individualNotes', + key: 'Individual Notes', + index: 7, + required: false, + parser: parseString, + validators: [], + }), + new TemplateColumn({ + id: 'consanguinity', + key: 'Consanguinity', + index: 8, + required: false, + parser: parseBoolean, + validators: [], + }), + new TemplateColumn({ + id: 'otherAffectedRelatives', + key: 'Other Affected Relatives', + index: 9, + required: false, + parser: parseBoolean, + validators: [], + }), + new TemplateColumn({ + id: 'expectedInheritanceMode', + key: 'Expected Mode of Inheritance', + index: 10, + required: false, + parser: parseStringArray, + validators: [validateModeOfInheritanceTerms], + }), + new TemplateColumn({ + id: 'fertilityMedications', + key: 'Fertility medications', + index: 11, + required: false, + parser: parseBoolean, + validators: [], + }), + new TemplateColumn({ + id: 'intrauterineInsemination', + key: 'Intrauterine insemination', + index: 12, + required: false, + parser: parseBoolean, + validators: [], + }), + new TemplateColumn({ + id: 'inVitroFertilization', + key: 'In vitro fertilization', + index: 13, + required: false, + parser: parseBoolean, + validators: [], + }), + new TemplateColumn({ + id: 'intraCytoplasmicSpermInjection', + key: 'Intra-cytoplasmic sperm injection', + index: 14, + required: false, + parser: parseBoolean, + validators: [], + }), + new TemplateColumn({ + id: 'gestationalSurrogacy', + key: 'Gestational surrogacy', + index: 15, + required: false, + parser: parseBoolean, + validators: [], + }), + new TemplateColumn({ + id: 'donorEgg', + key: 'Donor egg', + index: 16, + required: false, + parser: parseBoolean, + validators: [], + }), + new TemplateColumn({ + id: 'donorSperm', + key: 'Donor sperm', + index: 17, + required: false, + parser: parseBoolean, + validators: [], + }), + new TemplateColumn({ + id: 'maternalAncestry', + key: 'Maternal Ancestry', + index: 18, + required: false, + parser: parseStringArray, + validators: [], + }), + new TemplateColumn({ + id: 'paternalAncestry', + key: 'Paternal Ancestry', + index: 19, + required: false, + parser: parseStringArray, + validators: [], + }), + new TemplateColumn({ + id: 'preDiscoveryOmimDisorders', + key: 'Pre-discovery OMIM disorders', + index: 20, + required: false, + parser: parseStringArray, + validators: [validateOmimTerms], + }), + new TemplateColumn({ + id: 'previouslyTestedGenes', + key: 'Previously Tested Genes', + index: 21, + required: false, + parser: parseStringArray, + validators: [], + }), + new TemplateColumn({ + id: 'candidateGenes', + key: 'Candidate Genes', + index: 22, + required: false, + parser: parseStringArray, + validators: [], + }), +]) + +export default IndividualTemplateColumns diff --git a/ui/pages/Project/components/DataLoadingWizard/templates/Pedigree.js b/ui/pages/Project/components/DataLoadingWizard/templates/Pedigree.js new file mode 100644 index 0000000000..e3ccf7aa5d --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/templates/Pedigree.js @@ -0,0 +1,89 @@ +import TemplateColumn from './File/TemplateColumn' +import { parseString } from './utils/parsers' + +const PedigreeTemplateColumns = () => ([ + new TemplateColumn({ + id: 'familyId', + key: 'Family ID', + index: 0, + required: true, + parser: parseString, + validators: [ + v => ((!v) ? 'A family ID must be present.' : null), + ], + }), + new TemplateColumn({ + id: 'individualId', + key: 'Individual ID', + index: 1, + required: true, + parser: parseString, + validators: [ + v => ((!v) ? 'An individual ID must be present.' : null), + ], + }), + new TemplateColumn({ + id: 'paternalId', + key: 'Paternal ID', + index: 2, + required: false, + parser: parseString, + validators: [], + }), + new TemplateColumn({ + id: 'maternalId', + key: 'Maternal ID', + index: 3, + required: false, + parser: parseString, + validators: [], + }), + new TemplateColumn({ + id: 'sex', + key: 'Sex', + index: 4, + required: true, + parser: parseString, + validators: [ + (v) => { + if (!v) { + return ( + 'A character representing sex must be present. ' + + "The character '1' represents male, '2' represents female, " + + "and any other character besides '1' and '2' represents unknown." + ) + } + return null + }, + ], + }), + new TemplateColumn({ + id: 'affectedStatus', + key: 'Affected Status', + index: 5, + required: true, + parser: parseString, + validators: [ + (v) => { + if (!v) { + return ( + 'A character representing phenotype affect status must be present. ' + + "The characters '0' and '-9' represent missing, '1' represents unaffected and '2' represents affected. " + + 'Any other characters are interpreted as string phenotype values.' + ) + } + return null + }, + ], + }), + new TemplateColumn({ + id: 'notes', + key: 'Notes', + index: 6, + required: false, + parser: parseString, + validators: [], + }), +]) + +export default PedigreeTemplateColumns diff --git a/ui/pages/Project/components/DataLoadingWizard/templates/utils/parsers.js b/ui/pages/Project/components/DataLoadingWizard/templates/utils/parsers.js new file mode 100644 index 0000000000..935b50b7d8 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/templates/utils/parsers.js @@ -0,0 +1,83 @@ +/** + * @param {?string} value + * @returns {string|null} + */ +export const parseString = (value) => { + if (value?.toString()?.trim()) { + return value.toString().trim() + } + return null +} + +/** + * @param {?string} value + * @param {string} separator + * @returns {string[]} + */ +export const parseStringArray = (value, separator = ',') => { + if (!value) return [] + + if (value.toString().trim()) { + return Array.from( + new Set( + value + .toString() + .trim() + .split(separator) + .map(p => p?.trim() || null) + .filter(p => p?.trim() && p.trim().length > 0), + ), + ) + } + return [] +} + +/** + * @param {?string} value + * @returns {number|null} + */ +export const parseNumber = (value) => { + const n = Number(value?.toString()?.trim()) + if (Number.isFinite(n)) { + return n + } + return null +} + +/** + * @param {?string} value + * @returns {boolean|null} + */ +export const parseBoolean = (value) => { + const b = value?.toString()?.trim()?.toLowerCase() + if (value) { + switch (b) { + case 'true': return true + case 'false': return false + default: return null + } + } + return null +} + +/** + * @param {?string} value + * @returns {string|null} + */ +export const parseYear = (value) => { + const d = value?.toString()?.trim() + const match = d.match(/(?\d{4})/) + + if (match?.groups?.year) { + return match.groups.year + } + return null +} + +export default { + parseString, + parseStringArray, + parseNumber, + parseBoolean, + parseYear, +} diff --git a/ui/pages/Project/components/DataLoadingWizard/templates/utils/validators.js b/ui/pages/Project/components/DataLoadingWizard/templates/utils/validators.js new file mode 100644 index 0000000000..a9c4aaebe9 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/templates/utils/validators.js @@ -0,0 +1,126 @@ +import * as _ from 'lodash' +import { INHERITANCE_MODE_OPTIONS, ONSET_AGE_OPTIONS } from '../../../../constants' + +/** + * @param {*[]} collection + * @param {?string} message + * @returns {string[]} + */ +export const validateUnique = (collection, message = null) => { + const counts = _.countBy(collection) + + const errors = _.zip(collection, Object.values(counts)).map(([key, count]) => { + if (count > 1) { + return message || `'${key}' is not unique, and was counted ${count} times` + } + return null + }) + + return errors.filter(e => e != null) +} + +/** + * @param {?string} value + * @param {?string} message + * @returns {string|null} + */ +export const validateHpoTerm = (value, message = null) => { + if (!value) return null + + if (!value.toString().toUpperCase().match(/HP:\d{7}/)) { + return message || `'${value}' is not a valid HPO term` + } + return null +} + +/** + * @param {string[]} collection + * @param {?string} message + * @returns {string[]} + */ +export const validateHpoTerms = (collection, message = null) => ( + collection + .map(v => validateHpoTerm(v, message)) + .filter(e => e != null) +) + +/** + * @param {?string} value + * @param {?string} message + * @returns {string|null} + */ +export const validateOnsetCategory = (value, message = null) => { + if (!value) return null + + const categories = ONSET_AGE_OPTIONS.map(o => o.text) + + if (!categories.map(c => c.toLowerCase()).includes(value.toLowerCase())) { + return message || `'${value}' is not one of ${categories.join(', ')}` + } + + return null +} + +/** + * @param {?string} value + * @param {?string} message + * @returns {string|null} + */ +export const validateModeOfInheritanceTerm = (value, message = null) => { + if (!value) return null + + const categories = INHERITANCE_MODE_OPTIONS.map(o => o.text) + + if (!categories.map(c => c.toLowerCase()).includes(value.toLowerCase())) { + return message || `'${value}' is not one of ${categories.join(', ')}` + } + + return null +} + +/** + * @param {string[]} collection + * @param {?string} message + * @returns {string[]} + */ +export const validateModeOfInheritanceTerms = (collection, message) => ( + collection + .map(v => validateModeOfInheritanceTerm(v, message)) + .filter(e => e != null) +) + +/** + * @param {?string} value + * @param {?string} message + * @returns {string|null} + */ +export const validateOmimTerm = (value, message = null) => { + if (!value) return null + + if (!value.toString().toUpperCase().match(/^OMIM:\d+$/)) { + return message || `'${value}' is not a valid OMIM term` + } + return null +} + +/** + * @param {string[]} collection + * @param {?string} message + * @returns {string[]} + */ +export const validateOmimTerms = (collection, message) => ( + collection + .map(v => validateOmimTerm(v, message)) + .filter(e => e != null) +) + +export default { + validateUnique, + validateHpoTerm, + validateHpoTerms, + validateOnsetCategory, + validateModeOfInheritanceTerm, + validateModeOfInheritanceTerms, + validateOmimTerm, + validateOmimTerms, +} diff --git a/ui/pages/Project/components/DataLoadingWizard/ui/Centered.jsx b/ui/pages/Project/components/DataLoadingWizard/ui/Centered.jsx new file mode 100644 index 0000000000..0e82f3fd03 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/ui/Centered.jsx @@ -0,0 +1,8 @@ +import styled from 'styled-components' + +const Centered = styled.div` + display: flex; + justify-content: center; +` + +export default Centered diff --git a/ui/pages/Project/components/DataLoadingWizard/ui/FormSection.jsx b/ui/pages/Project/components/DataLoadingWizard/ui/FormSection.jsx new file mode 100644 index 0000000000..66af0e103e --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/ui/FormSection.jsx @@ -0,0 +1,8 @@ +import styled from 'styled-components' + +const FormSection = styled.div` + margin-top: 2em; + margin-bottom: 2em; +` + +export default FormSection diff --git a/ui/pages/Project/components/DataLoadingWizard/ui/FormStepButtons.jsx b/ui/pages/Project/components/DataLoadingWizard/ui/FormStepButtons.jsx new file mode 100644 index 0000000000..fec464d5ee --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/ui/FormStepButtons.jsx @@ -0,0 +1,70 @@ +import React from 'react' +import PropTypes from 'prop-types' +import { Button } from 'semantic-ui-react' +import styled from 'styled-components' + +const FormButtonGroup = styled.div` + display: flex; + justify-content: right; +` + +const FormStepButtons = ({ isLastStep, onNext, onBack, onSubmit, enableNext, enableSubmit, loading, size }) => { + const submitButton = ( + + ) + + return ( + + + + { + isLastStep ? submitButton : null + } + + ) +} + +FormStepButtons.propTypes = { + isLastStep: PropTypes.bool, + onNext: PropTypes.func, + onBack: PropTypes.func, + onSubmit: PropTypes.func, + enableNext: PropTypes.bool, + enableSubmit: PropTypes.bool, + loading: PropTypes.bool, + size: PropTypes.string, +} + +FormStepButtons.defaultProps = { + isLastStep: false, + onNext: () => {}, + onBack: () => {}, + onSubmit: () => {}, + enableNext: false, + enableSubmit: false, + loading: false, + size: 'small', +} + +export default FormStepButtons diff --git a/ui/pages/Project/components/DataLoadingWizard/ui/ReadableText.jsx b/ui/pages/Project/components/DataLoadingWizard/ui/ReadableText.jsx new file mode 100644 index 0000000000..175c5552f4 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/ui/ReadableText.jsx @@ -0,0 +1,19 @@ +import styled from 'styled-components' +import PropTypes from 'prop-types' + +const ReadableText = styled.p` + font-size: ${props => props.fontSize}; + line-height: ${props => props.lineHeight}; +` + +ReadableText.propTypes = { + fontSize: PropTypes.string, + lineHeight: PropTypes.number, +} + +ReadableText.defaultProps = { + fontSize: '16px', + lineHeight: 1.6, +} + +export default ReadableText diff --git a/ui/pages/Project/components/DataLoadingWizard/ui/Scrollable.jsx b/ui/pages/Project/components/DataLoadingWizard/ui/Scrollable.jsx new file mode 100644 index 0000000000..248e0bed38 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/ui/Scrollable.jsx @@ -0,0 +1,19 @@ +import styled from 'styled-components' +import PropTypes from 'prop-types' + +const Scrollable = styled.div` + overflow-x: ${props => props.x}; + overflow-y: ${props => props.y}; +` + +Scrollable.propTypes = { + x: PropTypes.oneOf(['visible', 'hidden', 'scroll', 'auto', 'initial', 'inherit']), + y: PropTypes.oneOf(['visible', 'hidden', 'scroll', 'auto', 'initial', 'inherit']), +} + +Scrollable.defaultProps = { + x: 'initial', + y: 'initial', +} + +export default Scrollable diff --git a/ui/pages/Project/components/DataLoadingWizard/ui/WithSpace.jsx b/ui/pages/Project/components/DataLoadingWizard/ui/WithSpace.jsx new file mode 100644 index 0000000000..208b6c8772 --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/ui/WithSpace.jsx @@ -0,0 +1,27 @@ +import styled from 'styled-components' +import PropTypes from 'prop-types' + +const WithSpace = styled.div` + ${props => `${props.type}-top: ${props.top}`}; + ${props => `${props.type}-bottom: ${props.bottom}`}; + ${props => `${props.type}-left: ${props.left}`}; + ${props => `${props.type}-right: ${props.right}`}; +` + +WithSpace.propTypes = { + type: PropTypes.oneOf(['padding', 'margin']), + top: PropTypes.string, + bottom: PropTypes.string, + left: PropTypes.string, + right: PropTypes.string, +} + +WithSpace.defaultProps = { + type: 'margin', + top: 'initial', + bottom: 'initial', + left: 'initial', + right: 'initial', +} + +export default WithSpace diff --git a/ui/pages/Project/components/DataLoadingWizard/ui/index.js b/ui/pages/Project/components/DataLoadingWizard/ui/index.js new file mode 100644 index 0000000000..5621e6685b --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizard/ui/index.js @@ -0,0 +1,15 @@ +import Centered from './Centered' +import FormSection from './FormSection' +import FormStepButtons from './FormStepButtons' +import ReadableText from './ReadableText' +import WithSpace from './WithSpace' +import Scrollable from './Scrollable' + +export { + Centered, + FormSection, + FormStepButtons, + ReadableText, + WithSpace, + Scrollable, +} diff --git a/ui/pages/Project/components/DataLoadingWizardPage.jsx b/ui/pages/Project/components/DataLoadingWizardPage.jsx new file mode 100644 index 0000000000..79bbb72cee --- /dev/null +++ b/ui/pages/Project/components/DataLoadingWizardPage.jsx @@ -0,0 +1,26 @@ +import React from 'react' +import PropTypes from 'prop-types' +import { connect } from 'react-redux' + +import { Error401 } from '../../../shared/components/page/Errors' +import { getCurrentProject } from '../selectors' + +import DataLoadingWizardForm from './DataLoadingWizardForm/DataLoadingWizardForm' + +const DataLoadingWizardPage = ({ project }) => { + if (!project.canEdit) { + return + } + + return +} + +DataLoadingWizardPage.propTypes = { + project: PropTypes.object, +} + +const mapStateToProps = state => ({ + project: getCurrentProject(state), +}) + +export default connect(mapStateToProps)(DataLoadingWizardPage)