Merge pull request #167 from UgnisSoftware/UGN-369

feat UGN-369 - Improve performance for large files

Author: masiulis

src/steps/UploadStep/components/DropZone.tsx

@@ -39,7 +39,12 @@ export const DropZone = ({ onContinue, isLoading }: DropZoneProps) => {
     onDrop: async ([file]) => {
       setLoading(true)
       const arrayBuffer = await readFileAsync(file)
-      const workbook = XLSX.read(arrayBuffer, { cellDates: true, dateNF: dateFormat, raw: parseRaw })
+      const workbook = XLSX.read(arrayBuffer, {
+        cellDates: true,
+        dateNF: dateFormat,
+        raw: parseRaw,
+        dense: true,
+      })
       setLoading(false)
       onContinue(workbook)
     },

src/steps/ValidationStep/ValidationStep.tsx

@@ -75,17 +75,25 @@ export const ValidationStep = <T extends string>({ initialData }: Props<T>) => {
 
   const rowKeyGetter = useCallback((row: Data<T> & Meta) => row.__index, [])
 
-  const submitData = () => {
-    const all = data.map(({ __index, __errors, ...value }) => ({ ...value })) as unknown as Data<T>[]
-    const validData = all.filter((value, index) => {
-      const originalValue = data[index]
-      if (originalValue?.__errors) {
-        return !Object.values(originalValue.__errors)?.filter((err) => err.level === "error").length
-      }
-      return true
-    })
-    const invalidData = all.filter((value) => !validData.includes(value))
-    onSubmit({ validData, invalidData, all: data })
+  const submitData = async () => {
+    const calculatedData = data.reduce(
+      (acc, value) => {
+        const { __index, __errors, ...values } = value
+        if (__errors) {
+          for (const key in __errors) {
+            if (__errors[key].level === "error") {
+              acc.invalidData.push(values as unknown as Data<T>)
+              return acc
+            }
+          }
+        }
+        acc.validData.push(values as unknown as Data<T>)
+        return acc
+      },
+      { validData: [] as Data<T>[], invalidData: [] as Data<T>[], all: data },
+    )
+    onSubmit(calculatedData)
+    setShowSubmitAlert(false)
     onClose()
   }
   const onContinue = () => {
@@ -104,14 +112,7 @@ export const ValidationStep = <T extends string>({ initialData }: Props<T>) => {
 
   return (
     <>
-      <SubmitDataAlert
-        isOpen={showSubmitAlert}
-        onClose={() => setShowSubmitAlert(false)}
-        onConfirm={() => {
-          setShowSubmitAlert(false)
-          submitData()
-        }}
-      />
+      <SubmitDataAlert isOpen={showSubmitAlert} onClose={() => setShowSubmitAlert(false)} onConfirm={submitData} />
       <ModalBody pb={0}>
         <Box display="flex" justifyContent="space-between" alignItems="center" mb="2rem" flexWrap="wrap" gap="8px">
           <Heading sx={styles.heading}>{translations.validationStep.title}</Heading>

src/stories/Default.stories.tsx

@@ -2,13 +2,14 @@ import { ReactSpreadsheetImport } from "../ReactSpreadsheetImport"
 import { Box, Link, Code, Button, useDisclosure } from "@chakra-ui/react"
 import { mockRsiValues } from "./mockRsiValues"
 import { useState } from "react"
+import type { Result } from "src/types"
 
 export default {
   title: "React spreadsheet import",
 }
 
 export const Basic = () => {
-  const [data, setData] = useState<any>(null)
+  const [data, setData] = useState<Result<any> | null>(null)
   const { isOpen, onOpen, onClose } = useDisclosure()
   return (
     <>
@@ -22,9 +23,9 @@ export const Basic = () => {
         Download example file
       </Link>
       <ReactSpreadsheetImport {...mockRsiValues} isOpen={isOpen} onClose={onClose} onSubmit={setData} />
-      {data && (
+      {!!data && (
         <Box pt={64} display="flex" gap="8px" flexDirection="column">
-          <b>Returned data:</b>
+          <b>Returned data (showing first 100 rows):</b>
           <Code
             display="flex"
             alignItems="center"
@@ -34,7 +35,17 @@ export const Basic = () => {
             color="white"
             p={32}
           >
-            <pre>{JSON.stringify(data, undefined, 4)}</pre>
+            <pre>
+              {JSON.stringify(
+                {
+                  validData: data.validData.slice(0, 100),
+                  invalidData: data.invalidData.slice(0, 100),
+                  all: data.all.slice(0, 100),
+                },
+                undefined,
+                4,
+              )}
+            </pre>
           </Code>
         </Box>
       )}