diff --git a/src/shared/metadata/fetch-metadata.js b/src/shared/metadata/fetch-metadata.js new file mode 100644 index 0000000000..08d623b763 --- /dev/null +++ b/src/shared/metadata/fetch-metadata.js @@ -0,0 +1,301 @@ +/* + * Client-side replacement for the bespoke `GET /api/dataEntry/metadata` endpoint. + * + * That endpoint walks the whole metadata entity-graph via lazy Hibernate + * associations, which triggers an N+1 query explosion (thousands of SQL + * statements) on large instances. Instead of one call to that endpoint, this + * module fetches each metadata type FLAT and IN BULK from the standard, + * cache-friendly metadata endpoints and stitches the result together on the + * client into the exact same shape the app already consumes. + * + * Both endpoints share the server's `FieldFilterService`, so we reuse the + * endpoint's original `fields=` strings verbatim: the per-type JSON is then + * produced by the same code path and is field-for-field identical. The four + * jobs the bespoke endpoint did that we must reproduce here are: + * + * 1. Transitive closure + dedup — walk data sets -> elements -> combos -> + * categories -> options -> option-combos -> option-sets and return seven + * de-duplicated arrays. We do this by collecting ids at each stage and + * fetching the next type by id (chunked to keep URLs bounded). + * 2. Data-write ACL filtering — the endpoint only returns data sets (and + * attribute category-options) the user may WRITE. Standard endpoints + * default to READ sharing, so we request the computed `access` field and + * filter on `access.data.write` client-side, then strip `access` again. + * 3. Org-unit associations — `organisationUnits~pluck[id]` on the data + * set, matching the endpoint's association output. + * 4. Server-computed fields — the endpoint populates the transient + * `explodedNumerator/Denominator` on indicators. This app renders + * indicators from the raw `numerator`/`denominator` and never reads the + * exploded fields, so we intentionally skip that computation. + * + * See DHIS2-21757. + */ + +// Field-filter strings, copied from the server's +// DefaultDataSetMetadataExportService so the per-type JSON matches exactly. +// +// `!href` is appended to each string: the standard list endpoints inject an +// `href` link that the bespoke endpoint does not, so we exclude it to keep the +// output byte-identical (the app never reads it). +const FIELDS_DATA_SETS = + ':simple,!href,categoryCombo[id],formType,dataEntryForm[id],' + + 'dataInputPeriods[period,openingDate,closingDate],' + + 'indicators~pluck[id],' + + 'compulsoryDataElementOperands[dataElement[id],categoryOptionCombo[id]],' + + 'sections[:simple,displayOptions,dataElements~pluck[id],indicators~pluck[id],' + + 'greyedFields[dataElement[id],categoryOptionCombo[id]]],' + + // added by the endpoint after field-filtering; requested directly here: + 'dataSetElements[dataElement[id],categoryCombo[id]],' + + 'organisationUnits~pluck[id]' + +const FIELDS_DATA_ELEMENTS = + ':identifiable,!href,displayName,displayShortName,displayFormName,' + + 'zeroIsSignificant,valueType,aggregationType,categoryCombo[id],optionSet[id],' + + 'commentOptionSet,description' + +// The bespoke endpoint additionally populates the transient +// `explodedNumerator`/`explodedDenominator` via ExpressionService. The standard +// /api/indicators endpoint does not compute them, and this app renders +// indicators from the raw `numerator`/`denominator` (indicators-table-body.jsx) +// and never reads the exploded fields, so we intentionally omit them. +const FIELDS_INDICATORS = ':simple,!href,indicatorType[factor]' + +const FIELDS_DATA_ELEMENT_CAT_COMBOS = + ':simple,!href,isDefault,categories~pluck[id],' + + 'categoryOptionCombos[id,code,name,displayName,categoryOptions~pluck[id]]' + +const FIELDS_DATA_SET_CAT_COMBOS = + ':simple,!href,isDefault,categories~pluck[id]' + +const FIELDS_CATEGORIES = ':simple,!href,categoryOptions~pluck[id]' + +const FIELDS_CATEGORY_OPTIONS = ':simple,!href,organisationUnits~pluck[id]' + +const FIELDS_OPTION_SETS = ':simple,!href,options[id,code,displayName]' + +// Max number of ids per `id:in:[...]` filter, to keep request URLs well under +// the servlet container's header-size limit (~8KB). 200 ids * ~12 chars is +// comfortably below that. +const ID_CHUNK_SIZE = 200 + +const chunk = (array, size) => { + const chunks = [] + for (let i = 0; i < array.length; i += size) { + chunks.push(array.slice(i, i + size)) + } + return chunks +} + +const uniq = (array) => Array.from(new Set(array)) + +/** + * Fetch every object of `resource` matching the given `ids`, flat and in bulk, + * chunking the id filter across multiple requests and concatenating the result. + * Returns [] when there are no ids (issues no request). + */ +const fetchByIds = async (engine, resource, ids, fields) => { + if (!ids.length) { + return [] + } + const chunks = chunk(ids, ID_CHUNK_SIZE) + const responses = await Promise.all( + chunks.map((idChunk) => + engine.query({ + [resource]: { + resource, + params: { + fields, + filter: `id:in:[${idChunk.join(',')}]`, + paging: false, + }, + }, + }) + ) + ) + return responses.flatMap((response) => response[resource][resource]) +} + +const canDataWrite = (object) => object?.access?.data?.write === true + +// The `access` field is only requested so we can apply data-write filtering; +// it is not part of the endpoint's response, so strip it before assembling. +const stripAccess = (object) => { + const rest = { ...object } + delete rest.access + return rest +} + +/** + * Reproduce `GET /api/dataEntry/metadata` using flat, per-type requests to the + * standard metadata endpoints. Returns the same seven-array object the endpoint + * returns (before the client-side `hashArraysInObject` transform). + * + * @param {object} engine an @dhis2/app-runtime data engine + * @returns {Promise} { dataSets, dataElements, indicators, + * categoryCombos, categories, categoryOptions, optionSets } + */ +export const fetchMetadata = async (engine) => { + // --- Stage 1: writable data sets (the roots of the closure) -------------- + // Request `access` so we can keep only data sets the user may write, exactly + // as the endpoint's getDataWriteAll(DataSet.class) does. + const dataSetsResponse = await engine.query({ + dataSets: { + resource: 'dataSets', + params: { + fields: `${FIELDS_DATA_SETS},access`, + paging: false, + }, + }, + }) + const dataSets = dataSetsResponse.dataSets.dataSets + .filter(canDataWrite) + .map(stripAccess) + + // Ids reachable directly from the writable data sets. + const dataElementIds = uniq( + dataSets.flatMap((ds) => + (ds.dataSetElements ?? []).map((dse) => dse.dataElement.id) + ) + ) + const indicatorIds = uniq(dataSets.flatMap((ds) => ds.indicators ?? [])) + // Per-data-set-element categoryCombo overrides (a data element's combo can + // be overridden per data set). These belong to the "data element" combo set. + const overrideCategoryComboIds = uniq( + dataSets.flatMap((ds) => + (ds.dataSetElements ?? []) + .map((dse) => dse.categoryCombo?.id) + .filter(Boolean) + ) + ) + // Each data set's own (attribute) categoryCombo. + const dataSetCategoryComboIds = uniq( + dataSets.map((ds) => ds.categoryCombo?.id).filter(Boolean) + ) + + // --- Stage 2: data elements & indicators (fetched by id, in parallel) ---- + const [dataElements, indicators] = await Promise.all([ + fetchByIds(engine, 'dataElements', dataElementIds, FIELDS_DATA_ELEMENTS), + fetchByIds(engine, 'indicators', indicatorIds, FIELDS_INDICATORS), + ]) + + // A data element's category combos = its own combo + any per-data-set + // overrides (matches DataElement.getCategoryCombos() server-side). + const dataElementOwnCategoryComboIds = dataElements + .map((de) => de.categoryCombo?.id) + .filter(Boolean) + const dataElementCategoryComboIds = uniq([ + ...dataElementOwnCategoryComboIds, + ...overrideCategoryComboIds, + ]) + + // Option sets referenced by data elements (value option set + comment + // option set), matching the endpoint's getOptionSets(dataElements). + const optionSetIds = uniq( + dataElements.flatMap((de) => + [de.optionSet?.id, de.commentOptionSet?.id].filter(Boolean) + ) + ) + + // --- Stage 3: category combos -------------------------------------------- + // Data-element combos carry their categoryOptionCombos; attribute (data set) + // combos do not. A combo used by both is emitted once, in the data-element + // group (the richer field set), exactly like the endpoint's removeAll(). + const dataSetOnlyCategoryComboIds = dataSetCategoryComboIds.filter( + (id) => !dataElementCategoryComboIds.includes(id) + ) + const [dataElementCategoryCombos, dataSetOnlyCategoryCombos] = + await Promise.all([ + fetchByIds( + engine, + 'categoryCombos', + dataElementCategoryComboIds, + FIELDS_DATA_ELEMENT_CAT_COMBOS + ), + fetchByIds( + engine, + 'categoryCombos', + dataSetOnlyCategoryComboIds, + FIELDS_DATA_SET_CAT_COMBOS + ), + ]) + const categoryCombos = [ + ...dataElementCategoryCombos, + ...dataSetOnlyCategoryCombos, + ] + + // --- Stage 4: categories -------------------------------------------------- + // Categories reached via data-element combos vs. via data set (attribute) + // combos are treated differently for category-option ACL (see stage 5), so + // track the two sets separately. Note: the data set category set uses the + // FULL set of data set combos (before the removeAll above), matching the + // server, where dataSetCategories is computed before dataSetCategoryCombos + // has the data-element combos removed. + const categoriesOf = (combos) => + uniq(combos.flatMap((cc) => cc.categories ?? [])) + + const dataElementCategoryIds = categoriesOf(dataElementCategoryCombos) + // For the attribute combos, look up each combo's categories from whichever + // group it was fetched in (a combo may be shared). + const categoryComboById = new Map( + categoryCombos.map((cc) => [cc.id, cc]) + ) + const dataSetCategoryIds = uniq( + dataSetCategoryComboIds.flatMap( + (id) => categoryComboById.get(id)?.categories ?? [] + ) + ) + const categoryIds = uniq([ + ...dataElementCategoryIds, + ...dataSetCategoryIds, + ]) + + const categories = await fetchByIds( + engine, + 'categories', + categoryIds, + FIELDS_CATEGORIES + ) + + // --- Stage 5: category options (with data-write ACL) --------------------- + // Options of data-element categories are included unconditionally; options + // of data set (attribute) categories are included only if the user may + // write them (getDataWriteCategoryOptions server-side). + const categoryById = new Map(categories.map((c) => [c.id, c])) + const optionsOf = (ids) => + ids.flatMap((id) => categoryById.get(id)?.categoryOptions ?? []) + + const dataElementOptionIds = new Set(optionsOf(dataElementCategoryIds)) + const dataSetOptionIds = new Set(optionsOf(dataSetCategoryIds)) + const allOptionIds = uniq([...dataElementOptionIds, ...dataSetOptionIds]) + + const fetchedOptions = await fetchByIds( + engine, + 'categoryOptions', + allOptionIds, + `${FIELDS_CATEGORY_OPTIONS},access` + ) + const categoryOptions = fetchedOptions + .filter( + (option) => + dataElementOptionIds.has(option.id) || canDataWrite(option) + ) + .map(stripAccess) + + // --- Stage 6: option sets ------------------------------------------------- + const optionSets = await fetchByIds( + engine, + 'optionSets', + optionSetIds, + FIELDS_OPTION_SETS + ) + + return { + dataSets, + dataElements, + indicators, + categoryCombos, + categories, + categoryOptions, + optionSets, + } +} diff --git a/src/shared/metadata/fetch-metadata.test.js b/src/shared/metadata/fetch-metadata.test.js new file mode 100644 index 0000000000..980e948516 --- /dev/null +++ b/src/shared/metadata/fetch-metadata.test.js @@ -0,0 +1,265 @@ +import { fetchMetadata } from './fetch-metadata.js' + +/* + * A mock @dhis2/app-runtime data engine that serves per-type fixtures from an + * in-memory "database", honouring the `filter: id:in:[...]` and returning the + * requested `fields` faithfully enough for the assembly logic. It records every + * request so tests can assert on query count / chunking. + */ +const createMockEngine = (db) => { + const calls = [] + + const query = (queries) => { + const result = {} + for (const [name, def] of Object.entries(queries)) { + const { resource, params = {} } = def + calls.push({ resource, params }) + + let rows = db[resource] ?? [] + + // Apply `id:in:[...]` filter if present. + const filter = params.filter + if (filter && filter.startsWith('id:in:[')) { + const ids = filter + .slice('id:in:['.length, -1) + .split(',') + .filter(Boolean) + const idSet = new Set(ids) + rows = rows.filter((row) => idSet.has(row.id)) + } + + result[name] = { [resource]: rows } + } + return Promise.resolve(result) + } + + return { engine: { query }, calls } +} + +describe('fetchMetadata', () => { + /* + * Fixture graph: + * + * dataSets: + * ds_write (access.data.write=true) -> keep + * ds_read (access.data.write=false) -> drop (read-only) + * + * ds_write: + * categoryCombo: cc_attr (attribute combo, no COCs) + * dataSetElements: + * de1 (no override -> uses de1.categoryCombo = cc_de) + * de2 (override -> cc_override) + * indicators: [ind1] + * + * de1.categoryCombo = cc_de (data-element combo, has COCs) + * de1.optionSet = os1, de1.commentOptionSet = os2 + * de2.categoryCombo = cc_de + * + * cc_de -> categories [cat_de] + * cc_override -> categories [cat_de] + * cc_attr -> categories [cat_attr] + * + * cat_de -> categoryOptions [co_de] (data-element category: no ACL) + * cat_attr -> categoryOptions [co_attr_ok, co_attr_no] (attribute: ACL applies) + * + * co_attr_ok.access.data.write = true -> keep + * co_attr_no.access.data.write = false -> drop + */ + const buildDb = () => ({ + dataSets: [ + { + id: 'ds_write', + displayName: 'Writable', + categoryCombo: { id: 'cc_attr' }, + organisationUnits: ['ou1'], + indicators: ['ind1'], + dataSetElements: [ + { dataElement: { id: 'de1' } }, + { + dataElement: { id: 'de2' }, + categoryCombo: { id: 'cc_override' }, + }, + ], + access: { data: { write: true } }, + }, + { + id: 'ds_read', + displayName: 'Read only', + categoryCombo: { id: 'cc_attr' }, + organisationUnits: ['ou1'], + indicators: ['ind_read'], + dataSetElements: [{ dataElement: { id: 'de_read' } }], + access: { data: { write: false } }, + }, + ], + dataElements: [ + { + id: 'de1', + displayName: 'DE 1', + categoryCombo: { id: 'cc_de' }, + optionSet: { id: 'os1' }, + commentOptionSet: { id: 'os2' }, + }, + { id: 'de2', displayName: 'DE 2', categoryCombo: { id: 'cc_de' } }, + { id: 'de_read', displayName: 'should not be fetched' }, + ], + indicators: [ + { id: 'ind1', displayName: 'Ind 1' }, + { id: 'ind_read', displayName: 'should not be fetched' }, + ], + categoryCombos: [ + { + id: 'cc_de', + isDefault: false, + categories: ['cat_de'], + categoryOptionCombos: [{ id: 'coc_de' }], + }, + { + id: 'cc_override', + isDefault: false, + categories: ['cat_de'], + categoryOptionCombos: [{ id: 'coc_override' }], + }, + { id: 'cc_attr', isDefault: false, categories: ['cat_attr'] }, + ], + categories: [ + { id: 'cat_de', categoryOptions: ['co_de'] }, + { id: 'cat_attr', categoryOptions: ['co_attr_ok', 'co_attr_no'] }, + ], + categoryOptions: [ + { id: 'co_de', displayName: 'Opt DE' }, + { + id: 'co_attr_ok', + displayName: 'Opt attr ok', + access: { data: { write: true } }, + }, + { + id: 'co_attr_no', + displayName: 'Opt attr no', + access: { data: { write: false } }, + }, + ], + optionSets: [ + { id: 'os1', options: [{ id: 'opt1' }] }, + { id: 'os2', options: [{ id: 'opt2' }] }, + ], + }) + + it('keeps only data sets the user can write, and strips the access field', async () => { + const { engine } = createMockEngine(buildDb()) + const result = await fetchMetadata(engine) + + expect(result.dataSets.map((ds) => ds.id)).toEqual(['ds_write']) + expect(result.dataSets[0]).not.toHaveProperty('access') + }) + + it('builds the transitive closure and dedups, excluding the read-only branch', async () => { + const { engine } = createMockEngine(buildDb()) + const result = await fetchMetadata(engine) + + expect(result.dataElements.map((de) => de.id).sort()).toEqual([ + 'de1', + 'de2', + ]) + expect(result.indicators.map((i) => i.id)).toEqual(['ind1']) + // cc_de + cc_override (data-element combos) + cc_attr (attribute combo) + expect(result.categoryCombos.map((cc) => cc.id).sort()).toEqual([ + 'cc_attr', + 'cc_de', + 'cc_override', + ]) + expect(result.categories.map((c) => c.id).sort()).toEqual([ + 'cat_attr', + 'cat_de', + ]) + expect(result.optionSets.map((os) => os.id).sort()).toEqual([ + 'os1', + 'os2', + ]) + // de_read / ind_read must never be fetched + expect(result.dataElements.map((de) => de.id)).not.toContain('de_read') + }) + + it('includes per-data-set-element categoryCombo overrides as data-element combos (with COCs)', async () => { + const { engine } = createMockEngine(buildDb()) + const result = await fetchMetadata(engine) + + const override = result.categoryCombos.find( + (cc) => cc.id === 'cc_override' + ) + expect(override).toBeDefined() + expect(override.categoryOptionCombos).toBeDefined() + }) + + it('applies data-write ACL only to attribute-category options, keeps data-element options unconditionally', async () => { + const { engine } = createMockEngine(buildDb()) + const result = await fetchMetadata(engine) + + const optionIds = result.categoryOptions.map((co) => co.id).sort() + // co_de: data-element category option -> kept regardless of access + // co_attr_ok: attribute option with write access -> kept + // co_attr_no: attribute option without write access -> dropped + expect(optionIds).toEqual(['co_attr_ok', 'co_de']) + result.categoryOptions.forEach((co) => + expect(co).not.toHaveProperty('access') + ) + }) + + it('an attribute option is kept regardless of access when it also belongs to a data-element category', async () => { + const db = buildDb() + // Make the data-element category also reference the un-writable attr option. + db.categories.find((c) => c.id === 'cat_de').categoryOptions = [ + 'co_de', + 'co_attr_no', + ] + const { engine } = createMockEngine(db) + const result = await fetchMetadata(engine) + + expect(result.categoryOptions.map((co) => co.id)).toContain('co_attr_no') + }) + + it('returns the seven expected top-level arrays', async () => { + const { engine } = createMockEngine(buildDb()) + const result = await fetchMetadata(engine) + + expect(Object.keys(result).sort()).toEqual([ + 'categories', + 'categoryCombos', + 'categoryOptions', + 'dataElements', + 'dataSets', + 'indicators', + 'optionSets', + ]) + }) + + it('issues no request for a type when the closure yields no ids of that type', async () => { + const db = buildDb() + // Remove all indicators from the writable data set. + db.dataSets.find((ds) => ds.id === 'ds_write').indicators = [] + const { engine, calls } = createMockEngine(db) + await fetchMetadata(engine) + + expect(calls.some((c) => c.resource === 'indicators')).toBe(false) + }) + + it('chunks large id filters into multiple bounded requests', async () => { + const db = buildDb() + // 250 data elements referenced by the writable data set -> 2 chunks of 200. + const manyDes = Array.from({ length: 250 }, (_, i) => ({ + id: `de_bulk_${i}`, + displayName: `Bulk ${i}`, + categoryCombo: { id: 'cc_de' }, + })) + db.dataElements.push(...manyDes) + db.dataSets.find((ds) => ds.id === 'ds_write').dataSetElements = manyDes.map( + (de) => ({ dataElement: { id: de.id } }) + ) + const { engine, calls } = createMockEngine(db) + const result = await fetchMetadata(engine) + + const dataElementCalls = calls.filter((c) => c.resource === 'dataElements') + expect(dataElementCalls.length).toBe(2) + expect(result.dataElements.length).toBe(250) + }) +}) diff --git a/src/shared/metadata/use-metadata.js b/src/shared/metadata/use-metadata.js index e9c8b967a2..7e20963b3e 100644 --- a/src/shared/metadata/use-metadata.js +++ b/src/shared/metadata/use-metadata.js @@ -1,5 +1,7 @@ +import { useDataEngine } from '@dhis2/app-runtime' import { useQuery } from '@tanstack/react-query' import { createSelector } from 'reselect' +import { fetchMetadata } from './fetch-metadata.js' import { hashArraysInObject } from './utils.js' const selectorFunction = createSelector( @@ -7,15 +9,26 @@ const selectorFunction = createSelector( (data) => hashArraysInObject(data) ) -const queryKey = [`/dataEntry/metadata`] - -const queryOpts = { - refetchOnMount: false, - select: selectorFunction, - staleTime: 24 * 60 * 60 * 1000, -} +// Previously this fetched the bespoke `/dataEntry/metadata` endpoint, which +// caused an N+1 query explosion server-side (DHIS2-21757). It now rebuilds the +// same metadata client-side from flat, per-type requests to the standard +// endpoints (see fetch-metadata.js). The return shape is unchanged, so every +// consumer and selector is untouched. +const queryKey = ['dataEntryMetadata'] export const useMetadata = () => { - const metadataQuery = useQuery(queryKey, queryOpts) + const engine = useDataEngine() + + const metadataQuery = useQuery(queryKey, () => fetchMetadata(engine), { + refetchOnMount: false, + select: selectorFunction, + staleTime: 24 * 60 * 60 * 1000, + // Persist the preloaded metadata to IndexedDB so it survives reloads and + // is available offline (DHIS2-21757: the endpoint's purpose is offline + // preloading; the previous query set no meta.persist, so metadata only + // ever survived in the service-worker runtime cache). + meta: { persist: true }, + }) + return metadataQuery }