From 1a2f3068e53cc5af36fea656011e520aeb6d5ea9 Mon Sep 17 00:00:00 2001 From: slaveeks Date: Tue, 5 Aug 2025 23:10:49 +0300 Subject: [PATCH 01/12] Moved merging to api --- package.json | 5 ++ src/models/eventsFactory.js | 40 ++++++++++++-- src/typeDefs/event.ts | 5 -- src/utils/merge.ts | 106 ++++++++++++++++++++++++++++++++++++ yarn.lock | 34 ++++++++++++ 5 files changed, 180 insertions(+), 10 deletions(-) create mode 100644 src/utils/merge.ts diff --git a/package.json b/package.json index f9b32ca1..78526165 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,8 @@ "devDependencies": { "@shelf/jest-mongodb": "^1.2.2", "@types/jest": "^26.0.8", + "@types/lodash.clonedeep": "^4.5.9", + "@types/lodash.mergewith": "^4.6.9", "eslint": "^6.7.2", "eslint-config-codex": "1.2.4", "eslint-plugin-import": "^2.19.1", @@ -38,6 +40,7 @@ "@graphql-tools/utils": "^8.9.0", "@hawk.so/nodejs": "^3.1.1", "@hawk.so/types": "^0.1.31", + "@n1ru4l/json-patch-plus": "^0.2.0", "@types/amqp-connection-manager": "^2.0.4", "@types/bson": "^4.0.5", "@types/debug": "^4.1.5", @@ -70,6 +73,8 @@ "graphql-upload": "^13", "jsonwebtoken": "^8.5.1", "lodash": "^4.17.15", + "lodash.clonedeep": "^4.5.0", + "lodash.mergewith": "^4.6.2", "migrate-mongo": "^7.0.1", "mime-types": "^2.1.25", "mongodb": "^3.7.3", diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index 1bac1551..c5888dcd 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -1,4 +1,5 @@ import { getMidnightWithTimezoneOffset, getUTCMidnight } from '../utils/dates'; +import { composeFullRepetitionEvent } from '../utils/merge'; import { groupBy } from '../utils/grouper'; import safe from 'safe-regex'; @@ -422,6 +423,19 @@ class EventsFactory extends Factory { .skip(skip) .toArray(); + console.log('repetitions', repetitions); + + console.log('eventOriginal', eventOriginal); + + + repetitions.forEach(repetition => { + repetition.payload = composeFullRepetitionEvent(eventOriginal, repetition).payload; + console.log(repetition); + if ('delta' in repetition) { + delete repetition.delta; + } + }); + const isLastPortion = repetitions.length < limit && skip === 0; /** @@ -455,10 +469,30 @@ class EventsFactory extends Factory { * @todo move to Repetitions(?) model */ async getEventRepetition(repetitionId) { - return this.getCollection(this.TYPES.REPETITIONS) + const repetition = await this.getCollection(this.TYPES.REPETITIONS) .findOne({ _id: ObjectID(repetitionId), }); + + if (!repetition) { + return null; + } + + const event = await this.findOneByQuery({ + groupHash: repetition.groupHash, + }); + + if (!event) { + return null; + } + + repetition.payload = composeFullRepetitionEvent(event, repetition).payload; + + if ('delta' in repetition) { + delete repetition.delta; + } + + return repetition; } /** @@ -469,10 +503,6 @@ class EventsFactory extends Factory { async getEventLastRepetition(eventId) { const repetitions = await this.getEventRepetitions(eventId, 1); - if (repetitions.length === 0) { - return null; - } - return repetitions.shift(); } diff --git a/src/typeDefs/event.ts b/src/typeDefs/event.ts index a7990679..af8ad7f9 100644 --- a/src/typeDefs/event.ts +++ b/src/typeDefs/event.ts @@ -255,11 +255,6 @@ type Repetition { """ payload: RepetitionPayload - """ - Delta of the event's payload, stringified JSON - """ - delta: String - """ Event timestamp """ diff --git a/src/utils/merge.ts b/src/utils/merge.ts new file mode 100644 index 00000000..048a711c --- /dev/null +++ b/src/utils/merge.ts @@ -0,0 +1,106 @@ +import mergeWith from 'lodash.mergewith'; +import cloneDeep from 'lodash.clonedeep'; +import { patch } from '@n1ru4l/json-patch-plus'; + +type HawkEvent = { + payload: { + [key: string]: any + } +} + +type HawkEventRepetition = { + payload: { + [key: string]: any + } + delta: string; +} + +/** + * One of the features of the events is that their repetition is the difference + * between the original, which greatly optimizes storage. So we need to restore + * the original repetition payload using the very first event and its difference + * between its repetition + * + * @param originalEvent - the very first event we received + * @param repetition - the difference with its repetition, for the repetition we want to display + * @returns fully assembled payload of the current repetition + */ +export function repetitionAssembler(originalEvent: Object, repetition: { [key: string]: any }): any { + const customizer = (originalParam: any, repetitionParam: any): any => { + if (repetitionParam === null) { + return originalParam; + } + + + if (typeof repetitionParam === 'object' && typeof originalParam === 'object') { + /** + * If original event has null but repetition has some value, we need to return repetition value + */ + if (originalParam === null) { + return repetitionParam; + /** + * Otherwise, we need to recursively merge original and repetition values + */ + } else { + return repetitionAssembler(originalParam, repetitionParam); + } + } + + return repetitionParam; + }; + + return mergeWith(cloneDeep(originalEvent), cloneDeep(repetition), customizer); + } + + +/** + * Helps to merge original event and repetition due to delta format, + * in case of old delta format, we need to patch the payload + * in case of new delta format, we need to assemble the payload + * + * @param originalEvent {HawkEvent} - The original event + * @param repetition {HawkEventRepetition} - The repetition to process + * @returns {HawkEvent} Updated event with processed repetition payload + */ +export function composeFullRepetitionEvent(originalEvent: HawkEvent, repetition: HawkEventRepetition | undefined): HawkEvent { + + console.log('originalEvent', originalEvent); + console.log('repetition', repetition); + + /** + * Make a deep copy of the original event, because we need to avoid mutating the original event + */ + const event = cloneDeep(originalEvent); + + if (!repetition) { + return event; + } + + /** + * New delta format (repetition.delta is not null) + */ + if (repetition.delta) { + event.payload = patch({ + left: event.payload, + delta: JSON.parse(repetition.delta) + }); + + return event; + } + + /** + * New delta format (repetition.payload is null) and repetition.delta is null (there is no delta between original and repetition) + */ + if (!repetition.payload) { + return event; + } + + /** + * Old delta format (repetition.payload is not null) + * @todo remove after July 5 2025 + */ + event.payload = repetitionAssembler(event.payload, repetition.payload); + + return event; + } + \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 9d2d9361..090af67e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -720,6 +720,11 @@ semver "^7.3.5" tar "^6.1.11" +"@n1ru4l/json-patch-plus@^0.2.0": + version "0.2.0" + resolved "https://registry.yarnpkg.com/@n1ru4l/json-patch-plus/-/json-patch-plus-0.2.0.tgz#b8fa09fd980c3460dfdc109a7c4cc5590157aa6b" + integrity sha512-pLkJy83/rVfDTyQgDSC8GeXAHEdXNHGNJrB1b7wAyGQu0iv7tpMXntKVSqj0+XKNVQbco40SZffNfVALzIt0SQ== + "@phc/format@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@phc/format/-/format-1.0.0.tgz#b5627003b3216dc4362125b13f48a4daa76680e4" @@ -1077,6 +1082,25 @@ "@types/koa-compose" "*" "@types/node" "*" +"@types/lodash.clonedeep@^4.5.9": + version "4.5.9" + resolved "https://registry.yarnpkg.com/@types/lodash.clonedeep/-/lodash.clonedeep-4.5.9.tgz#ea48276c7cc18d080e00bb56cf965bcceb3f0fc1" + integrity sha512-19429mWC+FyaAhOLzsS8kZUsI+/GmBAQ0HFiCPsKGU+7pBXOQWhyrY6xNNDwUSX8SMZMJvuFVMF9O5dQOlQK9Q== + dependencies: + "@types/lodash" "*" + +"@types/lodash.mergewith@^4.6.9": + version "4.6.9" + resolved "https://registry.yarnpkg.com/@types/lodash.mergewith/-/lodash.mergewith-4.6.9.tgz#7093028a36de3cae4495d03b9d92c351cab1f8bf" + integrity sha512-fgkoCAOF47K7sxrQ7Mlud2TH023itugZs2bUg8h/KzT+BnZNrR2jAOmaokbLunHNnobXVWOezAeNn/lZqwxkcw== + dependencies: + "@types/lodash" "*" + +"@types/lodash@*": + version "4.17.20" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.20.tgz#1ca77361d7363432d29f5e55950d9ec1e1c6ea93" + integrity sha512-H3MHACvFUEiujabxhaI/ImO6gUrd8oOurg7LQtS7mbwIXA/cUqWrvBsaeJ23aZEPk1TAYkurjfMbSELfoCXlGA== + "@types/long@^4.0.0": version "4.0.2" resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" @@ -4616,6 +4640,11 @@ lockfile@^1.0.4: dependencies: signal-exit "^3.0.2" +lodash.clonedeep@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" + integrity sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ== + lodash.includes@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f" @@ -4646,6 +4675,11 @@ lodash.isstring@^4.0.1: resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451" integrity sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw== +lodash.mergewith@^4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/lodash.mergewith/-/lodash.mergewith-4.6.2.tgz#617121f89ac55f59047c7aec1ccd6654c6590f55" + integrity sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ== + lodash.once@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" From ef976cee7bbf507a3879a44e58f489b2659af7f5 Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 00:09:28 +0300 Subject: [PATCH 02/12] fix merge --- src/utils/merge.ts | 117 +++++++++++++++++++++++++++------------------ 1 file changed, 70 insertions(+), 47 deletions(-) diff --git a/src/utils/merge.ts b/src/utils/merge.ts index 048a711c..f58673ab 100644 --- a/src/utils/merge.ts +++ b/src/utils/merge.ts @@ -3,16 +3,16 @@ import cloneDeep from 'lodash.clonedeep'; import { patch } from '@n1ru4l/json-patch-plus'; type HawkEvent = { - payload: { - [key: string]: any - } + payload: { + [key: string]: any + } } type HawkEventRepetition = { - payload: { - [key: string]: any - } - delta: string; + payload: { + [key: string]: any + } + delta: string; } /** @@ -27,31 +27,46 @@ type HawkEventRepetition = { */ export function repetitionAssembler(originalEvent: Object, repetition: { [key: string]: any }): any { const customizer = (originalParam: any, repetitionParam: any): any => { - if (repetitionParam === null) { - return originalParam; - } - - - if (typeof repetitionParam === 'object' && typeof originalParam === 'object') { - /** - * If original event has null but repetition has some value, we need to return repetition value - */ - if (originalParam === null) { - return repetitionParam; - /** - * Otherwise, we need to recursively merge original and repetition values - */ - } else { - return repetitionAssembler(originalParam, repetitionParam); + if (repetitionParam === null) { + return originalParam; } - } - - return repetitionParam; + + + if (typeof repetitionParam === 'object' && typeof originalParam === 'object') { + /** + * If original event has null but repetition has some value, we need to return repetition value + */ + if (originalParam === null) { + return repetitionParam; + /** + * Otherwise, we need to recursively merge original and repetition values + */ + } else { + return repetitionAssembler(originalParam, repetitionParam); + } + } + + return repetitionParam; }; - + return mergeWith(cloneDeep(originalEvent), cloneDeep(repetition), customizer); - } - +} + +function parsePayloadField(payload: any, field: string) { + if (payload && payload[field] && typeof payload[field] === 'string') { + payload[field] = JSON.parse(payload[field]); + } + + return payload; +} + +function stringifyPayloadField(payload: any, field: string) { + if (payload && payload[field]) { + payload[field] = JSON.stringify(payload[field]); + } + + return payload; +} /** * Helps to merge original event and repetition due to delta format, @@ -64,43 +79,51 @@ export function repetitionAssembler(originalEvent: Object, repetition: { [key: s */ export function composeFullRepetitionEvent(originalEvent: HawkEvent, repetition: HawkEventRepetition | undefined): HawkEvent { - console.log('originalEvent', originalEvent); - console.log('repetition', repetition); - /** * Make a deep copy of the original event, because we need to avoid mutating the original event */ const event = cloneDeep(originalEvent); - + if (!repetition) { - return event; + return event; } - + /** * New delta format (repetition.delta is not null) */ if (repetition.delta) { - event.payload = patch({ - left: event.payload, - delta: JSON.parse(repetition.delta) - }); - - return event; + /** + * Parse addons and context fields from string to object before patching + */ + event.payload = parsePayloadField(event.payload, 'addons'); + event.payload = parsePayloadField(event.payload, 'context'); + + event.payload = patch({ + left: event.payload, + delta: JSON.parse(repetition.delta) + }); + + /** + * Stringify addons and context fields from object to string after patching + */ + event.payload = stringifyPayloadField(event.payload, 'addons'); + event.payload = stringifyPayloadField(event.payload, 'context'); + + return event; } - + /** * New delta format (repetition.payload is null) and repetition.delta is null (there is no delta between original and repetition) */ if (!repetition.payload) { - return event; + return event; } - + /** * Old delta format (repetition.payload is not null) * @todo remove after July 5 2025 */ event.payload = repetitionAssembler(event.payload, repetition.payload); - + return event; - } - \ No newline at end of file +} From 70d4cc1c08df5b16cc1a4502c5ad5a517808343f Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 00:10:05 +0300 Subject: [PATCH 03/12] rm useless logs --- src/models/eventsFactory.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index c5888dcd..080c92ff 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -423,14 +423,9 @@ class EventsFactory extends Factory { .skip(skip) .toArray(); - console.log('repetitions', repetitions); - - console.log('eventOriginal', eventOriginal); - - repetitions.forEach(repetition => { repetition.payload = composeFullRepetitionEvent(eventOriginal, repetition).payload; - console.log(repetition); + if ('delta' in repetition) { delete repetition.delta; } From 995c18183edee42a42015c6478c4666acaba1a9e Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 00:10:26 +0300 Subject: [PATCH 04/12] fix lint --- src/utils/merge.ts | 140 ++++++++++++++++++++++----------------------- 1 file changed, 69 insertions(+), 71 deletions(-) diff --git a/src/utils/merge.ts b/src/utils/merge.ts index f58673ab..a79ac831 100644 --- a/src/utils/merge.ts +++ b/src/utils/merge.ts @@ -4,14 +4,14 @@ import { patch } from '@n1ru4l/json-patch-plus'; type HawkEvent = { payload: { - [key: string]: any - } + [key: string]: any; + }; } type HawkEventRepetition = { payload: { - [key: string]: any - } + [key: string]: any; + }; delta: string; } @@ -25,47 +25,46 @@ type HawkEventRepetition = { * @param repetition - the difference with its repetition, for the repetition we want to display * @returns fully assembled payload of the current repetition */ -export function repetitionAssembler(originalEvent: Object, repetition: { [key: string]: any }): any { - const customizer = (originalParam: any, repetitionParam: any): any => { - if (repetitionParam === null) { - return originalParam; - } - - - if (typeof repetitionParam === 'object' && typeof originalParam === 'object') { - /** - * If original event has null but repetition has some value, we need to return repetition value - */ - if (originalParam === null) { - return repetitionParam; - /** - * Otherwise, we need to recursively merge original and repetition values - */ - } else { - return repetitionAssembler(originalParam, repetitionParam); - } - } +export function repetitionAssembler(originalEvent: Record, repetition: { [key: string]: any }): any { + const customizer = (originalParam: any, repetitionParam: any): any => { + if (repetitionParam === null) { + return originalParam; + } + if (typeof repetitionParam === 'object' && typeof originalParam === 'object') { + /** + * If original event has null but repetition has some value, we need to return repetition value + */ + if (originalParam === null) { return repetitionParam; - }; + /** + * Otherwise, we need to recursively merge original and repetition values + */ + } else { + return repetitionAssembler(originalParam, repetitionParam); + } + } + + return repetitionParam; + }; - return mergeWith(cloneDeep(originalEvent), cloneDeep(repetition), customizer); + return mergeWith(cloneDeep(originalEvent), cloneDeep(repetition), customizer); } function parsePayloadField(payload: any, field: string) { - if (payload && payload[field] && typeof payload[field] === 'string') { - payload[field] = JSON.parse(payload[field]); - } + if (payload && payload[field] && typeof payload[field] === 'string') { + payload[field] = JSON.parse(payload[field]); + } - return payload; + return payload; } function stringifyPayloadField(payload: any, field: string) { - if (payload && payload[field]) { - payload[field] = JSON.stringify(payload[field]); - } + if (payload && payload[field]) { + payload[field] = JSON.stringify(payload[field]); + } - return payload; + return payload; } /** @@ -78,52 +77,51 @@ function stringifyPayloadField(payload: any, field: string) { * @returns {HawkEvent} Updated event with processed repetition payload */ export function composeFullRepetitionEvent(originalEvent: HawkEvent, repetition: HawkEventRepetition | undefined): HawkEvent { + /** + * Make a deep copy of the original event, because we need to avoid mutating the original event + */ + const event = cloneDeep(originalEvent); - /** - * Make a deep copy of the original event, because we need to avoid mutating the original event - */ - const event = cloneDeep(originalEvent); - - if (!repetition) { - return event; - } + if (!repetition) { + return event; + } + /** + * New delta format (repetition.delta is not null) + */ + if (repetition.delta) { /** - * New delta format (repetition.delta is not null) + * Parse addons and context fields from string to object before patching */ - if (repetition.delta) { - /** - * Parse addons and context fields from string to object before patching - */ - event.payload = parsePayloadField(event.payload, 'addons'); - event.payload = parsePayloadField(event.payload, 'context'); + event.payload = parsePayloadField(event.payload, 'addons'); + event.payload = parsePayloadField(event.payload, 'context'); - event.payload = patch({ - left: event.payload, - delta: JSON.parse(repetition.delta) - }); - - /** - * Stringify addons and context fields from object to string after patching - */ - event.payload = stringifyPayloadField(event.payload, 'addons'); - event.payload = stringifyPayloadField(event.payload, 'context'); - - return event; - } + event.payload = patch({ + left: event.payload, + delta: JSON.parse(repetition.delta), + }); /** - * New delta format (repetition.payload is null) and repetition.delta is null (there is no delta between original and repetition) + * Stringify addons and context fields from object to string after patching */ - if (!repetition.payload) { - return event; - } + event.payload = stringifyPayloadField(event.payload, 'addons'); + event.payload = stringifyPayloadField(event.payload, 'context'); - /** - * Old delta format (repetition.payload is not null) - * @todo remove after July 5 2025 - */ - event.payload = repetitionAssembler(event.payload, repetition.payload); + return event; + } + /** + * New delta format (repetition.payload is null) and repetition.delta is null (there is no delta between original and repetition) + */ + if (!repetition.payload) { return event; + } + + /** + * Old delta format (repetition.payload is not null) + * @todo remove after July 5 2025 + */ + event.payload = repetitionAssembler(event.payload, repetition.payload); + + return event; } From 60d22ed7f707654a8aef6ee3818e4c2ecd30649b Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 18:14:59 +0300 Subject: [PATCH 05/12] revevert factory changes --- src/models/eventsFactory.js | 37 ++++++------------------------------- src/typeDefs/event.ts | 6 ++++++ 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index 080c92ff..407200ac 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -1,5 +1,4 @@ import { getMidnightWithTimezoneOffset, getUTCMidnight } from '../utils/dates'; -import { composeFullRepetitionEvent } from '../utils/merge'; import { groupBy } from '../utils/grouper'; import safe from 'safe-regex'; @@ -423,14 +422,6 @@ class EventsFactory extends Factory { .skip(skip) .toArray(); - repetitions.forEach(repetition => { - repetition.payload = composeFullRepetitionEvent(eventOriginal, repetition).payload; - - if ('delta' in repetition) { - delete repetition.delta; - } - }); - const isLastPortion = repetitions.length < limit && skip === 0; /** @@ -464,30 +455,10 @@ class EventsFactory extends Factory { * @todo move to Repetitions(?) model */ async getEventRepetition(repetitionId) { - const repetition = await this.getCollection(this.TYPES.REPETITIONS) + return this.getCollection(this.TYPES.REPETITIONS) .findOne({ - _id: ObjectID(repetitionId), + _id: new ObjectID(repetitionId), }); - - if (!repetition) { - return null; - } - - const event = await this.findOneByQuery({ - groupHash: repetition.groupHash, - }); - - if (!event) { - return null; - } - - repetition.payload = composeFullRepetitionEvent(event, repetition).payload; - - if ('delta' in repetition) { - delete repetition.delta; - } - - return repetition; } /** @@ -498,6 +469,10 @@ class EventsFactory extends Factory { async getEventLastRepetition(eventId) { const repetitions = await this.getEventRepetitions(eventId, 1); + if (repetitions.length === 0) { + return null; + } + return repetitions.shift(); } diff --git a/src/typeDefs/event.ts b/src/typeDefs/event.ts index af8ad7f9..824bda03 100644 --- a/src/typeDefs/event.ts +++ b/src/typeDefs/event.ts @@ -255,6 +255,12 @@ type Repetition { """ payload: RepetitionPayload + """ + Delta of the event's payload, stringified JSON + """ + delta: String + + """ Event timestamp """ From 027ea78ae18f5e79940a5974b523c91a6c5bdc5c Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 18:15:48 +0300 Subject: [PATCH 06/12] fix --- src/models/eventsFactory.js | 2 +- src/typeDefs/event.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/models/eventsFactory.js b/src/models/eventsFactory.js index 407200ac..1bac1551 100644 --- a/src/models/eventsFactory.js +++ b/src/models/eventsFactory.js @@ -457,7 +457,7 @@ class EventsFactory extends Factory { async getEventRepetition(repetitionId) { return this.getCollection(this.TYPES.REPETITIONS) .findOne({ - _id: new ObjectID(repetitionId), + _id: ObjectID(repetitionId), }); } diff --git a/src/typeDefs/event.ts b/src/typeDefs/event.ts index 824bda03..a7990679 100644 --- a/src/typeDefs/event.ts +++ b/src/typeDefs/event.ts @@ -260,7 +260,6 @@ type Repetition { """ delta: String - """ Event timestamp """ From bddcecfd8be9a2a7171b0cddc33214c9d6e127ae Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 6 Aug 2025 15:17:08 +0000 Subject: [PATCH 07/12] Bump version up to 1.1.30 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 78526165..9eb9157d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.1.29", + "version": "1.1.30", "main": "index.ts", "license": "UNLICENSED", "scripts": { From 1c2ad412666a4d36797658e43bf41b1c109e9132 Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 18:26:31 +0300 Subject: [PATCH 08/12] use types from lib --- package.json | 2 +- src/utils/merge.ts | 16 ++-------------- yarn.lock | 8 ++++---- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index 78526165..603cff67 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "@graphql-tools/schema": "^8.5.1", "@graphql-tools/utils": "^8.9.0", "@hawk.so/nodejs": "^3.1.1", - "@hawk.so/types": "^0.1.31", + "@hawk.so/types": "^0.1.33", "@n1ru4l/json-patch-plus": "^0.2.0", "@types/amqp-connection-manager": "^2.0.4", "@types/bson": "^4.0.5", diff --git a/src/utils/merge.ts b/src/utils/merge.ts index a79ac831..28898556 100644 --- a/src/utils/merge.ts +++ b/src/utils/merge.ts @@ -1,19 +1,7 @@ import mergeWith from 'lodash.mergewith'; import cloneDeep from 'lodash.clonedeep'; import { patch } from '@n1ru4l/json-patch-plus'; - -type HawkEvent = { - payload: { - [key: string]: any; - }; -} - -type HawkEventRepetition = { - payload: { - [key: string]: any; - }; - delta: string; -} +import { GroupedEventDBScheme, RepetitionDBScheme } from '@hawk.so/types'; /** * One of the features of the events is that their repetition is the difference @@ -76,7 +64,7 @@ function stringifyPayloadField(payload: any, field: string) { * @param repetition {HawkEventRepetition} - The repetition to process * @returns {HawkEvent} Updated event with processed repetition payload */ -export function composeFullRepetitionEvent(originalEvent: HawkEvent, repetition: HawkEventRepetition | undefined): HawkEvent { +export function composeFullRepetitionEvent(originalEvent: GroupedEventDBScheme, repetition: RepetitionDBScheme | undefined): GroupedEventDBScheme { /** * Make a deep copy of the original event, because we need to avoid mutating the original event */ diff --git a/yarn.lock b/yarn.lock index 090af67e..80a104ae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -458,10 +458,10 @@ dependencies: "@types/mongodb" "^3.5.34" -"@hawk.so/types@^0.1.31": - version "0.1.31" - resolved "https://registry.yarnpkg.com/@hawk.so/types/-/types-0.1.31.tgz#fba2c3451e927558bfcc3b1d942baaf8e72ad214" - integrity sha512-o1LeA3JVIUPRSIZegKwAdl4noQ1KYxwr80eisJMlghP9knu6PbYw20rIMyan5qQ3epOWs8gO1CU3iwHZprFiCg== +"@hawk.so/types@^0.1.33": + version "0.1.33" + resolved "https://registry.yarnpkg.com/@hawk.so/types/-/types-0.1.33.tgz#feb077b699b3e0001552588a372e1efe6cd58f40" + integrity sha512-q3AdVxzQ8Qk8qyYiAcAacxNZXWTG/oLmVpjQlcLm2Eh5OJgpaZvH8hQCeRQ/ml1cqbYW8gUrRbMMCS2QOcwxEw== dependencies: "@types/mongodb" "^3.5.34" From e3b4d8a8e25f1a33c0405d1e66cf881eac91c92d Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 18:44:09 +0300 Subject: [PATCH 09/12] added merge tests --- test/utils/merge.test.ts | 366 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 366 insertions(+) create mode 100644 test/utils/merge.test.ts diff --git a/test/utils/merge.test.ts b/test/utils/merge.test.ts new file mode 100644 index 00000000..c065d338 --- /dev/null +++ b/test/utils/merge.test.ts @@ -0,0 +1,366 @@ +import { composeFullRepetitionEvent } from '../../src/utils/merge'; +import { GroupedEventDBScheme, RepetitionDBScheme } from '@hawk.so/types'; + +import { diff } from '@n1ru4l/json-patch-plus'; + +describe('composeFullRepetitionEvent', () => { + const mockOriginalEvent: GroupedEventDBScheme = { + groupHash: 'original-event-1', + totalCount: 1, + catcherType: 'javascript', + payload: { + title: 'Original message', + type: 'error', + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }, + usersAffected: 1, + visitedBy: [], + timestamp: 1640995200, // 2023-01-01T00:00:00Z + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('when repetition is undefined', () => { + it('should return a deep copy of the original event', () => { + /** + * Arrange + */ + const repetition = undefined; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result).toEqual(mockOriginalEvent); + expect(result).toMatchObject(mockOriginalEvent); + expect(result.payload).toMatchObject(mockOriginalEvent.payload); + }); + }); + + describe('when repetition.delta is provided (new delta format)', () => { + it('should parse addons and context, apply patch, and stringify fields back', () => { + /** + * Arrange + */ + const delta = diff({ + left: mockOriginalEvent.payload, + right: { + ...mockOriginalEvent.payload, + title: 'Updated message', + type: 'warning', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Updated message', + type: 'warning', + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + + it('should handle delta with new fields', () => { + /** + * Arrange + */ + const delta = diff({ + left: mockOriginalEvent.payload, + right: { + ...mockOriginalEvent.payload, + release: 'v1.0.0', + catcherVersion: '2.0.0', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Original message', + type: 'error', + release: 'v1.0.0', + catcherVersion: '2.0.0', + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + }); + + describe('when repetition.delta is undefined and repetition.payload is undefined', () => { + it('should return the original event unchanged', () => { + /** + * Arrange + */ + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: undefined, + payload: undefined, + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result).toEqual(mockOriginalEvent); + expect(result).not.toBe(mockOriginalEvent); // Должна быть глубокая копия + }); + }); + + describe('when repetition.delta is undefined and repetition.payload is provided (old delta format)', () => { + it('should use repetitionAssembler to merge payloads', () => { + /** + * Arrange + */ + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: undefined, + payload: { + title: 'Updated message', + type: 'warning', + release: 'v1.0.0', + }, + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Updated message', + type: 'warning', + release: 'v1.0.0', + // Addons and context should be, because old format doesn't remove fields + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + + it('should handle null values in repetition payload', () => { + /** + * Arrange + */ + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: undefined, + payload: { + title: 'Updated title', + type: 'info', + }, + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Updated title', // repetition value replaces original + type: 'info', + // Addons and context should be, because old format doesn't remove fields + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + + it('should preserve original value when repetition payload has null', () => { + /** + * Arrange + */ + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: undefined, + payload: { + title: null as any, + type: 'info', + }, + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Original message', // null в repetition должно сохранить оригинальное значение + type: 'info', + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + }); + + describe('edge cases', () => { + it('should handle empty payload in original event', () => { + /** + * Arrange + */ + const eventWithEmptyPayload: GroupedEventDBScheme = { + groupHash: 'event-4', + totalCount: 1, + catcherType: 'javascript', + payload: { + title: 'Empty event', + }, + usersAffected: 1, + visitedBy: [], + timestamp: 1640995200, + }; + + const delta = diff({ + left: eventWithEmptyPayload.payload, + right: { + ...eventWithEmptyPayload.payload, + title: 'New message', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'event-4', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(eventWithEmptyPayload, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'New message', + }); + }); + + it('should handle null payload in original event', () => { + /** + * Arrange + */ + const eventWithNullPayload: GroupedEventDBScheme = { + groupHash: 'event-5', + totalCount: 1, + catcherType: 'javascript', + payload: null as any, + usersAffected: 1, + visitedBy: [], + timestamp: 1640995200, + }; + + const delta = diff({ + left: eventWithNullPayload.payload, + right: { + title: 'New message', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'event-5', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(eventWithNullPayload, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'New message', + }); + }); + + it('should handle invalid JSON in addons or context', () => { + /** + * Arrange + */ + const eventWithInvalidJSON: GroupedEventDBScheme = { + groupHash: 'event-6', + totalCount: 1, + catcherType: 'javascript', + payload: { + title: 'Test', + addons: 'invalid json', + context: 'also invalid', + }, + usersAffected: 1, + visitedBy: [], + timestamp: 1640995200, + }; + + const delta = diff({ + left: eventWithInvalidJSON.payload, + right: { + ...eventWithInvalidJSON.payload, + title: 'Updated', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'event-6', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act & Assert + */ + expect(() => { + composeFullRepetitionEvent(eventWithInvalidJSON, repetition); + }).toThrow(); // Должно выбросить ошибку при парсинге невалидного JSON + }); + }); +}); \ No newline at end of file From a6b9f014d504dfb80bf87259c469854b4fae175f Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 19:43:59 +0300 Subject: [PATCH 10/12] fix code style --- src/utils/merge.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/utils/merge.ts b/src/utils/merge.ts index 28898556..af2029b8 100644 --- a/src/utils/merge.ts +++ b/src/utils/merge.ts @@ -9,11 +9,12 @@ import { GroupedEventDBScheme, RepetitionDBScheme } from '@hawk.so/types'; * the original repetition payload using the very first event and its difference * between its repetition * + * @deprecated remove after checking that all repetitions has no payloads in db * @param originalEvent - the very first event we received * @param repetition - the difference with its repetition, for the repetition we want to display * @returns fully assembled payload of the current repetition */ -export function repetitionAssembler(originalEvent: Record, repetition: { [key: string]: any }): any { +export function repetitionAssembler(originalEvent: GroupedEventDBScheme['payload'], repetition: GroupedEventDBScheme['payload']): GroupedEventDBScheme['payload'] { const customizer = (originalParam: any, repetitionParam: any): any => { if (repetitionParam === null) { return originalParam; @@ -39,7 +40,14 @@ export function repetitionAssembler(originalEvent: Record, repetiti return mergeWith(cloneDeep(originalEvent), cloneDeep(repetition), customizer); } -function parsePayloadField(payload: any, field: string) { +/** + * Parse addons and context fields from string to object, in db it stores as string + * + * @param payload - the payload of the event + * @param field - the field to parse, can be 'addons' or 'context' + * @returns the payload with parsed field + */ +function parsePayloadField(payload: GroupedEventDBScheme['payload'], field: 'addons' | 'context') { if (payload && payload[field] && typeof payload[field] === 'string') { payload[field] = JSON.parse(payload[field]); } @@ -47,7 +55,14 @@ function parsePayloadField(payload: any, field: string) { return payload; } -function stringifyPayloadField(payload: any, field: string) { +/** + * Stringify addons and context fields from object to string, in db it stores as string + * + * @param payload - the payload of the event + * @param field - the field to stringify, can be 'addons' or 'context' + * @returns the payload with stringified field + */ +function stringifyPayloadField(payload: GroupedEventDBScheme['payload'], field: 'addons' | 'context') { if (payload && payload[field]) { payload[field] = JSON.stringify(payload[field]); } @@ -107,7 +122,7 @@ export function composeFullRepetitionEvent(originalEvent: GroupedEventDBScheme, /** * Old delta format (repetition.payload is not null) - * @todo remove after July 5 2025 + * @todo remove after checking that all repetitions has no payloads in db */ event.payload = repetitionAssembler(event.payload, repetition.payload); From 1690cf8266ce36f0914d3031a8ab2268ed1064b7 Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 19:51:03 +0300 Subject: [PATCH 11/12] fix tests --- src/utils/merge.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/merge.ts b/src/utils/merge.ts index af2029b8..1663b5f4 100644 --- a/src/utils/merge.ts +++ b/src/utils/merge.ts @@ -49,7 +49,7 @@ export function repetitionAssembler(originalEvent: GroupedEventDBScheme['payload */ function parsePayloadField(payload: GroupedEventDBScheme['payload'], field: 'addons' | 'context') { if (payload && payload[field] && typeof payload[field] === 'string') { - payload[field] = JSON.parse(payload[field]); + payload[field] = JSON.parse(payload[field] as string); } return payload; From 9f6df5409812242c141413f2dfdc081539650838 Mon Sep 17 00:00:00 2001 From: slaveeks Date: Wed, 6 Aug 2025 20:01:04 +0300 Subject: [PATCH 12/12] set date --- src/utils/merge.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/merge.ts b/src/utils/merge.ts index 1663b5f4..090ada79 100644 --- a/src/utils/merge.ts +++ b/src/utils/merge.ts @@ -9,7 +9,7 @@ import { GroupedEventDBScheme, RepetitionDBScheme } from '@hawk.so/types'; * the original repetition payload using the very first event and its difference * between its repetition * - * @deprecated remove after checking that all repetitions has no payloads in db + * @deprecated remove after 6 september 2025 * @param originalEvent - the very first event we received * @param repetition - the difference with its repetition, for the repetition we want to display * @returns fully assembled payload of the current repetition @@ -122,7 +122,7 @@ export function composeFullRepetitionEvent(originalEvent: GroupedEventDBScheme, /** * Old delta format (repetition.payload is not null) - * @todo remove after checking that all repetitions has no payloads in db + * @todo remove after 6 september 2025 */ event.payload = repetitionAssembler(event.payload, repetition.payload);