diff --git a/.github/e2e-duration-seed.json b/.github/e2e-duration-seed.json new file mode 100644 index 00000000..173ba079 --- /dev/null +++ b/.github/e2e-duration-seed.json @@ -0,0 +1,448 @@ +{ + "schemaVersion": 1, + "source": { + "runId": 33400350423, + "headSha": "f3ee51e3cc08ac53af549c01805c3fbfb12dae5c" + }, + "tests": { + "aa3ad05f8c0e05077efb-aeb4b8febe5f73c5a6cb": 10547, + "aa3ad05f8c0e05077efb-ea59cf944d414ec07168": 11171, + "aa3ad05f8c0e05077efb-e431588d31fe862c6d07": 11844, + "aa3ad05f8c0e05077efb-5e6742020e17fdff3e40": 15925, + "aa3ad05f8c0e05077efb-a54eaf8e4a0b13bc34be": 15081, + "aa3ad05f8c0e05077efb-55a3d5476713b9251b97": 5422, + "aa3ad05f8c0e05077efb-245ed333dcf01ad3e4d1": 4074, + "aa3ad05f8c0e05077efb-b1ff5ecb6f716605102b": 2742, + "bce4dfba4bfbbb97b3db-f210b7ad6b42be69aa89": 4386, + "bce4dfba4bfbbb97b3db-4929a21d068009d5f700": 4387, + "bce4dfba4bfbbb97b3db-4b8645f81a53b5b82612": 3013, + "bce4dfba4bfbbb97b3db-1ecc858d302a71072157": 5308, + "bce4dfba4bfbbb97b3db-0a170b0e7696f4be8026": 6670, + "bce4dfba4bfbbb97b3db-4dadf98b680495b3ff54": 2746, + "bce4dfba4bfbbb97b3db-e57b3b4b13decc0e3b00": 4434, + "bce4dfba4bfbbb97b3db-82bf5ba4168fd65da829": 4089, + "bce4dfba4bfbbb97b3db-cc620355d991c5e621d6": 13120, + "bce4dfba4bfbbb97b3db-71400dd4b9626bc5c6ba": 3778, + "d4043a9dd1ab94463227-f149f305080f0c435ee7": 4507, + "d4043a9dd1ab94463227-af5c7ef85646ae767b54": 4230, + "d4043a9dd1ab94463227-dabd861c23085951cc48": 2782, + "d4043a9dd1ab94463227-4b7390ac0732232c4b3d": 1647, + "d4043a9dd1ab94463227-2efc7ca53f4b138cc546": 2789, + "d4043a9dd1ab94463227-97f0109d6cf393157b89": 1646, + "d4043a9dd1ab94463227-f7e0e9dfd353a34f0ad4": 1000, + "d4043a9dd1ab94463227-2e5eef2993475d2008aa": 1000, + "d4043a9dd1ab94463227-208705eabb0e39143250": 1000, + "d4043a9dd1ab94463227-f2d73f0cdcabef5b7c75": 1000, + "a19963100c65b4b43fa3-e86cf7f0adb0798c82a9": 17258, + "43e3b02573fb3cb2a894-7fa6d4fa172ae6433125": 3233, + "43e3b02573fb3cb2a894-eb8946df09684bf26074": 9454, + "43e3b02573fb3cb2a894-2561832458c475e4b0c1": 7472, + "43e3b02573fb3cb2a894-258bc48de0902e8c0d0f": 10094, + "8c21c95acf6b49c9b424-c97f557821edb78b610d": 4724, + "8c21c95acf6b49c9b424-f37f21438a285cdd9526": 7724, + "8c21c95acf6b49c9b424-0bb14f45ddae2c720e12": 6948, + "8c21c95acf6b49c9b424-970e5a9f46340e4b34bd": 7467, + "8c21c95acf6b49c9b424-36157b99251e3e7b3ea6": 5029, + "e0ec79b0c4f511f16424-613097fe783bc9ef0d61": 18193, + "e0ec79b0c4f511f16424-b43301a1dcb2da6d94c4": 9391, + "e0ec79b0c4f511f16424-0cb2b4fe7fc01fc1d9c6": 8333, + "e0ec79b0c4f511f16424-504a85fbe7deaaaeb7c8": 11548, + "e0ec79b0c4f511f16424-729b3d8fbb6a4da2e4be": 10903, + "e0ec79b0c4f511f16424-0ac00d5a8d5f9ce6428a": 8220, + "e0ec79b0c4f511f16424-8c7798d98eb084dcfa9a": 8780, + "e0ec79b0c4f511f16424-84370658f01edb595857": 7973, + "e0ec79b0c4f511f16424-b52a90f9f045eff1b725": 7780, + "e0ec79b0c4f511f16424-c1f2c7ca16d056392b7b": 6734, + "e0ec79b0c4f511f16424-112cc5a19447991625f0": 6758, + "e0ec79b0c4f511f16424-09d8b5a506463df483fb": 15722, + "e0ec79b0c4f511f16424-20d268f9bb34ccc97cdd": 7508, + "e0ec79b0c4f511f16424-e206f54e38602b7457d6": 7409, + "e0ec79b0c4f511f16424-f780fac2c39d1e0642b0": 6771, + "e0ec79b0c4f511f16424-8642986e4184b1056753": 9246, + "e0ec79b0c4f511f16424-f232e5688ea5068cbcab": 9895, + "e0ec79b0c4f511f16424-62b93d335efce307ea2e": 26905, + "e0ec79b0c4f511f16424-f166e1e2f1a128481fa0": 17371, + "6f04e1ee6ff12f188392-81d8c6b7cd82a3c9b613": 12968, + "6f04e1ee6ff12f188392-de316c025d04840fb399": 10001, + "6f04e1ee6ff12f188392-f304b13769ccddc35ce3": 15016, + "6f04e1ee6ff12f188392-d30fc524ea584f524167": 8929, + "6f04e1ee6ff12f188392-b19b898ebdd611d0c09d": 7322, + "6f04e1ee6ff12f188392-c09904eb3ccc2aceac6c": 7402, + "6f04e1ee6ff12f188392-6eaa4903c04b11c0df42": 32352, + "6f04e1ee6ff12f188392-1cefde4c0502b8296f62": 27989, + "6f04e1ee6ff12f188392-8364596cbf2ecc5d696d": 8725, + "6f04e1ee6ff12f188392-8ec5838765eafd90e6e6": 10259, + "6f04e1ee6ff12f188392-222cf6013ff99995a4a7": 9382, + "6f04e1ee6ff12f188392-5d7b22cf590633b640e6": 14156, + "4c3e36aab13a1592a9d5-da6bfa60c71daeadb30d": 6529, + "4c3e36aab13a1592a9d5-f8917237333e67a27a31": 6663, + "4c3e36aab13a1592a9d5-74c4954ef3068315a300": 9713, + "4c3e36aab13a1592a9d5-041334fa4e45f1c66f71": 13175, + "4c3e36aab13a1592a9d5-00617b7836270ab1a325": 1500, + "4c3e36aab13a1592a9d5-c9cd35a623ef0485a91c": 1186, + "4c3e36aab13a1592a9d5-0cacc5ecce865133ea1f": 1998, + "4c3e36aab13a1592a9d5-7ab8f728e52644910e81": 1000, + "4c3e36aab13a1592a9d5-154953e8d3373790f17a": 1317, + "4c3e36aab13a1592a9d5-5da5bcdad11e2ca89783": 6955, + "4c3e36aab13a1592a9d5-65f23a0bb655f85f408c": 19793, + "4c3e36aab13a1592a9d5-dd69694defe6302dd9a2": 7097, + "2637827af6d57af612e8-0ad44dde41a20d31ac31": 18887, + "2637827af6d57af612e8-4242e2a60023b9a5b6f3": 15016, + "2637827af6d57af612e8-fc47a868b32ccfba996a": 14513, + "2637827af6d57af612e8-577cfa763bb0e463a6ea": 8852, + "2637827af6d57af612e8-4fa7d9575260ef11b0ee": 8926, + "57ef374cf88f7974a124-f0be1e46044ece5eb82b": 1000, + "57ef374cf88f7974a124-e56260f3283a719d0100": 1000, + "57ef374cf88f7974a124-aef88d5078b2b457536a": 1000, + "3facdf5ef3bd951cb354-497d328037cb7b7bee7c": 6794, + "3facdf5ef3bd951cb354-8a73e4c0bd507e4aecf6": 6708, + "3facdf5ef3bd951cb354-fc2f5c25aa403c079104": 14200, + "fe534f0825407f213faa-25c9a61fd1601154e240": 5031, + "fe534f0825407f213faa-0dda63ab2826db3be6da": 20823, + "fe534f0825407f213faa-4d41daec4a23380b87f0": 5311, + "fe534f0825407f213faa-98211843289e9ad0fdb6": 3480, + "fe534f0825407f213faa-d7991b3501f2c34a0176": 11082, + "fe534f0825407f213faa-cd2028ac51ddec6a7b93": 7219, + "fe534f0825407f213faa-3c85e03efa0175967909": 5838, + "fe534f0825407f213faa-ac5dc31239ba1a27a248": 7259, + "fe534f0825407f213faa-eed1a7e6f32d8d120c90": 9792, + "8b82586345c5df7a3d44-9991bd591ef1b6369554": 9822, + "8b82586345c5df7a3d44-e3c55f90346a78ba5808": 6843, + "8b82586345c5df7a3d44-4cb98df18da3346b5f58": 3810, + "b14bed13436323dfa1ba-b0d91d0376a4f6760269": 12571, + "b14bed13436323dfa1ba-61f6350d50e484e3c9a3": 25414, + "b14bed13436323dfa1ba-37317e8d627ac3abeb28": 4144, + "b14bed13436323dfa1ba-bf3b109b8ca27ccc9a8c": 3961, + "b14bed13436323dfa1ba-c518d3e60b9424673d8e": 4154, + "b14bed13436323dfa1ba-cbe7137cb40e5794d936": 6963, + "b14bed13436323dfa1ba-cae9b7db2e38d2989bed": 7024, + "b14bed13436323dfa1ba-5ea7ce56f96d2a404267": 4984, + "b14bed13436323dfa1ba-614f0b3f1f91555a0722": 4749, + "b14bed13436323dfa1ba-15349f3d63a5abbb7efc": 4451, + "b14bed13436323dfa1ba-7f1bc0efda089ba712e2": 5043, + "b14bed13436323dfa1ba-86f02942e3be8d27813d": 6127, + "b14bed13436323dfa1ba-27546b8780963aa86834": 6149, + "b14bed13436323dfa1ba-e3a0feb01f979dc14ff7": 4868, + "b14bed13436323dfa1ba-0becc68e140d9e10d829": 4328, + "b14bed13436323dfa1ba-b88f8b990dd62652a876": 4238, + "b14bed13436323dfa1ba-750a2243f9d501f30e3e": 3443, + "c0011e267bb7a139fed4-9234a3b6a1c56792b95d": 24337, + "c0011e267bb7a139fed4-fe9501a90751fdd2353b": 30412, + "c0011e267bb7a139fed4-d0167dbcb79e413eb318": 19056, + "c0011e267bb7a139fed4-40fd9a8003003b77e0c3": 22170, + "c0011e267bb7a139fed4-c8846e2587777f89331d": 26649, + "c0011e267bb7a139fed4-9a30207cc1992d4a74e3": 18572, + "c0011e267bb7a139fed4-02059a8029aa1b1d07d1": 19732, + "c0011e267bb7a139fed4-2511d1250855c73c2316": 26307, + "c0011e267bb7a139fed4-69220ddf64bdc187f7c0": 20462, + "c0011e267bb7a139fed4-b53906e545163f237fc2": 21398, + "c0011e267bb7a139fed4-73820884f08153dc0776": 27456, + "c0011e267bb7a139fed4-f2d58f075cb3d24336f0": 18422, + "c0011e267bb7a139fed4-a00b624bf69a6f57a3b5": 19598, + "c0011e267bb7a139fed4-749190ef0277b2be1986": 19250, + "c0011e267bb7a139fed4-9fb7aa0e8379c7e0ebd8": 17640, + "c0011e267bb7a139fed4-fe5bdfce54de75441e42": 17971, + "c0011e267bb7a139fed4-b2ed77e6232463c73749": 17646, + "c0011e267bb7a139fed4-957137393e5c3b72f85d": 9837, + "c0011e267bb7a139fed4-4d0fbd1696cd3d6f54ab": 9689, + "c0011e267bb7a139fed4-b01b5f1458cff401849a": 10730, + "c0011e267bb7a139fed4-2cf121662ff384371bb3": 12087, + "31cca38a48e525ad44aa-a70e65e920090baf2cc5": 2052, + "31cca38a48e525ad44aa-e8d53f955619e178e648": 7542, + "31cca38a48e525ad44aa-c54c4139c96777d498e3": 7411, + "31cca38a48e525ad44aa-6865e9ce96208095e8a3": 7358, + "31cca38a48e525ad44aa-285588c53d2f075523c5": 7436, + "31cca38a48e525ad44aa-f268d862a1b426a053b9": 9972, + "52299a6368cebd707692-fbcff25cd422c66c7f4e": 6646, + "52299a6368cebd707692-f3c9d0c266df12c37ff5": 10038, + "52299a6368cebd707692-da244552089b0cc707b8": 5178, + "a0e54995ea868c771645-60f7d7d814a3c472d229": 3624, + "a0e54995ea868c771645-8041346bb7a6931a7ec6": 4198, + "a0e54995ea868c771645-1a1ae8183bc451d530c6": 5527, + "a0e54995ea868c771645-b7fa1bc553b6b3435c30": 4042, + "38ff7d31d451c7116c9b-3c32b3f76c1a64d97d57": 7489, + "a7e9410bcc1ef00e9656-a573a2a36ba1d52d7dce": 1000, + "a7e9410bcc1ef00e9656-d9763a85c28305121b18": 1000, + "a7e9410bcc1ef00e9656-3f5f3587ec650f5ea6c5": 1000, + "a7e9410bcc1ef00e9656-108789ca1bc58dc018e2": 1000, + "d877ffdf71ec720167ae-a772e81fbd16d188c36a": 10726, + "d877ffdf71ec720167ae-5a0fe0a79ee33201d000": 10714, + "d877ffdf71ec720167ae-3adef19e3468b998ac0c": 5319, + "d877ffdf71ec720167ae-92cca7b93353f754bc83": 7625, + "d877ffdf71ec720167ae-4540c2f85ff2c6b1d0d2": 11445, + "d877ffdf71ec720167ae-26d652ac7c33eb7c9b8f": 5800, + "d877ffdf71ec720167ae-c49a1812f2feb092b170": 5466, + "d877ffdf71ec720167ae-23f6432c38a605a459dd": 5288, + "d877ffdf71ec720167ae-d3ab69f35a3b347729ba": 5637, + "d877ffdf71ec720167ae-226b07519eaea2544710": 5333, + "d877ffdf71ec720167ae-de498129bcddf663d067": 4683, + "d877ffdf71ec720167ae-3f55da29b845b8ae7314": 5395, + "d877ffdf71ec720167ae-3c39a513ae5ac65be49b": 6307, + "d877ffdf71ec720167ae-f847648047d364a507c6": 3808, + "d877ffdf71ec720167ae-d6597aa4f1c592f910b0": 4635, + "d877ffdf71ec720167ae-2580abd9a91f5dd3f72d": 5676, + "d877ffdf71ec720167ae-569b8cbc0bc00cc054fe": 6918, + "d877ffdf71ec720167ae-416cfc12a541fcceab51": 6559, + "d877ffdf71ec720167ae-7cb127c2319066fde3b3": 4792, + "d877ffdf71ec720167ae-5dbb295e5ec7096a16d3": 17769, + "d877ffdf71ec720167ae-99be63fef3540d2c57c0": 17412, + "d877ffdf71ec720167ae-8279faa678c8b88af12d": 5867, + "d877ffdf71ec720167ae-1885033776157823bf89": 5107, + "1b3887103c26287817e9-5c3531e6096dd1529113": 5451, + "1b3887103c26287817e9-72360fd5b26f8b297ac0": 3140, + "1b3887103c26287817e9-c60b24b359b1dce88764": 2594, + "f7c608f214ee45b77db4-78e357cab6653e69a4b8": 3817, + "f7c608f214ee45b77db4-cc14c3d775f38eee992b": 3894, + "f7c608f214ee45b77db4-66f124074900953be807": 5886, + "f7c608f214ee45b77db4-6c5faffabdaca867e15e": 4913, + "f7c608f214ee45b77db4-8d1430703269bab04171": 4709, + "f7c608f214ee45b77db4-69297ff30ba9aaf7b8c5": 3791, + "f7c608f214ee45b77db4-43dc200fb3916ede35a0": 7078, + "f7c608f214ee45b77db4-485e87d35746ddf8f521": 5195, + "f7c608f214ee45b77db4-fd3d70d9748a98a8bf22": 1693, + "f7c608f214ee45b77db4-3444926d2ce44289169d": 16437, + "f7c608f214ee45b77db4-5176501f2de2d3e67944": 4761, + "4219922fea2e2bd3c691-20dea26d6b139e76cb38": 5393, + "4219922fea2e2bd3c691-e895f0e6278aa0e88ce1": 5260, + "4219922fea2e2bd3c691-313db2914dac1ba1e1ed": 3266, + "4219922fea2e2bd3c691-313e9a71e01cd36bd24c": 1000, + "4219922fea2e2bd3c691-e587d9ed2e2a3fc369d2": 6776, + "4219922fea2e2bd3c691-601cfa3e67a0299aae31": 1000, + "6e02b10b8c48e14325ce-ac48bce6e1472e41800b": 2778, + "6e02b10b8c48e14325ce-db2c64f0ae55d81638b3": 4112, + "6e02b10b8c48e14325ce-0a1f020bfe235e3237d7": 6463, + "583c5232e0c448f0dd4d-ae4de3b0b67f70b00f45": 1000, + "583c5232e0c448f0dd4d-abe58bba579536fe1dd7": 1000, + "583c5232e0c448f0dd4d-d57eb0dcde0dbfbdc292": 1000, + "583c5232e0c448f0dd4d-7250bfb02fafc2aeff8d": 1000, + "583c5232e0c448f0dd4d-9b0aaf7cb2034b39480e": 1000, + "583c5232e0c448f0dd4d-c66b906cf858ad3e0042": 1000, + "583c5232e0c448f0dd4d-6ba954456ae530aebcfc": 1000, + "583c5232e0c448f0dd4d-de98838dad4f30cf3568": 1000, + "583c5232e0c448f0dd4d-0c4988202ccf82c19f6e": 1019, + "2d5735934fcfb6f50388-6701d439a5fdf3e1ed0d": 13760, + "2d5735934fcfb6f50388-44195843f09a7915d54b": 3879, + "d4043a9dd1ab94463227-6221604b0c06494b6800": 1000, + "0dcc9561e04af1dfbd33-ce48c51175d81c49b3a3": 15354, + "0dcc9561e04af1dfbd33-86dd9a0dc5c7812bddd5": 16061, + "0dcc9561e04af1dfbd33-208c0c99158f96be4296": 12246, + "0dcc9561e04af1dfbd33-e6427a9562db6236a7f9": 12118, + "0dcc9561e04af1dfbd33-eea5fa6892829dd17987": 15347, + "0dcc9561e04af1dfbd33-f61eb427a41e8f25dfd0": 14555, + "0dcc9561e04af1dfbd33-64d7ed871f84ad3ab30b": 13624, + "0dcc9561e04af1dfbd33-0b25df00927154787216": 13144, + "0dcc9561e04af1dfbd33-442aa966001fec24320f": 12851, + "0dcc9561e04af1dfbd33-a82c4600f69976899c0a": 12580, + "0dcc9561e04af1dfbd33-630356f7fd1d27700b99": 12942, + "0dcc9561e04af1dfbd33-7d271a3bf7138857a889": 13368, + "0dcc9561e04af1dfbd33-65e63a975e4c795d7606": 9902, + "0dcc9561e04af1dfbd33-82ba310a6f3f825dce55": 11231, + "0dcc9561e04af1dfbd33-08b837eebb763a8e75ba": 11483, + "0dcc9561e04af1dfbd33-ff4ae29aebdcbc2c38fc": 11090, + "0dcc9561e04af1dfbd33-c7c8c7aadd0400281792": 9950, + "0dcc9561e04af1dfbd33-a4c4d9d62ccbe53d8e2d": 1000, + "0dcc9561e04af1dfbd33-848ffa4865e3666e12d6": 18943, + "0dcc9561e04af1dfbd33-a36e57164912aaee47c1": 24994, + "0dcc9561e04af1dfbd33-72d0a755adb16af5627a": 14020, + "ba846ca5ce61c11e304c-1bbae16cfbe6452d85a8": 6987, + "ba846ca5ce61c11e304c-acf2b47e1dbd03fea0ba": 10936, + "ba846ca5ce61c11e304c-d2f8baa4000d7c76a279": 5599, + "74d1f5db5cd4f0db0c8e-a374e33b472bc87a95a3": 4983, + "74d1f5db5cd4f0db0c8e-e51a12cabd4558c590d7": 4606, + "74d1f5db5cd4f0db0c8e-944716b2dc40faf47d5c": 3672, + "c83c749215335e4d58cf-7fa6b73980fe3ff71661": 7037, + "c83c749215335e4d58cf-8883ad8c7e4c7b379994": 6663, + "c8cd569b3399ecc87f89-7295f18e0386363e7d72": 4248, + "c8cd569b3399ecc87f89-9f6d0b7b305e84794404": 6268, + "d396f6276c2dfa296cb6-9f2a5bfdaca2c994a8d2": 3114, + "d396f6276c2dfa296cb6-2d6793a6ab3a9c1b4f66": 5052, + "d396f6276c2dfa296cb6-00b6a4edf47d065e01fe": 6569, + "d396f6276c2dfa296cb6-90046680b4350ad6a982": 4448, + "d396f6276c2dfa296cb6-934fe535c5ff1d3f295d": 2642, + "d396f6276c2dfa296cb6-fa7dde43685514b42aa4": 2578, + "d396f6276c2dfa296cb6-41439244d20686401fa5": 6694, + "d396f6276c2dfa296cb6-237ccba63a370a062ee4": 3746, + "d396f6276c2dfa296cb6-67f1ac0f81d6025e039e": 7456, + "d396f6276c2dfa296cb6-a84faa4a4c068c0de561": 5911, + "d396f6276c2dfa296cb6-92f72f8712f7c2671186": 3924, + "d396f6276c2dfa296cb6-dcbbc943ae5f1f1872f4": 3813, + "d396f6276c2dfa296cb6-4c16bbdec46071bfca84": 3554, + "d396f6276c2dfa296cb6-f4ab1ca4d7b99494faca": 3893, + "d396f6276c2dfa296cb6-71bcafa9264dc884b99c": 3313, + "d396f6276c2dfa296cb6-cca171b5380ac8b9b935": 3678, + "d396f6276c2dfa296cb6-9f3424c3bae60d9ffb69": 5078, + "d396f6276c2dfa296cb6-649c9e33bdd8c9b56570": 20890, + "4548c29124195ce38b0b-135fe04c81dfd510d6e3": 5900, + "2fd95dd700a123285254-71bc236bc38bb9f9a329": 3769, + "2fd95dd700a123285254-3237b374591c586f2371": 2968, + "2fd95dd700a123285254-c0acf4ef0227f6839310": 4810, + "2fd95dd700a123285254-365f15c2dd1fb5827141": 2709, + "1232c0efb955ebd14840-9581a0cbd0252a0bffd3": 2519, + "1232c0efb955ebd14840-897afe4cdf5ee662d3d8": 7168, + "1232c0efb955ebd14840-eec0469d4b3f58ff1f7b": 8747, + "1232c0efb955ebd14840-8cd0e4776b322df15b15": 5585, + "1232c0efb955ebd14840-84906b062915b6ccdc6f": 6246, + "1232c0efb955ebd14840-c03e128337974f19c9c3": 6547, + "1232c0efb955ebd14840-1005115e9d167dad8892": 4906, + "92f846591084a08aaff9-f1a76b394ba277d9472b": 4718, + "92f846591084a08aaff9-4bbbf050a5d22172674a": 5832, + "92f846591084a08aaff9-da09d5e4cc175febcb05": 4538, + "92f846591084a08aaff9-5bbf47b4c09784243b05": 4636, + "92f846591084a08aaff9-39b8dcbbd4dabb839fe5": 10874, + "d748ac400d08b85935ef-d53b41c81039c115bd51": 7778, + "d748ac400d08b85935ef-9c4bf1f9cb3fbb79afce": 10032, + "d748ac400d08b85935ef-a965ccf2e98055d20350": 18195, + "d748ac400d08b85935ef-56eaca019ca8a0353b90": 13070, + "d748ac400d08b85935ef-80b4116343a9f1491c05": 13066, + "d748ac400d08b85935ef-e703470b86596a618c24": 13419, + "d748ac400d08b85935ef-d92bf9f2b253e69eb0b1": 32415, + "d748ac400d08b85935ef-c5d456ba8abd738394d5": 41766, + "d748ac400d08b85935ef-6584cf53199cebbc9142": 10595, + "d748ac400d08b85935ef-908f6ba0c38e2c7c7f22": 3341, + "d748ac400d08b85935ef-ded4774a1b09861dee5a": 25481, + "d748ac400d08b85935ef-88cae0794cc039d88063": 26167, + "d748ac400d08b85935ef-61c3cc8e882a650a0669": 1000, + "d748ac400d08b85935ef-a1b18674401378dc01e8": 1000, + "d748ac400d08b85935ef-3905d0ca7a885f69ff06": 3203, + "4d12dc0369cb90bc5a69-2f47863b652c64646849": 8778, + "4d12dc0369cb90bc5a69-fbbcf493bd23eb4efde5": 7706, + "4d12dc0369cb90bc5a69-851f87c6a180c1ca6e33": 13835, + "4d12dc0369cb90bc5a69-3aa9421283c90eed6dac": 14053, + "4d12dc0369cb90bc5a69-4a706c4f4061ead9e0bf": 8016, + "1c8909c5413b5986f1cc-bec97492f59721d3ada9": 9315, + "1c8909c5413b5986f1cc-926564ef5ea04a923fe2": 11224, + "1c8909c5413b5986f1cc-cdf9d7bab68bea0a2866": 9628, + "10083587e442a46a627e-c471e97ca3641fa593bb": 8320, + "10083587e442a46a627e-e79ac4d0df7ea4babf59": 5653, + "10083587e442a46a627e-b3ac174c7367c50cfabd": 7360, + "10083587e442a46a627e-7223559d07267e4e48d7": 3686, + "10083587e442a46a627e-ce68e7711844ad1cc921": 5643, + "10083587e442a46a627e-907578807908d6b6db45": 5072, + "10083587e442a46a627e-b2cc667da8641af3cb10": 5799, + "10083587e442a46a627e-4d250e0e5c7cb259fef0": 5941, + "10083587e442a46a627e-e0a8c82b846c85134894": 5742, + "10083587e442a46a627e-dd9a49cd561c466c6601": 5115, + "10083587e442a46a627e-60cca78e61b4df8a9737": 5872, + "10083587e442a46a627e-ab1e6fbe14f6254b5eff": 6221, + "10083587e442a46a627e-c8711d65db481e9ad3c5": 5857, + "10083587e442a46a627e-c330dddba602bf7ac01d": 5873, + "10083587e442a46a627e-e5eb7c672eb44a909004": 5867, + "10083587e442a46a627e-c06dbd774747b025019f": 5902, + "10083587e442a46a627e-fd7daf200fb2b38ca130": 4377, + "10083587e442a46a627e-5dd08f5b0c6d7b0480e3": 3999, + "450147eb64b66e68c76a-d5fbafd89119ffb6f5ee": 1000, + "450147eb64b66e68c76a-ef6f6534bda863392672": 1000, + "450147eb64b66e68c76a-4ef3227e5e60452c5d51": 1000, + "450147eb64b66e68c76a-9e3542b5f9f013558cdd": 1000, + "5578aff731b4c2fd6476-756dcba093d6f1c0c7ee": 12436, + "5578aff731b4c2fd6476-f45776b035298099c771": 19534, + "5578aff731b4c2fd6476-9af8e86594f84ee2ccdb": 6067, + "5578aff731b4c2fd6476-0e59e784522549d70c7f": 13465, + "5578aff731b4c2fd6476-3a49c62c0d81812d3a6e": 9032, + "3c3389f956afd5d7b5be-3ec4b73a8e0acfe7378b": 5182, + "3c3389f956afd5d7b5be-20edfd9f27c46808ab2b": 8292, + "3c3389f956afd5d7b5be-04a5131be77557bc5466": 8026, + "3c3389f956afd5d7b5be-0e47332f61cb39e14b09": 12170, + "50fea97024d96f5f2763-414e86a98d0ff4eaf960": 1000, + "50fea97024d96f5f2763-b4e4b176d8f0fa0f5976": 2954, + "50fea97024d96f5f2763-fc374c50039ea091c419": 3362, + "50fea97024d96f5f2763-2e7243fabc3e34ff9044": 1000, + "50fea97024d96f5f2763-8dbe1d35e4153ba72868": 1448, + "50fea97024d96f5f2763-67b91eab68551e899cb2": 3891, + "2032af3e7446fbae9504-132f4817f5925f6f4732": 8041, + "2032af3e7446fbae9504-391e8e437d8093a77c40": 7766, + "2032af3e7446fbae9504-4ea5bdcdbb4e7186b564": 9908, + "2032af3e7446fbae9504-077408908c58246cd3e7": 8663, + "2032af3e7446fbae9504-0c557c69d1535e082ec5": 8774, + "2032af3e7446fbae9504-4eb0f34baa5d12deb3ae": 8841, + "2032af3e7446fbae9504-d25a48651be80050889a": 8953, + "2032af3e7446fbae9504-0485aae22eb87ed1c09f": 10530, + "2032af3e7446fbae9504-3d61cea895d30949f86b": 10658, + "d2f56d7d370f0ba02e91-36871a0975cbb87ce6bc": 7189, + "d2f56d7d370f0ba02e91-1461af053d2680d6974f": 8696, + "d2f56d7d370f0ba02e91-428826278c34d7991d12": 7954, + "d2f56d7d370f0ba02e91-e397f69a3e602ed1474d": 10722, + "d2f56d7d370f0ba02e91-755b9e23959cc95c4dac": 11606, + "d2f56d7d370f0ba02e91-d353f7ea03171ce9f299": 31945, + "d2f56d7d370f0ba02e91-451603b8d7336b62d776": 8195, + "d2f56d7d370f0ba02e91-1382384d563f6d3875e8": 9663, + "d2f56d7d370f0ba02e91-986bb5b7d7e9214bf592": 9784, + "d2f56d7d370f0ba02e91-8d8aba181bce63fbc017": 8672, + "d2f56d7d370f0ba02e91-1be4a2fd2763e42a2545": 8895, + "d2f56d7d370f0ba02e91-4dd669c65ae7d1e0af37": 8410, + "d2f56d7d370f0ba02e91-3274248ae79e9ee81661": 20101, + "d2f56d7d370f0ba02e91-ac7a4670b21299e52139": 20195, + "d2f56d7d370f0ba02e91-b8ad157396a572d5accb": 8698, + "d2f56d7d370f0ba02e91-4f0c7be13731868171de": 8937, + "d2f56d7d370f0ba02e91-31ad41b6d96d0950cab1": 9432, + "d2f56d7d370f0ba02e91-a1a82097472601383c0d": 8888, + "d2f56d7d370f0ba02e91-0f30599773fdcd2ee901": 9689, + "8aeeb858911acce61c44-61647a32c07fc2a545ae": 1000, + "8aeeb858911acce61c44-d48a596ffa5cedc90435": 1000, + "8aeeb858911acce61c44-093c841b7569d19bbee5": 1000, + "8aeeb858911acce61c44-f380269ed9fa4d255a4c": 1000, + "8aeeb858911acce61c44-d79c60dfa0f0822e2459": 1000, + "65cc2781c99e75c568c3-270dbcecdcb0fb651082": 4469, + "65cc2781c99e75c568c3-0b3bf4e548b17f98a9bf": 3705, + "65cc2781c99e75c568c3-6e86b960fce8cf236ad8": 5153, + "eccc8495635fa8823817-46debcd0747427b5ed66": 4020, + "eccc8495635fa8823817-21a18e656d8388cfb6e7": 7817, + "eccc8495635fa8823817-e94662cd61ce6360996c": 6756, + "eccc8495635fa8823817-3b0f1db91b05fa545ae7": 4697, + "eccc8495635fa8823817-322e1a8b3be405f9dfbb": 5732, + "eccc8495635fa8823817-6f4a86b292f5375e601f": 6763, + "eccc8495635fa8823817-45220bce19b29f824481": 3226, + "eccc8495635fa8823817-0e170e3dc73358816dc2": 16478, + "eccc8495635fa8823817-ba2c3f93324896bd0de9": 14251, + "eccc8495635fa8823817-7077b12683036b3f70dc": 15842, + "eccc8495635fa8823817-b68c636bcc4472b8bcee": 15136, + "eccc8495635fa8823817-d29bcf363efb6e75d865": 15564, + "9c5c505a22ce15e27478-aab64d1dcf0061f66c7e": 11515, + "9c5c505a22ce15e27478-f67f490d44bba80aab2d": 10965, + "9c5c505a22ce15e27478-9adf8ce0bec80ce41c88": 9467, + "9c5c505a22ce15e27478-b5b03ad9f794a1ae45ce": 10959, + "9c5c505a22ce15e27478-70a76796f98a80c5767f": 10515, + "9c5c505a22ce15e27478-2fc24809b2e1e5117fac": 10531, + "9c5c505a22ce15e27478-f3b424b46dce4b9bab77": 10552, + "9c5c505a22ce15e27478-dfd03e7f8d0d007fd393": 10541, + "9c5c505a22ce15e27478-9a0631594ae11ecc68bc": 10846, + "9c5c505a22ce15e27478-d344e71b731399ff26c1": 10810, + "9c5c505a22ce15e27478-ec4c3ee59a40455845a4": 8744, + "9c5c505a22ce15e27478-b352593adcb521ba47d8": 11237, + "9c5c505a22ce15e27478-cade55ba0440709c9093": 14724, + "9c5c505a22ce15e27478-d03ec619e0a201d70c56": 9120, + "9c5c505a22ce15e27478-852d9aeb939eb234193c": 9301, + "9c5c505a22ce15e27478-c6e968b13c365a524966": 9691, + "9c5c505a22ce15e27478-7dce5cc7183e100e7bda": 9756, + "9c5c505a22ce15e27478-b3eebc4e2f5c904a14b4": 9842, + "9c5c505a22ce15e27478-42689dd5c6309b417313": 9461, + "9c5c505a22ce15e27478-1aae155f9ada624d002a": 9161, + "9c5c505a22ce15e27478-fc329875d5fc01972d5e": 11905, + "9c5c505a22ce15e27478-592f35b0856c3f69023e": 11796, + "9c5c505a22ce15e27478-09673ed3b7a115411fe3": 5998, + "9c5c505a22ce15e27478-26a98a3e64f7336bb023": 8798, + "9c5c505a22ce15e27478-898ec7479a4a8ee8174e": 8711, + "9c5c505a22ce15e27478-98089f7e3be8c4451c4d": 9505, + "9c5c505a22ce15e27478-57b514ca0b48637e91c8": 9294, + "9c5c505a22ce15e27478-44463deb44a7bed10f9f": 7226, + "9c5c505a22ce15e27478-4e9828b7857fc70351b5": 7355, + "9c5c505a22ce15e27478-d54532bacc742607a7a1": 7450, + "9c5c505a22ce15e27478-8a12340fd87a72cff55a": 8513, + "9c5c505a22ce15e27478-5ac24209303331956272": 7876, + "9c5c505a22ce15e27478-83c6e5ba448b55b416bd": 8139, + "9c5c505a22ce15e27478-1b7d0929e9030907bd5f": 7260, + "7552a30c6376f66a76ac-3c2848cd985d6848a36c": 6093, + "7552a30c6376f66a76ac-3e7de8fff1c7f6a442b7": 8966, + "7552a30c6376f66a76ac-ad1d0ce83c9138350a05": 10326, + "7552a30c6376f66a76ac-c6e2a8534939df71bf0a": 11228, + "7552a30c6376f66a76ac-eb50d1f349b7cf7bdd92": 10162, + "7552a30c6376f66a76ac-42f5d8d0a7cb024a98be": 15515, + "7552a30c6376f66a76ac-f9e1eec3ec852a1f5c54": 12577, + "7552a30c6376f66a76ac-d83de1cd6e15cc8086b3": 9541, + "a19963100c65b4b43fa3-12289ee882758e164340": 5801, + "a19963100c65b4b43fa3-adfcd3f0c63b7e3deb2e": 11080, + "a19963100c65b4b43fa3-5ba0f1d6efb211301b82": 12976, + "a19963100c65b4b43fa3-1d2632500949d52a6a2c": 14591, + "a19963100c65b4b43fa3-0a533f15ced84c386ac9": 9313 + } +} diff --git a/.github/scripts/e2e-duration-history.mjs b/.github/scripts/e2e-duration-history.mjs new file mode 100644 index 00000000..e49484e4 --- /dev/null +++ b/.github/scripts/e2e-duration-history.mjs @@ -0,0 +1,91 @@ +#!/usr/bin/env node + +import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SCHEMA_VERSION = 1; +const PREVIOUS_WEIGHT = 0.7; + +function timingFiles(root) { + const files = []; + for (const entry of readdirSync(root)) { + const candidate = join(root, entry); + if (statSync(candidate).isDirectory()) files.push(...timingFiles(candidate)); + else if (entry.endsWith(".json")) files.push(candidate); + } + return files.sort(); +} +function normalizeBase(value) { + if (typeof value === "number") return { durationMs: value, samples: 1 }; + return { + durationMs: value?.durationMs, + samples: Number.isInteger(value?.samples) ? value.samples : 1, + }; +} + +export function mergeDurationHistory({ base, reports }) { + const tests = Object.fromEntries( + Object.entries(base?.tests ?? {}).map(([id, value]) => [id, normalizeBase(value)]), + ); + const observed = new Set(); + for (const { name, report } of reports) { + if (report?.schemaVersion !== SCHEMA_VERSION) throw new Error(`${name} has an unsupported schema`); + if (report.status !== "passed") throw new Error(`${name} did not record a clean Playwright run`); + for (const [id, timing] of Object.entries(report.tests ?? {})) { + if (observed.has(id)) throw new Error(`duplicate timing for test ${id}`); + observed.add(id); + if (timing.status !== "passed" || !Number.isFinite(timing.durationMs) || timing.durationMs <= 0) continue; + const previous = tests[id]; + tests[id] = previous && Number.isFinite(previous.durationMs) + ? { + durationMs: Math.round(previous.durationMs * PREVIOUS_WEIGHT + timing.durationMs * (1 - PREVIOUS_WEIGHT)), + samples: Math.min(20, previous.samples + 1), + } + : { durationMs: Math.round(timing.durationMs), samples: 1 }; + } + } + if (observed.size === 0) throw new Error("no passing test timings were found"); + return { + schemaVersion: SCHEMA_VERSION, + algorithm: "ewma-0.7", + tests: Object.fromEntries(Object.entries(tests).sort(([left], [right]) => left.localeCompare(right))), + }; +} + +function parseArguments(argv) { + const values = {}; + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith("--") || value === undefined) throw new Error(`invalid argument: ${key ?? ""}`); + values[key.slice(2)] = value; + } + return values; +} + +function main() { + const args = parseArguments(process.argv.slice(2)); + if (!args.base || !args.input || !args.output) { + throw new Error("usage: e2e-duration-history.mjs --base --input --output "); + } + const reports = timingFiles(args.input).map((file) => ({ + name: basename(file), + report: JSON.parse(readFileSync(file, "utf8")), + })); + const result = mergeDurationHistory({ + base: JSON.parse(readFileSync(args.base, "utf8")), + reports, + }); + writeFileSync(args.output, `${JSON.stringify(result, null, 2)}\n`); + process.stdout.write(`updated ${Object.keys(result.tests).length} duration records from ${reports.length} shards\n`); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + main(); + } catch (error) { + console.error(`e2e-duration-history: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/.github/scripts/e2e-duration-history.test.mjs b/.github/scripts/e2e-duration-history.test.mjs new file mode 100644 index 00000000..0f015046 --- /dev/null +++ b/.github/scripts/e2e-duration-history.test.mjs @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { mergeDurationHistory } from "./e2e-duration-history.mjs"; + +test("merges clean shard timings with a bounded moving average", () => { + const result = mergeDurationHistory({ + base: { tests: { existing: 10_000, old: { durationMs: 4_000, samples: 20 } } }, + reports: [ + { name: "shard-1.json", report: { schemaVersion: 1, status: "passed", tests: { + existing: { durationMs: 20_000, status: "passed" }, + unseen: { durationMs: 5_000, status: "passed" }, + skipped: { durationMs: 0, status: "skipped" }, + } } }, + { name: "shard-2.json", report: { schemaVersion: 1, status: "passed", tests: { + old: { durationMs: 6_000, status: "passed" }, + } } }, + ], + }); + assert.deepEqual(result.tests.existing, { durationMs: 13_000, samples: 2 }); + assert.deepEqual(result.tests.unseen, { durationMs: 5_000, samples: 1 }); + assert.deepEqual(result.tests.old, { durationMs: 4_600, samples: 20 }); + assert.equal(result.tests.skipped, undefined); +}); +test("rejects failed reports and duplicate coverage", () => { + assert.throws(() => mergeDurationHistory({ + base: { tests: {} }, + reports: [{ name: "failed.json", report: { schemaVersion: 1, status: "failed", tests: {} } }], + }), /did not record a clean/); + const report = { schemaVersion: 1, status: "passed", tests: { same: { durationMs: 1_000, status: "passed" } } }; + assert.throws(() => mergeDurationHistory({ + base: { tests: {} }, + reports: [{ name: "one.json", report }, { name: "two.json", report }], + }), /duplicate timing/); +}); diff --git a/.github/scripts/e2e-duration-plan.mjs b/.github/scripts/e2e-duration-plan.mjs new file mode 100644 index 00000000..6e474dcf --- /dev/null +++ b/.github/scripts/e2e-duration-plan.mjs @@ -0,0 +1,159 @@ +#!/usr/bin/env node + +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SCHEMA_VERSION = 1; +const DEFAULT_DURATION_MS = 8_000; +const MINIMUM_DURATION_MS = 1_000; + +function walkSuites(suites, ancestors = [], tests = []) { + for (const suite of suites ?? []) { + const nextAncestors = suite.title && suite.title !== suite.file + ? [...ancestors, suite.title] + : ancestors; + for (const spec of suite.specs ?? []) { + for (const project of spec.tests ?? []) { + tests.push({ + id: spec.id, + projectName: project.projectName, + file: spec.file, + line: spec.line, + column: spec.column, + path: nextAncestors.filter(Boolean), + title: spec.title, + tags: spec.tags ?? [], + }); + } + } + walkSuites(suite.suites, nextAncestors, tests); + } + return tests; +} + +export function inventoryTests(report, { project = "chromium", tag } = {}) { + const tests = walkSuites(report?.suites).filter((test) => + test.projectName === project && (!tag || test.tags.includes(tag)), + ); + const ids = new Set(); + const selectors = new Set(); + for (const test of tests) { + if (!test.id || ids.has(test.id)) { + throw new Error(`inventory contains a missing or duplicate test id: ${test.id ?? ""}`); + } + ids.add(test.id); + test.selector = [ + `[${test.projectName}]`, + `${test.file}:${test.line}:${test.column}`, + ...test.path, + test.title, + ].join(" › "); + if (selectors.has(test.selector)) { + throw new Error(`inventory contains a duplicate selector: ${test.selector}`); + } + selectors.add(test.selector); + } + if (tests.length === 0) throw new Error("inventory did not contain any matching tests"); + return tests; +} + +function durationFor(test, history) { + const value = history?.tests?.[test.id]; + const duration = typeof value === "number" ? value : value?.durationMs; + return Number.isFinite(duration) && duration > 0 + ? Math.max(MINIMUM_DURATION_MS, Math.round(duration)) + : DEFAULT_DURATION_MS; +} + +export function buildDurationPlan({ tests, history, shardCount }) { + if (!Number.isInteger(shardCount) || shardCount < 1) { + throw new Error("shard count must be a positive integer"); + } + const shards = Array.from({ length: shardCount }, (_, index) => ({ + shard: index + 1, + predictedWorkMs: 0, + tests: [], + })); + const weighted = tests + .map((test) => ({ ...test, durationMs: durationFor(test, history) })) + .sort((left, right) => right.durationMs - left.durationMs || left.id.localeCompare(right.id)); + + for (const test of weighted) { + shards.sort((left, right) => + left.predictedWorkMs - right.predictedWorkMs || + left.tests.length - right.tests.length || + left.shard - right.shard, + ); + shards[0].tests.push(test); + shards[0].predictedWorkMs += test.durationMs; + } + shards.sort((left, right) => left.shard - right.shard); + + const plannedIds = shards.flatMap((shard) => shard.tests.map((test) => test.id)); + if (plannedIds.length !== tests.length || new Set(plannedIds).size !== tests.length) { + throw new Error("duration plan did not assign every inventory test exactly once"); + } + return { + schemaVersion: SCHEMA_VERSION, + algorithm: "longest-predicted-first", + testCount: tests.length, + shardCount, + defaultDurationMs: DEFAULT_DURATION_MS, + shards, + }; +} + +export function writeDurationPlan(plan, outputDirectory) { + mkdirSync(outputDirectory, { recursive: true }); + const manifest = { + ...plan, + shards: plan.shards.map((shard) => ({ + shard: shard.shard, + predictedWorkMs: shard.predictedWorkMs, + testCount: shard.tests.length, + file: `shard-${shard.shard}.txt`, + })), + }; + for (const shard of plan.shards) { + writeFileSync( + resolve(outputDirectory, `shard-${shard.shard}.txt`), + `${shard.tests.map((test) => test.selector).join("\n")}\n`, + ); + } + writeFileSync(resolve(outputDirectory, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); + return manifest; +} + +function parseArguments(argv) { + const values = {}; + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith("--") || value === undefined) throw new Error(`invalid argument: ${key ?? ""}`); + values[key.slice(2)] = value; + } + return values; +} + +function main() { + const args = parseArguments(process.argv.slice(2)); + if (!args.inventory || !args.history || !args.output || !args.shards) { + throw new Error("usage: e2e-duration-plan.mjs --inventory --history --output --shards [--tag ]"); + } + const inventory = JSON.parse(readFileSync(args.inventory, "utf8")); + const history = JSON.parse(readFileSync(args.history, "utf8")); + const tests = inventoryTests(inventory, { tag: args.tag }); + const plan = buildDurationPlan({ tests, history, shardCount: Number(args.shards) }); + const manifest = writeDurationPlan(plan, args.output); + process.stdout.write(`${JSON.stringify(manifest, null, 2)}\n`); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + main(); + } catch (error) { + console.error(`e2e-duration-plan: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/.github/scripts/e2e-duration-plan.test.mjs b/.github/scripts/e2e-duration-plan.test.mjs new file mode 100644 index 00000000..5b1f9818 --- /dev/null +++ b/.github/scripts/e2e-duration-plan.test.mjs @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { buildDurationPlan, inventoryTests, writeDurationPlan } from "./e2e-duration-plan.mjs"; + +function inventory() { + return { + suites: [{ + title: "lesson.spec.ts", + file: "lesson.spec.ts", + suites: [{ + title: "lesson journey", + specs: [ + { id: "slow", title: "slow path", file: "lesson.spec.ts", line: 10, column: 3, tags: ["lane:critical"], tests: [{ projectName: "chromium" }] }, + { id: "medium", title: "medium path", file: "lesson.spec.ts", line: 20, column: 3, tags: [], tests: [{ projectName: "chromium" }] }, + { id: "fast", title: "fast path", file: "lesson.spec.ts", line: 30, column: 3, tags: ["lane:critical"], tests: [{ projectName: "chromium" }] }, + { id: "webkit", title: "other browser", file: "lesson.spec.ts", line: 40, column: 3, tags: [], tests: [{ projectName: "webkit" }] }, + ], + }], + }], + }; +} + +test("balances measured durations and assigns every Chromium test once", () => { + const tests = inventoryTests(inventory()); + const plan = buildDurationPlan({ + tests, + history: { tests: { slow: 20_000, medium: 11_000, fast: 9_000 } }, + shardCount: 2, + }); + assert.equal(plan.testCount, 3); + assert.deepEqual(plan.shards.map((shard) => shard.predictedWorkMs), [20_000, 20_000]); + assert.deepEqual( + plan.shards.flatMap((shard) => shard.tests.map((candidate) => candidate.id)).sort(), + ["fast", "medium", "slow"], + ); +}); + +test("uses a conservative default for unseen tests and can select a tagged lane", () => { + const tests = inventoryTests(inventory(), { tag: "lane:critical" }); + const plan = buildDurationPlan({ tests, history: { tests: { slow: 12_000 } }, shardCount: 2 }); + assert.deepEqual(plan.shards.map((shard) => shard.predictedWorkMs), [12_000, 8_000]); +}); + +test("writes Playwright test-list files plus a compact manifest", () => { + const output = mkdtempSync(join(tmpdir(), "e2e-duration-plan-")); + const plan = buildDurationPlan({ + tests: inventoryTests(inventory()), + history: { tests: {} }, + shardCount: 2, + }); + const manifest = writeDurationPlan(plan, output); + assert.equal(manifest.testCount, 3); + const selectors = [1, 2] + .flatMap((shard) => readFileSync(join(output, `shard-${shard}.txt`), "utf8").trim().split("\n")) + .sort(); + assert.deepEqual(selectors, [ + "[chromium] › lesson.spec.ts:10:3 › lesson journey › slow path", + "[chromium] › lesson.spec.ts:20:3 › lesson journey › medium path", + "[chromium] › lesson.spec.ts:30:3 › lesson journey › fast path", + ]); +}); + +test("rejects duplicate ids instead of silently dropping coverage", () => { + const report = inventory(); + report.suites[0].suites[0].specs[1].id = "slow"; + assert.throws(() => inventoryTests(report), /duplicate test id/); +}); diff --git a/.github/scripts/e2e-shard-capacity.test.mjs b/.github/scripts/e2e-shard-capacity.test.mjs index 8603d659..4d8791a1 100644 --- a/.github/scripts/e2e-shard-capacity.test.mjs +++ b/.github/scripts/e2e-shard-capacity.test.mjs @@ -63,12 +63,37 @@ test("tracked decision preserves the clean controlled benchmark evidence", () => }); test("blocking workflow uses the selected matrix and derives its denominator", () => { - const shardMatrix = workflow.match(/matrix:\n\s+shard: \[([^\]]+)]/)?.[1] + const exhaustiveJob = workflow.match(/\n e2e:\n([\s\S]+?)\n cross-browser-core:/)?.[1] ?? ""; + const shardMatrix = exhaustiveJob.match(/matrix:\n\s+shard: \[([^\]]+)]/)?.[1] .split(",") .map((value) => Number(value.trim())); assert.deepEqual(shardMatrix, Array.from({ length: record.selectedShards }, (_, index) => index + 1)); - assert.match(workflow, /--active-shards "\$\{\{ strategy\.job-total }}/); - assert.match(workflow, /--shard=\$\{\{ matrix\.shard }}\/\$\{\{ strategy\.job-total }}/); + assert.match(exhaustiveJob, /--active-shards "\$\{\{ strategy\.job-total }}/); + assert.match(workflow, /--output e2e\/duration-plan\/full[\s\S]+--shards 16/); + assert.match(exhaustiveJob, /--test-list=duration-plan-artifact\/full\/shard-\$\{\{ matrix\.shard }}\.txt/); + assert.match(exhaustiveJob, /name: Upload test-duration evidence/); +}); + +test("advisory critical coverage is split across two isolated duration-balanced jobs", () => { + assert.match(workflow, /--output e2e\/duration-plan\/critical[\s\S]+--shards 2[\s\S]+--tag lane:critical/); + assert.match(workflow, /critical-shadow:[\s\S]+matrix:\n\s+shard: \[1, 2]/); + assert.doesNotMatch(workflow, /critical-shadow-summary:/); + assert.match(workflow, /shadow-evidence:[\s\S]+needs: \[duration-plan, critical-shadow, e2e, cross-browser-core][\s\S]+files\.length!==2/); +}); + +test("duration planning receives the authenticated fixture environment required for discovery", () => { + const planningJob = workflow.match(/\n duration-plan:\n([\s\S]+?)\n prepare-backend:/)?.[1] ?? ""; + for (const variable of [ + "SUPABASE_URL", + "SUPABASE_ANON_KEY", + "SUPABASE_SERVICE_ROLE_KEY", + "VITE_SUPABASE_URL", + "VITE_SUPABASE_ANON_KEY", + "DATABASE_URL", + "BYOK_ENCRYPTION_KEY", + ]) { + assert.match(planningJob, new RegExp(`${variable}: \\$\\{\\{ secrets\\.${variable} }}`)); + } }); test("accepts the measured inventory and normal growth", () => { diff --git a/.github/scripts/pull-container-images.sh b/.github/scripts/pull-container-images.sh new file mode 100755 index 00000000..5be9fab6 --- /dev/null +++ b/.github/scripts/pull-container-images.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if (( $# == 0 )); then + echo "usage: pull-container-images.sh [...]" >&2 + exit 2 +fi +pids=() +for image in "$@"; do + docker pull "$image" & + pids+=("$!") +done + +failed=0 +for pid in "${pids[@]}"; do + if ! wait "$pid"; then + failed=1 + fi +done + +if (( failed != 0 )); then + echo "one or more container image pulls failed" >&2 + exit 1 +fi diff --git a/.github/scripts/pull-container-images.test.mjs b/.github/scripts/pull-container-images.test.mjs new file mode 100644 index 00000000..f9e797f8 --- /dev/null +++ b/.github/scripts/pull-container-images.test.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { chmodSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; + +const script = resolve(".github/scripts/pull-container-images.sh"); +const workflow = readFileSync(resolve(".github/workflows/e2e.yml"), "utf8"); +const compose = readFileSync(resolve("docker-compose.yml"), "utf8"); + +function fixture() { + const directory = mkdtempSync(join(tmpdir(), "parallel-pulls-")); + const docker = join(directory, "docker"); + const log = join(directory, "calls.log"); + writeFileSync(docker, `#!/usr/bin/env bash\necho "$2" >> "$PULL_LOG"\n[[ "$2" != "$FAIL_IMAGE" ]]\n`); + chmodSync(docker, 0o755); + return { directory, log }; +} + +test("pulls every supplied immutable image and succeeds when all pulls pass", () => { + const { directory, log } = fixture(); + const result = spawnSync(script, ["backend@sha256:a", "runner@sha256:b", "frontend@sha256:c"], { + env: { ...process.env, PATH: `${directory}:${process.env.PATH}`, PULL_LOG: log, FAIL_IMAGE: "" }, + }); + assert.equal(result.status, 0, result.stderr.toString()); + assert.deepEqual(readFileSync(log, "utf8").trim().split("\n").sort(), [ + "backend@sha256:a", + "frontend@sha256:c", + "runner@sha256:b", + ]); +}); +test("waits for every pull but fails closed when any one pull fails", () => { + const { directory, log } = fixture(); + const result = spawnSync(script, ["backend", "runner", "frontend"], { + env: { ...process.env, PATH: `${directory}:${process.env.PATH}`, PULL_LOG: log, FAIL_IMAGE: "runner" }, + }); + assert.equal(result.status, 1); + assert.match(result.stderr.toString(), /one or more container image pulls failed/); + assert.deepEqual(readFileSync(log, "utf8").trim().split("\n").sort(), ["backend", "frontend", "runner"]); +}); + +test("E2E pre-pulls the pinned socket proxy and probes it faster without changing local defaults", () => { + assert.match(workflow, /SOCKET_PROXY_IMAGE: tecnativa\/docker-socket-proxy:0\.3\.0/); + assert.match(workflow, /SOCKET_PROXY_HEALTH_INTERVAL: "1s"/); + assert.equal( + workflow.match(/pull-container-images\.sh[^\n]+"\$SOCKET_PROXY_IMAGE"/g)?.length, + 3, + ); + assert.match(compose, /image: \$\{SOCKET_PROXY_IMAGE:-tecnativa\/docker-socket-proxy:0\.3\.0}/); + assert.match(compose, /interval: \$\{SOCKET_PROXY_HEALTH_INTERVAL:-5s}/); +}); diff --git a/.github/scripts/security-workflow.test.mjs b/.github/scripts/security-workflow.test.mjs new file mode 100644 index 00000000..3e154a1d --- /dev/null +++ b/.github/scripts/security-workflow.test.mjs @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const workflow = readFileSync(new URL("../workflows/security.yml", import.meta.url), "utf8"); +const securityConfig = readFileSync(new URL("../../e2e/security-suite/playwright.config.ts", import.meta.url), "utf8"); +const sharedSetup = readFileSync(new URL("../../e2e/fixtures/boot.ts", import.meta.url), "utf8"); + +test("API-only security scenarios do not build or boot the frontend", () => { + assert.doesNotMatch(workflow, /docker compose build frontend/); + assert.doesNotMatch(workflow, /docker compose up[^\n]*backend frontend/); + assert.match(workflow, /docker compose up -d --no-build backend/); + assert.match(workflow, /docker compose up -d backend/); +}); + +test("API-only security scenarios do not provision unused browser dependencies", () => { + assert.doesNotMatch(workflow, /playwright install(?:-deps| --with-deps)/); + assert.doesNotMatch(workflow, /\.cache\/ms-playwright/); +}); + +test("API-only security mode skips only the frontend readiness probe", () => { + assert.match(securityConfig, /E2E_SKIP_FRONTEND_HEALTH \?\?= "1"/); + assert.match(sharedSetup, /E2E_SKIP_FRONTEND_HEALTH !== "1"/); + assert.match(sharedSetup, /ping\(`\$\{BACKEND}\/api\/health`, "backend"\)/); +}); + +test("host sentinel installs tcpdump only when the runner image lacks it", () => { + assert.match(workflow, /if ! command -v tcpdump/); + assert.match(workflow, /sudo -n tcpdump --version/); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7205b73f..5675ae1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: with: node-version: '20' - name: Test immutable manifest and VM promotion rollback - run: node --test .github/scripts/e2e-forwarded-ip.test.mjs .github/scripts/e2e-shadow-evidence.test.mjs .github/scripts/e2e-shard-benchmark.test.mjs .github/scripts/e2e-runtime-benchmark.test.mjs .github/scripts/e2e-shard-capacity.test.mjs .github/scripts/frontend-release-probe.test.mjs .github/scripts/production-synthetic.test.mjs .github/scripts/release-manifest.test.mjs .github/scripts/release-permissions.test.mjs .github/scripts/vm-promote-candidate.test.mjs scripts/agent-harness.test.mjs scripts/e2e-shadow-contract.test.mjs scripts/production-dependency-audit.test.mjs + run: node --test .github/scripts/e2e-duration-history.test.mjs .github/scripts/e2e-duration-plan.test.mjs .github/scripts/e2e-forwarded-ip.test.mjs .github/scripts/e2e-shadow-evidence.test.mjs .github/scripts/e2e-shard-benchmark.test.mjs .github/scripts/e2e-runtime-benchmark.test.mjs .github/scripts/e2e-shard-capacity.test.mjs .github/scripts/frontend-release-probe.test.mjs .github/scripts/production-synthetic.test.mjs .github/scripts/pull-container-images.test.mjs .github/scripts/release-manifest.test.mjs .github/scripts/release-permissions.test.mjs .github/scripts/security-workflow.test.mjs .github/scripts/vm-promote-candidate.test.mjs scripts/agent-harness.test.mjs scripts/e2e-shadow-contract.test.mjs scripts/production-dependency-audit.test.mjs - name: Validate agent harness contract run: node scripts/agent-harness.mjs doctor --ci - name: Audit production dependencies diff --git a/.github/workflows/e2e-shard-topology.yml b/.github/workflows/e2e-shard-topology.yml index a6585226..76aa9983 100644 --- a/.github/workflows/e2e-shard-topology.yml +++ b/.github/workflows/e2e-shard-topology.yml @@ -111,9 +111,7 @@ jobs: run: | set -euo pipefail if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ] && [ -n "$FRONTEND_IMAGE" ]; then - docker pull "$BACKEND_IMAGE" - docker pull "$RUNNER_IMAGE" - docker pull "$FRONTEND_IMAGE" + .github/scripts/pull-container-images.sh "$BACKEND_IMAGE" "$RUNNER_IMAGE" "$FRONTEND_IMAGE" docker compose up -d --no-build backend frontend else docker compose up -d backend frontend diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c786dc2b..3ba9666a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -50,8 +50,83 @@ env: CI_BACKEND_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-backend CI_RUNNER_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-runner CI_FRONTEND_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-frontend + SOCKET_PROXY_IMAGE: tecnativa/docker-socket-proxy:0.3.0 + SOCKET_PROXY_HEALTH_INTERVAL: "1s" jobs: + duration-plan: + name: Plan duration-balanced E2E shards + runs-on: ubuntu-latest + outputs: + contract-outcome: ${{ steps.contract.outcome }} + # Test discovery imports the same authenticated fixtures as execution. + # Keep its environment in parity with the browser jobs even though this + # planner never starts a browser or calls the services. + env: + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }} + SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} + VITE_SUPABASE_URL: ${{ secrets.VITE_SUPABASE_URL }} + VITE_SUPABASE_ANON_KEY: ${{ secrets.VITE_SUPABASE_ANON_KEY }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + BYOK_ENCRYPTION_KEY: ${{ secrets.BYOK_ENCRYPTION_KEY }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: e2e/package-lock.json + - name: Restore trusted duration history + id: duration-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: .e2e-duration + key: e2e-duration-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + e2e-duration-${{ runner.os }}- + - name: Install e2e deps + working-directory: e2e + run: npm ci + - name: Build coverage-complete duration plan + run: | + set -euo pipefail + mkdir -p .e2e-duration e2e/duration-plan/full e2e/duration-plan/critical + if [ ! -s .e2e-duration/history.json ]; then + cp .github/e2e-duration-seed.json .e2e-duration/history.json + fi + cp .e2e-duration/history.json e2e/duration-plan/history.json + cd e2e + npx playwright test --list --project=chromium --reporter=json > duration-plan/inventory.json + cd .. + node .github/scripts/e2e-duration-plan.mjs \ + --inventory e2e/duration-plan/inventory.json \ + --history .e2e-duration/history.json \ + --output e2e/duration-plan/full \ + --shards 16 + node .github/scripts/e2e-duration-plan.mjs \ + --inventory e2e/duration-plan/inventory.json \ + --history .e2e-duration/history.json \ + --output e2e/duration-plan/critical \ + --shards 2 \ + --tag lane:critical + - name: Validate critical metadata and frozen regression coverage + id: contract + continue-on-error: true + run: | + set -euo pipefail + cd e2e + npx playwright test --list --grep @lane:critical --reporter=json > duration-plan/critical-inventory.json + cd .. + node scripts/e2e-shadow-contract.mjs --inventory e2e/duration-plan/critical-inventory.json + - name: Upload duration plan + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: e2e-duration-plan + path: e2e/duration-plan + retention-days: 7 + if-no-files-found: error + prepare-backend: name: Prepare backend E2E image runs-on: ubuntu-latest @@ -200,16 +275,17 @@ jobs: fi critical-shadow: - name: Playwright critical lane (advisory) - needs: [prepare-backend, prepare-runner, prepare-frontend] + name: Playwright critical lane (advisory ${{ matrix.shard }}/2) + needs: [duration-plan, prepare-backend, prepare-runner, prepare-frontend] runs-on: ubuntu-latest timeout-minutes: 20 # Release 1D begins in shadow mode. The exhaustive Chromium shards below # remain the blocking source of truth until the measured exit gate passes. continue-on-error: true - outputs: - critical-outcome: ${{ steps.critical.outcome }} - contract-outcome: ${{ steps.contract.outcome }} + strategy: + fail-fast: false + matrix: + shard: [1, 2] env: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }} @@ -219,7 +295,7 @@ jobs: DATABASE_URL: ${{ secrets.DATABASE_URL }} BYOK_ENCRYPTION_KEY: ${{ secrets.BYOK_ENCRYPTION_KEY }} METRICS_TOKEN: e2e-test-metrics-token - E2E_USER_SUFFIX: critical-run${{ github.run_id }}-attempt${{ github.run_attempt }} + E2E_USER_SUFFIX: critical-${{ matrix.shard }}-run${{ github.run_id }}-attempt${{ github.run_attempt }} MAX_SESSIONS_PER_USER: "60" MAX_SESSIONS_GLOBAL: "200" DOCKER_EXEC_CONCURRENCY: "16" @@ -250,9 +326,7 @@ jobs: run: | set -euo pipefail if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ] && [ -n "$FRONTEND_IMAGE" ]; then - docker pull "$BACKEND_IMAGE" - docker pull "$RUNNER_IMAGE" - docker pull "$FRONTEND_IMAGE" + .github/scripts/pull-container-images.sh "$BACKEND_IMAGE" "$RUNNER_IMAGE" "$FRONTEND_IMAGE" "$SOCKET_PROXY_IMAGE" docker compose up -d --no-build backend frontend else docker compose up -d backend frontend @@ -262,15 +336,11 @@ jobs: working-directory: e2e run: npm ci - - name: Validate critical metadata and frozen regression coverage - id: contract - run: | - set -euo pipefail - mkdir -p e2e/shadow-results - cd e2e - npx playwright test --list --grep @lane:critical --reporter=json > shadow-results/critical-inventory.json - cd .. - node scripts/e2e-shadow-contract.mjs --inventory e2e/shadow-results/critical-inventory.json + - name: Download duration-balanced critical plan + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: e2e-duration-plan + path: e2e/duration-plan-artifact - name: Cache Playwright browser uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 @@ -280,34 +350,37 @@ jobs: restore-keys: | ${{ runner.os }}-playwright- - - name: Install Playwright browser + - name: Prepare Playwright browser runtime working-directory: e2e - run: npx playwright install --with-deps chromium + run: | + set -euo pipefail + npx playwright install chromium + node scripts/ensure-browser-runtime.mjs chromium - name: Run critical lane without retries id: critical continue-on-error: true working-directory: e2e - run: npx playwright test --grep @lane:critical --project=chromium --retries=0 + run: npx playwright test --test-list=duration-plan-artifact/critical/shard-${{ matrix.shard }}.txt --retries=0 - name: Record advisory outcome if: always() env: - CONTRACT_OUTCOME: ${{ steps.contract.outcome }} + CONTRACT_OUTCOME: ${{ needs.duration-plan.outputs.contract-outcome }} CRITICAL_OUTCOME: ${{ steps.critical.outcome }} run: | mkdir -p e2e/shadow-results node -e 'const fs=require("fs"); fs.writeFileSync("e2e/shadow-results/outcome.json", JSON.stringify({schemaVersion:1, contract:process.env.CONTRACT_OUTCOME, critical:process.env.CRITICAL_OUTCOME, blocking:false, retries:0}, null, 2)+"\n")' - name: Dump docker-compose logs on failure - if: steps.critical.outcome == 'failure' || steps.contract.outcome == 'failure' + if: steps.critical.outcome == 'failure' run: docker compose logs --no-color --tail=300 - name: Upload critical shadow evidence if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: playwright-critical-shadow + name: playwright-critical-shadow-${{ matrix.shard }} path: | e2e/shadow-results e2e/playwright-report @@ -321,7 +394,7 @@ jobs: e2e: name: Playwright (chromium) - needs: [prepare-backend, prepare-runner, prepare-frontend] + needs: [duration-plan, prepare-backend, prepare-runner, prepare-frontend] runs-on: ubuntu-latest # Sharding splits the exhaustive suite across isolated runners. The # capacity record owns the measured decision and the shard-1 guard below @@ -387,6 +460,7 @@ jobs: BACKEND_IMAGE: ${{ needs.prepare-backend.outputs.ref }} RUNNER_IMAGE: ${{ needs.prepare-runner.outputs.ref }} FRONTEND_IMAGE: ${{ needs.prepare-frontend.outputs.ref }} + E2E_TIMING_OUTPUT: duration-results/shard-${{ matrix.shard }}.json steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -411,9 +485,7 @@ jobs: run: | set -euo pipefail if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ] && [ -n "$FRONTEND_IMAGE" ]; then - docker pull "$BACKEND_IMAGE" - docker pull "$RUNNER_IMAGE" - docker pull "$FRONTEND_IMAGE" + .github/scripts/pull-container-images.sh "$BACKEND_IMAGE" "$RUNNER_IMAGE" "$FRONTEND_IMAGE" "$SOCKET_PROXY_IMAGE" docker compose up -d --no-build backend frontend else docker compose up -d backend frontend @@ -423,12 +495,18 @@ jobs: working-directory: e2e run: npm ci + - name: Download duration-balanced shard plan + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: e2e-duration-plan + path: e2e/duration-plan-artifact + - name: Enforce measured shard capacity if: matrix.shard == 1 working-directory: e2e run: | set -euo pipefail - total_tests=$(npx playwright test --list --project=chromium | sed -nE 's/^Total: ([0-9]+) tests.*/\1/p' | tail -1) + total_tests=$(node -e 'const fs=require("fs"); const value=JSON.parse(fs.readFileSync("duration-plan-artifact/full/manifest.json","utf8")); process.stdout.write(String(value.testCount))') test -n "$total_tests" node ../.github/scripts/e2e-shard-capacity.mjs \ --record ../.github/e2e-shard-capacity.json \ @@ -443,18 +521,28 @@ jobs: restore-keys: | ${{ runner.os }}-playwright- - - name: Install Playwright browsers + - name: Prepare Playwright browser runtime working-directory: e2e - run: npx playwright install --with-deps chromium + run: | + set -euo pipefail + npx playwright install chromium + node scripts/ensure-browser-runtime.mjs chromium - name: Run Playwright tests working-directory: e2e - # --shard partitions specs across the matrix — each runner - # picks up ~1/16 of the suite and runs it in parallel with the - # other shards. Each shard has its own docker-compose stack + - # 2 Playwright workers, so the effective parallelism is 32 - # without any single runner being starved. - run: npx playwright test --shard=${{ matrix.shard }}/${{ strategy.job-total }} + # Playwright's built-in sharding balances test count, not runtime. The + # plan assigns every test exactly once using trusted measured duration, + # while each isolated shard keeps the proven two-worker limit. + run: npx playwright test --test-list=duration-plan-artifact/full/shard-${{ matrix.shard }}.txt + + - name: Upload test-duration evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: e2e-timing-shard-${{ matrix.shard }} + path: e2e/duration-results/shard-${{ matrix.shard }}.json + retention-days: 7 + if-no-files-found: warn - name: Dump docker-compose logs on failure if: failure() @@ -534,9 +622,7 @@ jobs: run: | set -euo pipefail if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ] && [ -n "$FRONTEND_IMAGE" ]; then - docker pull "$BACKEND_IMAGE" - docker pull "$RUNNER_IMAGE" - docker pull "$FRONTEND_IMAGE" + .github/scripts/pull-container-images.sh "$BACKEND_IMAGE" "$RUNNER_IMAGE" "$FRONTEND_IMAGE" "$SOCKET_PROXY_IMAGE" docker compose up -d --no-build backend frontend else docker compose up -d backend frontend @@ -554,9 +640,12 @@ jobs: restore-keys: | ${{ runner.os }}-playwright-${{ matrix.browser }}- - - name: Install Playwright browser + - name: Prepare Playwright browser runtime working-directory: e2e - run: npx playwright install --with-deps ${{ matrix.browser }} + run: | + set -euo pipefail + npx playwright install ${{ matrix.browser }} + node scripts/ensure-browser-runtime.mjs ${{ matrix.browser }} - name: Run cross-browser critical journey working-directory: e2e @@ -588,14 +677,63 @@ jobs: if: always() run: docker compose down -v + duration-history: + name: Learn E2E test durations + if: needs.e2e.result == 'success' + needs: [e2e] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Download duration evidence and planning baseline + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: e2e-timing-shard-* + path: e2e/duration-results + merge-multiple: true + - name: Download duration plan baseline + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: e2e-duration-plan + path: e2e/duration-plan-artifact + - name: Update duration history from a clean exhaustive run + run: | + set -euo pipefail + mkdir -p .e2e-duration + node .github/scripts/e2e-duration-history.mjs \ + --base e2e/duration-plan-artifact/history.json \ + --input e2e/duration-results \ + --output .e2e-duration/history.json + - name: Save trusted duration history + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: .e2e-duration + key: e2e-duration-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + shadow-evidence: name: E2E shadow evidence if: always() - needs: [critical-shadow, e2e, cross-browser-core] + needs: [duration-plan, critical-shadow, e2e, cross-browser-core] runs-on: ubuntu-latest steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Download critical outcomes + continue-on-error: true + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: playwright-critical-shadow-* + path: critical-shadow + + - name: Aggregate critical outcome + id: critical-summary + run: | + set -euo pipefail + outcome=$(node -e 'const fs=require("fs"),path=require("path"); if(!fs.existsSync("critical-shadow")){process.stdout.write("failure");process.exit(0)} const files=fs.readdirSync("critical-shadow",{recursive:true}).filter(file=>file.endsWith("outcome.json")); const failed=files.length!==2||files.some(file=>JSON.parse(fs.readFileSync(path.join("critical-shadow",file),"utf8")).critical!=="success"); process.stdout.write(failed?"failure":"success")') + echo "critical-outcome=$outcome" >> "$GITHUB_OUTPUT" + - name: Collect run and job timing env: GH_TOKEN: ${{ github.token }} @@ -618,8 +756,8 @@ jobs: env: FULL_OUTCOME: ${{ needs.e2e.result }} CROSS_BROWSER_OUTCOME: ${{ needs.cross-browser-core.result }} - CRITICAL_OUTCOME: ${{ needs.critical-shadow.outputs.critical-outcome }} - CONTRACT_OUTCOME: ${{ needs.critical-shadow.outputs.contract-outcome }} + CRITICAL_OUTCOME: ${{ steps.critical-summary.outputs.critical-outcome }} + CONTRACT_OUTCOME: ${{ needs.duration-plan.outputs.contract-outcome }} run: | node .github/scripts/e2e-shadow-evidence.mjs \ --run e2e/shadow-results/run.json \ @@ -657,7 +795,6 @@ jobs: - critical-shadow - e2e - cross-browser-core - - shadow-evidence runs-on: ubuntu-latest env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index a29c3274..5c83faf4 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -29,6 +29,7 @@ on: - 'runner-image/**' - 'docker-compose*.yml' - 'e2e/security-suite/**' + - '.github/scripts/pull-container-images.sh' - '.github/workflows/security.yml' schedule: # Nightly run catches drift from upstream base-image updates that @@ -103,37 +104,27 @@ jobs: # belt-and-suspenders no-op on the default runner; keeps the # workflow portable to any runner image that strips it. run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends tcpdump >/dev/null + set -euo pipefail + if ! command -v tcpdump >/dev/null 2>&1; then + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends tcpdump >/dev/null + fi sudo -n tcpdump --version - name: Boot docker-compose stack run: | + set -euo pipefail if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ]; then - docker pull "$BACKEND_IMAGE" - docker pull "$RUNNER_IMAGE" - docker compose build frontend - docker compose up -d --no-build backend frontend + .github/scripts/pull-container-images.sh "$BACKEND_IMAGE" "$RUNNER_IMAGE" + docker compose up -d --no-build backend else - docker compose up -d backend frontend + docker compose up -d backend fi - name: Install e2e deps working-directory: e2e run: npm ci - - name: Cache Playwright browsers - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ hashFiles('e2e/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-playwright- - - - name: Install Playwright (no browser — API context only) - working-directory: e2e - run: npx playwright install-deps - - name: Run security suite working-directory: e2e # Use the suite-specific config so we don't pick up UI specs. diff --git a/backend/src/services/execution/backends/localDocker.test.ts b/backend/src/services/execution/backends/localDocker.test.ts index c07d6960..f0917c6b 100644 --- a/backend/src/services/execution/backends/localDocker.test.ts +++ b/backend/src/services/execution/backends/localDocker.test.ts @@ -2,12 +2,14 @@ import { describe, it, expect } from "vitest"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { PassThrough } from "node:stream"; import { LocalDockerBackend, ensureNoSymlinkInPath, joinHostPath, resolveRunnerWorkspacePermissions, truncateLongLines, + waitForDockerExecCompletion, MAX_LINE_BYTES, } from "./localDocker.js"; import type { SessionHandle } from "./types.js"; @@ -184,6 +186,34 @@ describe("LocalDockerBackend handle cast", () => { }); }); +describe("waitForDockerExecCompletion", () => { + it("resolves when a fast Docker command ended before completion was observed", async () => { + const stream = new PassThrough(); + const ended = new Promise((resolve) => stream.once("end", resolve)); + stream.resume(); + stream.end(); + await ended; + + await expect(waitForDockerExecCompletion(stream)).resolves.toBeUndefined(); + }); + + it("observes a Docker command that completes after registration", async () => { + const stream = new PassThrough(); + const completion = waitForDockerExecCompletion(stream); + stream.end(); + + await expect(completion).resolves.toBeUndefined(); + }); + + it("rejects a Docker stream error instead of reporting cancellation success", async () => { + const stream = new PassThrough(); + const completion = waitForDockerExecCompletion(stream); + stream.destroy(new Error("docker stream failed")); + + await expect(completion).rejects.toThrow("docker stream failed"); + }); +}); + describe("truncateLongLines", () => { // This is the second half of the two-tier output cap. The stream-level // 1 MB cap in `exec` prevents memory blowup; this per-line cap prevents diff --git a/backend/src/services/execution/backends/localDocker.ts b/backend/src/services/execution/backends/localDocker.ts index 355624b2..f6c0a99f 100644 --- a/backend/src/services/execution/backends/localDocker.ts +++ b/backend/src/services/execution/backends/localDocker.ts @@ -44,6 +44,59 @@ export interface LocalDockerBackendOptions { dockerExecConcurrency?: number; } +type DockerExecStream = NodeJS.ReadableStream & { + readonly readableEnded?: boolean; + readonly destroyed?: boolean; + readonly errored?: unknown; +}; + +/** + * Wait for a short-lived Docker exec stream without missing a completion that + * races ahead of listener registration. Dockerode can resolve exec.start() + * after a very fast command has already ended or closed its stream. + */ +export function waitForDockerExecCompletion(stream: DockerExecStream): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = () => { + stream.removeListener("end", complete); + stream.removeListener("close", complete); + stream.removeListener("error", fail); + }; + const complete = () => { + if (settled) return; + settled = true; + cleanup(); + resolve(); + }; + const fail = (error: unknown) => { + if (settled) return; + settled = true; + cleanup(); + reject(error instanceof Error ? error : new Error(String(error))); + }; + + stream.once("end", complete); + stream.once("close", complete); + stream.once("error", fail); + + // The terminal event may have fired before the listeners above were + // attached. Inspect state only after registration so neither ordering can + // leave cancellation waiting forever. + if (stream.errored) { + fail(stream.errored); + return; + } + if (stream.readableEnded || stream.destroyed) { + complete(); + return; + } + + // Consume any diagnostic bytes so a non-empty exec stream can reach end. + stream.resume(); + }); +} + export class LocalDockerBackend implements ExecutionBackend { readonly kind = "local-docker"; private docker: Docker; @@ -368,11 +421,7 @@ export class LocalDockerBackend implements ExecutionBackend { Tty: false, }); const stream = await cancelExec.start({ hijack: true }); - await new Promise((resolve, reject) => { - stream.once("end", resolve); - stream.once("close", resolve); - stream.once("error", reject); - }); + await waitForDockerExecCompletion(stream); } async writeFiles( diff --git a/docker-compose.yml b/docker-compose.yml index 4c84e32b..68396b1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ services: # to "can manage containers and exec in them." Map matches the API calls # LocalDockerBackend actually makes — see backends/localDocker.ts. socket-proxy: - image: tecnativa/docker-socket-proxy:0.3.0 + image: ${SOCKET_PROXY_IMAGE:-tecnativa/docker-socket-proxy:0.3.0} environment: CONTAINERS: "1" # create/start/stop/inspect/remove + self-inspect EXEC: "1" # exec create/start/inspect for runner commands @@ -33,7 +33,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:2375/_ping"] - interval: 5s + interval: ${SOCKET_PROXY_HEALTH_INTERVAL:-5s} timeout: 2s retries: 10 # Audit-v2 SRE-C3 (companion to the runner LogConfig in localDocker.ts): diff --git a/e2e/README.md b/e2e/README.md index ba5ff043..6b7a964b 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -162,3 +162,21 @@ the live Chromium inventory and fails closed when it reaches 467 tests or falls to 411, one measured shard-workload from the 439-test baseline. Re-run the benchmark and update the record at that point instead of guessing a new shard count or selecting tests away. + +The blocking 16-shard lane uses a duration-aware plan rather than Playwright's +test-count-only partition. `.github/e2e-duration-seed.json` is the cold-start +baseline from a clean 439-test run. Before each workflow, the planner enumerates +the current Chromium inventory and assigns the longest predicted test to the +least-loaded shard until every test appears exactly once. Unseen tests receive +an eight-second conservative estimate, so additions cannot disappear from the +suite. A successful exhaustive run publishes per-test timings; a separate +post-processing job updates a branch-scoped moving-average cache for the next +run. Forks can read the trusted default-branch history but cannot publish it. +The tracked seed remains the deterministic fallback if no cache is available. + +The 63-test advisory critical lane consumes the same inventory and duration +history but runs as two isolated shards. This preserves its frozen contract, +zero-retry posture, and complete coverage while removing one duplicated serial +bottleneck. Browser jobs still use two Playwright workers: the measured +three-worker candidate was faster but failed under resource contention, so +duration balancing adds no extra pressure inside an individual runner. diff --git a/e2e/fixtures/boot.ts b/e2e/fixtures/boot.ts index 0be4b4ac..e2126326 100644 --- a/e2e/fixtures/boot.ts +++ b/e2e/fixtures/boot.ts @@ -1,7 +1,7 @@ -// Global setup. Asserts the frontend + backend are reachable before any spec -// runs. We don't boot the stack ourselves — that's the developer's one-time -// `docker compose up -d` setup. Surfacing a clear error here is far more -// useful than each spec timing out on a connection refused. +// Global setup. Browser suites require frontend + backend reachability; +// API-only suites can explicitly skip the frontend probe. We don't boot the +// stack ourselves — that's the developer's one-time `docker compose up -d` +// setup. Surfacing a clear error here is more useful than per-spec timeouts. import { request } from "@playwright/test"; @@ -34,13 +34,16 @@ async function ping(url: string, label: string) { } export default async function globalSetup() { - await Promise.all([ - ping(FRONTEND, "frontend"), + const readinessChecks = [ // /api/health is the unauthenticated liveness probe. Post-20-P3 the // previous target (/api/ai/validate-key) is auth-gated, so a 401 leaks // into the ping path and obscures actual connectivity failures. ping(`${BACKEND}/api/health`, "backend"), - ]); + ]; + if (process.env.E2E_SKIP_FRONTEND_HEALTH !== "1") { + readinessChecks.push(ping(FRONTEND, "frontend")); + } + await Promise.all(readinessChecks); if (!process.env.SUPABASE_URL) { throw new Error( diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index fdd6548c..7d0deaba 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -22,6 +22,9 @@ const API_URL = process.env.E2E_API_URL ?? "http://localhost:4000"; const IS_CI = !!process.env.CI; const CROSS_BROWSER = process.env.E2E_CROSS_BROWSER === "1"; const RECORD_VIDEO = IS_CI || process.env.E2E_VIDEO === "1"; +const durationReporter = process.env.E2E_TIMING_OUTPUT + ? [[path.resolve(__dirname, "reporters/durationReporter.ts"), { outputFile: process.env.E2E_TIMING_OUTPUT }]] + : []; // GitHub-hosted Ubuntu and Playwright's official Linux container ship // different native fallback-font sets. Both are valid release renderers, but // their glyph metrics differ enough on phone layouts that one shared "linux" @@ -76,8 +79,8 @@ export default defineConfig({ timeout: 60_000, expect: { timeout: 10_000 }, reporter: IS_CI - ? [["html", { open: "never" }], ["github"], ["list"]] - : [["html", { open: "never" }], ["list"]], + ? [["html", { open: "never" }], ["github"], ["list"], ...durationReporter] + : [["html", { open: "never" }], ["list"], ...durationReporter], // Keep strict visual baselines per reviewed rendering environment. The // production app deliberately falls back to native fonts while its optional // brand-font stylesheet loads, so macOS, Playwright-container Linux, and diff --git a/e2e/reporters/durationReporter.ts b/e2e/reporters/durationReporter.ts new file mode 100644 index 00000000..24f99396 --- /dev/null +++ b/e2e/reporters/durationReporter.ts @@ -0,0 +1,35 @@ +import type { FullResult, Reporter, TestCase, TestResult } from "@playwright/test/reporter"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +type TimingRecord = { + durationMs: number; + status: TestResult["status"]; +}; + +export default class DurationReporter implements Reporter { + private readonly outputFile: string; + private readonly tests = new Map(); + + constructor(options: { outputFile?: string } = {}) { + const configured = options.outputFile ?? process.env.E2E_TIMING_OUTPUT; + if (!configured) throw new Error("duration reporter requires outputFile or E2E_TIMING_OUTPUT"); + this.outputFile = resolve(configured); + } + + onTestEnd(test: TestCase, result: TestResult) { + this.tests.set(test.id, { + durationMs: Math.max(0, Math.round(result.duration)), + status: result.status, + }); + } + + onEnd(result: FullResult) { + mkdirSync(dirname(this.outputFile), { recursive: true }); + writeFileSync(this.outputFile, `${JSON.stringify({ + schemaVersion: 1, + status: result.status, + tests: Object.fromEntries([...this.tests.entries()].sort(([left], [right]) => left.localeCompare(right))), + }, null, 2)}\n`); + } +} diff --git a/e2e/scripts/ensure-browser-runtime.mjs b/e2e/scripts/ensure-browser-runtime.mjs new file mode 100644 index 00000000..cf9af58e --- /dev/null +++ b/e2e/scripts/ensure-browser-runtime.mjs @@ -0,0 +1,63 @@ +import { spawnSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +const SUPPORTED_BROWSERS = new Set(["chromium", "firefox", "webkit"]); + +export async function ensureBrowserRuntime({ + browserName, + browserTypes, + installDependencies, +}) { + if (!SUPPORTED_BROWSERS.has(browserName) || !browserTypes[browserName]) { + throw new Error(`Unsupported Playwright browser: ${browserName}`); + } + + const launch = async () => { + const browser = await browserTypes[browserName].launch({ headless: true }); + await browser.close(); + }; + + try { + await launch(); + console.log(`[playwright-runtime] ${browserName} launched with runner-provided libraries`); + return { installedDependencies: false }; + } catch (firstError) { + console.warn( + `[playwright-runtime] ${browserName} preflight failed; installing OS dependencies`, + ); + await installDependencies(browserName, firstError); + await launch(); + console.log(`[playwright-runtime] ${browserName} launched after dependency installation`); + return { installedDependencies: true }; + } +} + +async function main() { + const browserName = process.argv[2]; + const playwright = await import("playwright"); + await ensureBrowserRuntime({ + browserName, + browserTypes: { + chromium: playwright.chromium, + firefox: playwright.firefox, + webkit: playwright.webkit, + }, + installDependencies: async (name) => { + const executable = process.platform === "win32" ? "npx.cmd" : "npx"; + const result = spawnSync(executable, ["playwright", "install-deps", name], { + stdio: "inherit", + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error(`playwright install-deps ${name} exited ${result.status}`); + } + }, + }); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/e2e/scripts/ensure-browser-runtime.test.mjs b/e2e/scripts/ensure-browser-runtime.test.mjs new file mode 100644 index 00000000..23573b61 --- /dev/null +++ b/e2e/scripts/ensure-browser-runtime.test.mjs @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { ensureBrowserRuntime } from "./ensure-browser-runtime.mjs"; + +function fakeBrowser(launches) { + return { + async launch() { + launches.count += 1; + const next = launches.results.shift(); + if (next instanceof Error) throw next; + return { + async close() { + launches.closed += 1; + }, + }; + }, + }; +} + +test("skips OS installation when the cached browser already launches", async () => { + const launches = { count: 0, closed: 0, results: [true] }; + let installs = 0; + const result = await ensureBrowserRuntime({ + browserName: "chromium", + browserTypes: { chromium: fakeBrowser(launches) }, + installDependencies: async () => { + installs += 1; + }, + }); + + assert.deepEqual(result, { installedDependencies: false }); + assert.equal(installs, 0); + assert.deepEqual(launches, { count: 1, closed: 1, results: [] }); +}); + +test("installs dependencies once and requires a successful second launch", async () => { + const launches = { count: 0, closed: 0, results: [new Error("missing library"), true] }; + const installs = []; + const result = await ensureBrowserRuntime({ + browserName: "webkit", + browserTypes: { webkit: fakeBrowser(launches) }, + installDependencies: async (name, firstError) => { + installs.push({ name, message: firstError.message }); + }, + }); + + assert.deepEqual(result, { installedDependencies: true }); + assert.deepEqual(installs, [{ name: "webkit", message: "missing library" }]); + assert.deepEqual(launches, { count: 2, closed: 1, results: [] }); +}); + +test("fails closed for an unknown browser", async () => { + await assert.rejects( + ensureBrowserRuntime({ + browserName: "chrome", + browserTypes: {}, + installDependencies: async () => {}, + }), + /Unsupported Playwright browser: chrome/, + ); +}); diff --git a/e2e/security-suite/README.md b/e2e/security-suite/README.md index 6bcf93b1..7f1d83ab 100644 --- a/e2e/security-suite/README.md +++ b/e2e/security-suite/README.md @@ -13,8 +13,9 @@ runtime monitoring (Falco/Tetragon) for a complete story. ## Run locally ```bash -# from repo root — boot the stack -docker compose up -d backend frontend +# from repo root — the API-only suite needs the backend and its runner/proxy +# dependencies; Compose starts those dependencies automatically. +docker compose up -d backend # tcpdump is optional locally. If absent, egress tests fall back to # the backend-level assertion only (no packet-level observer). diff --git a/e2e/security-suite/playwright.config.ts b/e2e/security-suite/playwright.config.ts index 317ddba7..8bd09f0f 100644 --- a/e2e/security-suite/playwright.config.ts +++ b/e2e/security-suite/playwright.config.ts @@ -12,6 +12,10 @@ dotenv.config({ path: path.resolve(__dirname, "..", "..", ".env") }); // The security suite is a standalone Playwright entrypoint, so it cannot rely // on the main config to establish a test-user namespace for local runs. process.env.E2E_USER_SUFFIX ??= `security-local-${process.pid}-${Date.now().toString(36)}`; +// Security scenarios use Playwright's APIRequestContext and never render the +// app. Keep the shared readiness hook, but scope it to the backend so this +// suite does not build and boot an unused frontend merely to satisfy a probe. +process.env.E2E_SKIP_FRONTEND_HEALTH ??= "1"; const IS_CI = !!process.env.CI; diff --git a/e2e/specs/admin-quality.spec.ts b/e2e/specs/admin-quality.spec.ts index 6e34620a..27064432 100644 --- a/e2e/specs/admin-quality.spec.ts +++ b/e2e/specs/admin-quality.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from "@playwright/test"; +import { expect, test, type Route } from "@playwright/test"; import { loginAsAdminTestUser, removeAdminEmailPreviewFixture, @@ -441,23 +441,38 @@ test.describe("Q7 calm and trustworthy admin operations", () => { }); test("a hung admin read becomes an explicit retry state and recovers", async ({ page }) => { - await page.route("**/api/admin/anon-summary", async (route) => { - // Playwright keeps the intercepted fetch pending until this handler - // releases it, even after the page's AbortController fires at 10 s. - // Release just after that boundary so the browser can surface the - // already-triggered timeout. client.test.ts enforces the exact 10 s - // timer; this journey owns the rendered recovery and retry contract. - await new Promise((resolve) => setTimeout(resolve, 11_000)); - await route.continue().catch(() => undefined); + let releaseHungRead!: () => void; + const hungRead = new Promise((resolve) => { + releaseHungRead = resolve; }); + const pendingRoutes = new Set>(); + const anonSummaryPattern = "**/api/admin/anon-summary"; + const holdAnonSummary = async (route: Route) => { + // Keep every intercepted read pending until the product's own timeout + // renders. client.test.ts owns the exact 10 s timer; this browser journey + // owns the visible recovery and retry contract without racing two clocks. + const continuation = hungRead.then(() => route.continue().catch(() => undefined)); + pendingRoutes.add(continuation); + try { + await continuation; + } finally { + pendingRoutes.delete(continuation); + } + }; + await page.route(anonSummaryPattern, holdAnonSummary); await page.goto("/admin/anon"); const alert = page.getByRole("alert"); - await expect(alert).toContainText("Trial-path data did not load", { timeout: 14_000 }); - await expect(alert).toContainText("admin request took too long"); const retry = page.getByRole("button", { name: "Try again" }); - await expect(retry).toBeVisible(); + try { + await expect(alert).toContainText("Trial-path data did not load", { timeout: 20_000 }); + await expect(alert).toContainText("admin request took too long"); + await expect(retry).toBeVisible(); + } finally { + releaseHungRead(); + await page.unroute(anonSummaryPattern, holdAnonSummary); + await Promise.allSettled([...pendingRoutes]); + } - await page.unroute("**/api/admin/anon-summary"); await retry.click(); await expect(page.getByRole("heading", { name: "Funnel (today, UTC)" })).toBeVisible({ timeout: 12_000, diff --git a/e2e/specs/multi-tab.spec.ts b/e2e/specs/multi-tab.spec.ts index 50489b32..2d5a22c6 100644 --- a/e2e/specs/multi-tab.spec.ts +++ b/e2e/specs/multi-tab.spec.ts @@ -122,10 +122,9 @@ test.describe("tab-sleep → wake rebind", () => { const reassigned = page.getByRole("alertdialog", { name: /workspace reassigned/i, }); - if (await reassigned.isVisible()) { - await reassigned.getByRole("button", { name: /got it/i }).click(); - await expect(reassigned).toBeHidden(); - } + await expect(reassigned).toBeVisible({ timeout: 10_000 }); + await reassigned.getByRole("button", { name: /got it/i }).click(); + await expect(reassigned).toBeHidden(); // After rebind, the Run button must still be enabled against the // new sessionId — if not, the learner is stuck until manual reload.