diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3c2b018b097..08f78554145 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,6 +23,6 @@ } } }, - "postCreateCommand": "python --version && java --version && pip install --upgrade pip && pip install pipenv && bash local/install_deps.bash", + "postCreateCommand": "python --version && java --version && bash local/install_deps.bash", "remoteUser": "vscode" } diff --git a/.gitignore b/.gitignore index 5d11f1093fc..de67af4b5b8 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,7 @@ /local/bin/android-sdk/* /local/storage /paramiko.log -/src/appengine/requirements.txt +/.venv /src/appengine/app.yaml /src/appengine/cron-service.yaml /src/appengine/cron.yaml @@ -43,7 +43,6 @@ /src/local/remote/generated.remmina /src/protos/ENV /src/third_party -/src/requirements.txt _test_data bazel-* /src/dist @@ -57,5 +56,5 @@ bazel-* **/.terraform/* # Ignore temporary build files. -docker/base/Pipfile -docker/base/Pipfile.lock +docker/base/pyproject.toml +docker/base/uv.lock diff --git a/AGENTS.md b/AGENTS.md index 8204672aaec..499428c6cc6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,15 +38,7 @@ asdf plugin add gcloud https://github.com/jthegedus/asdf-gcloud asdf install ``` -### 4. Install `pipenv` - -Once `asdf` has installed the correct python version, you need to install `pipenv`: - -```bash -python -m pip install pipenv -``` - -### 5. Install Project Dependencies +### 4. Install Project Dependencies Now you are ready to install the project's dependencies. Run the following command from the root of the repository: @@ -55,18 +47,17 @@ Now you are ready to install the project's dependencies. Run the following comma ``` ## Using Butler -Before each time you use the `butler.py` script, verify that you are inside the virtual environment, if not activate it by running `python -m pipenv shell`, or pre append any `python butler.py` call with -`pipenv run` +Before using the `butler.py` script, verify that you are inside the virtual environment (`source .venv/bin/activate`), or prepend any `butler.py` call with `uv run`. For instance: ```bash -pipenv run python butler.py lint +uv run butler.py lint ``` Is the same as running ```bash python butler.py lint ``` -if you are inside a virtual environment. +if you are inside an activated virtual environment. ## Testing diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 48fa5aac7fa..00000000000 --- a/Pipfile +++ /dev/null @@ -1,32 +0,0 @@ -[[source]] -url = "https://pypi.python.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -cryptography = "==37.0.4" -crcmod = "==1.7" -future = "==0.17.1" -protobuf = "==4.23.4" -psutil = "==5.9.4" -google-cloud-ndb = "==2.3.4" - -[dev-packages] -Fabric = "==1.14.1" -grpcio-tools = "==1.62.2" -gunicorn = "*" -isort = "*" -mypy-protobuf = "==3.4.0" -nodeenv = "==1.9.1" -parameterized = "==0.9.0" -paramiko = "==2.6.0" -pipenv = "==2022.8.5" -pyfakefs = "==4.4.0" -pylint = "~=2.4" -pylint-protobuf = "==0.21.0" -pyright = "==1.1.393" -pyyaml = "==6.0.2" -selenium = "==3.141.0" -twine = "*" -WebTest = "==3.0.0" -yapf = "==0.22.0" diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index e10f232ff68..00000000000 --- a/Pipfile.lock +++ /dev/null @@ -1,1613 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "f8858cd09afd214bef4a14a6a3f0ad7046dcb715d65cb1a22ae92a62d1cc8c8f" - }, - "pipfile-spec": 6, - "requires": {}, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.python.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "cachetools": { - "hashes": [ - "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", - "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a" - ], - "markers": "python_version >= '3.7'", - "version": "==5.5.2" - }, - "certifi": { - "hashes": [ - "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", - "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5" - ], - "markers": "python_version >= '3.7'", - "version": "==2025.8.3" - }, - "cffi": { - "hashes": [ - "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", - "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", - "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", - "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", - "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", - "sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2", - "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", - "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", - "sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65", - "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", - "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", - "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", - "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", - "sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a", - "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", - "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", - "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", - "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", - "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", - "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", - "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", - "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", - "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", - "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", - "sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165", - "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", - "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", - "sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c", - "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", - "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", - "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", - "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", - "sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63", - "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", - "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", - "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", - "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", - "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", - "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", - "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", - "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", - "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", - "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", - "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", - "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", - "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", - "sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322", - "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", - "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", - "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", - "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", - "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", - "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", - "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", - "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", - "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", - "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", - "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", - "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", - "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", - "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9", - "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", - "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", - "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", - "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", - "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", - "sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f", - "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", - "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", - "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", - "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", - "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", - "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", - "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", - "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", - "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", - "sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7", - "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", - "sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534", - "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", - "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", - "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", - "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", - "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf" - ], - "markers": "python_version >= '3.9'", - "version": "==2.0.0" - }, - "charset-normalizer": { - "hashes": [ - "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", - "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", - "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", - "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", - "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", - "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", - "sha256:0f2be7e0cf7754b9a30eb01f4295cc3d4358a479843b31f328afd210e2c7598c", - "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", - "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", - "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", - "sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432", - "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", - "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", - "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", - "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", - "sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19", - "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", - "sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e", - "sha256:252098c8c7a873e17dd696ed98bbe91dbacd571da4b87df3736768efa7a792e4", - "sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7", - "sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312", - "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", - "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", - "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", - "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", - "sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99", - "sha256:3653fad4fe3ed447a596ae8638b437f827234f01a8cd801842e43f3d0a6b281b", - "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", - "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", - "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", - "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", - "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", - "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", - "sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc", - "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", - "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", - "sha256:5b413b0b1bfd94dbf4023ad6945889f374cd24e3f62de58d6bb102c4d9ae534a", - "sha256:5d8d01eac18c423815ed4f4a2ec3b439d654e55ee4ad610e153cf02faf67ea40", - "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", - "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", - "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", - "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", - "sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05", - "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", - "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", - "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", - "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", - "sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34", - "sha256:8999f965f922ae054125286faf9f11bc6932184b93011d138925a1773830bbe9", - "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", - "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", - "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", - "sha256:a2d08ac246bb48479170408d6c19f6385fa743e7157d716e144cad849b2dd94b", - "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", - "sha256:b5e3b2d152e74e100a9e9573837aba24aab611d39428ded46f4e4022ea7d1942", - "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", - "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", - "sha256:c60e092517a73c632ec38e290eba714e9627abe9d301c8c8a12ec32c314a2a4b", - "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", - "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", - "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", - "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", - "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", - "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", - "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", - "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", - "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", - "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", - "sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca", - "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", - "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", - "sha256:d95bfb53c211b57198bb91c46dd5a2d8018b3af446583aab40074bf7988401cb", - "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", - "sha256:ec557499516fc90fd374bf2e32349a2887a876fbf162c160e3c01b6849eaf557", - "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", - "sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7", - "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", - "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", - "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9" - ], - "markers": "python_version >= '3.7'", - "version": "==3.4.3" - }, - "crcmod": { - "hashes": [ - "sha256:50586ab48981f11e5b117523d97bb70864a2a1af246cf6e4f5c4a21ef4611cd1", - "sha256:69a2e5c6c36d0f096a7beb4cd34e5f882ec5fd232efb710cdb85d4ff196bd52e", - "sha256:737fb308fa2ce9aed2e29075f0d5980d4a89bfbec48a368c607c5c63b3efb90e", - "sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e" - ], - "index": "pypi", - "version": "==1.7" - }, - "cryptography": { - "hashes": [ - "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", - "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", - "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", - "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", - "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", - "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", - "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", - "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", - "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", - "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", - "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", - "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", - "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", - "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", - "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", - "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", - "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", - "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", - "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", - "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", - "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", - "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==37.0.4" - }, - "future": { - "hashes": [ - "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" - ], - "index": "pypi", - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", - "version": "==0.17.1" - }, - "google-api-core": { - "extras": [ - "grpc" - ], - "hashes": [ - "sha256:8a2a56c1fef82987a524371f99f3bd0143702fecc670c72e600c1cda6bf8dbb7", - "sha256:d2aaa0b13c78c61cb3f4282c464c046e45fbd75755683c9c525e6e8f7ed0a5e8" - ], - "markers": "python_version >= '3.7'", - "version": "==2.25.1" - }, - "google-auth": { - "hashes": [ - "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca", - "sha256:500c3a29adedeb36ea9cf24b8d10858e152f2412e3ca37829b3fa18e33d63b77" - ], - "markers": "python_version >= '3.7'", - "version": "==2.40.3" - }, - "google-cloud-core": { - "hashes": [ - "sha256:1fab62d7102844b278fe6dead3af32408b1df3eb06f5c7e8634cbd40edc4da53", - "sha256:5130f9f4c14b4fafdff75c79448f9495cfade0d8775facf1b09c3bf67e027f6e" - ], - "markers": "python_version >= '3.7'", - "version": "==2.4.3" - }, - "google-cloud-datastore": { - "hashes": [ - "sha256:eee454dd4a55f5b327f9f344928ff1a09a6f77c23d5e3d908ad31a13cc2f4073", - "sha256:f303f27cd1983383f20bd227019cd8a7897419e0ec6b878367c58c66245f9d9b" - ], - "markers": "python_version >= '3.7'", - "version": "==2.21.0" - }, - "google-cloud-ndb": { - "hashes": [ - "sha256:6bf46c7331a58229408251652521f14355cf74ba58f5aa049c0664562d468a29", - "sha256:c2cdc38b22fa2d3f6984db6368b9be11902235ef46e2252a17f8083fe3176160" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.3.4" - }, - "googleapis-common-protos": { - "hashes": [ - "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", - "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8" - ], - "markers": "python_version >= '3.7'", - "version": "==1.70.0" - }, - "grpcio": { - "hashes": [ - "sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f", - "sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc", - "sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7", - "sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7", - "sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a", - "sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4", - "sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac", - "sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6", - "sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89", - "sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3", - "sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49", - "sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20", - "sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f", - "sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc", - "sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae", - "sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82", - "sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b", - "sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91", - "sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9", - "sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5", - "sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362", - "sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a", - "sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d", - "sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb", - "sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31", - "sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b", - "sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854", - "sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1", - "sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176", - "sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8", - "sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907", - "sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11", - "sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c", - "sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4", - "sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7", - "sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707", - "sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5", - "sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce", - "sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa", - "sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01", - "sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9", - "sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182", - "sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b", - "sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486", - "sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249", - "sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3", - "sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11", - "sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa", - "sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e", - "sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24", - "sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e" - ], - "markers": "python_version >= '3.9'", - "version": "==1.74.0" - }, - "grpcio-status": { - "hashes": [ - "sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485", - "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8" - ], - "markers": "python_version >= '3.6'", - "version": "==1.62.3" - }, - "idna": { - "hashes": [ - "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", - "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" - ], - "markers": "python_version >= '3.6'", - "version": "==3.10" - }, - "proto-plus": { - "hashes": [ - "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66", - "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012" - ], - "markers": "python_version >= '3.7'", - "version": "==1.26.1" - }, - "protobuf": { - "hashes": [ - "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474", - "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2", - "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b", - "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720", - "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12", - "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd", - "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0", - "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e", - "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9", - "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70", - "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff", - "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597", - "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==4.23.4" - }, - "psutil": { - "hashes": [ - "sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff", - "sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1", - "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62", - "sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549", - "sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08", - "sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7", - "sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e", - "sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe", - "sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24", - "sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad", - "sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94", - "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8", - "sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7", - "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4" - ], - "index": "pypi", - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==5.9.4" - }, - "pyasn1": { - "hashes": [ - "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", - "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034" - ], - "markers": "python_version >= '3.8'", - "version": "==0.6.1" - }, - "pyasn1-modules": { - "hashes": [ - "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", - "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6" - ], - "markers": "python_version >= '3.8'", - "version": "==0.4.2" - }, - "pycparser": { - "hashes": [ - "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", - "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934" - ], - "markers": "python_version >= '3.8'", - "version": "==2.23" - }, - "pymemcache": { - "hashes": [ - "sha256:27bf9bd1bbc1e20f83633208620d56de50f14185055e49504f4f5e94e94aff94", - "sha256:f507bc20e0dc8d562f8df9d872107a278df049fa496805c1431b926f3ddd0eab" - ], - "markers": "python_version >= '3.7'", - "version": "==4.0.0" - }, - "pytz": { - "hashes": [ - "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", - "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00" - ], - "version": "==2025.2" - }, - "redis": { - "hashes": [ - "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010", - "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f" - ], - "markers": "python_version >= '3.9'", - "version": "==6.4.0" - }, - "requests": { - "hashes": [ - "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", - "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf" - ], - "markers": "python_version >= '3.9'", - "version": "==2.32.5" - }, - "rsa": { - "hashes": [ - "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", - "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75" - ], - "markers": "python_version >= '3.6' and python_version < '4'", - "version": "==4.9.1" - }, - "urllib3": { - "hashes": [ - "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", - "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" - ], - "markers": "python_version >= '3.9'", - "version": "==2.5.0" - } - }, - "develop": { - "astroid": { - "hashes": [ - "sha256:1aa149fc5c6589e3d0ece885b4491acd80af4f087baafa3fb5203b113e68cd3c", - "sha256:6c107453dffee9055899705de3c9ead36e74119cee151e5a9aaf7f0b0e020a6a" - ], - "markers": "python_full_version >= '3.7.2'", - "version": "==2.15.8" - }, - "backports.tarfile": { - "hashes": [ - "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", - "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991" - ], - "markers": "python_version >= '3.8'", - "version": "==1.2.0" - }, - "bcrypt": { - "hashes": [ - "sha256:0042b2e342e9ae3d2ed22727c1262f76cc4f345683b5c1715f0250cf4277294f", - "sha256:0142b2cb84a009f8452c8c5a33ace5e3dfec4159e7735f5afe9a4d50a8ea722d", - "sha256:08bacc884fd302b611226c01014eca277d48f0a05187666bca23aac0dad6fe24", - "sha256:0d3efb1157edebfd9128e4e46e2ac1a64e0c1fe46fb023158a407c7892b0f8c3", - "sha256:0e30e5e67aed0187a1764911af023043b4542e70a7461ad20e837e94d23e1d6c", - "sha256:107d53b5c67e0bbc3f03ebf5b030e0403d24dda980f8e244795335ba7b4a027d", - "sha256:12fa6ce40cde3f0b899729dbd7d5e8811cb892d31b6f7d0334a1f37748b789fd", - "sha256:17a854d9a7a476a89dcef6c8bd119ad23e0f82557afbd2c442777a16408e614f", - "sha256:191354ebfe305e84f344c5964c7cd5f924a3bfc5d405c75ad07f232b6dffb49f", - "sha256:2ef6630e0ec01376f59a006dc72918b1bf436c3b571b80fa1968d775fa02fe7d", - "sha256:3004df1b323d10021fda07a813fd33e0fd57bef0e9a480bb143877f6cba996fe", - "sha256:335a420cfd63fc5bc27308e929bee231c15c85cc4c496610ffb17923abf7f231", - "sha256:33752b1ba962ee793fa2b6321404bf20011fe45b9afd2a842139de3011898fef", - "sha256:3a3fd2204178b6d2adcf09cb4f6426ffef54762577a7c9b54c159008cb288c18", - "sha256:3b8d62290ebefd49ee0b3ce7500f5dbdcf13b81402c05f6dafab9a1e1b27212f", - "sha256:3e36506d001e93bffe59754397572f21bb5dc7c83f54454c990c74a468cd589e", - "sha256:41261d64150858eeb5ff43c753c4b216991e0ae16614a308a15d909503617732", - "sha256:50e6e80a4bfd23a25f5c05b90167c19030cf9f87930f7cb2eacb99f45d1c3304", - "sha256:531457e5c839d8caea9b589a1bcfe3756b0547d7814e9ce3d437f17da75c32b0", - "sha256:55a935b8e9a1d2def0626c4269db3fcd26728cbff1e84f0341465c31c4ee56d8", - "sha256:57967b7a28d855313a963aaea51bf6df89f833db4320da458e5b3c5ab6d4c938", - "sha256:584027857bc2843772114717a7490a37f68da563b3620f78a849bcb54dc11e62", - "sha256:59e1aa0e2cd871b08ca146ed08445038f42ff75968c7ae50d2fdd7860ade2180", - "sha256:5bd3cca1f2aa5dbcf39e2aa13dd094ea181f48959e1071265de49cc2b82525af", - "sha256:5c1949bf259a388863ced887c7861da1df681cb2388645766c89fdfd9004c669", - "sha256:62f26585e8b219cdc909b6a0069efc5e4267e25d4a3770a364ac58024f62a761", - "sha256:67a561c4d9fb9465ec866177e7aebcad08fe23aaf6fbd692a6fab69088abfc51", - "sha256:6fb1fd3ab08c0cbc6826a2e0447610c6f09e983a281b919ed721ad32236b8b23", - "sha256:74a8d21a09f5e025a9a23e7c0fd2c7fe8e7503e4d356c0a2c1486ba010619f09", - "sha256:79e70b8342a33b52b55d93b3a59223a844962bef479f6a0ea318ebbcadf71505", - "sha256:7a4be4cbf241afee43f1c3969b9103a41b40bcb3a3f467ab19f891d9bc4642e4", - "sha256:7c03296b85cb87db865d91da79bf63d5609284fc0cab9472fdd8367bbd830753", - "sha256:842d08d75d9fe9fb94b18b071090220697f9f184d4547179b60734846461ed59", - "sha256:864f8f19adbe13b7de11ba15d85d4a428c7e2f344bac110f667676a0ff84924b", - "sha256:97eea7408db3a5bcce4a55d13245ab3fa566e23b4c67cd227062bb49e26c585d", - "sha256:a839320bf27d474e52ef8cb16449bb2ce0ba03ca9f44daba6d93fa1d8828e48a", - "sha256:afe327968aaf13fc143a56a3360cb27d4ad0345e34da12c7290f1b00b8fe9a8b", - "sha256:b4d4e57f0a63fd0b358eb765063ff661328f69a04494427265950c71b992a39a", - "sha256:b6354d3760fcd31994a14c89659dee887f1351a06e5dac3c1142307172a79f90", - "sha256:b693dbb82b3c27a1604a3dff5bfc5418a7e6a781bb795288141e5f80cf3a3492", - "sha256:bdc6a24e754a555d7316fa4774e64c6c3997d27ed2d1964d55920c7c227bc4ce", - "sha256:beeefe437218a65322fbd0069eb437e7c98137e08f22c4660ac2dc795c31f8bb", - "sha256:c5eeac541cefd0bb887a371ef73c62c3cd78535e4887b310626036a7c0a817bb", - "sha256:c950d682f0952bafcceaf709761da0a32a942272fad381081b51096ffa46cea1", - "sha256:d9af79d322e735b1fc33404b5765108ae0ff232d4b54666d46730f8ac1a43676", - "sha256:e53e074b120f2877a35cc6c736b8eb161377caae8925c17688bd46ba56daaa5b", - "sha256:e965a9c1e9a393b8005031ff52583cedc15b7884fce7deb8b0346388837d6cfe", - "sha256:f01e060f14b6b57bbb72fc5b4a83ac21c443c9a2ee708e04a10e9192f90a6281", - "sha256:f1e3ffa1365e8702dc48c8b360fef8d7afeca482809c5e45e653af82ccd088c1", - "sha256:f6746e6fec103fcd509b96bacdfdaa2fbde9a553245dbada284435173a6f1aef", - "sha256:f81b0ed2639568bf14749112298f9e4e2b28853dab50a8b357e31798686a036d" - ], - "markers": "python_version >= '3.8'", - "version": "==4.3.0" - }, - "beautifulsoup4": { - "hashes": [ - "sha256:5e70131382930e7c3de33450a2f54a63d5e4b19386eab43a5b34d594268f3695", - "sha256:642085eaa22233aceadff9c69651bc51e8bf3f874fb6d7104ece2beb24b47c4a" - ], - "markers": "python_full_version >= '3.7.0'", - "version": "==4.13.5" - }, - "certifi": { - "hashes": [ - "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", - "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5" - ], - "markers": "python_version >= '3.7'", - "version": "==2025.8.3" - }, - "cffi": { - "hashes": [ - "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", - "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", - "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", - "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", - "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", - "sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2", - "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", - "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", - "sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65", - "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", - "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", - "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", - "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", - "sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a", - "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", - "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", - "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", - "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", - "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", - "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", - "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", - "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", - "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", - "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", - "sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165", - "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", - "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", - "sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c", - "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", - "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", - "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", - "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", - "sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63", - "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", - "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", - "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", - "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", - "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", - "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", - "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", - "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", - "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", - "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", - "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", - "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", - "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", - "sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322", - "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", - "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", - "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", - "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", - "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", - "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", - "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", - "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", - "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", - "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", - "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", - "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", - "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", - "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9", - "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", - "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", - "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", - "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", - "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", - "sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f", - "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", - "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", - "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", - "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", - "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", - "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", - "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", - "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", - "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", - "sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7", - "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", - "sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534", - "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", - "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", - "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", - "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", - "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf" - ], - "markers": "python_version >= '3.9'", - "version": "==2.0.0" - }, - "charset-normalizer": { - "hashes": [ - "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", - "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", - "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", - "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", - "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", - "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", - "sha256:0f2be7e0cf7754b9a30eb01f4295cc3d4358a479843b31f328afd210e2c7598c", - "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", - "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", - "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", - "sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432", - "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", - "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", - "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", - "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", - "sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19", - "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", - "sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e", - "sha256:252098c8c7a873e17dd696ed98bbe91dbacd571da4b87df3736768efa7a792e4", - "sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7", - "sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312", - "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", - "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", - "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", - "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", - "sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99", - "sha256:3653fad4fe3ed447a596ae8638b437f827234f01a8cd801842e43f3d0a6b281b", - "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", - "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", - "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", - "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", - "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", - "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", - "sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc", - "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", - "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", - "sha256:5b413b0b1bfd94dbf4023ad6945889f374cd24e3f62de58d6bb102c4d9ae534a", - "sha256:5d8d01eac18c423815ed4f4a2ec3b439d654e55ee4ad610e153cf02faf67ea40", - "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", - "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", - "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", - "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", - "sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05", - "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", - "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", - "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", - "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", - "sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34", - "sha256:8999f965f922ae054125286faf9f11bc6932184b93011d138925a1773830bbe9", - "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", - "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", - "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", - "sha256:a2d08ac246bb48479170408d6c19f6385fa743e7157d716e144cad849b2dd94b", - "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", - "sha256:b5e3b2d152e74e100a9e9573837aba24aab611d39428ded46f4e4022ea7d1942", - "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", - "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", - "sha256:c60e092517a73c632ec38e290eba714e9627abe9d301c8c8a12ec32c314a2a4b", - "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", - "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", - "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", - "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", - "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", - "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", - "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", - "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", - "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", - "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", - "sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca", - "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", - "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", - "sha256:d95bfb53c211b57198bb91c46dd5a2d8018b3af446583aab40074bf7988401cb", - "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", - "sha256:ec557499516fc90fd374bf2e32349a2887a876fbf162c160e3c01b6849eaf557", - "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", - "sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7", - "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", - "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", - "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9" - ], - "markers": "python_version >= '3.7'", - "version": "==3.4.3" - }, - "cryptography": { - "hashes": [ - "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", - "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", - "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", - "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", - "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", - "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", - "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", - "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", - "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", - "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", - "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", - "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", - "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", - "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", - "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", - "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", - "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", - "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", - "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", - "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", - "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", - "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==37.0.4" - }, - "dill": { - "hashes": [ - "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", - "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049" - ], - "markers": "python_version >= '3.8'", - "version": "==0.4.0" - }, - "distlib": { - "hashes": [ - "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", - "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d" - ], - "version": "==0.4.0" - }, - "docutils": { - "hashes": [ - "sha256:4ed966a0e96a0477d852f7af31bdcb3adc049fbb35ccba358c2ea8a03287615e", - "sha256:ba9d57750e92331ebe7c08a1bbf7a7f8143b86c476acd51528b042216a6aad0f" - ], - "markers": "python_version >= '3.9'", - "version": "==0.22" - }, - "fabric": { - "hashes": [ - "sha256:2bb6c6922cbdfe35884c937bfcff63c70750c18456b0707898112c5ceaab38c6", - "sha256:66097883bb3e5beecacae92b82b2bd489d10a8fd4f06ce1cb27019de2e6d76a8" - ], - "index": "pypi", - "version": "==1.14.1" - }, - "filelock": { - "hashes": [ - "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", - "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d" - ], - "markers": "python_version >= '3.9'", - "version": "==3.19.1" - }, - "grpcio": { - "hashes": [ - "sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f", - "sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc", - "sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7", - "sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7", - "sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a", - "sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4", - "sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac", - "sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6", - "sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89", - "sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3", - "sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49", - "sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20", - "sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f", - "sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc", - "sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae", - "sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82", - "sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b", - "sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91", - "sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9", - "sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5", - "sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362", - "sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a", - "sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d", - "sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb", - "sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31", - "sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b", - "sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854", - "sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1", - "sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176", - "sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8", - "sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907", - "sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11", - "sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c", - "sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4", - "sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7", - "sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707", - "sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5", - "sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce", - "sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa", - "sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01", - "sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9", - "sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182", - "sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b", - "sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486", - "sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249", - "sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3", - "sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11", - "sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa", - "sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e", - "sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24", - "sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e" - ], - "markers": "python_version >= '3.9'", - "version": "==1.74.0" - }, - "grpcio-tools": { - "hashes": [ - "sha256:04a394cf5e51ba9be412eb9f6c482b6270bd81016e033e8eb7d21b8cc28fe8b5", - "sha256:04c607029ae3660fb1624ed273811ffe09d57d84287d37e63b5b802a35897329", - "sha256:10cc3321704ecd17c93cf68c99c35467a8a97ffaaed53207e9b2da6ae0308ee1", - "sha256:1679b4903aed2dc5bd8cb22a452225b05dc8470a076f14fd703581efc0740cdb", - "sha256:184b4174d4bd82089d706e8223e46c42390a6ebac191073b9772abc77308f9fa", - "sha256:19216e1fb26dbe23d12a810517e1b3fbb8d4f98b1a3fbebeec9d93a79f092de4", - "sha256:19ea69e41c3565932aa28a202d1875ec56786aea46a2eab54a3b28e8a27f9517", - "sha256:1d768a5c07279a4c461ebf52d0cec1c6ca85c6291c71ec2703fe3c3e7e28e8c4", - "sha256:1fe08d2038f2b7c53259b5c49e0ad08c8e0ce2b548d8185993e7ef67e8592cca", - "sha256:217c2ee6a7ce519a55958b8622e21804f6fdb774db08c322f4c9536c35fdce7c", - "sha256:2ce149ea55eadb486a7fb75a20f63ef3ac065ee6a0240ed25f3549ce7954c653", - "sha256:2ed775e844566ce9ce089be9a81a8b928623b8ee5820f5e4d58c1a9d33dfc5ae", - "sha256:3708a747aa4b6b505727282ca887041174e146ae030ebcadaf4c1d346858df62", - "sha256:3a8d6f07e64c0c7756f4e0c4781d9d5a2b9cc9cbd28f7032a6fb8d4f847d0445", - "sha256:3c53b221378b035ae2f1881cbc3aca42a6075a8e90e1a342c2f205eb1d1aa6a1", - "sha256:40cd4eeea4b25bcb6903b82930d579027d034ba944393c4751cdefd9c49e6989", - "sha256:45db5da2bcfa88f2b86b57ef35daaae85c60bd6754a051d35d9449c959925b57", - "sha256:462e0ab8dd7c7b70bfd6e3195eebc177549ede5cf3189814850c76f9a340d7ce", - "sha256:47117c8a7e861382470d0e22d336e5a91fdc5f851d1db44fa784b9acea190d87", - "sha256:472505d030135d73afe4143b0873efe0dcb385bd6d847553b4f3afe07679af00", - "sha256:4f6f32d39283ea834a493fccf0ebe9cfddee7577bdcc27736ad4be1732a36399", - "sha256:4f955702dc4b530696375251319d05223b729ed24e8673c2129f7a75d2caefbb", - "sha256:4f989e5cebead3ae92c6abf6bf7b19949e1563a776aea896ac5933f143f0c45d", - "sha256:58cbb24b3fa6ae35aa9c210fcea3a51aa5fef0cd25618eb4fd94f746d5a9b703", - "sha256:5b07b5874187e170edfbd7aa2ca3a54ebf3b2952487653e8c0b0d83601c33035", - "sha256:5fd5e1582b678e6b941ee5f5809340be5e0724691df5299aae8226640f94e18f", - "sha256:6413581e14a80e0b4532577766cf0586de4dd33766a31b3eb5374a746771c07d", - "sha256:72b61332f1b439c14cbd3815174a8f1d35067a02047c32decd406b3a09bb9890", - "sha256:759c60f24c33a181bbbc1232a6752f9b49fbb1583312a4917e2b389fea0fb0f2", - "sha256:76eb459bdf3fb666e01883270beee18f3f11ed44488486b61cd210b4e0e17cc1", - "sha256:7a49bccae1c7d154b78e991885c3111c9ad8c8fa98e91233de425718f47c6139", - "sha256:7d8b4e00c3d7237b92260fc18a561cd81f1da82e8be100db1b7d816250defc66", - "sha256:7ea369c4d1567d1acdf69c8ea74144f4ccad9e545df7f9a4fc64c94fa7684ba3", - "sha256:8214820990d01b52845f9fbcb92d2b7384a0c321b303e3ac614c219dc7d1d3af", - "sha256:8c616d0ad872e3780693fce6a3ac8ef00fc0963e6d7815ce9dcfae68ba0fc287", - "sha256:987e774f74296842bbffd55ea8826370f70c499e5b5f71a8cf3103838b6ee9c3", - "sha256:9be84ff6d47fd61462be7523b49d7ba01adf67ce4e1447eae37721ab32464dd8", - "sha256:9d41e0e47dd075c075bb8f103422968a65dd0d8dc8613288f573ae91eb1053ba", - "sha256:9f1ba79a253df9e553d20319c615fa2b429684580fa042dba618d7f6649ac7e4", - "sha256:a34d97c62e61bfe9e6cff0410fe144ac8cca2fc979ad0be46b7edf026339d161", - "sha256:a7e439476b29d6dac363b321781a113794397afceeb97dad85349db5f1cb5e9a", - "sha256:ab84bae88597133f6ea7a2bdc57b2fda98a266fe8d8d4763652cbefd20e73ad7", - "sha256:b6746bc823958499a3cf8963cc1de00072962fb5e629f26d658882d3f4c35095", - "sha256:b8574469ecc4ff41d6bb95f44e0297cdb0d95bade388552a9a444db9cd7485cd", - "sha256:bdc5dd3f57b5368d5d661d5d3703bcaa38bceca59d25955dff66244dbc987271", - "sha256:c195d74fe98541178ece7a50dad2197d43991e0f77372b9a88da438be2486f12", - "sha256:c384c838b34d1b67068e51b5bbe49caa6aa3633acd158f1ab16b5da8d226bc53", - "sha256:c48fabe40b9170f4e3d7dd2c252e4f1ff395dc24e49ac15fc724b1b6f11724da", - "sha256:cbb8453ae83a1db2452b7fe0f4b78e4a8dd32be0f2b2b73591ae620d4d784d3d", - "sha256:d58389fe8be206ddfb4fa703db1e24c956856fcb9a81da62b13577b3a8f7fda7", - "sha256:d82f681c9a9d933a9d8068e8e382977768e7779ddb8870fa0cf918d8250d1532", - "sha256:e33b59fb3efdddeb97ded988a871710033e8638534c826567738d3edce528752", - "sha256:ec674b4440ef4311ac1245a709e87b36aca493ddc6850eebe0b278d1f2b6e7d1", - "sha256:fa107460c842e4c1a6266150881694fefd4f33baa544ea9489601810c2210ef8" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.62.2" - }, - "gunicorn": { - "hashes": [ - "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", - "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==23.0.0" - }, - "id": { - "hashes": [ - "sha256:292cb8a49eacbbdbce97244f47a97b4c62540169c976552e497fd57df0734c1d", - "sha256:f1434e1cef91f2cbb8a4ec64663d5a23b9ed43ef44c4c957d02583d61714c658" - ], - "markers": "python_version >= '3.8'", - "version": "==1.5.0" - }, - "idna": { - "hashes": [ - "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", - "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" - ], - "markers": "python_version >= '3.6'", - "version": "==3.10" - }, - "importlib-metadata": { - "hashes": [ - "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", - "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd" - ], - "markers": "python_version >= '3.9'", - "version": "==8.7.0" - }, - "isort": { - "hashes": [ - "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109", - "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6" - ], - "index": "pypi", - "markers": "python_full_version >= '3.8.0'", - "version": "==5.13.2" - }, - "jaraco.classes": { - "hashes": [ - "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", - "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790" - ], - "markers": "python_version >= '3.8'", - "version": "==3.4.0" - }, - "jaraco.context": { - "hashes": [ - "sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3", - "sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4" - ], - "markers": "python_version >= '3.8'", - "version": "==6.0.1" - }, - "jaraco.functools": { - "hashes": [ - "sha256:227ff8ed6f7b8f62c56deff101545fa7543cf2c8e7b82a7c2116e672f29c26e8", - "sha256:cfd13ad0dd2c47a3600b439ef72d8615d482cedcff1632930d6f28924d92f294" - ], - "markers": "python_version >= '3.9'", - "version": "==4.3.0" - }, - "jeepney": { - "hashes": [ - "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", - "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732" - ], - "markers": "python_version >= '3.7'", - "version": "==0.9.0" - }, - "keyring": { - "hashes": [ - "sha256:0b39998aa941431eb3d9b0d4b2460bc773b9df6fed7621c2dfb291a7e0187a66", - "sha256:552a3f7af126ece7ed5c89753650eec89c7eaae8617d0aa4d9ad2b75111266bd" - ], - "markers": "python_version >= '3.9'", - "version": "==25.6.0" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:029d2b355076710505c9545aef5ab3f750d89779310e26ddf2b7b23f6ea03cd8", - "sha256:08c465fb5cd23527512f9bd7b4c7ba6cec33e28aad36fbbe46bf7b858f9f3f7f", - "sha256:0a83c6f7a6b2bfc11ef3ed67f8cbe99f8ff500b05655d8e7df9aab993a6abc95", - "sha256:1192e8c2f1031a6ff453ee40213afa01ba765b3dc861302cd91dbdb2e2660b00", - "sha256:14e348185adbd03ec17d051e169ec45686dcd840a3779c9d4c10aabe2ca6e1c0", - "sha256:15400b18893f345857b9e18b9bd87bd06aba84af6ed086187add70aeaa3f93f1", - "sha256:1cf69cd1a6c7fe2dbcc3edaa017cf010f4192e53796538cc7d5e1fedbfa4bcff", - "sha256:1f5a462d92fd0cfb82f1fab28b51bfb209fabbe6aabf7f0d51472c0c124c0c61", - "sha256:256262384ebd2a77b023ad02fbcc9326282bcfd16484d5531154b02bc304f4c5", - "sha256:31020c84005d3daa4cc0fa5a310af2066efe6b0d82aeebf9ab199292652ff036", - "sha256:338ab2f132276203e404951205fe80c3fd59429b3a724e7b662b2eb539bb1be9", - "sha256:3605b632e82a1cbc32a1e5034278a64db555b3496e0795723ee697006b980508", - "sha256:3d3964fbd326578bcdfffd017ef101b6fb0484f34e731fe060ba9b8816498c36", - "sha256:424a8ab6695400845c39f13c685050eab69fa0bbac5790b201cd27375e5e41d7", - "sha256:4a79b909aa16bde8ae606f06e6bbc9d3219d2e57fb3e0076e17879072b742c65", - "sha256:4ab2c584e3cc8be0dfca422e05ad30a9abe3555ce63e9ab7a559f62f8dbc6ff9", - "sha256:53c7fd99eb156bbb82cbc5d5188891d8fdd805ba6c1e3b92b90092da2a837073", - "sha256:563d2ec8e4d4b68ee7848c5ab4d6057a6d703cb7963b342968bb8758dda33a23", - "sha256:61d5e3310a4aa5792c2b599a7a78ccf8687292c8eb09cf187cca8f09cf6a7519", - "sha256:6763941dbf97eea6b90f5b06eb4da9418cc088fce0e3883f5816090f9afcde4a", - "sha256:67f07ab742f1adfb3966c40f630baaa7902be4222a17941f3d85fd1dae5565ff", - "sha256:717484c309df78cedf48396e420fa57fc8a2b1f06ea889df7248fdd156e58847", - "sha256:75ba769017b944fcacbf6a80c18b2761a1795b03f8899acdad1f1c39db4409be", - "sha256:7601ec171c7e8584f8ff3f4e440aa2eebf93e854f04639263875b8c2971f819f", - "sha256:7b22c2bbfb155706b928ac4d74c1a63ac8552a55ba7fff4445155523ea4067e1", - "sha256:800f32b00a47c27446a2b767df7538e6c66a3488632c402b4fb2224f9794f3c0", - "sha256:81d1852fb30fab81696f93db1b1e55a5d1ff7940838191062f5f56987d5fcc3e", - "sha256:86fd61cb2ba249b9f436d789d1356deae69ad3231dc3c0f17293ac535162672e", - "sha256:8c40b3c9faee2e32bfce0df4ae63f4e73529766893258eca78548bac801c8f66", - "sha256:8ee0d6027b760a11cc18281e702c0309dd92da458a74b4c15025d7fc490deede", - "sha256:997b1d6e10ecc6fb6fe0f2c959791ae59599f41da61d652f6c903d1ee58b7370", - "sha256:a61095f5d9d1a743e1e20ec6d6db6c2ca511961777257ebd9b288951b23b44fa", - "sha256:a6b7ea5ea1ffe15059eb44bcbcb258f97bcb40e139b88152c40d07b1a1dfc9ac", - "sha256:ae575ad9b674d0029fc077c5231b3bc6b433a3d1a62a8c363df96974b5534728", - "sha256:be5fe974e39ceb0d6c9db0663c0464669cf866b2851c73971409b9566e880eab", - "sha256:be9045646d83f6c2664c1330904b245ae2371b5c57a3195e4028aedc9f999655", - "sha256:c1ca33565f698ac1aece152a10f432415d1a2aa9a42dfe23e5ba2bc255ab91f6", - "sha256:c3b2e0af1f7f77c4263759c4824316ce458fabe0fceadcd24ef8ca08b2d1e402", - "sha256:c4fcbe74fb85df8ba7825fa05eddca764138da752904b378f0ae5ab33a36c308", - "sha256:c9defba70ab943f1df98a656247966d7729da2fe9c2d5d85346464bf320820a3", - "sha256:cc6e3614eca88b1c8a625fc0a47d0d745e7c3255b21dac0e30b3037c5e3deeb8", - "sha256:d01c7819a410f7c255b20799b65d36b414379a30c6f1684c7bd7eb6777338c1b", - "sha256:efff4375a8c52f55a145dc8487a2108c2140f0bec4151ab4e1843e52eb9987ad", - "sha256:fdc70d81235fc586b9e3d1aeef7d1553259b62ecaae9db2167a5d2550dcc391a" - ], - "markers": "python_version >= '3.9'", - "version": "==1.12.0" - }, - "markdown-it-py": { - "hashes": [ - "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", - "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3" - ], - "markers": "python_version >= '3.10'", - "version": "==4.0.0" - }, - "mccabe": { - "hashes": [ - "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", - "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" - ], - "markers": "python_version >= '3.6'", - "version": "==0.7.0" - }, - "mdurl": { - "hashes": [ - "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", - "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" - ], - "markers": "python_version >= '3.7'", - "version": "==0.1.2" - }, - "more-itertools": { - "hashes": [ - "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", - "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd" - ], - "markers": "python_version >= '3.9'", - "version": "==10.8.0" - }, - "mypy-protobuf": { - "hashes": [ - "sha256:7d75a079651b105076776a35a5405e3fa773b8a167118f1b712e443e9a6c18a2", - "sha256:da33dfde7547ff57e5ba5564126cbfa114f14413b2fa50759b1fa5de1e4ab511" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==3.4.0" - }, - "nh3": { - "hashes": [ - "sha256:0649464ac8eee018644aacbc103874ccbfac80e3035643c3acaab4287e36e7f5", - "sha256:16f8670201f7e8e0e05ed1a590eb84bfa51b01a69dd5caf1d3ea57733de6a52f", - "sha256:1adeb1062a1c2974bc75b8d1ecb014c5fd4daf2df646bbe2831f7c23659793f9", - "sha256:37d3003d98dedca6cd762bf88f2e70b67f05100f6b949ffe540e189cc06887f9", - "sha256:389d93d59b8214d51c400fb5b07866c2a4f79e4e14b071ad66c92184fec3a392", - "sha256:3f1b4f8a264a0c86ea01da0d0c390fe295ea0bcacc52c2103aca286f6884f518", - "sha256:423201bbdf3164a9e09aa01e540adbb94c9962cc177d5b1cbb385f5e1e79216e", - "sha256:634e34e6162e0408e14fb61d5e69dbaea32f59e847cfcfa41b66100a6b796f62", - "sha256:6d68fa277b4a3cf04e5c4b84dd0c6149ff7d56c12b3e3fab304c525b850f613d", - "sha256:7275fdffaab10cc5801bf026e3c089d8de40a997afc9e41b981f7ac48c5aa7d5", - "sha256:7852f038a054e0096dac12b8141191e02e93e0b4608c4b993ec7d4ffafea4e49", - "sha256:7c915060a2c8131bef6a29f78debc29ba40859b6dbe2362ef9e5fd44f11487c2", - "sha256:80fe20171c6da69c7978ecba33b638e951b85fb92059259edd285ff108b82a6d", - "sha256:a537ece1bf513e5a88d8cff8a872e12fe8d0f42ef71dd15a5e7520fecd191bbb", - "sha256:af5aa8127f62bbf03d68f67a956627b1bd0469703a35b3dad28d0c1195e6c7fb", - "sha256:b0612ccf5de8a480cf08f047b08f9d3fecc12e63d2ee91769cb19d7290614c23", - "sha256:ba0caa8aa184196daa6e574d997a33867d6d10234018012d35f86d46024a2a95", - "sha256:bae63772408fd63ad836ec569a7c8f444dd32863d0c67f6e0b25ebbd606afa95", - "sha256:c7a32a7f0d89f7d30cb8f4a84bdbd56d1eb88b78a2434534f62c71dac538c450", - "sha256:ce5e7185599f89b0e391e2f29cc12dc2e206167380cea49b33beda4891be2fe1", - "sha256:d8ba24cb31525492ea71b6aac11a4adac91d828aadeff7c4586541bf5dc34d2f", - "sha256:d97d3efd61404af7e5721a0e74d81cdbfc6e5f97e11e731bb6d090e30a7b62b2", - "sha256:e90883f9f85288f423c77b3f5a6f4486375636f25f793165112679a7b6363b35", - "sha256:e9e6a7e4d38f7e8dda9edd1433af5170c597336c1a74b4693c5cb75ab2b30f2a", - "sha256:ec6cfdd2e0399cb79ba4dcffb2332b94d9696c52272ff9d48a630c5dca5e325a", - "sha256:f416c35efee3e6a6c9ab7716d9e57aa0a49981be915963a82697952cba1353e1" - ], - "markers": "python_version >= '3.8'", - "version": "==0.3.0" - }, - "nodeenv": { - "hashes": [ - "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", - "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9" - ], - "index": "pypi", - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", - "version": "==1.9.1" - }, - "packaging": { - "hashes": [ - "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", - "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" - ], - "markers": "python_version >= '3.8'", - "version": "==25.0" - }, - "parameterized": { - "hashes": [ - "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b", - "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==0.9.0" - }, - "paramiko": { - "hashes": [ - "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", - "sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041" - ], - "index": "pypi", - "version": "==2.6.0" - }, - "pipenv": { - "hashes": [ - "sha256:6cc26b3b2095c2fbc71a5157443e98d80f779b0613e201bf77f745258ee036fb", - "sha256:e9333cdd365aeb9c3f9460ac2d84c4750e3ffa8e766fe3d5de03d4496b55d851" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2022.8.5" - }, - "platformdirs": { - "hashes": [ - "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", - "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf" - ], - "markers": "python_version >= '3.9'", - "version": "==4.4.0" - }, - "protobuf": { - "hashes": [ - "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474", - "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2", - "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b", - "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720", - "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12", - "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd", - "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0", - "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e", - "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9", - "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70", - "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff", - "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597", - "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==4.23.4" - }, - "pycparser": { - "hashes": [ - "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", - "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934" - ], - "markers": "python_version >= '3.8'", - "version": "==2.23" - }, - "pyfakefs": { - "hashes": [ - "sha256:082d863e0e2a74351f697da404e329a91e18e5055942e59d1b836e8459b2c94c", - "sha256:1ac3b2845dabe69af56c20691b9347914581195ccdde352535fb7d4ff0055c19" - ], - "index": "pypi", - "markers": "python_version >= '3.5'", - "version": "==4.4.0" - }, - "pygments": { - "hashes": [ - "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", - "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b" - ], - "markers": "python_version >= '3.8'", - "version": "==2.19.2" - }, - "pylint": { - "hashes": [ - "sha256:27a8d4c7ddc8c2f8c18aa0050148f89ffc09838142193fdbe98f172781a3ff87", - "sha256:f4fcac7ae74cfe36bc8451e931d8438e4a476c20314b1101c458ad0f05191fad" - ], - "index": "pypi", - "markers": "python_full_version >= '3.7.2'", - "version": "==2.17.7" - }, - "pylint-protobuf": { - "hashes": [ - "sha256:b334ba32f40a579d1f8b44f3ab7a495f2553b204df6f628ee20e53db4d5a9b17", - "sha256:c643f796db9d0c85fc597803bc92c920c0a3eb1f02acff210518589aef79112f" - ], - "index": "pypi", - "version": "==0.21.0" - }, - "pynacl": { - "hashes": [ - "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858", - "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d", - "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93", - "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1", - "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92", - "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff", - "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba", - "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394", - "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", - "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543" - ], - "markers": "python_version >= '3.6'", - "version": "==1.5.0" - }, - "pyright": { - "hashes": [ - "sha256:8320629bb7a44ca90944ba599390162bf59307f3d9fb6e27da3b7011b8c17ae5", - "sha256:aeeb7ff4e0364775ef416a80111613f91a05c8e01e58ecfefc370ca0db7aed9c" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.1.393" - }, - "pyyaml": { - "hashes": [ - "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff", - "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", - "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", - "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e", - "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", - "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", - "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", - "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", - "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", - "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", - "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a", - "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", - "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", - "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8", - "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", - "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19", - "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", - "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a", - "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", - "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", - "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", - "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631", - "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d", - "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", - "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", - "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", - "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", - "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", - "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", - "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706", - "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", - "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", - "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", - "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083", - "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", - "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", - "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", - "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f", - "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725", - "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", - "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", - "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", - "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", - "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", - "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5", - "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d", - "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290", - "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", - "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", - "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", - "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", - "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12", - "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==6.0.2" - }, - "readme-renderer": { - "hashes": [ - "sha256:2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151", - "sha256:8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1" - ], - "markers": "python_version >= '3.9'", - "version": "==44.0" - }, - "requests": { - "hashes": [ - "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", - "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf" - ], - "markers": "python_version >= '3.9'", - "version": "==2.32.5" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", - "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.0.0" - }, - "rfc3986": { - "hashes": [ - "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", - "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c" - ], - "markers": "python_version >= '3.7'", - "version": "==2.0.0" - }, - "rich": { - "hashes": [ - "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", - "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8" - ], - "markers": "python_full_version >= '3.8.0'", - "version": "==14.1.0" - }, - "secretstorage": { - "hashes": [ - "sha256:0e3b6265c2c63509fb7415717607e4b2c9ab767b7f344a57473b779ca13bd02e", - "sha256:c46e216d6815aff8a8a18706a2fbfd8d53fcbb0dce99301881687a1b0289ef7c" - ], - "markers": "python_version >= '3.10'", - "version": "==3.4.0" - }, - "selenium": { - "hashes": [ - "sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c", - "sha256:deaf32b60ad91a4611b98d8002757f29e6f2c2d5fcaf202e1c9ad06d6772300d" - ], - "index": "pypi", - "version": "==3.141.0" - }, - "setuptools": { - "hashes": [ - "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", - "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c" - ], - "markers": "python_version >= '3.9'", - "version": "==80.9.0" - }, - "soupsieve": { - "hashes": [ - "sha256:0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c", - "sha256:e2dd4a40a628cb5f28f6d4b0db8800b8f581b65bb380b97de22ba5ca8d72572f" - ], - "markers": "python_version >= '3.9'", - "version": "==2.8" - }, - "tomlkit": { - "hashes": [ - "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1", - "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0" - ], - "markers": "python_version >= '3.8'", - "version": "==0.13.3" - }, - "twine": { - "hashes": [ - "sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8", - "sha256:e5ed0d2fd70c9959770dce51c8f39c8945c574e18173a7b81802dab51b4b75cf" - ], - "index": "pypi", - "markers": "python_version >= '3.9'", - "version": "==6.2.0" - }, - "types-protobuf": { - "hashes": [ - "sha256:5584c39f7e36104b5f8bdfd31815fa1d5b7b3455a79ddddc097b62320f4b1841", - "sha256:faacbbe87bd8cba4472361c0bd86f49296bd36f7761e25d8ada4f64767c1bde9" - ], - "markers": "python_version >= '3.9'", - "version": "==6.30.2.20250822" - }, - "typing-extensions": { - "hashes": [ - "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", - "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548" - ], - "markers": "python_version >= '3.9'", - "version": "==4.15.0" - }, - "urllib3": { - "hashes": [ - "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", - "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" - ], - "markers": "python_version >= '3.9'", - "version": "==2.5.0" - }, - "virtualenv": { - "hashes": [ - "sha256:341f5afa7eee943e4984a9207c025feedd768baff6753cd660c857ceb3e36026", - "sha256:44815b2c9dee7ed86e387b842a84f20b93f7f417f95886ca1996a72a4138eb1a" - ], - "markers": "python_version >= '3.8'", - "version": "==20.34.0" - }, - "virtualenv-clone": { - "hashes": [ - "sha256:418ee935c36152f8f153c79824bb93eaf6f0f7984bae31d3f48f350b9183501a", - "sha256:44d5263bceed0bac3e1424d64f798095233b64def1c5689afa43dc3223caf5b0" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.5.7" - }, - "waitress": { - "hashes": [ - "sha256:682aaaf2af0c44ada4abfb70ded36393f0e307f4ab9456a215ce0020baefc31f", - "sha256:c56d67fd6e87c2ee598b76abdd4e96cfad1f24cacdea5078d382b1f9d7b5ed2e" - ], - "markers": "python_full_version >= '3.9.0'", - "version": "==3.0.2" - }, - "webob": { - "hashes": [ - "sha256:45e34c58ed0c7e2ecd238ffd34432487ff13d9ad459ddfd77895e67abba7c1f9", - "sha256:ad6078e2edb6766d1334ec3dee072ac6a7f95b1e32ce10def8ff7f0f02d56589" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==1.8.9" - }, - "webtest": { - "hashes": [ - "sha256:2a001a9efa40d2a7e5d9cd8d1527c75f41814eb6afce2c3d207402547b1e5ead", - "sha256:54bd969725838d9861a9fa27f8d971f79d275d94ae255f5c501f53bb6d9929eb" - ], - "index": "pypi", - "markers": "python_version >= '3.6' and python_version < '4'", - "version": "==3.0.0" - }, - "wrapt": { - "hashes": [ - "sha256:02b551d101f31694fc785e58e0720ef7d9a10c4e62c1c9358ce6f63f23e30a56", - "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", - "sha256:0610b46293c59a3adbae3dee552b648b984176f8562ee0dba099a56cfbe4df1f", - "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", - "sha256:0b1831115c97f0663cb77aa27d381237e73ad4f721391a9bfb2fe8bc25fa6e77", - "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", - "sha256:0f5f51a6466667a5a356e6381d362d259125b57f059103dd9fdc8c0cf1d14139", - "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7", - "sha256:1f0b2f40cf341ee8cc1a97d51ff50dddb9fcc73241b9143ec74b30fc4f44f6cb", - "sha256:1f23fa283f51c890eda8e34e4937079114c74b4c81d2b2f1f1d94948f5cc3d7f", - "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", - "sha256:249f88ed15503f6492a71f01442abddd73856a0032ae860de6d75ca62eed8067", - "sha256:24c2ed34dc222ed754247a2702b1e1e89fdbaa4016f324b4b8f1a802d4ffe87f", - "sha256:273a736c4645e63ac582c60a56b0acb529ef07f78e08dc6bfadf6a46b19c0da7", - "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", - "sha256:30ce38e66630599e1193798285706903110d4f057aab3168a34b7fdc85569afc", - "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05", - "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd", - "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", - "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", - "sha256:3e62d15d3cfa26e3d0788094de7b64efa75f3a53875cdbccdf78547aed547a81", - "sha256:41b1d2bc74c2cac6f9074df52b2efbef2b30bdfe5f40cb78f8ca22963bc62977", - "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", - "sha256:46acc57b331e0b3bcb3e1ca3b421d65637915cfcd65eb783cb2f78a511193f9b", - "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", - "sha256:507553480670cab08a800b9463bdb881b2edeed77dc677b0a5915e6106e91a58", - "sha256:53e5e39ff71b3fc484df8a522c933ea2b7cdd0d5d15ae82e5b23fde87d44cbd8", - "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77", - "sha256:5531d911795e3f935a9c23eb1c8c03c211661a5060aab167065896bbf62a5f85", - "sha256:55cbbc356c2842f39bcc553cf695932e8b30e30e797f961860afb308e6b1bb7c", - "sha256:59923aa12d0157f6b82d686c3fd8e1166fa8cdfb3e17b42ce3b6147ff81528df", - "sha256:5a03a38adec8066d5a37bea22f2ba6bbf39fcdefbe2d91419ab864c3fb515454", - "sha256:5a7b3c1ee8265eb4c8f1b7d29943f195c00673f5ab60c192eba2d4a7eae5f46a", - "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", - "sha256:5ea5eb3c0c071862997d6f3e02af1d055f381b1d25b286b9d6644b79db77657c", - "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", - "sha256:656873859b3b50eeebe6db8b1455e99d90c26ab058db8e427046dbc35c3140a5", - "sha256:65d1d00fbfb3ea5f20add88bbc0f815150dbbde3b026e6c24759466c8b5a9ef9", - "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", - "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", - "sha256:70d86fa5197b8947a2fa70260b48e400bf2ccacdcab97bb7de47e3d1e6312225", - "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", - "sha256:73d496de46cd2cdbdbcce4ae4bcdb4afb6a11234a1df9c085249d55166b95116", - "sha256:7425ac3c54430f5fc5e7b6f41d41e704db073309acfc09305816bc6a0b26bb16", - "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", - "sha256:758895b01d546812d1f42204bd443b8c433c44d090248bf22689df673ccafe00", - "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2", - "sha256:7e18f01b0c3e4a07fe6dfdb00e29049ba17eadbc5e7609a2a3a4af83ab7d710a", - "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", - "sha256:88bbae4d40d5a46142e70d58bf664a89b6b4befaea7b2ecc14e03cedb8e06c04", - "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1", - "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", - "sha256:a36692b8491d30a8c75f1dfee65bef119d6f39ea84ee04d9f9311f83c5ad9390", - "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0", - "sha256:a7c06742645f914f26c7f1fa47b8bc4c91d222f76ee20116c43d5ef0912bba2d", - "sha256:a9a2203361a6e6404f80b99234fe7fb37d1fc73487b5a78dc1aa5b97201e0f22", - "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", - "sha256:ad85e269fe54d506b240d2d7b9f5f2057c2aa9a2ea5b32c66f8902f768117ed2", - "sha256:af338aa93554be859173c39c85243970dc6a289fa907402289eeae7543e1ae18", - "sha256:afd964fd43b10c12213574db492cb8f73b2f0826c8df07a68288f8f19af2ebe6", - "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311", - "sha256:c31eebe420a9a5d2887b13000b043ff6ca27c452a9a22fa71f35f118e8d4bf89", - "sha256:caea3e9c79d5f0d2c6d9ab96111601797ea5da8e6d0723f77eabb0d4068d2b2f", - "sha256:cf30f6e3c077c8e6a9a7809c94551203c8843e74ba0c960f4a98cd80d4665d39", - "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", - "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5", - "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", - "sha256:df7d30371a2accfe4013e90445f6388c570f103d61019b6b7c57e0265250072a", - "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", - "sha256:e1a4120ae5705f673727d3253de3ed0e016f7cd78dc463db1b31e2463e1f3cf6", - "sha256:e228514a06843cae89621384cfe3a80418f3c04aadf8a3b14e46a7be704e4235", - "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", - "sha256:e6b13af258d6a9ad602d57d889f83b9d5543acd471eee12eb51f5b01f8eb1bc2", - "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418", - "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", - "sha256:ed7c635ae45cfbc1a7371f708727bf74690daedc49b4dba310590ca0bd28aa8a", - "sha256:f38e60678850c42461d4202739f9bf1e3a737c7ad283638251e79cc49effb6b6", - "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", - "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775", - "sha256:fbd3c8319de8e1dc79d346929cd71d523622da527cca14e0c1d257e31c2b8b10", - "sha256:fd341868a4b6714a5962c1af0bd44f7c404ef78720c7de4892901e540417111c" - ], - "markers": "python_version >= '3.8'", - "version": "==1.17.3" - }, - "yapf": { - "hashes": [ - "sha256:6567745f0b6656f9c33a73c56a393071c699e6284a70d793798ab6e3769d25ec", - "sha256:a98a6eacca64d2b920558f4a2f78150db9474de821227e60deaa29f186121c63" - ], - "index": "pypi", - "version": "==0.22.0" - }, - "zipp": { - "hashes": [ - "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", - "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166" - ], - "markers": "python_version >= '3.9'", - "version": "==3.23.0" - } - } -} diff --git a/cli/casp/src/casp/utils/container.py b/cli/casp/src/casp/utils/container.py index 29e7678428c..4b8d773820f 100644 --- a/cli/casp/src/casp/utils/container.py +++ b/cli/casp/src/casp/utils/container.py @@ -32,7 +32,7 @@ # The base command prefix for executing ClusterFuzz butler commands. # This ensures that commands are run with the correct Python environment # and logging settings within the container. -_COMMAND_PREFIX = 'pipenv run python butler.py --local-logging' +_COMMAND_PREFIX = 'uv run butler.py --local-logging' def build_butler_command(subcommand: str, **kwargs: str) -> list[str]: diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 6e5a9e6ef67..bcede7960f9 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -29,19 +29,7 @@ steps: mv tmp/.git .git rm -rf tmp - name: 'gcr.io/clusterfuzz-images/ci:d41d82f-202601161634' - args: ['pipenv', 'sync', '--dev', '--python=3.11'] - env: - - PIPENV_VENV_IN_PROJECT=1 -- name: 'gcr.io/clusterfuzz-images/ci:d41d82f-202601161634' - args: ['pipenv', 'run', 'setup'] - env: - - PIPENV_VENV_IN_PROJECT=1 -- name: 'gcr.io/clusterfuzz-images/ci:d41d82f-202601161634' - args: ['pipenv', 'run', 'python', 'butler.py', 'lint'] - env: - - PIPENV_VENV_IN_PROJECT=1 -- name: 'gcr.io/clusterfuzz-images/ci:d41d82f-202601161634' - args: ['pipenv', 'run', 'local/tests/run_all_tests'] + args: ['local/tests/ci_tests_inner.bash', 'local/tests/run_all_tests'] env: - TEST_BLOBS_BUCKET=clusterfuzz-ci-blobs - TEST_BUCKET=clusterfuzz-ci-test @@ -49,7 +37,6 @@ steps: - TEST_QUARANTINE_BUCKET=clusterfuzz-ci-quarantine - TEST_BACKUP_BUCKET=clusterfuzz-ci-backup - TEST_COVERAGE_BUCKET=clusterfuzz-ci-coverage - - PIPENV_VENV_IN_PROJECT=1 timeout: 5400s options: machineType: E2_HIGHCPU_8 diff --git a/configs/test/bot/setup/android.bash b/configs/test/bot/setup/android.bash index fb98a4c498d..f4f3e1ece4d 100755 --- a/configs/test/bot/setup/android.bash +++ b/configs/test/bot/setup/android.bash @@ -96,16 +96,21 @@ else fi unzip -q clusterfuzz-source.zip -echo "Installing ClusterFuzz package dependencies using pipenv." +echo "Installing ClusterFuzz package dependencies using uv." cd clusterfuzz if ! python3 -m pip > /dev/null ; then curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python3 get-pip.py fi -python3 -m pip install --upgrade pipenv -pipenv --python 3.11 -pipenv sync -source "$(pipenv --venv)/bin/activate" +if ! command -v pipx &> /dev/null; then + python3 -m pip install --upgrade pipx==1.10.0 +fi +if ! command -v uv &> /dev/null; then + pipx install uv==0.12.3 +fi +export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")" +uv sync +source .venv/bin/activate if [ -z "$ANDROID_SERIAL" ]; then echo "No \$ANDROID_SERIAL set. Will automatically detect devices and start ClusterFuzz for each." diff --git a/configs/test/bot/setup/linux.bash b/configs/test/bot/setup/linux.bash index df7466bbe2a..5d4a9d92a4f 100755 --- a/configs/test/bot/setup/linux.bash +++ b/configs/test/bot/setup/linux.bash @@ -91,15 +91,20 @@ else fi unzip -q clusterfuzz-source.zip -echo "Installing ClusterFuzz package dependencies using pipenv." +echo "Installing ClusterFuzz package dependencies using uv." cd clusterfuzz if ! python3 -m pip > /dev/null ; then curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python3 get-pip.py fi -python3 -m pip install --upgrade pipenv -pipenv --python 3.11 -pipenv sync -source "$(pipenv --venv)/bin/activate" +if ! command -v pipx &> /dev/null; then + python3 -m pip install --upgrade pipx==1.10.0 +fi +if ! command -v uv &> /dev/null; then + pipx install uv==0.12.3 +fi +export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")" +uv sync +source .venv/bin/activate run_bot & diff --git a/configs/test/bot/setup/mac.bash b/configs/test/bot/setup/mac.bash index d6ed279a8fe..eacff79f67c 100755 --- a/configs/test/bot/setup/mac.bash +++ b/configs/test/bot/setup/mac.bash @@ -84,16 +84,21 @@ else fi unzip -q clusterfuzz-source.zip -echo "Installing ClusterFuzz package dependencies using pipenv." +echo "Installing ClusterFuzz package dependencies using uv." cd clusterfuzz if ! python3 -m pip > /dev/null ; then curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python3 get-pip.py fi -python3 -m pip install --upgrade pipenv -pipenv --python 3.11 -pipenv sync -source "$(pipenv --venv)/bin/activate" +if ! command -v pipx &> /dev/null; then + python3 -m pip install --upgrade pipx==1.10.0 +fi +if ! command -v uv &> /dev/null; then + pipx install uv==0.12.3 +fi +export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")" +uv sync +source .venv/bin/activate echo "Running ClusterFuzz." GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" ROOT_DIR="$ROOT_DIR" PYTHONPATH="$PYTHONPATH" GSUTIL_PATH="$GSUTIL_PATH" python $ROOT_DIR/src/python/bot/startup/run.py & diff --git a/configs/test/gce/windows-init.ps1 b/configs/test/gce/windows-init.ps1 index 87893ed5fb1..91decceeab0 100755 --- a/configs/test/gce/windows-init.ps1 +++ b/configs/test/gce/windows-init.ps1 @@ -172,7 +172,9 @@ cmd /c c:\python27\python -m pip install -U wheel cmd /c c:\python27\python -m pip install crcmod==1.7 cryptography==3.3.2 pyOpenSSL==17.4.0 pywinauto==0.6.4 psutil==5.4.7 future==0.17.1 cmd /c c:\python311\python -m pip install -U pip -cmd /c c:\python311\python -m pip install pipenv +cmd /c c:\python311\python -m pip install pipx==1.10.0 +cmd /c c:\python311\python -m pipx ensurepath +cmd /c c:\python311\python -m pipx install uv==0.12.3 # Install NodeJS. $fileName = "$tmp\nodejs.zip" @@ -301,13 +303,11 @@ Set-Content $packageSetupFilePath "Skipped package install" # Schedule chkdsk on every reboot. echo y | chkdsk C: /F /I /C -# Install Pipfile dependencies -$env:Path += ";c:\python311;c:\python311\scripts" +# Install dependencies using uv +$pipxBinDir = cmd /c c:\python311\python -m pipx environment --value PIPX_BIN_DIR +$env:Path += ";c:\python311;c:\python311\scripts;$pipxBinDir" cd c:\clusterfuzz -cmd /c c:\python311\scripts\pipenv install --deploy --system - -# Can't be managed by pipenv due to https://github.com/pypa/pipenv/issues/3193. -cmd /c c:\python311\python -m pip install pywinauto==0.6.8 +cmd /c uv sync # Run the scripts. Write-Host "Run scripts" diff --git a/deploy/cloudbuild.yaml b/deploy/cloudbuild.yaml index 2b9c48085c2..493f4643a44 100644 --- a/deploy/cloudbuild.yaml +++ b/deploy/cloudbuild.yaml @@ -66,9 +66,11 @@ steps: # Install required deps for performing butler deploy bash ./local/install_deps.bash - + + export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")" + # Get into the venv - source "$$(python3.11 -m pipenv --venv)/bin/activate" + source .venv/bin/activate # Check if the _PROJECT substitution was provided. if [ -z "${_PROJECT}" ]; then diff --git a/docker/base/.gitignore b/docker/base/.gitignore index ee826c1b540..26599f89bf1 100644 --- a/docker/base/.gitignore +++ b/docker/base/.gitignore @@ -1,2 +1,2 @@ -Pipfile -Pipfile.lock +pyproject.toml +uv.lock diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 1d2003160a0..e6efe4ca80f 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -124,7 +124,11 @@ RUN curl -sS https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz | tar -C cd /tmp/Python-3.11.4 && \ ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \ rm -rf /tmp/Python-3.11.4 /tmp/Python-3.11.4.tar.xz -RUN pip3.11 --no-cache-dir install pipenv==2022.8.5 +# Symlink uv binary installed by pipx to /usr/local/bin so it is on system PATH, +# and symlink python3.11 to /usr/bin/python3.11 for system tools expecting it in /usr/bin. +RUN pip3.11 --no-cache-dir install pipx==1.10.0 +RUN pipx install uv==0.12.3 +RUN ln -s $(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "/root/.local/bin")/uv /usr/local/bin/uv RUN ln -s /usr/local/bin/python3.11 /usr/bin/python3.11 # Install Node.js @@ -144,7 +148,7 @@ RUN echo "deb https://packages.cloud.google.com/apt cloud-sdk main" \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ | apt-key add - && \ apt-get update -y && \ - apt-get install -y google-cloud-sdk + apt-get install -y google-cloud-cli # Common environment variables. ENV USER=clusterfuzz @@ -169,10 +173,10 @@ RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 ENV PYTHONIOENCODING UTF-8 -COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh Pipfile Pipfile.lock start.sh /data/ +COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh pyproject.toml uv.lock start.sh /data/ RUN cd /data && \ - # Make pip3.11 the default so that pipenv install --system works. + # Make pip3.11 the default so that pip install works. mv /usr/local/bin/pip3.11 /usr/local/bin/pip && \ - python3.11 -m pipenv install --deploy --system + uv pip install --system -e /data CMD ["bash", "-ex", "/data/start.sh"] diff --git a/docker/base/ubuntu-20-04.Dockerfile b/docker/base/ubuntu-20-04.Dockerfile index 8cfd8755f03..5719958bdfe 100644 --- a/docker/base/ubuntu-20-04.Dockerfile +++ b/docker/base/ubuntu-20-04.Dockerfile @@ -124,7 +124,11 @@ RUN curl -sS https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz | tar -C cd /tmp/Python-3.11.4 && \ ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \ rm -rf /tmp/Python-3.11.4 /tmp/Python-3.11.4.tar.xz -RUN pip3.11 --no-cache-dir install pipenv==2022.8.5 +# Symlink uv binary installed by pipx to /usr/local/bin so it is on system PATH, +# and symlink python3.11 to /usr/bin/python3.11 for system tools expecting it in /usr/bin. +RUN pip3.11 --no-cache-dir install pipx==1.10.0 +RUN pipx install uv==0.12.3 +RUN ln -s $(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "/root/.local/bin")/uv /usr/local/bin/uv RUN ln -s /usr/local/bin/python3.11 /usr/bin/python3.11 # Install Node.js @@ -144,7 +148,7 @@ RUN echo "deb https://packages.cloud.google.com/apt cloud-sdk main" \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ | apt-key add - && \ apt-get update -y && \ - apt-get install -y google-cloud-sdk + apt-get install -y google-cloud-cli # Common environment variables. ENV USER=clusterfuzz @@ -169,10 +173,10 @@ RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 ENV PYTHONIOENCODING UTF-8 -COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh Pipfile Pipfile.lock start.sh /data/ +COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh pyproject.toml uv.lock start.sh /data/ RUN cd /data && \ - # Make pip3.11 the default so that pipenv install --system works. + # Make pip3.11 the default so that pip install works. mv /usr/local/bin/pip3.11 /usr/local/bin/pip && \ - python3.11 -m pipenv install --deploy --system + uv pip install --system -e /data CMD ["bash", "-ex", "/data/start.sh"] diff --git a/docker/base/ubuntu-24-04.Dockerfile b/docker/base/ubuntu-24-04.Dockerfile index d0ccf0d7f6d..1839662b5f8 100644 --- a/docker/base/ubuntu-24-04.Dockerfile +++ b/docker/base/ubuntu-24-04.Dockerfile @@ -100,7 +100,9 @@ RUN curl -sS https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz | tar -C cd /tmp/Python-3.11.4 && \ ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \ rm -rf /tmp/Python-3.11.4 /tmp/Python-3.11.4.tar.xz -RUN pip3.11 --no-cache-dir install pipenv==2022.8.5 +RUN pip3.11 --no-cache-dir install pipx==1.10.0 +RUN pipx install uv==0.12.3 +RUN ln -s $(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "/root/.local/bin")/uv /usr/local/bin/uv RUN ln -s /usr/local/bin/python3.11 /usr/bin/python3.11 && \ ln -s /usr/bin/python3.11 /usr/bin/python3 @@ -123,7 +125,7 @@ RUN echo "deb https://packages.cloud.google.com/apt cloud-sdk main" \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ | apt-key add - && \ apt-get update -y && \ - apt-get install -y google-cloud-sdk + apt-get install -y google-cloud-cli # Common environment variables. ENV USER=clusterfuzz @@ -150,10 +152,10 @@ ENV PYTHONIOENCODING UTF-8 ENV BASE_OS_VERSION=ubuntu-24-04 -COPY Pipfile Pipfile.lock setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh start.sh /data/ +COPY pyproject.toml uv.lock setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh start.sh /data/ RUN cd /data && \ - # Make pip3.11 the default so that pipenv install --system works. + # Make pip3.11 the default so that pip install works. mv /usr/local/bin/pip3.11 /usr/local/bin/pip && \ - python3.11 -m pipenv install --deploy --system + uv pip install --system -e /data CMD ["bash", "-ex", "/data/start.sh"] diff --git a/docker/build.sh b/docker/build.sh index 29e1945149a..a54afa66d28 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -32,12 +32,12 @@ IMAGES=( VERSION_TAG="latest" GIT_HASH_ARG="" PUSH="true" -NEEDS_ROOT_PIPFILE=false +NEEDS_ROOT_PYPROJECT=false -# Set up a trap to clean up Pipfiles on exit. +# Set up a trap to clean up pyproject files on exit. function cleanup() { - if [[ "$NEEDS_ROOT_PIPFILE" == "true" ]]; then - rm -f base/Pipfile base/Pipfile.lock + if [[ "$NEEDS_ROOT_PYPROJECT" == "true" ]]; then + rm -f base/pyproject.toml base/uv.lock fi } trap cleanup EXIT @@ -102,19 +102,19 @@ for image_name in "${IMAGES[@]}"; do image_with_version_tag="$image_name:$VERSION_TAG" image_with_stamp="$image_name:$stamp" - # Copy Pipfile to base for the ubuntu-24.04 build, as it's + # Copy pyproject.toml to base for the ubuntu-24.04 build, as it's # needed but not in the build context. if [[ "$image_dir" == "base" && "$dockerfile" == *"ubuntu-24-04"* ]]; then - NEEDS_ROOT_PIPFILE=true - cp ../Pipfile ../Pipfile.lock base/ + NEEDS_ROOT_PYPROJECT=true + cp ../pyproject.toml ../uv.lock base/ fi docker build -t "$image_with_version_tag" -f "$dockerfile" "$image_dir" # Clean up the copied files. - if [[ "$NEEDS_ROOT_PIPFILE" == "true" ]]; then - rm base/Pipfile base/Pipfile.lock - NEEDS_ROOT_PIPFILE=false + if [[ "$NEEDS_ROOT_PYPROJECT" == "true" ]]; then + rm base/pyproject.toml base/uv.lock + NEEDS_ROOT_PYPROJECT=false fi docker tag "$image_with_version_tag" "$image_with_stamp" diff --git a/docker/build_on_container_builder.sh b/docker/build_on_container_builder.sh index 6a2e1316279..54610c29c29 100755 --- a/docker/build_on_container_builder.sh +++ b/docker/build_on_container_builder.sh @@ -24,5 +24,5 @@ if [[ -n "$git_diff" ]]; then exit 1 fi -cp ../Pipfile* base/ +cp ../pyproject.toml ../uv.lock base/ gcloud builds submit . --project=clusterfuzz-images --substitutions=_GIT_HASH=$(git rev-parse HEAD | head -c7) diff --git a/docs/getting-started/prerequisites.md b/docs/getting-started/prerequisites.md index eba001bd30d..c88e82aef1e 100644 --- a/docs/getting-started/prerequisites.md +++ b/docs/getting-started/prerequisites.md @@ -110,20 +110,20 @@ gcloud auth login [set up ClusterFuzz in production]: {{ "/production-setup/" | relative_url }} [running ClusterFuzz locally]: {{ "/getting-started/local-instance/" | relative_url }} -## Loading pipenv +## Loading the Virtual Environment -After you run the `local/install_deps.bash` script, activate pipenv by running the following command: +After you run the `local/install_deps.bash` script, activate the virtual environment by running: ```bash -pipenv shell +source .venv/bin/activate ``` -This loads all the Python dependencies in the current environment. +This loads all the Python dependencies in the current environment. You can also run commands directly without activation using `uv run butler.py `. You can verify that everything works by running: ```bash -python butler.py --help +uv run butler.py --help ``` ### Debugging Common Dependency Issues @@ -137,32 +137,16 @@ $ PYTHON=python3.11 ./local/install_deps.bash Then run: ``` -$ pipenv shell +$ source .venv/bin/activate $ python --version ``` -The version should be the one you built from source and used to build your dependencies. If it is not, set the pipenv shell version of Python: - -``` -$ pipenv install --python 3.11 -``` - Assuming your shell has the expected version of Python assigned, the following command should run all appengine tests: ``` -python3.11 butler.py py_unittest -t appengine -``` - -If, at this point, you see an error to the effect that the config.yaml is missing, sync the pipenv environment’s dependencies with the outer environment’s dependencies: - -``` -$ cd src; pipenv sync +uv run butler.py py_unittest -t appengine ``` -#### Refreshing the Python Environment - -You might get stuck in a mode where the python version is the global value, rather than the version you set for the pipenv environment - even when apparently leaving the virtual environment through the `exit`command. If this happens, you may still be in a virtual environment (especially if you see `(clusterfuzz)`before the command prompt). To escape this virtual environment, use the command `deactivate`, then run `python3.11 -m pipenv shell` again to start the pipenv shell with the correct python version. - ## ClusterFuzz Development Tips - Before committing, run `python butler.py format` to clean up formatting diff --git a/local/Brewfile b/local/Brewfile index 842052f1e5a..7d3c4127d6e 100644 --- a/local/Brewfile +++ b/local/Brewfile @@ -1,7 +1,7 @@ brew "golang" brew "node" brew "nodeenv" -brew "pipenv" +brew "pipx" brew "pkill" brew "python3" brew "xz" diff --git a/local/README.md b/local/README.md index a7a462899b8..b75fc1cc9b0 100644 --- a/local/README.md +++ b/local/README.md @@ -20,11 +20,10 @@ This project uses a `.tool-versions` file. Follow these steps to install the cor ```bash asdf install - python3.11 -m pip install pipenv ``` 4. **Install Deps** - Run this command inside the project directory. This script will use `pipenv` to set up a virtual environment and install all the required packages specified in the `Pipfile`, ensuring your local setup is consistent with the project's requirements. + Run this command inside the project directory. This script will set up a virtual environment with `uv` and install all the required packages, ensuring your local setup is consistent with the project's requirements. ```bash ./local/install_deps.bash diff --git a/local/install_deps_linux.bash b/local/install_deps_linux.bash index e690ad93fe7..c2f7d9f7796 100755 --- a/local/install_deps_linux.bash +++ b/local/install_deps_linux.bash @@ -80,9 +80,10 @@ sudo apt-get install -y \ blackbox \ curl \ libpython3-all-dev \ - pipenv \ + pipx \ python3-dev \ python3-pip \ + python3-venv \ unzip \ xvfb diff --git a/local/install_deps_macos.bash b/local/install_deps_macos.bash index 9ff9446e8e6..d8d182b7df7 100755 --- a/local/install_deps_macos.bash +++ b/local/install_deps_macos.bash @@ -26,9 +26,13 @@ fi brew bundle --file=$(dirname "$0")/Brewfile -pipenv --python 3.11 -pipenv sync --dev -source "$(pipenv --venv)/bin/activate" +if ! command -v uv &> /dev/null; then + pipx install uv==0.12.3 +fi +export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")" + +uv sync +source .venv/bin/activate # Install other dependencies (e.g. bower). nodeenv -p --prebuilt @@ -50,6 +54,6 @@ set +x echo " Installation succeeded! -Please load environment by running 'pipenv shell'. +Please load environment by running 'source .venv/bin/activate' or use 'uv run butler.py'. " diff --git a/local/install_python_deps_linux.bash b/local/install_python_deps_linux.bash index e9d736618e6..ced6eb7e40f 100755 --- a/local/install_python_deps_linux.bash +++ b/local/install_python_deps_linux.bash @@ -14,16 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Setup pipenv and install python dependencies. -echo If this fails, you may need to build older Python from source -if $PYTHON -m pipenv --venv > /dev/null 2>&1; then - # Remove existing pipenv virtual environment. - $PYTHON -m pipenv --rm +# Setup uv and install python dependencies. +if ! command -v uv &> /dev/null; then + pipx install uv==0.12.3 fi +export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")" -$PYTHON -m pipenv --python $PYTHON -$PYTHON -m pipenv sync --dev -source "$(${PYTHON} -m pipenv --venv)/bin/activate" +uv sync +source .venv/bin/activate if [ $install_android_emulator ]; then ANDROID_SDK_INSTALL_DIR=local/bin/android-sdk @@ -54,9 +52,7 @@ fi # Install other dependencies (e.g. bower). nodeenv -p --prebuilt -# Re-activate virtual environment after nodeenv modifies activate scripts and PATH -# so that global node npm packages (like bower) install into the virtual environment bin directory. -source "$(${PYTHON} -m pipenv --venv)/bin/activate" +source .venv/bin/activate # Unsafe perm flag allows bower and polymer-bundler install for root users as well. npm install --unsafe-perm -g bower polymer-bundler bower --allow-root install @@ -68,6 +64,6 @@ set +x echo " Installation succeeded! -Please load environment by running "$PYTHON -m pipenv shell". +Please load environment by running \"source .venv/bin/activate\". " diff --git a/local/install_python_from_source.bash b/local/install_python_from_source.bash index 6472bc592d7..dfe72e22309 100644 --- a/local/install_python_from_source.bash +++ b/local/install_python_from_source.bash @@ -21,7 +21,9 @@ make -j$(nproc) make install curl -O https://bootstrap.pypa.io/get-pip.py $HOME/.localpython/bin/python3 get-pip.py -$HOME/.localpython/bin/python3.11 -m pip install pipenv +$HOME/.localpython/bin/python3.11 -m pip install pipx==1.10.0 +$HOME/.localpython/bin/python3.11 -m pipx install uv==0.12.3 +export PATH="$PATH:$($HOME/.localpython/bin/python3.11 -m pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")" # Copy distutils to this new install git clone --branch v3.11.9 https://github.com/python/cpython.git diff --git a/local/tests/ci_tests.bash b/local/tests/ci_tests.bash index e32e93cae09..e9bd1f2af30 100755 --- a/local/tests/ci_tests.bash +++ b/local/tests/ci_tests.bash @@ -16,27 +16,8 @@ IMAGE=gcr.io/clusterfuzz-images/ci:bc696af-202407021946 -docker run -i --rm \ - -e PIPENV_VENV_IN_PROJECT=1 \ - -e IS_GITHUB_ACTIONS=true \ - -v $(pwd):/workspace \ - $IMAGE \ - pipenv sync --dev --python=python3.11 -docker run -i --rm \ - -e PIPENV_VENV_IN_PROJECT=1 \ - -e IS_GITHUB_ACTIONS=true \ - -v $(pwd):/workspace \ - $IMAGE \ - pipenv run setup -docker run -i --rm \ - -e PIPENV_VENV_IN_PROJECT=1 \ - -e IS_GITHUB_ACTIONS=true \ - -v $(pwd):/workspace \ - $IMAGE \ - pipenv run python butler.py lint docker run -i --rm --privileged --cap-add=all \ - -e PIPENV_VENV_IN_PROJECT=1 \ -e IS_GITHUB_ACTIONS=true \ -v $(pwd):/workspace \ $IMAGE \ - pipenv run local/tests/run_tests + local/tests/ci_tests_inner.bash diff --git a/local/tests/ci_tests_inner.bash b/local/tests/ci_tests_inner.bash new file mode 100755 index 00000000000..a997a92ea18 --- /dev/null +++ b/local/tests/ci_tests_inner.bash @@ -0,0 +1,26 @@ +#!/bin/bash -ex +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TEST_CMD=${1:-local/tests/run_tests} + +pip install pipx==1.10.0 +pipx install uv==0.12.3 +export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")" + +uv sync +uv run butler.py bootstrap +uv run butler.py lint +uv run $TEST_CMD diff --git a/local/tests/install_deps_test_linux.bash b/local/tests/install_deps_test_linux.bash index 7f41e90256c..ed18f9cd509 100755 --- a/local/tests/install_deps_test_linux.bash +++ b/local/tests/install_deps_test_linux.bash @@ -32,7 +32,11 @@ trap cleanup EXIT git clone "$REPO_DIR" "$TEMP_DIR/clusterfuzz" cd "$TEMP_DIR/clusterfuzz" -# Verify initial clean state (no vendored directories). +# Verify initial clean state (no venv or vendored directories). +if [ -d .venv ]; then + echo "ERROR: .venv should not exist in fresh checkout." + exit 1 +fi if [ -d src/third_party ]; then echo "ERROR: src/third_party should not exist in fresh checkout." exit 1 @@ -46,9 +50,8 @@ fi ./local/install_deps.bash # Verify virtual environment and python installation. -VENV_DIR=$(python3 -m pipenv --venv) -if [ ! -f "$VENV_DIR/bin/python" ]; then - echo "ERROR: Pipenv virtual environment python was not created." +if [ ! -f .venv/bin/python ]; then + echo "ERROR: .venv/bin/python was not created." exit 1 fi @@ -71,10 +74,10 @@ if [ ! -d src/appengine/private/bower_components ]; then fi # Verify linting passes in fresh environment. -pipenv run python butler.py lint +uv run butler.py lint # Verify running core and appengine unit tests passes in fresh environment. -pipenv run python butler.py py_unittest -t core -p deploy_test.py -pipenv run python butler.py py_unittest -t appengine -p home_test.py +uv run butler.py py_unittest -t core -p deploy_test.py +uv run butler.py py_unittest -t appengine -p home_test.py echo "SUCCESS: Fresh checkout setup using local/install_deps.bash verified!" diff --git a/local/tests/kubernetes_e2e_test.bash b/local/tests/kubernetes_e2e_test.bash index 16a75bdc0a6..4c9864cabba 100755 --- a/local/tests/kubernetes_e2e_test.bash +++ b/local/tests/kubernetes_e2e_test.bash @@ -16,14 +16,8 @@ # This script is for running the Kubernetes end-to-end test in CI. -pip install pipenv - -# Install dependencies. -pipenv --python 3.11 -pipenv install - ./local/install_deps.bash # Run the test. export K8S_E2E=1 -pipenv run python butler.py py_unittest -t core -p k8s_service_e2e_test.py +uv run butler.py py_unittest -t core -p k8s_service_e2e_test.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..c085527116a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,180 @@ +[project] +name = "clusterfuzz" +version = "0.1.0" +description = "ClusterFuzz" +readme = "README.md" +requires-python = ">=3.11,<3.12" + +[[tool.uv.index]] +name = "pypi" +url = "https://pypi.org/simple" +default = true + +# We do something a bit peculiar that probably predates Python dependency +# managers. Instead of installing dependencies into a virtual environment and +# executing within said environment, dependencies are vendored (as source or +# binary distributions) into `src/third_party` and `src/appengine/third_party`. +# The resulting codebase is then shipped as is to bots for execution. +# +# In practice `butler.py bootstrap` runs `uv export` to convert this file into +# several `requirements.txt` passed to `(uv) pip install --target $3P_DIR`. +# We then rely on PYTHONPATH shenanigans to get project code to find dependency +# packages. +# +# TODO: This whole setup should likely be replaced by using `uv` and virtualenvs +# directly, both within and without docker images. It used to be the case that +# docker images were updated much less frequently than the source code, but +# this is not the case anymore - each release of ClusterFuzz runs +# `local/install_deps.bash` to build the image, which includes installing +# Python dependencies. Still, some bots run ClusterFuzz natively, which includes +# another peculiarity: the self-update mechanism in `update_task.py`. Someone +# would need to figure out how to mesh that with a virtualenv-based release. +dependencies = [] + +[dependency-groups] +dev = [ + # Development, testing, and linting tools (from ./Pipfile) + "Fabric==1.14.1", + "grpcio-tools==1.62.2", + "gunicorn", + "isort", + "mypy-protobuf==3.4.0", + "nodeenv==1.9.1", + "parameterized==0.9.0", + "paramiko==2.6.0", + "pyfakefs==4.4.0", + "pylint~=2.4", + "pylint-protobuf==0.21.0", + "pyright==1.1.411", + "pywinauto==0.6.8; sys_platform == 'win32'", + "PyYAML==6.0.1", + "selenium==3.141.0", + "setuptools==80.9.0", + "twine", + "ty==0.0.69", + "WebTest==3.0.0", + "yapf==0.22.0", +] + +[project.optional-dependencies] +core = [ + # Core bot & engine runtime dependencies (vendored to src/third_party) + "aiohttp==3.10.5", + "antlr4-python3-runtime==4.7.2", + "bcrypt==4.1.2", + "certifi==2024.2.2", + "cffi==1.15.1", + "crcmod==1.7", + "cryptography==37.0.4", + "future==0.17.1", + "google-api-core==2.11.1", + "google-api-python-client==2.93.0", + "google-auth==2.22.0", + "google-auth-oauthlib==0.4.1", + "google-cloud-batch==0.11.0", + "google-cloud-core==2.3.3", + "google-cloud-datastore==2.16.1", + "google-cloud-logging==3.6.0", + "google-cloud-monitoring==2.15.1", + "google-cloud-ndb==2.2.1", + "google-cloud-profiler==4.1.0", + "google-cloud-secret-manager==2.17.0", + "google-cloud-storage==2.10.0", + "google-crc32c==1.5.0", + "grpcio==1.62.2", + "httplib2==0.19.0", + "jira==2.0.0", + "Jinja2==3.1.4", + "kubernetes==34.1.0", + "mozprocess==1.3.1", + "oauth2client==4.1.3", + "protobuf==4.23.4", + "psutil==5.9.4", + "pygithub==1.55", + "pyOpenSSL==22.0.0", + "python-dateutil==2.8.1", + "PyYAML==6.0.1", + "pytz==2023.3", + "redis==4.6.0", + "requests==2.21.0", + "sendgrid==6.0.4", + "wrapt==1.16.0", +] + +appengine = [ + # App Engine specific dependencies (vendored to src/appengine/third_party) + "charset-normalizer==3.3.2", + "firebase-admin==6.2.0", + "Flask==2.2.2", + "itsdangerous==2.0.1", + "parameterized==0.9.0", + "PyJWT==2.7.0", + "requests-toolbelt==0.9.1", + "werkzeug==2.2.2", +] + +# Note: platform-specific dependencies are binary wheels downloaded via `pip download` +# for target platform/ABI (see common.py `_install_platform_pip`) rather than vendored +# via `uv pip install --target`. +platform-specific = [ + # Platform-specific packages (replacing src/platform_requirements.txt) + "grpcio==1.62.2", + "kubernetes==34.1.0", + "protobuf==4.23.4", + "psutil==5.9.4", +] + +[tool.ty.rules] +deprecated = "ignore" +unused-type-ignore-comment = "ignore" + +[tool.ty.src] +include = [ + "src/clusterfuzz", + "src/appengine", +] +exclude = [ + "src/appengine/__init__.py", + "src/appengine/clusterfuzz", + "src/appengine/handlers/__init__.py", + "src/appengine/handlers/cron/__init__.py", + "src/appengine/templates", + "src/appengine/third_party", + "src/clusterfuzz/_internal/bot/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/afl/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/centipede/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/googlefuzztest/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/honggfuzz/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/__init__.py", + "src/clusterfuzz/_internal/bot/minimizer/__init__.py", + "src/clusterfuzz/_internal/bot/tasks/__init__.py", + "src/clusterfuzz/_internal/bot/tokenizer/__init__.py", + "src/clusterfuzz/_internal/bot/tokenizer/grammars/HTMLLexer.py", + "src/clusterfuzz/_internal/bot/tokenizer/grammars/JavaScriptBaseLexer.py", + "src/clusterfuzz/_internal/bot/tokenizer/grammars/JavaScriptLexer.py", + "src/clusterfuzz/_internal/bot/tokenizer/grammars/__init__.py", + "src/clusterfuzz/_internal/bot/untrusted_runner/__init__.py", + "src/clusterfuzz/_internal/bot/webserver/__init__.py", + "src/clusterfuzz/_internal/config/__init__.py", + "src/clusterfuzz/_internal/datastore/__init__.py", + "src/clusterfuzz/_internal/fuzzer_utils/__init__.py", + "src/clusterfuzz/_internal/google_cloud_utils/__init__.py", + "src/clusterfuzz/_internal/metrics/__init__.py", + "src/clusterfuzz/_internal/protos", + "src/clusterfuzz/_internal/system/__init__.py", + "src/clusterfuzz/_internal/tests", + "src/clusterfuzz/environment/__init__.py", + "src/clusterfuzz/fuzz/__init__.py", + "src/third_party", +] + +[tool.ty.environment] +python-version = "3.11" +extra-paths = [ + "src", + "src/appengine", + "src/third_party", + "src/appengine/third_party", + "src/appengine/libs", +] diff --git a/pyrightconfig.json b/pyrightconfig.json index 4b3fd677127..6ed8a9ecc9a 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,14 +1,62 @@ { + "stubPath": "typings", "include": [ - "src/clusterfuzz" + "src/clusterfuzz", + "src/appengine" ], "ignore": [ + "src/appengine/__init__.py", + "src/appengine/clusterfuzz", + "src/appengine/handlers/__init__.py", + "src/appengine/handlers/cron/__init__.py", + "src/appengine/templates", + "src/appengine/third_party", + "src/clusterfuzz/_internal/bot/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/afl/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/centipede/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/googlefuzztest/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/honggfuzz/__init__.py", + "src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/__init__.py", + "src/clusterfuzz/_internal/bot/minimizer/__init__.py", + "src/clusterfuzz/_internal/bot/tasks/__init__.py", + "src/clusterfuzz/_internal/bot/tokenizer/__init__.py", + "src/clusterfuzz/_internal/bot/tokenizer/grammars/HTMLLexer.py", + "src/clusterfuzz/_internal/bot/tokenizer/grammars/JavaScriptBaseLexer.py", + "src/clusterfuzz/_internal/bot/tokenizer/grammars/JavaScriptLexer.py", + "src/clusterfuzz/_internal/bot/tokenizer/grammars/__init__.py", + "src/clusterfuzz/_internal/bot/untrusted_runner/__init__.py", + "src/clusterfuzz/_internal/bot/webserver/__init__.py", + "src/clusterfuzz/_internal/config/__init__.py", + "src/clusterfuzz/_internal/datastore/__init__.py", + "src/clusterfuzz/_internal/fuzzer_utils/__init__.py", + "src/clusterfuzz/_internal/google_cloud_utils/__init__.py", + "src/clusterfuzz/_internal/metrics/__init__.py", + "src/clusterfuzz/_internal/protos", + "src/clusterfuzz/_internal/system/__init__.py", "src/clusterfuzz/_internal/tests", + "src/clusterfuzz/environment/__init__.py", + "src/clusterfuzz/fuzz/__init__.py", "src/third_party" ], - "executionEnvironments": [{ - "root": "src", - "extraPaths": [ "src/third_party" ], - "pythonVersion": "3.11" - }] + "executionEnvironments": [ + { + "root": "src/appengine", + "extraPaths": [ + "src/appengine/third_party", + "src/appengine/libs", + "src", + "src/third_party" + ], + "pythonVersion": "3.11" + }, + { + "root": "src", + "extraPaths": [ + "src/appengine", + "src/third_party" + ], + "pythonVersion": "3.11" + } + ] } diff --git a/src/Pipfile b/src/Pipfile deleted file mode 100644 index 2a3efcb090f..00000000000 --- a/src/Pipfile +++ /dev/null @@ -1,57 +0,0 @@ -[[source]] -url = "https://pypi.python.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -antlr4-python3-runtime = "==4.7.2" -bcrypt = "==4.1.2" -certifi = "==2024.2.2" -cffi = "==1.15.1" -cryptography = "==37.0.4" -future = "==0.17.1" -google-api-core = "==2.11.1" -google-api-python-client = "==2.93.0" -google-auth = "==2.22.0" -google-auth-oauthlib = "==0.4.1" -google-cloud-batch = "==0.11.0" -google-cloud-core = "==2.3.3" -google-cloud-datastore = "==2.16.1" -google-cloud-logging = "==3.6.0" -google-cloud-monitoring = "==2.15.1" -google-cloud-ndb = "==2.2.1" -google-cloud-profiler = "==4.1.0" -google-cloud-secret-manager = "==2.17.0" -google-cloud-storage = "==2.10.0" -google-crc32c = "==1.5.0" -grpcio = "==1.62.2" -httplib2 = "==0.19.0" -jira = "==2.0.0" -Jinja2 = "==3.1.4" -kubernetes = "==34.1.0" -mozprocess = "==1.3.1" -oauth2client = "==4.1.3" -psutil = "==5.9.4" -protobuf = "==4.23.4" -pygithub = "==1.55" -pyOpenSSL = "==22.0.0" -python-dateutil = "==2.8.1" -PyYAML = "==6.0.1" -pytz = "==2023.3" -redis = "==4.6.0" -requests = "==2.21.0" -sendgrid = "==6.0.4" -wrapt = "==1.16.0" -# Hack: We are using this to specify App Engine packages, -# since there is no support for custom package sets. -aiohttp = "==3.10.5" - -[dev-packages] -charset-normalizer = "==3.3.2" -firebase-admin = "==6.2.0" -Flask = "==2.2.2" -itsdangerous = "==2.0.1" -PyJWT = "==2.7.0" -requests-toolbelt = "==0.9.1" -werkzeug = "==2.2.2" -parameterized = "==0.9.0" diff --git a/src/Pipfile.lock b/src/Pipfile.lock deleted file mode 100644 index e0891e25016..00000000000 --- a/src/Pipfile.lock +++ /dev/null @@ -1,2602 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "3c4997a996de56b08da9534d8179e1e1c23b386e88a8b4c71443f2588f39c7cd" - }, - "pipfile-spec": 6, - "requires": {}, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.python.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "aiohappyeyeballs": { - "hashes": [ - "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", - "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8" - ], - "markers": "python_version >= '3.9'", - "version": "==2.6.1" - }, - "aiohttp": { - "hashes": [ - "sha256:02594361128f780eecc2a29939d9dfc870e17b45178a867bf61a11b2a4367277", - "sha256:03f2645adbe17f274444953bdea69f8327e9d278d961d85657cb0d06864814c1", - "sha256:074d1bff0163e107e97bd48cad9f928fa5a3eb4b9d33366137ffce08a63e37fe", - "sha256:0912b8a8fadeb32ff67a3ed44249448c20148397c1ed905d5dac185b4ca547bb", - "sha256:0d277cfb304118079e7044aad0b76685d30ecb86f83a0711fc5fb257ffe832ca", - "sha256:0d93400c18596b7dc4794d48a63fb361b01a0d8eb39f28800dc900c8fbdaca91", - "sha256:123dd5b16b75b2962d0fff566effb7a065e33cd4538c1692fb31c3bda2bfb972", - "sha256:17e997105bd1a260850272bfb50e2a328e029c941c2708170d9d978d5a30ad9a", - "sha256:18a01eba2574fb9edd5f6e5fb25f66e6ce061da5dab5db75e13fe1558142e0a3", - "sha256:1923a5c44061bffd5eebeef58cecf68096e35003907d8201a4d0d6f6e387ccaa", - "sha256:1942244f00baaacaa8155eca94dbd9e8cc7017deb69b75ef67c78e89fdad3c77", - "sha256:1b2c16a919d936ca87a3c5f0e43af12a89a3ce7ccbce59a2d6784caba945b68b", - "sha256:1c19de68896747a2aa6257ae4cf6ef59d73917a36a35ee9d0a6f48cff0f94db8", - "sha256:1e72589da4c90337837fdfe2026ae1952c0f4a6e793adbbfbdd40efed7c63599", - "sha256:22c0a23a3b3138a6bf76fc553789cb1a703836da86b0f306b6f0dc1617398abc", - "sha256:2c634a3207a5445be65536d38c13791904fda0748b9eabf908d3fe86a52941cf", - "sha256:2d21ac12dc943c68135ff858c3a989f2194a709e6e10b4c8977d7fcd67dfd511", - "sha256:2f1f1c75c395991ce9c94d3e4aa96e5c59c8356a15b1c9231e783865e2772699", - "sha256:305be5ff2081fa1d283a76113b8df7a14c10d75602a38d9f012935df20731487", - "sha256:33e6bc4bab477c772a541f76cd91e11ccb6d2efa2b8d7d7883591dfb523e5987", - "sha256:349ef8a73a7c5665cca65c88ab24abe75447e28aa3bc4c93ea5093474dfdf0ff", - "sha256:380f926b51b92d02a34119d072f178d80bbda334d1a7e10fa22d467a66e494db", - "sha256:38172a70005252b6893088c0f5e8a47d173df7cc2b2bd88650957eb84fcf5022", - "sha256:391cc3a9c1527e424c6865e087897e766a917f15dddb360174a70467572ac6ce", - "sha256:3a1c32a19ee6bbde02f1cb189e13a71b321256cc1d431196a9f824050b160d5a", - "sha256:4120d7fefa1e2d8fb6f650b11489710091788de554e2b6f8347c7a20ceb003f5", - "sha256:424ae21498790e12eb759040bbb504e5e280cab64693d14775c54269fd1d2bb7", - "sha256:44b324a6b8376a23e6ba25d368726ee3bc281e6ab306db80b5819999c737d820", - "sha256:4790f0e15f00058f7599dab2b206d3049d7ac464dc2e5eae0e93fa18aee9e7bf", - "sha256:4aff049b5e629ef9b3e9e617fa6e2dfeda1bf87e01bcfecaf3949af9e210105e", - "sha256:4b38b1570242fbab8d86a84128fb5b5234a2f70c2e32f3070143a6d94bc854cf", - "sha256:4d46c7b4173415d8e583045fbc4daa48b40e31b19ce595b8d92cf639396c15d5", - "sha256:4f1c9866ccf48a6df2b06823e6ae80573529f2af3a0992ec4fe75b1a510df8a6", - "sha256:4f7acae3cf1a2a2361ec4c8e787eaaa86a94171d2417aae53c0cca6ca3118ff6", - "sha256:54d9ddea424cd19d3ff6128601a4a4d23d54a421f9b4c0fff740505813739a91", - "sha256:58718e181c56a3c02d25b09d4115eb02aafe1a732ce5714ab70326d9776457c3", - "sha256:5ede29d91a40ba22ac1b922ef510aab871652f6c88ef60b9dcdf773c6d32ad7a", - "sha256:61645818edd40cc6f455b851277a21bf420ce347baa0b86eaa41d51ef58ba23d", - "sha256:66bf9234e08fe561dccd62083bf67400bdbf1c67ba9efdc3dac03650e97c6088", - "sha256:673f988370f5954df96cc31fd99c7312a3af0a97f09e407399f61583f30da9bc", - "sha256:676f94c5480d8eefd97c0c7e3953315e4d8c2b71f3b49539beb2aa676c58272f", - "sha256:6c225286f2b13bab5987425558baa5cbdb2bc925b2998038fa028245ef421e75", - "sha256:7384d0b87d4635ec38db9263e6a3f1eb609e2e06087f0aa7f63b76833737b471", - "sha256:7e2fe37ac654032db1f3499fe56e77190282534810e2a8e833141a021faaab0e", - "sha256:7f2bfc0032a00405d4af2ba27f3c429e851d04fad1e5ceee4080a1c570476697", - "sha256:7f6b639c36734eaa80a6c152a238242bedcee9b953f23bb887e9102976343092", - "sha256:814375093edae5f1cb31e3407997cf3eacefb9010f96df10d64829362ae2df69", - "sha256:8224f98be68a84b19f48e0bdc14224b5a71339aff3a27df69989fa47d01296f3", - "sha256:898715cf566ec2869d5cb4d5fb4be408964704c46c96b4be267442d265390f32", - "sha256:8989f46f3d7ef79585e98fa991e6ded55d2f48ae56d2c9fa5e491a6e4effb589", - "sha256:8ba01ebc6175e1e6b7275c907a3a36be48a2d487549b656aa90c8a910d9f3178", - "sha256:8c5c6fa16412b35999320f5c9690c0f554392dc222c04e559217e0f9ae244b92", - "sha256:8c6a4e5e40156d72a40241a25cc226051c0a8d816610097a8e8f517aeacd59a2", - "sha256:8eaf44ccbc4e35762683078b72bf293f476561d8b68ec8a64f98cf32811c323e", - "sha256:8fb4fc029e135859f533025bc82047334e24b0d489e75513144f25408ecaf058", - "sha256:9093a81e18c45227eebe4c16124ebf3e0d893830c6aca7cc310bfca8fe59d857", - "sha256:94c4381ffba9cc508b37d2e536b418d5ea9cfdc2848b9a7fea6aebad4ec6aac1", - "sha256:94fac7c6e77ccb1ca91e9eb4cb0ac0270b9fb9b289738654120ba8cebb1189c6", - "sha256:95c4dc6f61d610bc0ee1edc6f29d993f10febfe5b76bb470b486d90bbece6b22", - "sha256:975218eee0e6d24eb336d0328c768ebc5d617609affaca5dbbd6dd1984f16ed0", - "sha256:ad146dae5977c4dd435eb31373b3fe9b0b1bf26858c6fc452bf6af394067e10b", - "sha256:afe16a84498441d05e9189a15900640a2d2b5e76cf4efe8cbb088ab4f112ee57", - "sha256:b1c43eb1ab7cbf411b8e387dc169acb31f0ca0d8c09ba63f9eac67829585b44f", - "sha256:b90078989ef3fc45cf9221d3859acd1108af7560c52397ff4ace8ad7052a132e", - "sha256:b98e698dc34966e5976e10bbca6d26d6724e6bdea853c7c10162a3235aba6e16", - "sha256:ba5a8b74c2a8af7d862399cdedce1533642fa727def0b8c3e3e02fcb52dca1b1", - "sha256:c31ad0c0c507894e3eaa843415841995bf8de4d6b2d24c6e33099f4bc9fc0d4f", - "sha256:c3b9162bab7e42f21243effc822652dc5bb5e8ff42a4eb62fe7782bcbcdfacf6", - "sha256:c58c6837a2c2a7cf3133983e64173aec11f9c2cd8e87ec2fdc16ce727bcf1a04", - "sha256:c83f7a107abb89a227d6c454c613e7606c12a42b9a4ca9c5d7dad25d47c776ae", - "sha256:cde98f323d6bf161041e7627a5fd763f9fd829bcfcd089804a5fdce7bb6e1b7d", - "sha256:ce91db90dbf37bb6fa0997f26574107e1b9d5ff939315247b7e615baa8ec313b", - "sha256:d00f3c5e0d764a5c9aa5a62d99728c56d455310bcc288a79cab10157b3af426f", - "sha256:d17920f18e6ee090bdd3d0bfffd769d9f2cb4c8ffde3eb203777a3895c128862", - "sha256:d55f011da0a843c3d3df2c2cf4e537b8070a419f891c930245f05d329c4b0689", - "sha256:d742c36ed44f2798c8d3f4bc511f479b9ceef2b93f348671184139e7d708042c", - "sha256:d9a487ef090aea982d748b1b0d74fe7c3950b109df967630a20584f9a99c0683", - "sha256:d9ef084e3dc690ad50137cc05831c52b6ca428096e6deb3c43e95827f531d5ef", - "sha256:da452c2c322e9ce0cfef392e469a26d63d42860f829026a63374fde6b5c5876f", - "sha256:dc4826823121783dccc0871e3f405417ac116055bf184ac04c36f98b75aacd12", - "sha256:de7a5299827253023c55ea549444e058c0eb496931fa05d693b95140a947cb73", - "sha256:e04a1f2a65ad2f93aa20f9ff9f1b672bf912413e5547f60749fa2ef8a644e061", - "sha256:e1ca1ef5ba129718a8fc827b0867f6aa4e893c56eb00003b7367f8a733a9b072", - "sha256:ee40b40aa753d844162dcc80d0fe256b87cba48ca0054f64e68000453caead11", - "sha256:f071854b47d39591ce9a17981c46790acb30518e2f83dfca8db2dfa091178691", - "sha256:f29930bc2921cef955ba39a3ff87d2c4398a0394ae217f41cb02d5c26c8b1b77", - "sha256:f489a2c9e6455d87eabf907ac0b7d230a9786be43fbe884ad184ddf9e9c1e385", - "sha256:f5bf3ead3cb66ab990ee2561373b009db5bc0e857549b6c9ba84b20bc462e172", - "sha256:f6f18898ace4bcd2d41a122916475344a87f1dfdec626ecde9ee802a711bc569", - "sha256:f8112fb501b1e0567a1251a2fd0747baae60a4ab325a871e975b7bb67e59221f", - "sha256:fd31f176429cecbc1ba499d4aba31aaccfea488f418d60376b911269d3b883c5" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==3.10.5" - }, - "aiosignal": { - "hashes": [ - "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", - "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7" - ], - "markers": "python_version >= '3.9'", - "version": "==1.4.0" - }, - "antlr4-python3-runtime": { - "hashes": [ - "sha256:168cdcec8fb9152e84a87ca6fd261b3d54c8f6358f42ab3b813b14a7193bb50b" - ], - "index": "pypi", - "version": "==4.7.2" - }, - "attrs": { - "hashes": [ - "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11", - "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373" - ], - "markers": "python_version >= '3.9'", - "version": "==25.4.0" - }, - "bcrypt": { - "hashes": [ - "sha256:02d9ef8915f72dd6daaef40e0baeef8a017ce624369f09754baf32bb32dba25f", - "sha256:1c28973decf4e0e69cee78c68e30a523be441972c826703bb93099868a8ff5b5", - "sha256:2a298db2a8ab20056120b45e86c00a0a5eb50ec4075b6142db35f593b97cb3fb", - "sha256:33313a1200a3ae90b75587ceac502b048b840fc69e7f7a0905b5f87fac7a1258", - "sha256:3566a88234e8de2ccae31968127b0ecccbb4cddb629da744165db72b58d88ca4", - "sha256:387e7e1af9a4dd636b9505a465032f2f5cb8e61ba1120e79a0e1cd0b512f3dfc", - "sha256:44290ccc827d3a24604f2c8bcd00d0da349e336e6503656cb8192133e27335e2", - "sha256:57fa9442758da926ed33a91644649d3e340a71e2d0a5a8de064fb621fd5a3326", - "sha256:68e3c6642077b0c8092580c819c1684161262b2e30c4f45deb000c38947bf483", - "sha256:69057b9fc5093ea1ab00dd24ede891f3e5e65bee040395fb1e66ee196f9c9b4a", - "sha256:6cad43d8c63f34b26aef462b6f5e44fdcf9860b723d2453b5d391258c4c8e966", - "sha256:71b8be82bc46cedd61a9f4ccb6c1a493211d031415a34adde3669ee1b0afbb63", - "sha256:732b3920a08eacf12f93e6b04ea276c489f1c8fb49344f564cca2adb663b3e4c", - "sha256:9800ae5bd5077b13725e2e3934aa3c9c37e49d3ea3d06318010aa40f54c63551", - "sha256:a97e07e83e3262599434816f631cc4c7ca2aa8e9c072c1b1a7fec2ae809a1d2d", - "sha256:ac621c093edb28200728a9cca214d7e838529e557027ef0581685909acd28b5e", - "sha256:b8df79979c5bae07f1db22dcc49cc5bccf08a0380ca5c6f391cbb5790355c0b0", - "sha256:b90e216dc36864ae7132cb151ffe95155a37a14e0de3a8f64b49655dd959ff9c", - "sha256:ba4e4cc26610581a6329b3937e02d319f5ad4b85b074846bf4fef8a8cf51e7bb", - "sha256:ba55e40de38a24e2d78d34c2d36d6e864f93e0d79d0b6ce915e4335aa81d01b1", - "sha256:be3ab1071662f6065899fe08428e45c16aa36e28bc42921c4901a191fda6ee42", - "sha256:d75fc8cd0ba23f97bae88a6ec04e9e5351ff3c6ad06f38fe32ba50cbd0d11946", - "sha256:e51c42750b7585cee7892c2614be0d14107fad9581d1738d954a262556dd1aab", - "sha256:ea505c97a5c465ab8c3ba75c0805a102ce526695cd6818c6de3b1a38f6f60da1", - "sha256:eb3bd3321517916696233b5e0c67fd7d6281f0ef48e66812db35fc963a422a1c", - "sha256:f70d9c61f9c4ca7d57f3bfe88a5ccf62546ffbadf3681bb1e268d9d2e41c91a7", - "sha256:fbe188b878313d01b7718390f31528be4010fed1faa798c5a1d0469c9c48c369" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==4.1.2" - }, - "cachetools": { - "hashes": [ - "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", - "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a" - ], - "markers": "python_version >= '3.7'", - "version": "==5.5.2" - }, - "certifi": { - "hashes": [ - "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", - "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==2024.2.2" - }, - "cffi": { - "hashes": [ - "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", - "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", - "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", - "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", - "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", - "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", - "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", - "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", - "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", - "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", - "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", - "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", - "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", - "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", - "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", - "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", - "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", - "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", - "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", - "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", - "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", - "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", - "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", - "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", - "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", - "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", - "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", - "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", - "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", - "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", - "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", - "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", - "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", - "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", - "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", - "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", - "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", - "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", - "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", - "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", - "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", - "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", - "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", - "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", - "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", - "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", - "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", - "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", - "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", - "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", - "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", - "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", - "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", - "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", - "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", - "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", - "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", - "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", - "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", - "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", - "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", - "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", - "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", - "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" - ], - "index": "pypi", - "version": "==1.15.1" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "cryptography": { - "hashes": [ - "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", - "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", - "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", - "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", - "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", - "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", - "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", - "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", - "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", - "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", - "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", - "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", - "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", - "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", - "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", - "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", - "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", - "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", - "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", - "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", - "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", - "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==37.0.4" - }, - "defusedxml": { - "hashes": [ - "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", - "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==0.7.1" - }, - "deprecated": { - "hashes": [ - "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", - "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.3.1" - }, - "distro": { - "hashes": [ - "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", - "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2" - ], - "markers": "python_version >= '3.6'", - "version": "==1.9.0" - }, - "durationpy": { - "hashes": [ - "sha256:1fa6893409a6e739c9c72334fc65cca1f355dbdd93405d30f726deb5bde42fba", - "sha256:3b41e1b601234296b4fb368338fdcd3e13e0b4fb5b67345948f4f2bf9868b286" - ], - "version": "==0.10" - }, - "frozenlist": { - "hashes": [ - "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", - "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", - "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", - "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", - "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", - "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", - "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84", - "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", - "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", - "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", - "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967", - "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", - "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4", - "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", - "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", - "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9", - "sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3", - "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd", - "sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087", - "sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068", - "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7", - "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", - "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", - "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f", - "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25", - "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", - "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", - "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", - "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", - "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", - "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", - "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", - "sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675", - "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", - "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", - "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", - "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", - "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", - "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", - "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", - "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", - "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", - "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c", - "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", - "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", - "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", - "sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5", - "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", - "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", - "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", - "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", - "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", - "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", - "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", - "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", - "sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95", - "sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1", - "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", - "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", - "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6", - "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", - "sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459", - "sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a", - "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", - "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", - "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", - "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", - "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186", - "sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6", - "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", - "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e", - "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52", - "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", - "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450", - "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", - "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", - "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", - "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", - "sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178", - "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695", - "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", - "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", - "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", - "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", - "sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61", - "sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca", - "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad", - "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b", - "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a", - "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", - "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", - "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011", - "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", - "sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103", - "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b", - "sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda", - "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", - "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", - "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e", - "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", - "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef", - "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d", - "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567", - "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", - "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", - "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", - "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", - "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", - "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", - "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a", - "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", - "sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47", - "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", - "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", - "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f", - "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff", - "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", - "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", - "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", - "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581", - "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", - "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565", - "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", - "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92", - "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2", - "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", - "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", - "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93", - "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", - "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd" - ], - "markers": "python_version >= '3.9'", - "version": "==1.8.0" - }, - "future": { - "hashes": [ - "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" - ], - "index": "pypi", - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.17.1" - }, - "google-api-core": { - "extras": [ - "grpc" - ], - "hashes": [ - "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a", - "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.11.1" - }, - "google-api-python-client": { - "hashes": [ - "sha256:62ee28e96031a10a1c341f226a75ac6a4f16bdb1d888dc8222b2cdca133d0031", - "sha256:f34abb671afd488bd19d30721ea20fb30d3796ddd825d6f91f26d8c718a9f07d" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.93.0" - }, - "google-auth": { - "hashes": [ - "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce", - "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==2.22.0" - }, - "google-auth-httplib2": { - "hashes": [ - "sha256:177898a0175252480d5ed916aeea183c2df87c1f9c26705d74ae6b951c268b0b", - "sha256:426167e5df066e3f5a0fc7ea18768c08e7296046594ce4c8c409c2457dd1f776" - ], - "markers": "python_version >= '3.7'", - "version": "==0.3.0" - }, - "google-auth-oauthlib": { - "hashes": [ - "sha256:88d2cd115e3391eb85e1243ac6902e76e77c5fe438b7276b297fbe68015458dd", - "sha256:a92a0f6f41a0fb6138454fbc02674e64f89d82a244ea32f98471733c8ef0e0e1" - ], - "index": "pypi", - "version": "==0.4.1" - }, - "google-cloud-appengine-logging": { - "hashes": [ - "sha256:84b705a69e4109fc2f68dfe36ce3df6a34d5c3d989eee6d0ac1b024dda0ba6f5", - "sha256:a4ce9ce94a9fd8c89ed07fa0b06fcf9ea3642f9532a1be1a8c7b5f82c0a70ec6" - ], - "markers": "python_version >= '3.7'", - "version": "==1.8.0" - }, - "google-cloud-audit-log": { - "hashes": [ - "sha256:6b88e2349df45f8f4cc0993b687109b1388da1571c502dc1417efa4b66ec55e0", - "sha256:8467d4dcca9f3e6160520c24d71592e49e874838f174762272ec10e7950b6feb" - ], - "markers": "python_version >= '3.7'", - "version": "==0.4.0" - }, - "google-cloud-batch": { - "hashes": [ - "sha256:d2daf19987e72369e9d33f1b634f0963ec90eb08f9d3c5d0a1ab940c0bd74d51", - "sha256:f102a7230dd201236e1e2804d4ca025995bd2141887df8a95a64cc419b33b476" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==0.11.0" - }, - "google-cloud-core": { - "hashes": [ - "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb", - "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.3.3" - }, - "google-cloud-datastore": { - "hashes": [ - "sha256:153d8dac05f0248d7f7d6f0986a31f8fb22d3cdfb764718eac0f28e3cadc8144", - "sha256:710ef27ebdfb50340f4671c532ad4c1525665985e9421984ffcd6b96b57e34bb" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.16.1" - }, - "google-cloud-logging": { - "hashes": [ - "sha256:42134223956850ddd64877c88042d31f78658e7b067a5a8e3dd28236b71f3c32", - "sha256:6beb843cb88a9bc2f6df94ce69048243ebb360ee06979f5311ea7a0ec09bd097" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==3.6.0" - }, - "google-cloud-monitoring": { - "hashes": [ - "sha256:086a845a34a6e105ab8a3a4809392ea6367a65b28b7561c26101534a8705ed37", - "sha256:4394e5e031f30d622a24739678ec48a45400ead94af9a6032cbff7f66194dc12" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.15.1" - }, - "google-cloud-ndb": { - "hashes": [ - "sha256:39f5a81afa426b3acb81bcc696c84253bba81908cef40dd9fb730b6708d0ffaa", - "sha256:e849b48c984f31616e99e1fcc797d1deb1d883a6f38f58f77768b814fb84b57e" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.2.1" - }, - "google-cloud-profiler": { - "hashes": [ - "sha256:2d90f9c6d4c075ad6d43752ae39424c3d0bad63e6549a2c761881f9a235067ae" - ], - "index": "pypi", - "version": "==4.1.0" - }, - "google-cloud-secret-manager": { - "hashes": [ - "sha256:2bf58be569710773f0757d7a788e2860767aa1c148a5255e53b228da3fbcf446", - "sha256:8254e2960c06a8dc91aeac3c894afba0893a674582f0e0ecfc22f894e6173c2f" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.17.0" - }, - "google-cloud-storage": { - "hashes": [ - "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7", - "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.10.0" - }, - "google-crc32c": { - "hashes": [ - "sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a", - "sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876", - "sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c", - "sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289", - "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298", - "sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02", - "sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f", - "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2", - "sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a", - "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb", - "sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210", - "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5", - "sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee", - "sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c", - "sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a", - "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314", - "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd", - "sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65", - "sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37", - "sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4", - "sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13", - "sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894", - "sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31", - "sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e", - "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709", - "sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740", - "sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc", - "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d", - "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c", - "sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c", - "sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d", - "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906", - "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61", - "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57", - "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c", - "sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a", - "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438", - "sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946", - "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7", - "sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96", - "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091", - "sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae", - "sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d", - "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88", - "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2", - "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd", - "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541", - "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728", - "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178", - "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968", - "sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346", - "sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8", - "sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93", - "sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7", - "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273", - "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462", - "sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94", - "sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd", - "sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e", - "sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57", - "sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b", - "sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9", - "sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a", - "sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100", - "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325", - "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183", - "sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556", - "sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.5.0" - }, - "google-resumable-media": { - "hashes": [ - "sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582", - "sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae" - ], - "markers": "python_version >= '3.7'", - "version": "==2.8.0" - }, - "googleapis-common-protos": { - "extras": [ - "grpc" - ], - "hashes": [ - "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038", - "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5" - ], - "markers": "python_version >= '3.7'", - "version": "==1.72.0" - }, - "grpc-google-iam-v1": { - "hashes": [ - "sha256:7a7f697e017a067206a3dfef44e4c634a34d3dee135fe7d7a4613fe3e59217e6", - "sha256:879ac4ef33136c5491a6300e27575a9ec760f6cdf9a2518798c1b8977a5dc389" - ], - "markers": "python_version >= '3.7'", - "version": "==0.14.3" - }, - "grpcio": { - "hashes": [ - "sha256:07ce1f775d37ca18c7a141300e5b71539690efa1f51fe17f812ca85b5e73262f", - "sha256:112eaa7865dd9e6d7c0556c8b04ae3c3a2dc35d62ad3373ab7f6a562d8199200", - "sha256:162ccf61499c893831b8437120600290a99c0bc1ce7b51f2c8d21ec87ff6af8b", - "sha256:16da954692fd61aa4941fbeda405a756cd96b97b5d95ca58a92547bba2c1624f", - "sha256:17708db5b11b966373e21519c4c73e5a750555f02fde82276ea2a267077c68ad", - "sha256:1bcfe5070e4406f489e39325b76caeadab28c32bf9252d3ae960c79935a4cc36", - "sha256:1c1bb80299bdef33309dff03932264636450c8fdb142ea39f47e06a7153d3063", - "sha256:2507006c8a478f19e99b6fe36a2464696b89d40d88f34e4b709abe57e1337467", - "sha256:262cda97efdabb20853d3b5a4c546a535347c14b64c017f628ca0cc7fa780cc6", - "sha256:26f415f40f4a93579fd648f48dca1c13dfacdfd0290f4a30f9b9aeb745026811", - "sha256:2a0204532aa2f1afd467024b02b4069246320405bc18abec7babab03e2644e75", - "sha256:2e72ddfee62430ea80133d2cbe788e0d06b12f865765cb24a40009668bd8ea05", - "sha256:3abe6838196da518863b5d549938ce3159d809218936851b395b09cad9b5d64a", - "sha256:3ad00f3f0718894749d5a8bb0fa125a7980a2f49523731a9b1fabf2b3522aa43", - "sha256:3c3ed41f4d7a3aabf0f01ecc70d6b5d00ce1800d4af652a549de3f7cf35c4abd", - "sha256:404d3b4b6b142b99ba1cff0b2177d26b623101ea2ce51c25ef6e53d9d0d87bcc", - "sha256:41955b641c34db7d84db8d306937b72bc4968eef1c401bea73081a8d6c3d8033", - "sha256:53d3a59a10af4c2558a8e563aed9f256259d2992ae0d3037817b2155f0341de1", - "sha256:55ddaf53474e8caeb29eb03e3202f9d827ad3110475a21245f3c7712022882a9", - "sha256:589ea8e75de5fd6df387de53af6c9189c5231e212b9aa306b6b0d4f07520fbb9", - "sha256:5dab7ac2c1e7cb6179c6bfad6b63174851102cbe0682294e6b1d6f0981ad7138", - "sha256:65034473fc09628a02fb85f26e73885cf1ed39ebd9cf270247b38689ff5942c5", - "sha256:66344ea741124c38588a664237ac2fa16dfd226964cca23ddc96bd4accccbde5", - "sha256:6e784f60e575a0de554ef9251cbc2ceb8790914fe324f11e28450047f264ee6f", - "sha256:80407bc007754f108dc2061e37480238b0dc1952c855e86a4fc283501ee6bb5d", - "sha256:82af3613a219512a28ee5c95578eb38d44dd03bca02fd918aa05603c41018051", - "sha256:88b4f9ee77191dcdd8810241e89340a12cbe050be3e0d5f2f091c15571cd3930", - "sha256:99701979bcaaa7de8d5f60476487c5df8f27483624f1f7e300ff4669ee44d1f2", - "sha256:a1511a303f8074f67af4119275b4f954189e8313541da7b88b1b3a71425cdb10", - "sha256:a5eb4844e5e60bf2c446ef38c5b40d7752c6effdee882f716eb57ae87255d20a", - "sha256:a75af2fc7cb1fe25785be7bed1ab18cef959a376cdae7c6870184307614caa3f", - "sha256:a90ac47a8ce934e2c8d71e317d2f9e7e6aaceb2d199de940ce2c2eb611b8c0f4", - "sha256:aa787b83a3cd5e482e5c79be030e2b4a122ecc6c5c6c4c42a023a2b581fdf17b", - "sha256:aaae70364a2d1fb238afd6cc9fcb10442b66e397fd559d3f0968d28cc3ac929c", - "sha256:af15e9efa4d776dfcecd1d083f3ccfb04f876d613e90ef8432432efbeeac689d", - "sha256:af7dc3f7a44f10863b1b0ecab4078f0a00f561aae1edbd01fd03ad4dcf61c9e9", - "sha256:b7ec9e2f8ffc8436f6b642a10019fc513722858f295f7efc28de135d336ac189", - "sha256:b94d41b7412ef149743fbc3178e59d95228a7064c5ab4760ae82b562bdffb199", - "sha256:c1624aa686d4b36790ed1c2e2306cc3498778dffaf7b8dd47066cf819028c3ad", - "sha256:c5ffeb269f10cedb4f33142b89a061acda9f672fd1357331dbfd043422c94e9e", - "sha256:c6ad9c39704256ed91a1cffc1379d63f7d0278d6a0bad06b0330f5d30291e3a3", - "sha256:c772f225483905f675cb36a025969eef9712f4698364ecd3a63093760deea1bc", - "sha256:c77618071d96b7a8be2c10701a98537823b9c65ba256c0b9067e0594cdbd954d", - "sha256:c79b518c56dddeec79e5500a53d8a4db90da995dfe1738c3ac57fe46348be049", - "sha256:cfd23ad29bfa13fd4188433b0e250f84ec2c8ba66b14a9877e8bce05b524cf54", - "sha256:d0695ae31a89f1a8fc8256050329a91a9995b549a88619263a594ca31b76d756", - "sha256:d2c1771d0ee3cf72d69bb5e82c6a82f27fbd504c8c782575eddb7839729fbaad", - "sha256:da6a7b6b938c15fa0f0568e482efaae9c3af31963eec2da4ff13a6d8ec2888e4", - "sha256:db068bbc9b1fa16479a82e1ecf172a93874540cb84be69f0b9cb9b7ac3c82670", - "sha256:db707e3685ff16fc1eccad68527d072ac8bdd2e390f6daa97bc394ea7de4acea", - "sha256:e2cc8a308780edbe2c4913d6a49dbdb5befacdf72d489a368566be44cadaef1a", - "sha256:f27246d7da7d7e3bd8612f63785a7b0c39a244cf14b8dd9dd2f2fab939f2d7f1", - "sha256:f4aa94361bb5141a45ca9187464ae81a92a2a135ce2800b2203134f7a1a1d479", - "sha256:fa63245271920786f4cb44dcada4983a3516be8f470924528cf658731864c14b" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.62.2" - }, - "grpcio-status": { - "hashes": [ - "sha256:206ddf0eb36bc99b033f03b2c8e95d319f0044defae9b41ae21408e7e0cda48f", - "sha256:62e1bfcb02025a1cd73732a2d33672d3e9d0df4d21c12c51e0bbcaf09bab742a" - ], - "version": "==1.62.2" - }, - "httplib2": { - "hashes": [ - "sha256:749c32603f9bf16c1277f59531d502e8f1c2ca19901ae653b49c4ed698f0820e", - "sha256:e0d428dad43c72dbce7d163b7753ffc7a39c097e6788ef10f4198db69b92f08e" - ], - "index": "pypi", - "version": "==0.19.0" - }, - "idna": { - "hashes": [ - "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", - "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" - ], - "version": "==2.8" - }, - "jinja2": { - "hashes": [ - "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369", - "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==3.1.4" - }, - "jira": { - "hashes": [ - "sha256:9adeead4d5f5a6aff74c630787f8bd2d4b0e154f3a3036641298064e91b2d25d", - "sha256:e2a94adff98e45b29ded030adc76103eab34fa7d4d57303f211f572bedba0e93" - ], - "index": "pypi", - "version": "==2.0.0" - }, - "kubernetes": { - "hashes": [ - "sha256:8fe8edb0b5d290a2f3ac06596b23f87c658977d46b5f8df9d0f4ea83d0003912", - "sha256:bffba2272534e224e6a7a74d582deb0b545b7c9879d2cd9e4aae9481d1f2cc2a" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==34.1.0" - }, - "markupsafe": { - "hashes": [ - "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", - "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", - "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", - "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", - "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", - "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c", - "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", - "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", - "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", - "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", - "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", - "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26", - "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", - "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", - "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", - "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", - "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", - "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", - "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", - "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", - "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", - "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", - "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", - "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", - "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", - "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758", - "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", - "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8", - "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", - "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", - "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", - "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", - "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", - "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", - "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", - "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", - "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2", - "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", - "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", - "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", - "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", - "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", - "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", - "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", - "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", - "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e", - "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", - "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", - "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", - "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", - "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", - "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", - "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b", - "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", - "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", - "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", - "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d", - "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", - "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", - "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", - "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", - "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", - "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", - "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", - "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", - "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", - "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", - "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", - "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", - "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", - "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", - "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", - "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", - "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7", - "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", - "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", - "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", - "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", - "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", - "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42", - "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", - "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", - "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", - "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", - "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", - "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", - "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc", - "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", - "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50" - ], - "markers": "python_version >= '3.9'", - "version": "==3.0.3" - }, - "mozfile": { - "hashes": [ - "sha256:3b0afcda2fa8b802ef657df80a56f21619008f61fcc14b756124028d7b7adf5c", - "sha256:92ca1a786abbdf5e6a7aada62d3a4e28f441ef069c7623223add45268e53c789" - ], - "version": "==3.0.0" - }, - "mozinfo": { - "hashes": [ - "sha256:5d2b8a5f1b362692f221e33eb3ff47454a580db1a1384614cdc637b31131b438", - "sha256:90e0cfb377fc2cc3fad023d38c1f6d60a9135400ff5684a04abf79ca5cc3c521" - ], - "version": "==1.2.3" - }, - "mozprocess": { - "hashes": [ - "sha256:7dc38ec3c11693e9944ade1558392c04f37fd8df68d3ec7a20372dfe96b2e5bb", - "sha256:ae343fabb72840195278b73ad426fb04d35d66b15a664f5c47221f4e4f0842e5" - ], - "index": "pypi", - "version": "==1.3.1" - }, - "multidict": { - "hashes": [ - "sha256:03ca744319864e92721195fa28c7a3b2bc7b686246b35e4078c1e4d0eb5466d3", - "sha256:040f393368e63fb0f3330e70c26bfd336656bed925e5cbe17c9da839a6ab13ec", - "sha256:05047ada7a2fde2631a0ed706f1fd68b169a681dfe5e4cf0f8e4cb6618bbc2cd", - "sha256:0591b48acf279821a579282444814a2d8d0af624ae0bc600aa4d1b920b6e924b", - "sha256:07f5594ac6d084cbb5de2df218d78baf55ef150b91f0ff8a21cc7a2e3a5a58eb", - "sha256:08325c9e5367aa379a3496aa9a022fe8837ff22e00b94db256d3a1378c76ab32", - "sha256:08d4379f9744d8f78d98c8673c06e202ffa88296f009c71bbafe8a6bf847d01f", - "sha256:0934f3843a1860dd465d38895c17fce1f1cb37295149ab05cd1b9a03afacb2a7", - "sha256:096f52730c3fb8ed419db2d44391932b63891b2c5ed14850a7e215c0ba9ade36", - "sha256:09929cab6fcb68122776d575e03c6cc64ee0b8fca48d17e135474b042ce515cd", - "sha256:0a13fb8e748dfc94749f622de065dd5c1def7e0d2216dba72b1d8069a389c6ff", - "sha256:0db4956f82723cc1c270de9c6e799b4c341d327762ec78ef82bb962f79cc07d8", - "sha256:123e2a72e20537add2f33a79e605f6191fba2afda4cbb876e35c1a7074298a7d", - "sha256:14c9e076eede3b54c636f8ce1c9c252b5f057c62131211f0ceeec273810c9721", - "sha256:171b73bd4ee683d307599b66793ac80981b06f069b62eea1c9e29c9241aa66b0", - "sha256:18706cc31dbf402a7945916dd5cddf160251b6dab8a2c5f3d6d5a55949f676b3", - "sha256:19a1d55338ec1be74ef62440ca9e04a2f001a04d0cc49a4983dc320ff0f3212d", - "sha256:2049be98fb57a31b4ccf870bf377af2504d4ae35646a19037ec271e4c07998aa", - "sha256:2090d3718829d1e484706a2f525e50c892237b2bf9b17a79b059cb98cddc2f10", - "sha256:2397ab4daaf2698eb51a76721e98db21ce4f52339e535725de03ea962b5a3202", - "sha256:23bfeee5316266e5ee2d625df2d2c602b829435fc3a235c2ba2131495706e4a0", - "sha256:27e0b36c2d388dc7b6ced3406671b401e84ad7eb0656b8f3a2f46ed0ce483718", - "sha256:28b37063541b897fd6a318007373930a75ca6d6ac7c940dbe14731ffdd8d498e", - "sha256:295a92a76188917c7f99cda95858c822f9e4aae5824246bba9b6b44004ddd0a6", - "sha256:29fe6740ebccba4175af1b9b87bf553e9c15cd5868ee967e010efcf94e4fd0f1", - "sha256:2a7baa46a22e77f0988e3b23d4ede5513ebec1929e34ee9495be535662c0dfe2", - "sha256:2d2cfeec3f6f45651b3d408c4acec0ebf3daa9bc8a112a084206f5db5d05b754", - "sha256:2f67396ec0310764b9222a1728ced1ab638f61aadc6226f17a71dd9324f9a99c", - "sha256:30d193c6cc6d559db42b6bcec8a5d395d34d60c9877a0b71ecd7c204fcf15390", - "sha256:31bae522710064b5cbeddaf2e9f32b1abab70ac6ac91d42572502299e9953128", - "sha256:329aa225b085b6f004a4955271a7ba9f1087e39dcb7e65f6284a988264a63912", - "sha256:363eb68a0a59bd2303216d2346e6c441ba10d36d1f9969fcb6f1ba700de7bb5c", - "sha256:394fc5c42a333c9ffc3e421a4c85e08580d990e08b99f6bf35b4132114c5dcb3", - "sha256:3996b50c3237c4aec17459217c1e7bbdead9a22a0fcd3c365564fbd16439dde6", - "sha256:39f1719f57adbb767ef592a50ae5ebb794220d1188f9ca93de471336401c34d2", - "sha256:3b29b980d0ddbecb736735ee5bef69bb2ddca56eff603c86f3f29a1128299b4f", - "sha256:3ba3ef510467abb0667421a286dc906e30eb08569365f5cdb131d7aff7c2dd84", - "sha256:3bab1e4aff7adaa34410f93b1f8e57c4b36b9af0426a76003f441ee1d3c7e842", - "sha256:3d7b6ccce016e29df4b7ca819659f516f0bc7a4b3efa3bb2012ba06431b044f9", - "sha256:3da4fb467498df97e986af166b12d01f05d2e04f978a9c1c680ea1988e0bc4b6", - "sha256:3e56d780c238f9e1ae66a22d2adf8d16f485381878250db8d496623cd38b22bd", - "sha256:3e8bfdd0e487acf992407a140d2589fe598238eaeffa3da8448d63a63cd363f8", - "sha256:44b546bd3eb645fd26fb949e43c02a25a2e632e2ca21a35e2e132c8105dc8599", - "sha256:478cc36476687bac1514d651cbbaa94b86b0732fb6855c60c673794c7dd2da62", - "sha256:490dab541a6a642ce1a9d61a4781656b346a55c13038f0b1244653828e3a83ec", - "sha256:4a0df7ff02397bb63e2fd22af2c87dfa39e8c7f12947bc524dbdc528282c7e34", - "sha256:4b73189894398d59131a66ff157837b1fafea9974be486d036bb3d32331fdbf0", - "sha256:4b7a9db5a870f780220e931d0002bbfd88fb53aceb6293251e2c839415c1b20e", - "sha256:4c09703000a9d0fa3c3404b27041e574cc7f4df4c6563873246d0e11812a94b6", - "sha256:4d409aa42a94c0b3fa617708ef5276dfe81012ba6753a0370fcc9d0195d0a1fc", - "sha256:4d72a9a2d885f5c208b0cb91ff2ed43636bb7e345ec839ff64708e04f69a13cc", - "sha256:4ef089f985b8c194d341eb2c24ae6e7408c9a0e2e5658699c92f497437d88c3c", - "sha256:51cb455de290ae462593e5b1cb1118c5c22ea7f0d3620d9940bf695cea5a4bd7", - "sha256:521f33e377ff64b96c4c556b81c55d0cfffb96a11c194fd0c3f1e56f3d8dd5a4", - "sha256:53a42d364f323275126aff81fb67c5ca1b7a04fda0546245730a55c8c5f24bc4", - "sha256:5aa873cbc8e593d361ae65c68f85faadd755c3295ea2c12040ee146802f23b38", - "sha256:654030da3197d927f05a536a66186070e98765aa5142794c9904555d3a9d8fb5", - "sha256:661709cdcd919a2ece2234f9bae7174e5220c80b034585d7d8a755632d3e2111", - "sha256:680878b9f3d45c31e1f730eef731f9b0bc1da456155688c6745ee84eb818e90e", - "sha256:6843b28b0364dc605f21481c90fadb5f60d9123b442eb8a726bb74feef588a84", - "sha256:68af405971779d8b37198726f2b6fe3955db846fee42db7a4286fc542203934c", - "sha256:6b4c3d199f953acd5b446bf7c0de1fe25d94e09e79086f8dc2f48a11a129cdf1", - "sha256:6bdce131e14b04fd34a809b6380dbfd826065c3e2fe8a50dbae659fa0c390546", - "sha256:716133f7d1d946a4e1b91b1756b23c088881e70ff180c24e864c26192ad7534a", - "sha256:749a72584761531d2b9467cfbdfd29487ee21124c304c4b6cb760d8777b27f9c", - "sha256:7516c579652f6a6be0e266aec0acd0db80829ca305c3d771ed898538804c2036", - "sha256:79dcf9e477bc65414ebfea98ffd013cb39552b5ecd62908752e0e413d6d06e38", - "sha256:7a0222514e8e4c514660e182d5156a415c13ef0aabbd71682fc714e327b95e99", - "sha256:7b022717c748dd1992a83e219587aabe45980d88969f01b316e78683e6285f64", - "sha256:7bf77f54997a9166a2f5675d1201520586439424c2511723a7312bdb4bcc034e", - "sha256:7e73299c99939f089dd9b2120a04a516b95cdf8c1cd2b18c53ebf0de80b1f18f", - "sha256:7ef6b61cad77091056ce0e7ce69814ef72afacb150b7ac6a3e9470def2198159", - "sha256:7f5170993a0dd3ab871c74f45c0a21a4e2c37a2f2b01b5f722a2ad9c6650469e", - "sha256:803d685de7be4303b5a657b76e2f6d1240e7e0a8aa2968ad5811fa2285553a12", - "sha256:8891681594162635948a636c9fe0ff21746aeb3dd5463f6e25d9bea3a8a39ca1", - "sha256:8a19cdb57cd3df4cd865849d93ee14920fb97224300c88501f16ecfa2604b4e0", - "sha256:8a3862568a36d26e650a19bb5cbbba14b71789032aebc0423f8cc5f150730184", - "sha256:8b55d5497b51afdfde55925e04a022f1de14d4f4f25cdfd4f5d9b0aa96166851", - "sha256:8cfc12a8630a29d601f48d47787bd7eb730e475e83edb5d6c5084317463373eb", - "sha256:9281bf5b34f59afbc6b1e477a372e9526b66ca446f4bf62592839c195a718b32", - "sha256:92abb658ef2d7ef22ac9f8bb88e8b6c3e571671534e029359b6d9e845923eb1b", - "sha256:94218fcec4d72bc61df51c198d098ce2b378e0ccbac41ddbed5ef44092913288", - "sha256:95b5ffa4349df2887518bb839409bcf22caa72d82beec453216802f475b23c81", - "sha256:9600082733859f00d79dee64effc7aef1beb26adb297416a4ad2116fd61374bd", - "sha256:960c60b5849b9b4f9dcc9bea6e3626143c252c74113df2c1540aebce70209b45", - "sha256:9b2fd74c52accced7e75de26023b7dccee62511a600e62311b918ec5c168fc2a", - "sha256:9c0359b1ec12b1d6849c59f9d319610b7f20ef990a6d454ab151aa0e3b9f78ca", - "sha256:9cf41880c991716f3c7cec48e2f19ae4045fc9db5fc9cff27347ada24d710bb5", - "sha256:9d14baca2ee12c1a64740d4531356ba50b82543017f3ad6de0deb943c5979abb", - "sha256:9f474ad5acda359c8758c8accc22032c6abe6dc87a8be2440d097785e27a9349", - "sha256:9fb0211dfc3b51efea2f349ec92c114d7754dd62c01f81c3e32b765b70c45c9b", - "sha256:9fe04da3f79387f450fd0061d4dd2e45a72749d31bf634aecc9e27f24fdc4b3f", - "sha256:9ff96e8815eecacc6645da76c413eb3b3d34cfca256c70b16b286a687d013c32", - "sha256:a027ec240fe73a8d6281872690b988eed307cd7d91b23998ff35ff577ca688b5", - "sha256:a048ce45dcdaaf1defb76b2e684f997fb5abf74437b6cb7b22ddad934a964e34", - "sha256:a265acbb7bb33a3a2d626afbe756371dce0279e7b17f4f4eda406459c2b5ff1c", - "sha256:a35c5fc61d4f51eb045061e7967cfe3123d622cd500e8868e7c0c592a09fedc4", - "sha256:a37bd74c3fa9d00be2d7b8eca074dc56bd8077ddd2917a839bd989612671ed17", - "sha256:a60a4d75718a5efa473ebd5ab685786ba0c67b8381f781d1be14da49f1a2dc60", - "sha256:a6ef16328011d3f468e7ebc326f24c1445f001ca1dec335b2f8e66bed3006394", - "sha256:a90af66facec4cebe4181b9e62a68be65e45ac9b52b67de9eec118701856e7ff", - "sha256:ad9ce259f50abd98a1ca0aa6e490b58c316a0fce0617f609723e40804add2c00", - "sha256:afa8a2978ec65d2336305550535c9c4ff50ee527914328c8677b3973ade52b85", - "sha256:b15b3afff74f707b9275d5ba6a91ae8f6429c3ffb29bbfd216b0b375a56f13d7", - "sha256:b284e319754366c1aee2267a2036248b24eeb17ecd5dc16022095e747f2f4304", - "sha256:b2d7f80c4e1fd010b07cb26820aae86b7e73b681ee4889684fb8d2d4537aab13", - "sha256:b3bc26a951007b1057a1c543af845f1c7e3e71cc240ed1ace7bf4484aa99196e", - "sha256:b3e34f3a1b8131ba06f1a73adab24f30934d148afcd5f5de9a73565a4404384e", - "sha256:b4121773c49a0776461f4a904cdf6264c88e42218aaa8407e803ca8025872792", - "sha256:b61189b29081a20c7e4e0b49b44d5d44bb0dc92be3c6d06a11cc043f81bf9329", - "sha256:b6234e14f9314731ec45c42fc4554b88133ad53a09092cc48a88e771c125dadb", - "sha256:b8512bac933afc3e45fb2b18da8e59b78d4f408399a960339598374d4ae3b56b", - "sha256:ba672b26069957ee369cfa7fc180dde1fc6f176eaf1e6beaf61fbebbd3d9c000", - "sha256:bee7c0588aa0076ce77c0ea5d19a68d76ad81fcd9fe8501003b9a24f9d4000f6", - "sha256:c04a328260dfd5db8c39538f999f02779012268f54614902d0afc775d44e0a62", - "sha256:c1dcc7524066fa918c6a27d61444d4ee7900ec635779058571f70d042d86ed63", - "sha256:c6e99d9a65ca282e578dfea819cfa9c0a62b2499d8677392e09feaf305e9e6f5", - "sha256:ca43bdfa5d37bd6aee89d85e1d0831fb86e25541be7e9d376ead1b28974f8e5e", - "sha256:caf53b15b1b7df9fbd0709aa01409000a2b4dd03a5f6f5cc548183c7c8f8b63c", - "sha256:cc41db090ed742f32bd2d2c721861725e6109681eddf835d0a82bd3a5c382827", - "sha256:cd240939f71c64bd658f186330603aac1a9a81bf6273f523fca63673cb7378a8", - "sha256:ce8fdc2dca699f8dbf055a61d73eaa10482569ad20ee3c36ef9641f69afa8c91", - "sha256:d1bed1b467ef657f2a0ae62844a607909ef1c6889562de5e1d505f74457d0b96", - "sha256:d1d964afecdf3a8288789df2f5751dc0a8261138c3768d9af117ed384e538fad", - "sha256:d4393e3581e84e5645506923816b9cc81f5609a778c7e7534054091acc64d1c6", - "sha256:d874eb056410ca05fed180b6642e680373688efafc7f077b2a2f61811e873a40", - "sha256:db99677b4457c7a5c5a949353e125ba72d62b35f74e26da141530fbb012218a7", - "sha256:dd32a49400a2c3d52088e120ee00c1e3576cbff7e10b98467962c74fdb762ed4", - "sha256:df0e3bf7993bdbeca5ac25aa859cf40d39019e015c9c91809ba7093967f7a648", - "sha256:e011555abada53f1578d63389610ac8a5400fc70ce71156b0aa30d326f1a5064", - "sha256:e2862408c99f84aa571ab462d25236ef9cb12a602ea959ba9c9009a54902fc73", - "sha256:e3aa16de190d29a0ea1b48253c57d99a68492c8dd8948638073ab9e74dc9410b", - "sha256:e93a0617cd16998784bf4414c7e40f17a35d2350e5c6f0bd900d3a8e02bd3762", - "sha256:ea3334cabe4d41b7ccd01e4d349828678794edbc2d3ae97fc162a3312095092e", - "sha256:eb866162ef2f45063acc7a53a88ef6fe8bf121d45c30ea3c9cd87ce7e191a8d4", - "sha256:ec81878ddf0e98817def1e77d4f50dae5ef5b0e4fe796fae3bd674304172416e", - "sha256:efbb54e98446892590dc2458c19c10344ee9a883a79b5cec4bc34d6656e8d546", - "sha256:f0e77e3c0008bc9316e662624535b88d360c3a5d3f81e15cf12c139a75250046", - "sha256:f0feece2ef8ebc42ed9e2e8c78fc4aa3cf455733b507c09ef7406364c94376c6", - "sha256:f470f68adc395e0183b92a2f4689264d1ea4b40504a24d9882c27375e6662bb9", - "sha256:f844a1bbf1d207dd311a56f383f7eda2d0e134921d45751842d8235e7778965d", - "sha256:f8a93b1c0ed2d04b97a5e9336fd2d33371b9a6e29ab7dd6503d63407c20ffbaf", - "sha256:f8e5c0031b90ca9ce555e2e8fd5c3b02a25f14989cbc310701823832c99eb687", - "sha256:fb287618b9c7aa3bf8d825f02d9201b2f13078a5ed3b293c8f4d953917d84d5e", - "sha256:fbafe31d191dfa7c4c51f7a6149c9fb7e914dcf9ffead27dcfd9f1ae382b3885", - "sha256:fbd18dc82d7bf274b37aa48d664534330af744e03bccf696d6f4c6042e7d19e7" - ], - "markers": "python_version >= '3.9'", - "version": "==6.7.0" - }, - "oauth2client": { - "hashes": [ - "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac", - "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6" - ], - "index": "pypi", - "version": "==4.1.3" - }, - "oauthlib": { - "extras": [ - "signedtoken" - ], - "hashes": [ - "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", - "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1" - ], - "markers": "python_version >= '3.8'", - "version": "==3.3.1" - }, - "pbr": { - "hashes": [ - "sha256:b46004ec30a5324672683ec848aed9e8fc500b0d261d40a3229c2d2bbfcedc29", - "sha256:ff223894eb1cd271a98076b13d3badff3bb36c424074d26334cd25aebeecea6b" - ], - "markers": "python_version >= '2.6'", - "version": "==7.0.3" - }, - "propcache": { - "hashes": [ - "sha256:0002004213ee1f36cfb3f9a42b5066100c44276b9b72b4e1504cddd3d692e86e", - "sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4", - "sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be", - "sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3", - "sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85", - "sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b", - "sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367", - "sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf", - "sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393", - "sha256:182b51b421f0501952d938dc0b0eb45246a5b5153c50d42b495ad5fb7517c888", - "sha256:1cdb7988c4e5ac7f6d175a28a9aa0c94cb6f2ebe52756a3c0cda98d2809a9e37", - "sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8", - "sha256:1f0978529a418ebd1f49dad413a2b68af33f85d5c5ca5c6ca2a3bed375a7ac60", - "sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1", - "sha256:296f4c8ed03ca7476813fe666c9ea97869a8d7aec972618671b33a38a5182ef4", - "sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717", - "sha256:2b16ec437a8c8a965ecf95739448dd938b5c7f56e67ea009f4300d8df05f32b7", - "sha256:2bb07ffd7eaad486576430c89f9b215f9e4be68c4866a96e97db9e97fead85dc", - "sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe", - "sha256:357f5bb5c377a82e105e44bd3d52ba22b616f7b9773714bff93573988ef0a5fb", - "sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75", - "sha256:364426a62660f3f699949ac8c621aad6977be7126c5807ce48c0aeb8e7333ea6", - "sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e", - "sha256:3d233076ccf9e450c8b3bc6720af226b898ef5d051a2d145f7d765e6e9f9bcff", - "sha256:3d902a36df4e5989763425a8ab9e98cd8ad5c52c823b34ee7ef307fd50582566", - "sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12", - "sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367", - "sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874", - "sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf", - "sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566", - "sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a", - "sha256:4b536b39c5199b96fc6245eb5fb796c497381d3942f169e44e8e392b29c9ebcc", - "sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a", - "sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1", - "sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6", - "sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61", - "sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726", - "sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49", - "sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44", - "sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af", - "sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa", - "sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153", - "sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc", - "sha256:5d4e2366a9c7b837555cf02fb9be2e3167d333aff716332ef1b7c3a142ec40c5", - "sha256:5fd37c406dd6dc85aa743e214cef35dc54bbdd1419baac4f6ae5e5b1a2976938", - "sha256:60a8fda9644b7dfd5dece8c61d8a85e271cb958075bfc4e01083c148b61a7caf", - "sha256:66c1f011f45a3b33d7bcb22daed4b29c0c9e2224758b6be00686731e1b46f925", - "sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8", - "sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c", - "sha256:67fad6162281e80e882fb3ec355398cf72864a54069d060321f6cd0ade95fe85", - "sha256:6918ecbd897443087a3b7cd978d56546a812517dcaaca51b49526720571fa93e", - "sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0", - "sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1", - "sha256:71b749281b816793678ae7f3d0d84bd36e694953822eaad408d682efc5ca18e0", - "sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992", - "sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db", - "sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f", - "sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d", - "sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1", - "sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e", - "sha256:8c9b3cbe4584636d72ff556d9036e0c9317fa27b3ac1f0f558e7e84d1c9c5900", - "sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89", - "sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a", - "sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b", - "sha256:948dab269721ae9a87fd16c514a0a2c2a1bdb23a9a61b969b0f9d9ee2968546f", - "sha256:981333cb2f4c1896a12f4ab92a9cc8f09ea664e9b7dbdc4eff74627af3a11c0f", - "sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1", - "sha256:99d43339c83aaf4d32bda60928231848eee470c6bda8d02599cc4cebe872d183", - "sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66", - "sha256:9a52009f2adffe195d0b605c25ec929d26b36ef986ba85244891dee3b294df21", - "sha256:9d2b6caef873b4f09e26ea7e33d65f42b944837563a47a94719cc3544319a0db", - "sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded", - "sha256:a0ee98db9c5f80785b266eb805016e36058ac72c51a064040f2bc43b61101cdb", - "sha256:a129e76735bc792794d5177069691c3217898b9f5cee2b2661471e52ffe13f19", - "sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0", - "sha256:a9695397f85973bb40427dedddf70d8dc4a44b22f1650dd4af9eedf443d45165", - "sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778", - "sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455", - "sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f", - "sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b", - "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", - "sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81", - "sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859", - "sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c", - "sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835", - "sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393", - "sha256:c30b53e7e6bda1d547cabb47c825f3843a0a1a42b0496087bb58d8fedf9f41b5", - "sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641", - "sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144", - "sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74", - "sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db", - "sha256:cbc3b6dfc728105b2a57c06791eb07a94229202ea75c59db644d7d496b698cac", - "sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403", - "sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9", - "sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f", - "sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311", - "sha256:d82ad62b19645419fe79dd63b3f9253e15b30e955c0170e5cebc350c1844e581", - "sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36", - "sha256:daede9cd44e0f8bdd9e6cc9a607fc81feb80fae7a5fc6cecaff0e0bb32e42d00", - "sha256:db65d2af507bbfbdcedb254a11149f894169d90488dd3e7190f7cdcb2d6cd57a", - "sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f", - "sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2", - "sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7", - "sha256:e53f3a38d3510c11953f3e6a33f205c6d1b001129f972805ca9b42fc308bc239", - "sha256:e9b0d8d0845bbc4cfcdcbcdbf5086886bc8157aa963c31c777ceff7846c77757", - "sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72", - "sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9", - "sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4", - "sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24", - "sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207", - "sha256:f10207adf04d08bec185bae14d9606a1444715bc99180f9331c9c02093e1959e", - "sha256:f1d2f90aeec838a52f1c1a32fe9a619fefd5e411721a9117fbf82aea638fe8a1", - "sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d", - "sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37", - "sha256:f93243fdc5657247533273ac4f86ae106cc6445a0efacb9a1bfe982fcfefd90c", - "sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e", - "sha256:fc38cba02d1acba4e2869eef1a57a43dfbd3d49a59bf90dda7444ec2be6a5570", - "sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af", - "sha256:fd138803047fb4c062b1c1dd95462f5209456bfab55c734458f15d11da288f8f", - "sha256:fd2dbc472da1f772a4dae4fa24be938a6c544671a912e30529984dd80400cd88", - "sha256:fd6f30fdcf9ae2a70abd34da54f18da086160e4d7d9251f81f3da0ff84fc5a48", - "sha256:fe49d0a85038f36ba9e3ffafa1103e61170b28e95b16622e11be0a0ea07c6781" - ], - "markers": "python_version >= '3.9'", - "version": "==0.4.1" - }, - "proto-plus": { - "hashes": [ - "sha256:1baa7f81cf0f8acb8bc1f6d085008ba4171eaf669629d1b6d1673b21ed1c0a82", - "sha256:873af56dd0d7e91836aee871e5799e1c6f1bda86ac9a983e0bb9f0c266a568c4" - ], - "markers": "python_version >= '3.7'", - "version": "==1.27.0" - }, - "protobuf": { - "hashes": [ - "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474", - "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2", - "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b", - "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720", - "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12", - "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd", - "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0", - "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e", - "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9", - "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70", - "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff", - "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597", - "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==4.23.4" - }, - "psutil": { - "hashes": [ - "sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff", - "sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1", - "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62", - "sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549", - "sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08", - "sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7", - "sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e", - "sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe", - "sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24", - "sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad", - "sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94", - "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8", - "sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7", - "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4" - ], - "index": "pypi", - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==5.9.4" - }, - "pyasn1": { - "hashes": [ - "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", - "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b" - ], - "markers": "python_version >= '3.8'", - "version": "==0.6.2" - }, - "pyasn1-modules": { - "hashes": [ - "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", - "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6" - ], - "markers": "python_version >= '3.8'", - "version": "==0.4.2" - }, - "pycparser": { - "hashes": [ - "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", - "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934" - ], - "markers": "python_version >= '3.8'", - "version": "==2.23" - }, - "pygithub": { - "hashes": [ - "sha256:1bbfff9372047ff3f21d5cd8e07720f3dbfdaf6462fcaed9d815f528f1ba7283", - "sha256:2caf0054ea079b71e539741ae56c5a95e073b81fa472ce222e81667381b9601b" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==1.55" - }, - "pyjwt": { - "hashes": [ - "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", - "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb" - ], - "markers": "python_version >= '3.9'", - "version": "==2.10.1" - }, - "pymemcache": { - "hashes": [ - "sha256:27bf9bd1bbc1e20f83633208620d56de50f14185055e49504f4f5e94e94aff94", - "sha256:f507bc20e0dc8d562f8df9d872107a278df049fa496805c1431b926f3ddd0eab" - ], - "markers": "python_version >= '3.7'", - "version": "==4.0.0" - }, - "pynacl": { - "hashes": [ - "sha256:04f20784083014e265ad58c1b2dd562c3e35864b5394a14ab54f5d150ee9e53e", - "sha256:10d755cf2a455d8c0f8c767a43d68f24d163b8fe93ccfaabfa7bafd26be58d73", - "sha256:140373378e34a1f6977e573033d1dd1de88d2a5d90ec6958c9485b2fd9f3eb90", - "sha256:16c60daceee88d04f8d41d0a4004a7ed8d9a5126b997efd2933e08e93a3bd850", - "sha256:16dd347cdc8ae0b0f6187a2608c0af1c8b7ecbbe6b4a06bff8253c192f696990", - "sha256:25720bad35dfac34a2bcdd61d9e08d6bfc6041bebc7751d9c9f2446cf1e77d64", - "sha256:2d6cd56ce4998cb66a6c112fda7b1fdce5266c9f05044fa72972613bef376d15", - "sha256:347dcddce0b4d83ed3f32fd00379c83c425abee5a9d2cd0a2c84871334eaff64", - "sha256:4853c154dc16ea12f8f3ee4b7e763331876316cc3a9f06aeedf39bcdca8f9995", - "sha256:49c336dd80ea54780bcff6a03ee1a476be1612423010472e60af83452aa0f442", - "sha256:4a25cfede801f01e54179b8ff9514bd7b5944da560b7040939732d1804d25419", - "sha256:51fed9fe1bec9e7ff9af31cd0abba179d0e984a2960c77e8e5292c7e9b7f7b5d", - "sha256:536703b8f90e911294831a7fbcd0c062b837f3ccaa923d92a6254e11178aaf42", - "sha256:5789f016e08e5606803161ba24de01b5a345d24590a80323379fc4408832d290", - "sha256:6b08eab48c9669d515a344fb0ef27e2cbde847721e34bba94a343baa0f33f1f4", - "sha256:6b393bc5e5a0eb86bb85b533deb2d2c815666665f840a09e0aa3362bb6088736", - "sha256:84709cea8f888e618c21ed9a0efdb1a59cc63141c403db8bf56c469b71ad56f2", - "sha256:8bfaa0a28a1ab718bad6239979a5a57a8d1506d0caf2fba17e524dbb409441cf", - "sha256:bbcc4452a1eb10cd5217318c822fde4be279c9de8567f78bad24c773c21254f8", - "sha256:cb36deafe6e2bce3b286e5d1f3e1c246e0ccdb8808ddb4550bb2792f2df298f2", - "sha256:cf831615cc16ba324240de79d925eacae8265b7691412ac6b24221db157f6bd1", - "sha256:dcdeb41c22ff3c66eef5e63049abf7639e0db4edee57ba70531fc1b6b133185d", - "sha256:dea103a1afcbc333bc0e992e64233d360d393d1e63d0bc88554f572365664348", - "sha256:ef214b90556bb46a485b7da8258e59204c244b1b5b576fb71848819b468c44a7", - "sha256:f3482abf0f9815e7246d461fab597aa179b7524628a4bc36f86a7dc418d2608d", - "sha256:f46386c24a65383a9081d68e9c2de909b1834ec74ff3013271f1bca9c2d233eb", - "sha256:f4b3824920e206b4f52abd7de621ea7a44fd3cb5c8daceb7c3612345dfc54f2e" - ], - "markers": "python_version >= '3.8'", - "version": "==1.6.0" - }, - "pyopenssl": { - "hashes": [ - "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf", - "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==22.0.0" - }, - "pyparsing": { - "hashes": [ - "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", - "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.4.7" - }, - "python-dateutil": { - "hashes": [ - "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", - "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" - ], - "index": "pypi", - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.8.1" - }, - "python-http-client": { - "hashes": [ - "sha256:ad371d2bbedc6ea15c26179c6222a78bc9308d272435ddf1d5c84f068f249a36", - "sha256:bf841ee45262747e00dec7ee9971dfb8c7d83083f5713596488d67739170cea0" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.3.7" - }, - "pytz": { - "hashes": [ - "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", - "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb" - ], - "index": "pypi", - "version": "==2023.3" - }, - "pyyaml": { - "hashes": [ - "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", - "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", - "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", - "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", - "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", - "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", - "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", - "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", - "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", - "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", - "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", - "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", - "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", - "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", - "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", - "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", - "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", - "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", - "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", - "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", - "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", - "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", - "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", - "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", - "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", - "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", - "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", - "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", - "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", - "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef", - "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", - "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", - "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", - "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", - "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", - "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", - "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", - "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", - "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", - "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", - "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", - "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", - "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", - "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", - "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", - "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", - "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", - "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", - "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", - "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", - "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==6.0.1" - }, - "redis": { - "hashes": [ - "sha256:585dc516b9eb042a619ef0a39c3d7d55fe81bdb4df09a52c9cdde0d07bf1aa7d", - "sha256:e2b03db868160ee4591de3cb90d40ebb50a90dd302138775937f6a42b7ed183c" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==4.6.0" - }, - "requests": { - "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" - ], - "index": "pypi", - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.21.0" - }, - "requests-oauthlib": { - "hashes": [ - "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", - "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9" - ], - "markers": "python_version >= '3.4'", - "version": "==2.0.0" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", - "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.0.0" - }, - "rsa": { - "hashes": [ - "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", - "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75" - ], - "markers": "python_version >= '3.6' and python_version < '4'", - "version": "==4.9.1" - }, - "sendgrid": { - "hashes": [ - "sha256:351a7fc501d2b9d5afdcbc70a02490917057d6ce5cc22c558cadfc16229f157b", - "sha256:e1f93c72b3db3bd00d86f79ee926a093ee7e65533936a140855916569b08e0b0" - ], - "index": "pypi", - "version": "==6.0.4" - }, - "setuptools": { - "hashes": [ - "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", - "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c" - ], - "markers": "python_version >= '3.9'", - "version": "==80.9.0" - }, - "six": { - "hashes": [ - "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", - "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.17.0" - }, - "uritemplate": { - "hashes": [ - "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", - "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686" - ], - "markers": "python_version >= '3.9'", - "version": "==4.2.0" - }, - "urllib3": { - "hashes": [ - "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", - "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", - "version": "==1.24.3" - }, - "websocket-client": { - "hashes": [ - "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", - "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef" - ], - "markers": "python_version >= '3.9'", - "version": "==1.9.0" - }, - "wrapt": { - "hashes": [ - "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc", - "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81", - "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09", - "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e", - "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca", - "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0", - "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb", - "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487", - "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40", - "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c", - "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060", - "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202", - "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41", - "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9", - "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b", - "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664", - "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d", - "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362", - "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00", - "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc", - "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1", - "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267", - "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956", - "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966", - "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1", - "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228", - "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72", - "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d", - "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292", - "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0", - "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0", - "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36", - "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c", - "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5", - "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f", - "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73", - "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b", - "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2", - "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593", - "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39", - "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389", - "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf", - "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf", - "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89", - "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c", - "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c", - "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f", - "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440", - "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465", - "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136", - "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b", - "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8", - "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3", - "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8", - "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6", - "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e", - "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f", - "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c", - "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e", - "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8", - "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2", - "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020", - "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35", - "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d", - "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3", - "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537", - "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809", - "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d", - "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a", - "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==1.16.0" - }, - "yarl": { - "hashes": [ - "sha256:01e73b85a5434f89fc4fe27dcda2aff08ddf35e4d47bbbea3bdcd25321af538a", - "sha256:029866bde8d7b0878b9c160e72305bbf0a7342bcd20b9999381704ae03308dc8", - "sha256:078278b9b0b11568937d9509b589ee83ef98ed6d561dfe2020e24a9fd08eaa2b", - "sha256:078a8aefd263f4d4f923a9677b942b445a2be970ca24548a8102689a3a8ab8da", - "sha256:07a524d84df0c10f41e3ee918846e1974aba4ec017f990dc735aad487a0bdfdf", - "sha256:088e4e08f033db4be2ccd1f34cf29fe994772fb54cfe004bbf54db320af56890", - "sha256:0b5bcc1a9c4839e7e30b7b30dd47fe5e7e44fb7054ec29b5bb8d526aa1041093", - "sha256:0cf71bf877efeac18b38d3930594c0948c82b64547c1cf420ba48722fe5509f6", - "sha256:0d6e6885777af0f110b0e5d7e5dda8b704efed3894da26220b7f3d887b839a79", - "sha256:0dd9a702591ca2e543631c2a017e4a547e38a5c0f29eece37d9097e04a7ac683", - "sha256:10619d9fdee46d20edc49d3479e2f8269d0779f1b031e6f7c2aa1c76be04b7ed", - "sha256:131a085a53bfe839a477c0845acf21efc77457ba2bcf5899618136d64f3303a2", - "sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff", - "sha256:139718f35149ff544caba20fce6e8a2f71f1e39b92c700d8438a0b1d2a631a02", - "sha256:14291620375b1060613f4aab9ebf21850058b6b1b438f386cc814813d901c60b", - "sha256:1834bb90991cc2999f10f97f5f01317f99b143284766d197e43cd5b45eb18d03", - "sha256:1ab72135b1f2db3fed3997d7e7dc1b80573c67138023852b6efb336a5eae6511", - "sha256:1e7ce67c34138a058fd092f67d07a72b8e31ff0c9236e751957465a24b28910c", - "sha256:1e8fbaa7cec507aa24ea27a01456e8dd4b6fab829059b69844bd348f2d467124", - "sha256:22965c2af250d20c873cdbee8ff958fb809940aeb2e74ba5f20aaf6b7ac8c70c", - "sha256:22b029f2881599e2f1b06f8f1db2ee63bd309e2293ba2d566e008ba12778b8da", - "sha256:243dda95d901c733f5b59214d28b0120893d91777cb8aa043e6ef059d3cddfe2", - "sha256:2ca6fd72a8cd803be290d42f2dec5cdcd5299eeb93c2d929bf060ad9efaf5de0", - "sha256:2e4e1f6f0b4da23e61188676e3ed027ef0baa833a2e633c29ff8530800edccba", - "sha256:31f0b53913220599446872d757257be5898019c85e7971599065bc55065dc99d", - "sha256:334b8721303e61b00019474cc103bdac3d7b1f65e91f0bfedeec2d56dfe74b53", - "sha256:33e32a0dd0c8205efa8e83d04fc9f19313772b78522d1bdc7d9aed706bfd6138", - "sha256:34b36c2c57124530884d89d50ed2c1478697ad7473efd59cfd479945c95650e4", - "sha256:3aa27acb6de7a23785d81557577491f6c38a5209a254d1191519d07d8fe51748", - "sha256:3b06bcadaac49c70f4c88af4ffcfbe3dc155aab3163e75777818092478bcbbe7", - "sha256:3b7c88eeef021579d600e50363e0b6ee4f7f6f728cd3486b9d0f3ee7b946398d", - "sha256:3e2daa88dc91870215961e96a039ec73e4937da13cf77ce17f9cad0c18df3503", - "sha256:3ea66b1c11c9150f1372f69afb6b8116f2dd7286f38e14ea71a44eee9ec51b9d", - "sha256:42188e6a615c1a75bcaa6e150c3fe8f3e8680471a6b10150c5f7e83f47cc34d2", - "sha256:433885ab5431bc3d3d4f2f9bd15bfa1614c522b0f1405d62c4f926ccd69d04fa", - "sha256:437840083abe022c978470b942ff832c3940b2ad3734d424b7eaffcd07f76737", - "sha256:4398557cbf484207df000309235979c79c4356518fd5c99158c7d38203c4da4f", - "sha256:45c2842ff0e0d1b35a6bf1cd6c690939dacb617a70827f715232b2e0494d55d1", - "sha256:47743b82b76d89a1d20b83e60d5c20314cbd5ba2befc9cda8f28300c4a08ed4d", - "sha256:4792b262d585ff0dff6bcb787f8492e40698443ec982a3568c2096433660c694", - "sha256:47d8a5c446df1c4db9d21b49619ffdba90e77c89ec6e283f453856c74b50b9e3", - "sha256:47fdb18187e2a4e18fda2c25c05d8251a9e4a521edaed757fef033e7d8498d9a", - "sha256:4c52a6e78aef5cf47a98ef8e934755abf53953379b7d53e68b15ff4420e6683d", - "sha256:4dcc74149ccc8bba31ce1944acee24813e93cfdee2acda3c172df844948ddf7b", - "sha256:50678a3b71c751d58d7908edc96d332af328839eea883bb554a43f539101277a", - "sha256:51af598701f5299012b8416486b40fceef8c26fc87dc6d7d1f6fc30609ea0aa6", - "sha256:594fcab1032e2d2cc3321bb2e51271e7cd2b516c7d9aee780ece81b07ff8244b", - "sha256:595697f68bd1f0c1c159fcb97b661fc9c3f5db46498043555d04805430e79bea", - "sha256:59c189e3e99a59cf8d83cbb31d4db02d66cda5a1a4374e8a012b51255341abf5", - "sha256:5a3bf7f62a289fa90f1990422dc8dff5a458469ea71d1624585ec3a4c8d6960f", - "sha256:5c401e05ad47a75869c3ab3e35137f8468b846770587e70d71e11de797d113df", - "sha256:5cdac20da754f3a723cceea5b3448e1a2074866406adeb4ef35b469d089adb8f", - "sha256:5d0fcda9608875f7d052eff120c7a5da474a6796fe4d83e152e0e4d42f6d1a9b", - "sha256:5dbeefd6ca588b33576a01b0ad58aa934bc1b41ef89dee505bf2932b22ddffba", - "sha256:62441e55958977b8167b2709c164c91a6363e25da322d87ae6dd9c6019ceecf9", - "sha256:663e1cadaddae26be034a6ab6072449a8426ddb03d500f43daf952b74553bba0", - "sha256:669930400e375570189492dc8d8341301578e8493aec04aebc20d4717f899dd6", - "sha256:68986a61557d37bb90d3051a45b91fa3d5c516d177dfc6dd6f2f436a07ff2b6b", - "sha256:6944b2dc72c4d7f7052683487e3677456050ff77fcf5e6204e98caf785ad1967", - "sha256:6a635ea45ba4ea8238463b4f7d0e721bad669f80878b7bfd1f89266e2ae63da2", - "sha256:6c5010a52015e7c70f86eb967db0f37f3c8bd503a695a49f8d45700144667708", - "sha256:6dcbb0829c671f305be48a7227918cfcd11276c2d637a8033a99a02b67bf9eda", - "sha256:70dfd4f241c04bd9239d53b17f11e6ab672b9f1420364af63e8531198e3f5fe8", - "sha256:719ae08b6972befcba4310e49edb1161a88cdd331e3a694b84466bd938a6ab10", - "sha256:75976c6945d85dbb9ee6308cd7ff7b1fb9409380c82d6119bd778d8fcfe2931c", - "sha256:7861058d0582b847bc4e3a4a4c46828a410bca738673f35a29ba3ca5db0b473b", - "sha256:792a2af6d58177ef7c19cbf0097aba92ca1b9cb3ffdd9c7470e156c8f9b5e028", - "sha256:8009b3173bcd637be650922ac455946197d858b3630b6d8787aa9e5c4564533e", - "sha256:80ddf7a5f8c86cb3eb4bc9028b07bbbf1f08a96c5c0bc1244be5e8fefcb94147", - "sha256:8218f4e98d3c10d683584cb40f0424f4b9fd6e95610232dd75e13743b070ee33", - "sha256:84fc3ec96fce86ce5aa305eb4aa9358279d1aa644b71fab7b8ed33fe3ba1a7ca", - "sha256:852863707010316c973162e703bddabec35e8757e67fcb8ad58829de1ebc8590", - "sha256:8884d8b332a5e9b88e23f60bb166890009429391864c685e17bd73a9eda9105c", - "sha256:8dee9c25c74997f6a750cd317b8ca63545169c098faee42c84aa5e506c819b53", - "sha256:939fe60db294c786f6b7c2d2e121576628468f65453d86b0fe36cb52f987bd74", - "sha256:99b6fc1d55782461b78221e95fc357b47ad98b041e8e20f47c1411d0aacddc60", - "sha256:9d7672ecf7557476642c88497c2f8d8542f8e36596e928e9bcba0e42e1e7d71f", - "sha256:9f6d73c1436b934e3f01df1e1b21ff765cd1d28c77dfb9ace207f746d4610ee1", - "sha256:9fb17ea16e972c63d25d4a97f016d235c78dd2344820eb35bc034bc32012ee27", - "sha256:a49370e8f711daec68d09b821a34e1167792ee2d24d405cbc2387be4f158b520", - "sha256:a4fcfc8eb2c34148c118dfa02e6427ca278bfd0f3df7c5f99e33d2c0e81eae3e", - "sha256:a899cbd98dce6f5d8de1aad31cb712ec0a530abc0a86bd6edaa47c1090138467", - "sha256:a9b1ba5610a4e20f655258d5a1fdc7ebe3d837bb0e45b581398b99eb98b1f5ca", - "sha256:af74f05666a5e531289cb1cc9c883d1de2088b8e5b4de48004e5ca8a830ac859", - "sha256:b0748275abb8c1e1e09301ee3cf90c8a99678a4e92e4373705f2a2570d581273", - "sha256:b266bd01fedeffeeac01a79ae181719ff848a5a13ce10075adbefc8f1daee70e", - "sha256:b4f15793aa49793ec8d1c708ab7f9eded1aa72edc5174cae703651555ed1b601", - "sha256:b580e71cac3f8113d3135888770903eaf2f507e9421e5697d6ee6d8cd1c7f054", - "sha256:b6a6f620cfe13ccec221fa312139135166e47ae169f8253f72a0abc0dae94376", - "sha256:b790b39c7e9a4192dc2e201a282109ed2985a1ddbd5ac08dc56d0e121400a8f7", - "sha256:b85b982afde6df99ecc996990d4ad7ccbdbb70e2a4ba4de0aecde5922ba98a0b", - "sha256:b8a0588521a26bf92a57a1705b77b8b59044cdceccac7151bd8d229e66b8dedb", - "sha256:ba440ae430c00eee41509353628600212112cd5018d5def7e9b05ea7ac34eb65", - "sha256:bca03b91c323036913993ff5c738d0842fc9c60c4648e5c8d98331526df89784", - "sha256:bebf8557577d4401ba8bd9ff33906f1376c877aa78d1fe216ad01b4d6745af71", - "sha256:bec03d0d388060058f5d291a813f21c011041938a441c593374da6077fe21b1b", - "sha256:bf4a21e58b9cde0e401e683ebd00f6ed30a06d14e93f7c8fd059f8b6e8f87b6a", - "sha256:c0232bce2170103ec23c454e54a57008a9a72b5d1c3105dc2496750da8cfa47c", - "sha256:c4647674b6150d2cae088fc07de2738a84b8bcedebef29802cf0b0a82ab6face", - "sha256:c7044802eec4524fde550afc28edda0dd5784c4c45f0be151a2d3ba017daca7d", - "sha256:c7bd6683587567e5a49ee6e336e0612bec8329be1b7d4c8af5687dcdeb67ee1e", - "sha256:ca1f59c4e1ab6e72f0a23c13fca5430f889634166be85dbf1013683e49e3278e", - "sha256:cb95a9b1adaa48e41815a55ae740cfda005758104049a640a398120bf02515ca", - "sha256:cfebc0ac8333520d2d0423cbbe43ae43c8838862ddb898f5ca68565e395516e9", - "sha256:d332fc2e3c94dad927f2112395772a4e4fedbcf8f80efc21ed7cdfae4d574fdb", - "sha256:d3e32536234a95f513bd374e93d717cf6b2231a791758de6c509e3653f234c95", - "sha256:d5372ca1df0f91a86b047d1277c2aaf1edb32d78bbcefffc81b40ffd18f027ed", - "sha256:d77e1b2c6d04711478cb1c4ab90db07f1609ccf06a287d5607fcd90dc9863acf", - "sha256:d947071e6ebcf2e2bee8fce76e10faca8f7a14808ca36a910263acaacef08eca", - "sha256:dd7afd3f8b0bfb4e0d9fc3c31bfe8a4ec7debe124cfd90619305def3c8ca8cd2", - "sha256:de6b9a04c606978fdfe72666fa216ffcf2d1a9f6a381058d4378f8d7b1e5de62", - "sha256:e1651bf8e0398574646744c1885a41198eba53dc8a9312b954073f845c90a8df", - "sha256:e1b329cb8146d7b736677a2440e422eadd775d1806a81db2d4cded80a48efc1a", - "sha256:e1b51bebd221006d3d2f95fbe124b22b247136647ae5dcc8c7acafba66e5ee67", - "sha256:e340382d1afa5d32b892b3ff062436d592ec3d692aeea3bef3a5cfe11bbf8c6f", - "sha256:e4b582bab49ac33c8deb97e058cd67c2c50dac0dd134874106d9c774fd272529", - "sha256:e51ac5435758ba97ad69617e13233da53908beccc6cfcd6c34bbed8dcbede486", - "sha256:e5542339dcf2747135c5c85f68680353d5cb9ffd741c0f2e8d832d054d41f35a", - "sha256:e6438cc8f23a9c1478633d216b16104a586b9761db62bfacb6425bac0a36679e", - "sha256:e81fda2fb4a07eda1a2252b216aa0df23ebcd4d584894e9612e80999a78fd95b", - "sha256:ea70f61a47f3cc93bdf8b2f368ed359ef02a01ca6393916bc8ff877427181e74", - "sha256:ebd4549b108d732dba1d4ace67614b9545b21ece30937a63a65dd34efa19732d", - "sha256:efb07073be061c8f79d03d04139a80ba33cbd390ca8f0297aae9cce6411e4c6b", - "sha256:f0d97c18dfd9a9af4490631905a3f131a8e4c9e80a39353919e2cfed8f00aedc", - "sha256:f1e09112a2c31ffe8d80be1b0988fa6a18c5d5cad92a9ffbb1c04c91bfe52ad2", - "sha256:f3d7a87a78d46a2e3d5b72587ac14b4c16952dd0887dbb051451eceac774411e", - "sha256:f4afb5c34f2c6fecdcc182dfcfc6af6cccf1aa923eed4d6a12e9d96904e1a0d8", - "sha256:f6d2cb59377d99718913ad9a151030d6f83ef420a2b8f521d94609ecc106ee82", - "sha256:f87ac53513d22240c7d59203f25cc3beac1e574c6cd681bbfd321987b69f95fd", - "sha256:ff86011bd159a9d2dfc89c34cfd8aff12875980e3bd6a39ff097887520e60249" - ], - "markers": "python_version >= '3.9'", - "version": "==1.22.0" - } - }, - "develop": { - "cachecontrol": { - "hashes": [ - "sha256:b7ac014ff72ee199b5f8af1de29d60239954f223e948196fa3d84adaffc71d2b", - "sha256:e6220afafa4c22a47dd0badb319f84475d79108100d04e26e8542ef7d3ab05a1" - ], - "markers": "python_version >= '3.10'", - "version": "==0.14.4" - }, - "cachetools": { - "hashes": [ - "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", - "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a" - ], - "markers": "python_version >= '3.7'", - "version": "==5.5.2" - }, - "certifi": { - "hashes": [ - "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", - "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==2024.2.2" - }, - "cffi": { - "hashes": [ - "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", - "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", - "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", - "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", - "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", - "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", - "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", - "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", - "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", - "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", - "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", - "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", - "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", - "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", - "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", - "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", - "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", - "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", - "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", - "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", - "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", - "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", - "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", - "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", - "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", - "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", - "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", - "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", - "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", - "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", - "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", - "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", - "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", - "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", - "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", - "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", - "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", - "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", - "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", - "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", - "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", - "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", - "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", - "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", - "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", - "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", - "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", - "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", - "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", - "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", - "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", - "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", - "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", - "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", - "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", - "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", - "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", - "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", - "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", - "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", - "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", - "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", - "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", - "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" - ], - "index": "pypi", - "version": "==1.15.1" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "charset-normalizer": { - "hashes": [ - "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", - "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", - "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", - "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", - "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", - "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", - "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", - "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", - "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", - "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", - "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", - "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", - "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", - "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", - "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", - "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", - "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", - "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", - "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", - "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", - "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", - "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", - "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", - "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", - "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", - "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", - "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", - "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", - "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", - "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", - "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", - "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", - "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", - "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", - "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", - "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", - "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", - "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", - "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", - "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", - "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", - "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", - "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", - "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", - "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", - "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", - "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", - "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", - "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", - "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", - "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", - "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", - "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", - "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", - "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", - "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", - "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", - "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", - "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", - "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", - "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", - "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", - "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", - "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", - "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", - "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", - "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", - "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", - "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", - "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", - "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", - "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", - "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", - "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", - "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", - "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", - "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", - "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", - "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", - "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", - "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", - "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", - "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", - "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", - "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", - "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", - "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", - "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", - "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", - "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" - ], - "index": "pypi", - "markers": "python_full_version >= '3.7.0'", - "version": "==3.3.2" - }, - "click": { - "hashes": [ - "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", - "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6" - ], - "markers": "python_version >= '3.10'", - "version": "==8.3.1" - }, - "cryptography": { - "hashes": [ - "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", - "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", - "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", - "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", - "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", - "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", - "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", - "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", - "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", - "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", - "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", - "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", - "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", - "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", - "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", - "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", - "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", - "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", - "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", - "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", - "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", - "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==37.0.4" - }, - "firebase-admin": { - "hashes": [ - "sha256:e3b334d18bbea039f2f3e8a792ad6870d2a7cc79a13ed10659dedd63f5b475e4", - "sha256:e3c42351fb6194d7279a6fd9209a947005fb4ee7e9037d19762e6cb3da4a82e1" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==6.2.0" - }, - "flask": { - "hashes": [ - "sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b", - "sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.2.2" - }, - "google-api-core": { - "extras": [ - "grpc" - ], - "hashes": [ - "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a", - "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.11.1" - }, - "google-api-python-client": { - "hashes": [ - "sha256:62ee28e96031a10a1c341f226a75ac6a4f16bdb1d888dc8222b2cdca133d0031", - "sha256:f34abb671afd488bd19d30721ea20fb30d3796ddd825d6f91f26d8c718a9f07d" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.93.0" - }, - "google-auth": { - "hashes": [ - "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce", - "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==2.22.0" - }, - "google-auth-httplib2": { - "hashes": [ - "sha256:177898a0175252480d5ed916aeea183c2df87c1f9c26705d74ae6b951c268b0b", - "sha256:426167e5df066e3f5a0fc7ea18768c08e7296046594ce4c8c409c2457dd1f776" - ], - "markers": "python_version >= '3.7'", - "version": "==0.3.0" - }, - "google-cloud-core": { - "hashes": [ - "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb", - "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.3.3" - }, - "google-cloud-firestore": { - "hashes": [ - "sha256:19f2326cb466b0d52aed9fabbd89758be431f6ce18c422966cfdb8326b424314", - "sha256:a9cffba7cdc6101111d6d54cde22d521c98f9e7d415e67486b137fa16f06aa03" - ], - "markers": "platform_python_implementation != 'PyPy'", - "version": "==2.23.0" - }, - "google-cloud-storage": { - "hashes": [ - "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7", - "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.10.0" - }, - "google-crc32c": { - "hashes": [ - "sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a", - "sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876", - "sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c", - "sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289", - "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298", - "sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02", - "sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f", - "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2", - "sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a", - "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb", - "sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210", - "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5", - "sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee", - "sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c", - "sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a", - "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314", - "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd", - "sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65", - "sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37", - "sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4", - "sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13", - "sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894", - "sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31", - "sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e", - "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709", - "sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740", - "sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc", - "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d", - "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c", - "sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c", - "sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d", - "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906", - "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61", - "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57", - "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c", - "sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a", - "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438", - "sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946", - "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7", - "sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96", - "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091", - "sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae", - "sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d", - "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88", - "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2", - "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd", - "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541", - "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728", - "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178", - "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968", - "sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346", - "sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8", - "sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93", - "sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7", - "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273", - "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462", - "sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94", - "sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd", - "sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e", - "sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57", - "sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b", - "sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9", - "sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a", - "sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100", - "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325", - "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183", - "sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556", - "sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.5.0" - }, - "google-resumable-media": { - "hashes": [ - "sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582", - "sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae" - ], - "markers": "python_version >= '3.7'", - "version": "==2.8.0" - }, - "googleapis-common-protos": { - "extras": [ - "grpc" - ], - "hashes": [ - "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038", - "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5" - ], - "markers": "python_version >= '3.7'", - "version": "==1.72.0" - }, - "grpcio": { - "hashes": [ - "sha256:07ce1f775d37ca18c7a141300e5b71539690efa1f51fe17f812ca85b5e73262f", - "sha256:112eaa7865dd9e6d7c0556c8b04ae3c3a2dc35d62ad3373ab7f6a562d8199200", - "sha256:162ccf61499c893831b8437120600290a99c0bc1ce7b51f2c8d21ec87ff6af8b", - "sha256:16da954692fd61aa4941fbeda405a756cd96b97b5d95ca58a92547bba2c1624f", - "sha256:17708db5b11b966373e21519c4c73e5a750555f02fde82276ea2a267077c68ad", - "sha256:1bcfe5070e4406f489e39325b76caeadab28c32bf9252d3ae960c79935a4cc36", - "sha256:1c1bb80299bdef33309dff03932264636450c8fdb142ea39f47e06a7153d3063", - "sha256:2507006c8a478f19e99b6fe36a2464696b89d40d88f34e4b709abe57e1337467", - "sha256:262cda97efdabb20853d3b5a4c546a535347c14b64c017f628ca0cc7fa780cc6", - "sha256:26f415f40f4a93579fd648f48dca1c13dfacdfd0290f4a30f9b9aeb745026811", - "sha256:2a0204532aa2f1afd467024b02b4069246320405bc18abec7babab03e2644e75", - "sha256:2e72ddfee62430ea80133d2cbe788e0d06b12f865765cb24a40009668bd8ea05", - "sha256:3abe6838196da518863b5d549938ce3159d809218936851b395b09cad9b5d64a", - "sha256:3ad00f3f0718894749d5a8bb0fa125a7980a2f49523731a9b1fabf2b3522aa43", - "sha256:3c3ed41f4d7a3aabf0f01ecc70d6b5d00ce1800d4af652a549de3f7cf35c4abd", - "sha256:404d3b4b6b142b99ba1cff0b2177d26b623101ea2ce51c25ef6e53d9d0d87bcc", - "sha256:41955b641c34db7d84db8d306937b72bc4968eef1c401bea73081a8d6c3d8033", - "sha256:53d3a59a10af4c2558a8e563aed9f256259d2992ae0d3037817b2155f0341de1", - "sha256:55ddaf53474e8caeb29eb03e3202f9d827ad3110475a21245f3c7712022882a9", - "sha256:589ea8e75de5fd6df387de53af6c9189c5231e212b9aa306b6b0d4f07520fbb9", - "sha256:5dab7ac2c1e7cb6179c6bfad6b63174851102cbe0682294e6b1d6f0981ad7138", - "sha256:65034473fc09628a02fb85f26e73885cf1ed39ebd9cf270247b38689ff5942c5", - "sha256:66344ea741124c38588a664237ac2fa16dfd226964cca23ddc96bd4accccbde5", - "sha256:6e784f60e575a0de554ef9251cbc2ceb8790914fe324f11e28450047f264ee6f", - "sha256:80407bc007754f108dc2061e37480238b0dc1952c855e86a4fc283501ee6bb5d", - "sha256:82af3613a219512a28ee5c95578eb38d44dd03bca02fd918aa05603c41018051", - "sha256:88b4f9ee77191dcdd8810241e89340a12cbe050be3e0d5f2f091c15571cd3930", - "sha256:99701979bcaaa7de8d5f60476487c5df8f27483624f1f7e300ff4669ee44d1f2", - "sha256:a1511a303f8074f67af4119275b4f954189e8313541da7b88b1b3a71425cdb10", - "sha256:a5eb4844e5e60bf2c446ef38c5b40d7752c6effdee882f716eb57ae87255d20a", - "sha256:a75af2fc7cb1fe25785be7bed1ab18cef959a376cdae7c6870184307614caa3f", - "sha256:a90ac47a8ce934e2c8d71e317d2f9e7e6aaceb2d199de940ce2c2eb611b8c0f4", - "sha256:aa787b83a3cd5e482e5c79be030e2b4a122ecc6c5c6c4c42a023a2b581fdf17b", - "sha256:aaae70364a2d1fb238afd6cc9fcb10442b66e397fd559d3f0968d28cc3ac929c", - "sha256:af15e9efa4d776dfcecd1d083f3ccfb04f876d613e90ef8432432efbeeac689d", - "sha256:af7dc3f7a44f10863b1b0ecab4078f0a00f561aae1edbd01fd03ad4dcf61c9e9", - "sha256:b7ec9e2f8ffc8436f6b642a10019fc513722858f295f7efc28de135d336ac189", - "sha256:b94d41b7412ef149743fbc3178e59d95228a7064c5ab4760ae82b562bdffb199", - "sha256:c1624aa686d4b36790ed1c2e2306cc3498778dffaf7b8dd47066cf819028c3ad", - "sha256:c5ffeb269f10cedb4f33142b89a061acda9f672fd1357331dbfd043422c94e9e", - "sha256:c6ad9c39704256ed91a1cffc1379d63f7d0278d6a0bad06b0330f5d30291e3a3", - "sha256:c772f225483905f675cb36a025969eef9712f4698364ecd3a63093760deea1bc", - "sha256:c77618071d96b7a8be2c10701a98537823b9c65ba256c0b9067e0594cdbd954d", - "sha256:c79b518c56dddeec79e5500a53d8a4db90da995dfe1738c3ac57fe46348be049", - "sha256:cfd23ad29bfa13fd4188433b0e250f84ec2c8ba66b14a9877e8bce05b524cf54", - "sha256:d0695ae31a89f1a8fc8256050329a91a9995b549a88619263a594ca31b76d756", - "sha256:d2c1771d0ee3cf72d69bb5e82c6a82f27fbd504c8c782575eddb7839729fbaad", - "sha256:da6a7b6b938c15fa0f0568e482efaae9c3af31963eec2da4ff13a6d8ec2888e4", - "sha256:db068bbc9b1fa16479a82e1ecf172a93874540cb84be69f0b9cb9b7ac3c82670", - "sha256:db707e3685ff16fc1eccad68527d072ac8bdd2e390f6daa97bc394ea7de4acea", - "sha256:e2cc8a308780edbe2c4913d6a49dbdb5befacdf72d489a368566be44cadaef1a", - "sha256:f27246d7da7d7e3bd8612f63785a7b0c39a244cf14b8dd9dd2f2fab939f2d7f1", - "sha256:f4aa94361bb5141a45ca9187464ae81a92a2a135ce2800b2203134f7a1a1d479", - "sha256:fa63245271920786f4cb44dcada4983a3516be8f470924528cf658731864c14b" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.62.2" - }, - "grpcio-status": { - "hashes": [ - "sha256:206ddf0eb36bc99b033f03b2c8e95d319f0044defae9b41ae21408e7e0cda48f", - "sha256:62e1bfcb02025a1cd73732a2d33672d3e9d0df4d21c12c51e0bbcaf09bab742a" - ], - "version": "==1.62.2" - }, - "httplib2": { - "hashes": [ - "sha256:749c32603f9bf16c1277f59531d502e8f1c2ca19901ae653b49c4ed698f0820e", - "sha256:e0d428dad43c72dbce7d163b7753ffc7a39c097e6788ef10f4198db69b92f08e" - ], - "index": "pypi", - "version": "==0.19.0" - }, - "idna": { - "hashes": [ - "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", - "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" - ], - "version": "==2.8" - }, - "itsdangerous": { - "hashes": [ - "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c", - "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==2.0.1" - }, - "jinja2": { - "hashes": [ - "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369", - "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==3.1.4" - }, - "markupsafe": { - "hashes": [ - "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", - "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", - "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", - "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", - "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", - "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c", - "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", - "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", - "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", - "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", - "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", - "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26", - "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", - "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", - "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", - "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", - "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", - "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", - "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", - "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", - "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", - "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", - "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", - "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", - "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", - "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758", - "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", - "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8", - "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", - "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", - "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", - "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", - "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", - "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", - "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", - "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", - "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2", - "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", - "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", - "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", - "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", - "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", - "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", - "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", - "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", - "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e", - "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", - "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", - "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", - "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", - "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", - "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", - "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b", - "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", - "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", - "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", - "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d", - "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", - "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", - "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", - "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", - "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", - "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", - "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", - "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", - "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", - "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", - "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", - "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", - "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", - "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", - "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", - "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", - "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7", - "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", - "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", - "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", - "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", - "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", - "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42", - "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", - "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", - "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", - "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", - "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", - "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", - "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc", - "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", - "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50" - ], - "markers": "python_version >= '3.9'", - "version": "==3.0.3" - }, - "msgpack": { - "hashes": [ - "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2", - "sha256:04fb995247a6e83830b62f0b07bf36540c213f6eac8e851166d8d86d83cbd014", - "sha256:180759d89a057eab503cf62eeec0aa61c4ea1200dee709f3a8e9397dbb3b6931", - "sha256:1d1418482b1ee984625d88aa9585db570180c286d942da463533b238b98b812b", - "sha256:1de460f0403172cff81169a30b9a92b260cb809c4cb7e2fc79ae8d0510c78b6b", - "sha256:1fdf7d83102bf09e7ce3357de96c59b627395352a4024f6e2458501f158bf999", - "sha256:1fff3d825d7859ac888b0fbda39a42d59193543920eda9d9bea44d958a878029", - "sha256:283ae72fc89da59aa004ba147e8fc2f766647b1251500182fac0350d8af299c0", - "sha256:2929af52106ca73fcb28576218476ffbb531a036c2adbcf54a3664de124303e9", - "sha256:2e86a607e558d22985d856948c12a3fa7b42efad264dca8a3ebbcfa2735d786c", - "sha256:350ad5353a467d9e3b126d8d1b90fe05ad081e2e1cef5753f8c345217c37e7b8", - "sha256:354e81bcdebaab427c3df4281187edc765d5d76bfb3a7c125af9da7a27e8458f", - "sha256:365c0bbe981a27d8932da71af63ef86acc59ed5c01ad929e09a0b88c6294e28a", - "sha256:372839311ccf6bdaf39b00b61288e0557916c3729529b301c52c2d88842add42", - "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e", - "sha256:41d1a5d875680166d3ac5c38573896453bbbea7092936d2e107214daf43b1d4f", - "sha256:42eefe2c3e2af97ed470eec850facbe1b5ad1d6eacdbadc42ec98e7dcf68b4b7", - "sha256:446abdd8b94b55c800ac34b102dffd2f6aa0ce643c55dfc017ad89347db3dbdb", - "sha256:454e29e186285d2ebe65be34629fa0e8605202c60fbc7c4c650ccd41870896ef", - "sha256:4efd7b5979ccb539c221a4c4e16aac1a533efc97f3b759bb5a5ac9f6d10383bf", - "sha256:5559d03930d3aa0f3aacb4c42c776af1a2ace2611871c84a75afe436695e6245", - "sha256:5928604de9b032bc17f5099496417f113c45bc6bc21b5c6920caf34b3c428794", - "sha256:59415c6076b1e30e563eb732e23b994a61c159cec44deaf584e5cc1dd662f2af", - "sha256:5a46bf7e831d09470ad92dff02b8b1ac92175ca36b087f904a0519857c6be3ff", - "sha256:602b6740e95ffc55bfb078172d279de3773d7b7db1f703b2f1323566b878b90e", - "sha256:61c8aa3bd513d87c72ed0b37b53dd5c5a0f58f2ff9f26e1555d3bd7948fb7296", - "sha256:67016ae8c8965124fdede9d3769528ad8284f14d635337ffa6a713a580f6c030", - "sha256:6bde749afe671dc44893f8d08e83bf475a1a14570d67c4bb5cec5573463c8833", - "sha256:6c15b7d74c939ebe620dd8e559384be806204d73b4f9356320632d783d1f7939", - "sha256:70a0dff9d1f8da25179ffcf880e10cf1aad55fdb63cd59c9a49a1b82290062aa", - "sha256:70c5a7a9fea7f036b716191c29047374c10721c389c21e9ffafad04df8c52c90", - "sha256:7bc8813f88417599564fafa59fd6f95be417179f76b40325b500b3c98409757c", - "sha256:80a0ff7d4abf5fecb995fcf235d4064b9a9a8a40a3ab80999e6ac1e30b702717", - "sha256:86f8136dfa5c116365a8a651a7d7484b65b13339731dd6faebb9a0242151c406", - "sha256:897c478140877e5307760b0ea66e0932738879e7aa68144d9b78ea4c8302a84a", - "sha256:8b696e83c9f1532b4af884045ba7f3aa741a63b2bc22617293a2c6a7c645f251", - "sha256:8e22ab046fa7ede9e36eeb4cfad44d46450f37bb05d5ec482b02868f451c95e2", - "sha256:94fd7dc7d8cb0a54432f296f2246bc39474e017204ca6f4ff345941d4ed285a7", - "sha256:99e2cb7b9031568a2a5c73aa077180f93dd2e95b4f8d3b8e14a73ae94a9e667e", - "sha256:9ade919fac6a3e7260b7f64cea89df6bec59104987cbea34d34a2fa15d74310b", - "sha256:9fba231af7a933400238cb357ecccf8ab5d51535ea95d94fc35b7806218ff844", - "sha256:a465f0dceb8e13a487e54c07d04ae3ba131c7c5b95e2612596eafde1dccf64a9", - "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87", - "sha256:a668204fa43e6d02f89dbe79a30b0d67238d9ec4c5bd8a940fc3a004a47b721b", - "sha256:a7787d353595c7c7e145e2331abf8b7ff1e6673a6b974ded96e6d4ec09f00c8c", - "sha256:a8f6e7d30253714751aa0b0c84ae28948e852ee7fb0524082e6716769124bc23", - "sha256:ad09b984828d6b7bb52d1d1d0c9be68ad781fa004ca39216c8a1e63c0f34ba3c", - "sha256:bafca952dc13907bdfdedfc6a5f579bf4f292bdd506fadb38389afa3ac5b208e", - "sha256:be52a8fc79e45b0364210eef5234a7cf8d330836d0a64dfbb878efa903d84620", - "sha256:be5980f3ee0e6bd44f3a9e9dea01054f175b50c3e6cdb692bc9424c0bbb8bf69", - "sha256:c63eea553c69ab05b6747901b97d620bb2a690633c77f23feb0c6a947a8a7b8f", - "sha256:d198d275222dc54244bf3327eb8cbe00307d220241d9cec4d306d49a44e85f68", - "sha256:d62ce1f483f355f61adb5433ebfd8868c5f078d1a52d042b0a998682b4fa8c27", - "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46", - "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa", - "sha256:e23ce8d5f7aa6ea6d2a2b326b4ba46c985dbb204523759984430db7114f8aa00", - "sha256:e64c8d2f5e5d5fda7b842f55dec6133260ea8f53c4257d64494c534f306bf7a9", - "sha256:e69b39f8c0aa5ec24b57737ebee40be647035158f14ed4b40e6f150077e21a84", - "sha256:ea5405c46e690122a76531ab97a079e184c0daf491e588592d6a23d3e32af99e", - "sha256:f2cb069d8b981abc72b41aea1c580ce92d57c673ec61af4c500153a626cb9e20", - "sha256:fac4be746328f90caa3cd4bc67e6fe36ca2bf61d5c6eb6d895b6527e3f05071e", - "sha256:fffee09044073e69f2bad787071aeec727183e7580443dfeb8556cbf1978d162" - ], - "markers": "python_version >= '3.9'", - "version": "==1.1.2" - }, - "parameterized": { - "hashes": [ - "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b", - "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==0.9.0" - }, - "proto-plus": { - "hashes": [ - "sha256:1baa7f81cf0f8acb8bc1f6d085008ba4171eaf669629d1b6d1673b21ed1c0a82", - "sha256:873af56dd0d7e91836aee871e5799e1c6f1bda86ac9a983e0bb9f0c266a568c4" - ], - "markers": "python_version >= '3.7'", - "version": "==1.27.0" - }, - "protobuf": { - "hashes": [ - "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474", - "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2", - "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b", - "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720", - "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12", - "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd", - "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0", - "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e", - "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9", - "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70", - "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff", - "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597", - "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==4.23.4" - }, - "pyasn1": { - "hashes": [ - "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", - "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b" - ], - "markers": "python_version >= '3.8'", - "version": "==0.6.2" - }, - "pyasn1-modules": { - "hashes": [ - "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", - "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6" - ], - "markers": "python_version >= '3.8'", - "version": "==0.4.2" - }, - "pycparser": { - "hashes": [ - "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", - "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934" - ], - "markers": "python_version >= '3.8'", - "version": "==2.23" - }, - "pyjwt": { - "hashes": [ - "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", - "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb" - ], - "markers": "python_version >= '3.9'", - "version": "==2.10.1" - }, - "pyparsing": { - "hashes": [ - "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", - "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.4.7" - }, - "requests": { - "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" - ], - "index": "pypi", - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.21.0" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", - "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.0.0" - }, - "rsa": { - "hashes": [ - "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", - "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75" - ], - "markers": "python_version >= '3.6' and python_version < '4'", - "version": "==4.9.1" - }, - "six": { - "hashes": [ - "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", - "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.17.0" - }, - "uritemplate": { - "hashes": [ - "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", - "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686" - ], - "markers": "python_version >= '3.9'", - "version": "==4.2.0" - }, - "urllib3": { - "hashes": [ - "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", - "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", - "version": "==1.24.3" - }, - "werkzeug": { - "hashes": [ - "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f", - "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.2.2" - } - } -} diff --git a/src/appengine/handlers/auth.py b/src/appengine/handlers/auth.py index fa9f9755d27..a3a98f99d45 100644 --- a/src/appengine/handlers/auth.py +++ b/src/appengine/handlers/auth.py @@ -29,7 +29,7 @@ class Handler(base_handler.Handler): """Auth page.""" @handler.get(handler.HTML) - def get(self, extra=None): + def get(self, extra: str | None = None) -> Response: """Handle a get request.""" # We use `request.url` which is already the full URL. del extra diff --git a/src/appengine/handlers/base_handler.py b/src/appengine/handlers/base_handler.py index dc69a51e5aa..89f7f19e02e 100644 --- a/src/appengine/handlers/base_handler.py +++ b/src/appengine/handlers/base_handler.py @@ -14,7 +14,7 @@ """The superclass of all handlers.""" import base64 -import cgi +import cgi # pylint: disable=deprecated-module import datetime import json import logging @@ -22,6 +22,10 @@ import re import sys import traceback +from typing import Any +from typing import Callable +from typing import cast +from typing import Optional import urllib.parse from flask import redirect as flask_redirect @@ -31,6 +35,7 @@ from google.cloud import ndb import jinja2 import jira +import jira.resources from clusterfuzz._internal.base import utils from clusterfuzz._internal.config import db_config @@ -44,19 +49,19 @@ # Pattern from # https://github.com/google/closure-library/blob/ # 3037e09cc471bfe99cb8f0ee22d9366583a20c28/closure/goog/html/safeurl.js -_SAFE_URL_PATTERN = re.compile( +_SAFE_URL_PATTERN: re.Pattern = re.compile( r'^(?:(?:https?|mailto|ftp):|[^:/?#]*(?:[/?#]|$))', flags=re.IGNORECASE) -def add_jinja2_filter(name, fn): +def add_jinja2_filter(name: str, fn: Callable[..., Any]) -> None: _JINJA_ENVIRONMENT.filters[name] = fn class JsonEncoder(json.JSONEncoder): """Json encoder.""" - _EPOCH = datetime.datetime.utcfromtimestamp(0) + _EPOCH: datetime.datetime = datetime.datetime.utcfromtimestamp(0) - def default(self, o): # pylint: disable=arguments-differ,method-hidden + def default(self, o: Any) -> Any: # pylint: disable=arguments-differ,method-hidden if isinstance(o, ndb.Model): dict_obj = o.to_dict() dict_obj['id'] = o.key.id() @@ -76,32 +81,32 @@ def default(self, o): # pylint: disable=arguments-differ,method-hidden return json.JSONEncoder.default(self, o) -def format_time(dt): +def format_time(dt: datetime.datetime) -> str: """Format datetime object for display.""" return '{t.day} {t:%b} {t:%y} {t:%X} PDT'.format(t=dt) -def splitlines(text): +def splitlines(text: str) -> list[str]: """Split text into lines.""" return text.splitlines() -def split_br(text): +def split_br(text: str) -> list[str]: return re.split(r'\s*
\s*', text, flags=re.IGNORECASE) -def encode_json(value): +def encode_json(value: Any) -> str: """Dump base64-encoded JSON string (to avoid XSS).""" return base64.b64encode(json.dumps( value, cls=JsonEncoder).encode('utf-8')).decode('utf-8') -_JINJA_ENVIRONMENT = jinja2.Environment( +_JINJA_ENVIRONMENT: jinja2.Environment = jinja2.Environment( loader=jinja2.FileSystemLoader( os.path.join(os.path.dirname(__file__), '..', 'templates')), extensions=[], autoescape=True) -_MENU_ITEMS = [] +_MENU_ITEMS: list['_MenuItem'] = [] add_jinja2_filter('json', encode_json) add_jinja2_filter('format_time', format_time) @@ -110,17 +115,17 @@ def encode_json(value): add_jinja2_filter('polymer_tag', lambda v: '{{%s}}' % v) -def add_menu(name, href): +def add_menu(name: str, href: str) -> None: """Add menu item to the main navigation.""" _MENU_ITEMS.append(_MenuItem(name, href)) -def make_login_url(dest_url): +def make_login_url(dest_url: str) -> str: """Make the switch account url.""" return '/login?' + urllib.parse.urlencode({'dest': dest_url}) -def make_logout_url(dest_url): +def make_logout_url(dest_url: str) -> str: """Make the switch account url.""" return '/logout?' + urllib.parse.urlencode({ 'csrf_token': form.generate_csrf_token(), @@ -128,7 +133,7 @@ def make_logout_url(dest_url): }) -def check_redirect_url(url): +def check_redirect_url(url: str) -> None: """Check redirect URL is safe.""" if not _SAFE_URL_PATTERN.match(url): raise helpers.EarlyExitError('Invalid redirect.', 403) @@ -137,23 +142,24 @@ def check_redirect_url(url): class _MenuItem: """A menu item used for rendering an item in the main navigation.""" - def __init__(self, name, href): + def __init__(self, name: str, href: str) -> None: self.name = name self.href = href class Handler(MethodView): """A superclass for all handlers. It contains many convenient methods.""" + is_json: bool = False - def is_cron(self): + def is_cron(self) -> bool: """Return true if the request is from a cron job.""" return bool(request.headers.get('X-Appengine-Cron')) - def should_render_json(self): + def should_render_json(self) -> bool: return (self.is_json or 'application/json' in request.headers.get('accept', '')) - def render_forbidden(self, message): + def render_forbidden(self, message: str) -> Response: """Write HTML response for 403.""" login_url = make_login_url(dest_url=request.url) user_email = helpers.get_user_email() @@ -171,7 +177,7 @@ def render_forbidden(self, message): } return self.render('error-403.html', template_values, 403) - def _add_security_response_headers(self, response): + def _add_security_response_headers(self, response: Response) -> Response: """Add security-related headers to response.""" response.headers['Strict-Transport-Security'] = ( 'max-age=2592000; includeSubdomains') @@ -179,7 +185,10 @@ def _add_security_response_headers(self, response): response.headers['X-Frame-Options'] = 'deny' return response - def render(self, path, values=None, status=200): + def render(self, + path: str, + values: Optional[dict[str, Any]] = None, + status: int = 200) -> Response: """Write HTML response.""" if values is None: values = {} @@ -210,10 +219,10 @@ def render(self, path, values=None, status=200): return response # pylint: disable=unused-argument - def before_render_json(self, values, status): + def before_render_json(self, values: Any, status: int) -> None: """A hook for modifying values before render_json.""" - def render_json(self, values, status=200): + def render_json(self, values: Any, status: int = 200) -> Response: """Write JSON response.""" response = Response() response = self._add_security_response_headers(response) @@ -223,7 +232,7 @@ def render_json(self, values, status=200): response.status_code = status return response - def handle_exception(self, exception): + def handle_exception(self, exception: Exception) -> Optional[Response]: """Catch exception and format it properly.""" try: status = 500 @@ -256,7 +265,7 @@ def handle_exception(self, exception): return None - def handle_exception_exception(self): + def handle_exception_exception(self) -> Response: """Catch exception in handle_exception and format it properly.""" exception = sys.exc_info()[1] values = {'message': str(exception), 'traceDump': traceback.format_exc()} @@ -265,13 +274,13 @@ def handle_exception_exception(self): return self.render_json(values, 500) return self.render('error.html', values, 500) - def redirect(self, url, **kwargs): + def redirect(self, url: str, **kwargs: Any) -> Response: """Check vaid url and redirect to it, if valid.""" url = str(url) check_redirect_url(url) - return flask_redirect(url, **kwargs) + return cast(Response, flask_redirect(url, **kwargs)) - def dispatch_request(self, *args, **kwargs): + def dispatch_request(self, *args: Any, **kwargs: Any) -> Any: """Dispatch a request and postprocess.""" self.is_json = False try: @@ -282,18 +291,19 @@ def dispatch_request(self, *args, **kwargs): class GcsUploadHandler(Handler): """A handler which uploads files to GCS.""" + upload: Optional[storage.GcsBlobInfo] = None - def dispatch_request(self, *args, **kwargs): + def dispatch_request(self, *args: Any, **kwargs: Any) -> Any: """Dispatch a request and postprocess.""" self.upload = None return super().dispatch_request(*args, **kwargs) - def get_upload(self): + def get_upload(self) -> Optional[storage.GcsBlobInfo]: """Get uploads.""" if self.upload: return self.upload - upload_key = request.get('upload_key') + upload_key = cast(Any, request).get('upload_key') if not upload_key: return None diff --git a/src/appengine/handlers/bots.py b/src/appengine/handlers/bots.py index f41d78ce80a..9e7fd02a1bc 100644 --- a/src/appengine/handlers/bots.py +++ b/src/appengine/handlers/bots.py @@ -14,6 +14,9 @@ """Handlers for the bot list.""" import datetime +from typing import Any +from typing import cast +from typing import Sequence from flask import request @@ -27,15 +30,15 @@ from libs import helpers from libs.query import datastore_query -PAGE_SIZE = 10 -MORE_LIMIT = 50 - PAGE_SIZE # exactly 5 pages +PAGE_SIZE: int = 10 +MORE_LIMIT: int = 50 - PAGE_SIZE # exactly 5 pages -FILTERS = [ +FILTERS: list[filters.Filter] = [ filters.Keyword([], 'keywords', 'q'), ] -def _get_alive_cutoff(): +def _get_alive_cutoff() -> datetime.datetime: """Get the time before which we consider bots to be dead.""" seconds_to_wait_for_dead_bot = ( tasks.TASK_LEASE_SECONDS + tasks.TASK_COMPLETION_BUFFER + @@ -45,11 +48,13 @@ def _get_alive_cutoff(): return alive_cutoff -def _convert_heartbeats_to_dicts(heartbeats): +def _convert_heartbeats_to_dicts( + heartbeats: Sequence[data_types.Heartbeat]) -> list[dict[str, Any]]: """Format heartbeats for template.""" alive_cutoff = _get_alive_cutoff() - result = [] + result: list[dict[str, Any]] = [] for heartbeat in heartbeats: + last_beat_time = cast(datetime.datetime, heartbeat.last_beat_time) result.append({ 'bot_name': heartbeat.bot_name, @@ -66,27 +71,28 @@ def _convert_heartbeats_to_dicts(heartbeats): utils.utc_datetime_to_timestamp(heartbeat.last_beat_time) if heartbeat.last_beat_time else '', 'alive': - 'alive' if heartbeat.last_beat_time > alive_cutoff else 'dead' + 'alive' if last_beat_time > alive_cutoff else 'dead' }) return result -def get_results(): +def get_results() -> tuple[dict[str, Any], dict[str, Any]]: """Get results for the bots page.""" # Return bots sorted alphabetically by bot_name query = datastore_query.Query(data_types.Heartbeat) query.order('bot_name', is_desc=False) - params = dict(request.iterparams()) + request_any = cast(Any, request) + params = dict(request_any.iterparams()) filters.add(query, params, FILTERS) - page = helpers.cast(request.get('page', 1), int, "'page' is not an int.") + page = helpers.cast(request_any.get('page', 1), int, "'page' is not an int.") items, total_pages, total_items, has_more = query.fetch_page( page=page, page_size=PAGE_SIZE, projection=None, more_limit=MORE_LIMIT) items = _convert_heartbeats_to_dicts(items) helpers.log('Bots', helpers.VIEW_OPERATION) - result = { + result: dict[str, Any] = { 'hasMore': has_more, 'items': items, 'page': page, @@ -104,7 +110,7 @@ class Handler(base_handler.Handler): @handler.oauth @handler.check_admin_access_if_oss_fuzz @handler.check_user_access(need_privileged_access=False) - def get(self): + def get(self) -> base_handler.Response: """Render the bot list HTML.""" result, params = get_results() return self.render('bots.html', { @@ -119,7 +125,7 @@ class JsonHandler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.check_admin_access_if_oss_fuzz @handler.check_user_access(need_privileged_access=False) - def post(self): + def post(self) -> base_handler.Response: """Get and render the bots in JSON.""" result, _ = get_results() return self.render_json(result) @@ -129,7 +135,7 @@ class DeadBotsHandler(base_handler.Handler): """Output dead bots as json.""" @handler.get(handler.JSON) - def get(self): + def get(self) -> base_handler.Response: """Render dead bots as json (used by automated scripts).""" # This a publicly exposed chromium-specific page. @@ -138,10 +144,12 @@ def get(self): else: raise helpers.EarlyExitError('Dead bots list unavailable.', 400) - result = {} + result: dict[str, str] = {} alive_cutoff = _get_alive_cutoff() for heartbeat in heartbeats: - if heartbeat.last_beat_time <= alive_cutoff: - result[heartbeat.bot_name] = 'dead' + last_beat_time = cast(datetime.datetime, heartbeat.last_beat_time) + bot_name = cast(str, heartbeat.bot_name) + if last_beat_time <= alive_cutoff: + result[bot_name] = 'dead' return self.render_json(result) diff --git a/src/appengine/handlers/commit_range.py b/src/appengine/handlers/commit_range.py index a0e771770a9..98ecdde2ca1 100644 --- a/src/appengine/handlers/commit_range.py +++ b/src/appengine/handlers/commit_range.py @@ -14,8 +14,12 @@ """Handler for the regression page.""" import json +from typing import Any +from typing import cast +from typing import Mapping from flask import request +from flask import Response from clusterfuzz._internal.google_cloud_utils import big_query from handlers import base_handler @@ -63,7 +67,8 @@ class RevisionFilter(filters.Filter): """Filter for revision.""" - def add(self, query, params): + def add(self, query: big_query_query.Query, + params: Mapping[str, Any]) -> None: """Set query according to revision and type params.""" if not params.get('revision'): raise helpers.EarlyExitError('Please specify the revision.', 400) @@ -79,7 +84,8 @@ def add(self, query, params): class KeywordFilter(filters.Filter): """Filter for keyword.""" - def add(self, query, params): + def add(self, query: big_query_query.Query, + params: Mapping[str, Any]) -> None: """Set query according to search param.""" value = params.get('q', '') if filters.is_empty(value): @@ -95,7 +101,8 @@ def add(self, query, params): class IncludeZeroFilter(filters.Filter): """Filter for including a range starting with 0.""" - def add(self, query, params): + def add(self, query: big_query_query.Query, + params: Mapping[str, Any]) -> None: """Set query based on if range should include 0 or not.""" value = params.get('includeZero', False) prefix = params['type'] @@ -107,8 +114,10 @@ def add(self, query, params): class TypeFilter(filters.Filter): """Filter for type.""" - def add(self, query, params): # pylint: disable=unused-argument + def add(self, query: big_query_query.Query, + params: Mapping[str, Any]) -> None: """Validate type param is either regression or fixed.""" + # pylint: disable=unused-argument value = params.get('type', 'regression') if value not in ['fixed', 'regression']: @@ -116,7 +125,7 @@ def add(self, query, params): # pylint: disable=unused-argument "'type' can only be either 'fixed' or 'regression'.", 400) -FILTERS = [ +FILTERS: list[filters.Filter] = [ TypeFilter(), RevisionFilter(), KeywordFilter(), @@ -124,7 +133,8 @@ def add(self, query, params): # pylint: disable=unused-argument ] -def get(params, query, offset, limit): +def get(params: Mapping[str, Any], query: big_query_query.Query, offset: int, + limit: int) -> tuple[list[dict[str, Any]], int]: """Get the data from BigQuery.""" sql = SQL.format( table_id='%ss' % params['type'], @@ -137,11 +147,13 @@ def get(params, query, offset, limit): return result.rows, result.total_count -def get_result(): +def get_result() -> tuple[dict[str, Any], dict[str, Any]]: """Get the result for the crash stats page.""" - params = dict(request.iterparams()) + request_any = cast(Any, request) + params: dict[str, Any] = dict(request_any.iterparams()) params['type'] = params.get('type', 'regression') - page = helpers.cast(request.get('page') or 1, int, "'page' is not an int.") + page = helpers.cast( + request_any.get('page') or 1, int, "'page' is not an int.") is_revision_empty = 'revision' not in params @@ -151,7 +163,7 @@ def get_result(): if is_revision_empty: total_count = 0 - rows = [] + rows: list[dict[str, Any]] = [] else: filters.add(query, params, FILTERS) rows, total_count = get( @@ -161,7 +173,7 @@ def get_result(): limit=PAGE_SIZE) helpers.log('Regression', helpers.VIEW_OPERATION) - result = { + result: dict[str, Any] = { 'totalPages': (total_count // PAGE_SIZE) + 1, 'page': page, 'pageSize': PAGE_SIZE, @@ -178,7 +190,7 @@ class Handler(base_handler.Handler): @handler.unsupported_on_local_server @handler.get(handler.HTML) @handler.oauth - def get(self): + def get(self) -> Response: """Get and render the commit range in HTML.""" result, params = get_result() return self.render('commit_range.html', { @@ -194,13 +206,13 @@ class JsonHandler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.oauth @handler.allowed_cors - def post(self): + def post(self) -> Response: """Get and render the commit range in JSON.""" result, params = get_result() result['params'] = params return self.render_json(result) @handler.allowed_cors - def options(self): + def options(self) -> str: """Responds with CORS headers.""" return '' diff --git a/src/appengine/handlers/configuration.py b/src/appengine/handlers/configuration.py index 3a6b2046cee..1afa8c69a21 100644 --- a/src/appengine/handlers/configuration.py +++ b/src/appengine/handlers/configuration.py @@ -13,6 +13,9 @@ # limitations under the License. """Configuration Manager.""" +from typing import Any +from typing import cast + from flask import request from clusterfuzz._internal.base import utils @@ -23,7 +26,7 @@ from libs import handler from libs import helpers -USER_PERMISSION_ENTITY_KINDS = [ +USER_PERMISSION_ENTITY_KINDS: list[dict[str, Any]] = [ { 'name': 'fuzzer', 'value': data_types.PermissionEntityKind.FUZZER, @@ -38,7 +41,7 @@ }, ] -USER_PERMISSION_AUTO_CC_TYPES = [ +USER_PERMISSION_AUTO_CC_TYPES: list[dict[str, Any]] = [ { 'name': 'none', 'value': data_types.AutoCCType.NONE, @@ -54,7 +57,7 @@ ] -def get_value_by_name(item_list, name): +def get_value_by_name(item_list: list[dict[str, Any]], name: str) -> Any: """Return value for entry whose name matches the one in item list.""" for item in item_list: if item['name'] == name: @@ -68,7 +71,7 @@ class Handler(base_handler.Handler): @handler.get(handler.HTML) @handler.check_admin_access - def get(self): + def get(self) -> base_handler.Response: """Handle a get request.""" external_user_permissions = list( data_types.ExternalUserPermission.query().order( @@ -94,44 +97,46 @@ def get(self): @handler.post(handler.FORM, handler.HTML) @handler.check_admin_access @handler.require_csrf_token - def post(self): + def post(self) -> base_handler.Response: """Handle a post request.""" config = db_config.get() if not config: config = data_types.Config() - previous_hash = request.get('previous_hash') + request_any = cast(Any, request) + previous_hash = request_any.get('previous_hash') if config.previous_hash and config.previous_hash != previous_hash: raise helpers.EarlyExitError( 'Your change conflicts with another configuration update. ' 'Please refresh and try again.', 500) - build_apiary_service_account_private_key = request.get( + build_apiary_service_account_private_key = request_any.get( 'build_apiary_service_account_private_key') - bug_report_url = request.get('bug_report_url') - client_credentials = request.get('client_credentials') - jira_url = request.get('jira_url') - jira_credentials = request.get('jira_credentials') - component_repository_mappings = request.get('component_repository_mappings') - contact_string = request.get('contact_string') - documentation_url = request.get('documentation_url') - github_credentials = request.get('github_credentials') - oss_fuzz_robot_github_personal_access_token = request.get( + bug_report_url = request_any.get('bug_report_url') + client_credentials = request_any.get('client_credentials') + jira_url = request_any.get('jira_url') + jira_credentials = request_any.get('jira_credentials') + component_repository_mappings = request_any.get( + 'component_repository_mappings') + contact_string = request_any.get('contact_string') + documentation_url = request_any.get('documentation_url') + github_credentials = request_any.get('github_credentials') + oss_fuzz_robot_github_personal_access_token = request_any.get( 'oss_fuzz_robot_github_personal_access_token') - platform_group_mappings = request.get('platform_group_mappings') - privileged_users = request.get('privileged_users') - privileged_groups = request.get('privileged_groups') - blacklisted_users = request.get('blacklisted_users') - relax_security_bug_restrictions = request.get( + platform_group_mappings = request_any.get('platform_group_mappings') + privileged_users = request_any.get('privileged_users') + privileged_groups = request_any.get('privileged_groups') + blacklisted_users = request_any.get('blacklisted_users') + relax_security_bug_restrictions = request_any.get( 'relax_security_bug_restrictions') - relax_testcase_restrictions = request.get('relax_testcase_restrictions') - reproduction_help_url = request.get('reproduction_help_url') - test_account_email = request.get('test_account_email') - test_account_password = request.get('test_account_password') - wifi_ssid = request.get('wifi_ssid') - wifi_password = request.get('wifi_password') - sendgrid_api_key = request.get('sendgrid_api_key') - sendgrid_sender = request.get('sendgrid_sender') + relax_testcase_restrictions = request_any.get('relax_testcase_restrictions') + reproduction_help_url = request_any.get('reproduction_help_url') + test_account_email = request_any.get('test_account_email') + test_account_password = request_any.get('test_account_password') + wifi_ssid = request_any.get('wifi_ssid') + wifi_password = request_any.get('wifi_password') + sendgrid_api_key = request_any.get('sendgrid_api_key') + sendgrid_sender = request_any.get('sendgrid_sender') config.build_apiary_service_account_private_key = ( build_apiary_service_account_private_key) @@ -186,13 +191,14 @@ class AddExternalUserPermission(base_handler.Handler): @handler.post(handler.FORM, handler.HTML) @handler.check_admin_access @handler.require_csrf_token - def post(self): + def post(self) -> base_handler.Response: """Handle a post request.""" - email = utils.normalize_email(request.get('email')) - entity_kind = request.get('entity_kind') - entity_name = request.get('entity_name') - is_prefix = request.get('is_prefix') - auto_cc = request.get('auto_cc') + request_any = cast(Any, request) + email = utils.normalize_email(request_any.get('email')) + entity_kind = request_any.get('entity_kind') + entity_name = request_any.get('entity_name') + is_prefix = request_any.get('is_prefix') + auto_cc = request_any.get('auto_cc') if not email: raise helpers.EarlyExitError('No email provided.', 400) @@ -254,11 +260,12 @@ class DeleteExternalUserPermission(base_handler.Handler): @handler.post(handler.FORM, handler.HTML) @handler.check_admin_access @handler.require_csrf_token - def post(self): + def post(self) -> base_handler.Response: """Handle a post request.""" - email = request.get('email') - entity_kind = request.get('entity_kind') - entity_name = request.get('entity_name') + request_any = cast(Any, request) + email = request_any.get('email') + entity_kind = request_any.get('entity_kind') + entity_name = request_any.get('entity_name') if not email: raise helpers.EarlyExitError('No email provided.', 400) diff --git a/src/appengine/handlers/corpora.py b/src/appengine/handlers/corpora.py index 14bd2f1b131..8e62d8944ab 100644 --- a/src/appengine/handlers/corpora.py +++ b/src/appengine/handlers/corpora.py @@ -13,6 +13,9 @@ # limitations under the License. """Manage corpora.""" +from typing import Any +from typing import cast + from flask import request from google.cloud import ndb @@ -32,7 +35,7 @@ class Handler(base_handler.Handler): @handler.oauth @handler.check_admin_access_if_oss_fuzz @handler.check_user_access(need_privileged_access=False) - def get(self): + def get(self) -> base_handler.Response: """Handle a get request.""" data_bundles = list(data_types.DataBundle.query().order( data_types.DataBundle.name)) @@ -54,9 +57,9 @@ class CreateHandler(base_handler.Handler): @handler.post(handler.FORM, handler.HTML) @handler.check_user_access(need_privileged_access=True) @handler.require_csrf_token - def post(self): + def post(self) -> base_handler.Response: """Handle a post request.""" - name = request.get('name') + name = cast(Any, request).get('name') if not name: raise helpers.EarlyExitError('Please give this corpus a name!', 400) @@ -100,7 +103,7 @@ class DeleteHandler(base_handler.Handler): @handler.post(handler.FORM, handler.HTML) @handler.check_user_access(need_privileged_access=True) @handler.require_csrf_token - def post(self): + def post(self) -> base_handler.Response: """Handle a post request.""" key = helpers.get_integer_key(request) diff --git a/src/appengine/handlers/coverage_report.py b/src/appengine/handlers/coverage_report.py index 7c67cf55935..91fdbbad93e 100644 --- a/src/appengine/handlers/coverage_report.py +++ b/src/appengine/handlers/coverage_report.py @@ -26,28 +26,30 @@ VALID_DATE_REGEX = re.compile(r'^([0-9\-]+|latest)$') -def _get_project_report_url(job, date): +def _get_project_report_url(job: str, date: str) -> str | None: """Return url for the report requested.""" project = data_handler.get_project_name(job) if not project: return None + coverage_date: datetime.date | None if date == 'latest': - date = None + coverage_date = None else: try: - date = datetime.datetime.strptime(date, '%Y-%m-%d').date() + coverage_date = datetime.datetime.strptime(date, '%Y-%m-%d').date() except: raise helpers.EarlyExitError('Invalid date.', 400) - info = fuzzer_stats.get_coverage_info(project, date) + info = fuzzer_stats.get_coverage_info(project, coverage_date) if not info: return None return info.html_report_url -def get_report_url(report_type, argument, date): +def get_report_url(report_type: str | None, argument: str | None, + date: str | None) -> str | None: """Get report url for a redirect from the coverage report handler.""" # It's very easy to add support for per fuzzer reports, but we don't need it. if report_type != 'job': @@ -72,7 +74,11 @@ class Handler(base_handler.Handler): # pylint: disable=unused-argument @handler.get(handler.HTML) @handler.oauth - def get(self, report_type=None, argument=None, date=None, extra=None): + def get(self, + report_type: str | None = None, + argument: str | None = None, + date: str | None = None, + extra: str | None = None) -> base_handler.Response: """Handle a get request.""" report_url = get_report_url(report_type, argument, date) if report_url: diff --git a/src/appengine/handlers/crash_query.py b/src/appengine/handlers/crash_query.py index 7427ab644df..75f54ea1d3c 100644 --- a/src/appengine/handlers/crash_query.py +++ b/src/appengine/handlers/crash_query.py @@ -13,7 +13,11 @@ # limitations under the License. """Handler for the crash query api.""" +from typing import Any +from typing import cast + from flask import request +from flask import Response from clusterfuzz._internal.crash_analysis import crash_analyzer from clusterfuzz._internal.crash_analysis.stack_parsing import stack_analyzer @@ -30,14 +34,14 @@ class Handler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.oauth - def post(self): + def post(self) -> Response: """Handle a post request.""" if not auth.get_current_user(): raise helpers.AccessDeniedError() - project = request.get('project') - fuzz_target = request.get('fuzz_target') - stacktrace = request.get('stacktrace') + project = cast(Any, request).get('project') + fuzz_target = cast(Any, request).get('fuzz_target') + stacktrace = cast(Any, request).get('stacktrace') state = stack_analyzer.get_crash_data( stacktrace, diff --git a/src/appengine/handlers/crash_stats.py b/src/appengine/handlers/crash_stats.py index ab3d51b1d25..c88a63555a4 100644 --- a/src/appengine/handlers/crash_stats.py +++ b/src/appengine/handlers/crash_stats.py @@ -14,6 +14,9 @@ """Handler for the crash stats page.""" import json +from typing import Any +from typing import cast +from typing import Mapping from flask import request @@ -28,15 +31,15 @@ from libs import handler from libs import helpers -PAGE_SIZE = 10 -DEFAULT_DAYS_FOR_BY_HOURS = 3 -DEFAULT_DAYS_FOR_BY_DAYS = 7 +PAGE_SIZE: int = 10 +DEFAULT_DAYS_FOR_BY_HOURS: int = 3 +DEFAULT_DAYS_FOR_BY_DAYS: int = 7 class FuzzerFilter(filters.Filter): """Filter for fuzzer.""" - def add(self, query, params): + def add(self, query: Any, params: Mapping[str, Any]) -> None: """Set query according to fuzzer param.""" value = params.get('fuzzer', '') if filters.is_empty(value): @@ -51,7 +54,7 @@ def add(self, query, params): class PlatformFilter(filters.Filter): """Filter for platform.""" - def add(self, query, params): + def add(self, query: Any, params: Mapping[str, Any]) -> None: """Set query according to platform param.""" value = params.get('platform', '') if filters.is_empty(value): @@ -66,7 +69,7 @@ def add(self, query, params): class TimeFilter(filters.Filter): """Filter for start and end hour.""" - def add(self, query, params): + def add(self, query: Any, params: Mapping[str, Any]) -> None: """Set query according to end and hours params.""" if 'end' in params: end = helpers.cast(params['end'], int, "'end' must be an integer") @@ -82,9 +85,10 @@ def add(self, query, params): DEFAULT_DAYS_FOR_BY_HOURS if block == 'hour' else DEFAULT_DAYS_FOR_BY_DAYS) - params['end'] = str(end) - params['days'] = str(days) - params['block'] = str(block) + params_dict = cast(dict[str, Any], params) + params_dict['end'] = str(end) + params_dict['days'] = str(days) + params_dict['block'] = str(block) query.set_time_params(end, days, block) @@ -92,7 +96,7 @@ def add(self, query, params): class KeywordFilter(filters.Filter): """Filter for keyword.""" - def add(self, query, params): + def add(self, query: Any, params: Mapping[str, Any]) -> None: """Set query according to search param.""" value = params.get('q', '') if filters.is_empty(value): @@ -105,11 +109,11 @@ def add(self, query, params): json.dumps('%%%s%%' % keyword.lower()))) -GROUP_FILTERS = [ +GROUP_FILTERS: list[filters.Filter] = [ filters.Boolean('is_new', 'new'), ] -FILTERS = [ +FILTERS: list[filters.Filter] = [ TimeFilter(), filters.Boolean('security_flag', 'security'), filters.Boolean('reproducible_flag', 'reproducible'), @@ -121,8 +125,8 @@ def add(self, query, params): ] -def query_testcase(project_name, crash_type, crash_state, security_flag, - is_open): +def query_testcase(project_name: str, crash_type: str, crash_state: str, + security_flag: bool, is_open: bool) -> Any: """Start a query for an associated testcase.""" return data_types.Testcase.query( data_types.Testcase.project_name == project_name, @@ -138,9 +142,9 @@ def query_testcase(project_name, crash_type, crash_state, security_flag, ]) -def attach_testcases(rows): +def attach_testcases(rows: list[dict[str, Any]]) -> None: """Attach testcase to each crash.""" - testcases = {} + testcases: dict[int, dict[str, Any]] = {} for index, row in enumerate(rows): testcases[index] = { 'open_testcase': @@ -171,10 +175,12 @@ def attach_testcases(rows): row['testcase'] = testcase -def get_result(): +def get_result() -> tuple[dict[str, Any], dict[str, Any]]: """Get the result for the crash stats page.""" - params = dict(request.iterparams()) - page = helpers.cast(request.get('page') or 1, int, "'page' is not an int.") + request_any = cast(Any, request) + params: dict[str, Any] = dict(request_any.iterparams()) + page = helpers.cast( + request_any.get('page') or 1, int, "'page' is not an int.") group_by = params.get('group', 'platform') params['group'] = group_by sort_by = params.get('sort', 'total_count') @@ -216,7 +222,7 @@ def get_result(): return result, params -def get_all_platforms(): +def get_all_platforms() -> list[str]: """Get all platforms including parent platform.""" items = data_types.Testcase.query( projection=[data_types.Testcase.platform], distinct=True) @@ -233,7 +239,7 @@ class Handler(base_handler.Handler): @handler.get(handler.HTML) @handler.unsupported_on_local_server @handler.oauth - def get(self): + def get(self) -> base_handler.Response: """Get and render the crash stats in HTML.""" result, params = get_result() field_values = { @@ -262,7 +268,7 @@ class JsonHandler(base_handler.Handler): """Handler that gets the crash stats when user interacts with the page.""" @handler.post(handler.JSON, handler.JSON) - def post(self): + def post(self) -> base_handler.Response: """Get and render the crash stats in JSON.""" result, _ = get_result() return self.render_json(result) diff --git a/src/appengine/handlers/cron/cleanup.py b/src/appengine/handlers/cron/cleanup.py index 4a1aeeac363..37aa9a5ffc7 100644 --- a/src/appengine/handlers/cron/cleanup.py +++ b/src/appengine/handlers/cron/cleanup.py @@ -22,5 +22,5 @@ class Handler(base_handler.Handler): """Cleanup.""" @handler.cron() - def get(self): + def get(self) -> None: cleanup.main() diff --git a/src/appengine/handlers/cron/predator_pull.py b/src/appengine/handlers/cron/predator_pull.py index 9e30b7927eb..dc54be410af 100644 --- a/src/appengine/handlers/cron/predator_pull.py +++ b/src/appengine/handlers/cron/predator_pull.py @@ -22,6 +22,6 @@ class Handler(base_handler.Handler): """Handler to periodically gather new results from Predator requests.""" @handler.cron() - def get(self): + def get(self) -> None: """Process a GET request.""" predator_pull.main() diff --git a/src/appengine/handlers/cron/triage.py b/src/appengine/handlers/cron/triage.py index 525a52740e0..457261f7c86 100644 --- a/src/appengine/handlers/cron/triage.py +++ b/src/appengine/handlers/cron/triage.py @@ -22,5 +22,5 @@ class Handler(base_handler.Handler): """Triage testcases.""" @handler.cron() - def get(self): + def get(self) -> None: triage.main() diff --git a/src/appengine/handlers/domain_verifier.py b/src/appengine/handlers/domain_verifier.py index 4ec4ca8791c..38c6c3d5ba6 100644 --- a/src/appengine/handlers/domain_verifier.py +++ b/src/appengine/handlers/domain_verifier.py @@ -13,6 +13,8 @@ # limitations under the License. """Domain verifier.""" +from typing import cast + from clusterfuzz._internal.config import local_config from handlers import base_handler from libs import helpers @@ -21,9 +23,9 @@ class Handler(base_handler.Handler): """Serve google.*.html domain verification file.""" - def get(self, tag=None): + def get(self, tag: str | None = None) -> str: """Handle a get request.""" - tag = 'google' + tag + '.html' + tag = 'google' + cast(str, tag) + '.html' verification_tag = local_config.GAEConfig().get('domain_verification_tag') if verification_tag != tag: raise helpers.EarlyExitError('Not found.', 404) diff --git a/src/appengine/handlers/download.py b/src/appengine/handlers/download.py index bf66f4c88f2..8a3459fa9e3 100644 --- a/src/appengine/handlers/download.py +++ b/src/appengine/handlers/download.py @@ -14,32 +14,37 @@ """Download files from GCS.""" import os +from typing import cast import urllib.parse from flask import request +from flask import Response from clusterfuzz._internal.base import dates from clusterfuzz._internal.base import errors from clusterfuzz._internal.base import utils from clusterfuzz._internal.datastore import data_handler +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.google_cloud_utils import blobs +from clusterfuzz._internal.google_cloud_utils import storage from clusterfuzz._internal.issue_management import issue_tracker_utils +from clusterfuzz._internal.issue_management.issue_tracker import Issue from handlers import base_handler from libs import access from libs import gcs from libs import helpers -_OSS_FUZZ_REPRODUCER_DELAY = 30 +_OSS_FUZZ_REPRODUCER_DELAY: int = 30 class Handler(base_handler.Handler, gcs.SignedGcsHandler): """Download a file from GCS.""" def _send_blob(self, - blob_info, - testcase_id, - is_minimized=False, - fuzzer_binary_name=None): + blob_info: storage.GcsBlobInfo, + testcase_id: str | int | None, + is_minimized: bool = False, + fuzzer_binary_name: str | None = None) -> Response: """Send the blob.""" minimized_string = 'minimized-' if is_minimized else '' fuzzer_binary_string = ( @@ -63,7 +68,9 @@ def _send_blob(self, response.headers['Content-disposition'] = content_disposition return response - def check_public_testcase(self, issue, blob_info, testcase): + def check_public_testcase(self, issue: Issue | None, + blob_info: storage.GcsBlobInfo, + testcase: data_types.Testcase) -> bool: """Check public testcase. |issue| is the already-fetched issue (or None).""" if blob_info.key() != testcase.minimized_keys: return False @@ -86,7 +93,9 @@ def check_public_testcase(self, issue, blob_info, testcase): return True - def check_derestricted_testcase(self, issue, blob_info, testcase): + def check_derestricted_testcase(self, issue: Issue | None, + blob_info: storage.GcsBlobInfo, + testcase: data_types.Testcase) -> bool: """Check if a testcase's associated bug has been derestricted (made public). For Chromium deployments, checks if the corresponding bug tracker issue has @@ -106,7 +115,7 @@ def check_derestricted_testcase(self, issue, blob_info, testcase): return issue.is_unrestricted - def get(self, resource=None): + def get(self, resource: str | None = None) -> Response: """Handle a get request with resource.""" testcase = None testcase_id = request.args.get('testcase_id') @@ -129,7 +138,7 @@ def get(self, resource=None): if testcase: fuzzer_binary_name = testcase.get_metadata('fuzzer_binary_name') - resource = str(urllib.parse.unquote(resource)) + resource = str(urllib.parse.unquote(cast(str, resource))) blob_info = blobs.get_blob_info(resource) if not blob_info: raise helpers.EarlyExitError('File does not exist.', 400) @@ -138,7 +147,8 @@ def get(self, resource=None): testcase.minimized_keys != blob_info.key()): raise helpers.EarlyExitError('Invalid testcase.', 400) - is_minimized = testcase and blob_info.key() == testcase.minimized_keys + is_minimized = cast(bool, testcase and + blob_info.key() == testcase.minimized_keys) if not testcase: # Non-testcase blob. General access is sufficient. @@ -161,7 +171,8 @@ def get(self, resource=None): if testcase.bug_information: issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase( testcase) - issue = issue_tracker.get_issue(testcase.bug_information) + if issue_tracker: + issue = issue_tracker.get_issue(testcase.bug_information) if utils.is_oss_fuzz() and self.check_public_testcase( issue, blob_info, testcase): diff --git a/src/appengine/handlers/external_update.py b/src/appengine/handlers/external_update.py index 22012ca32ba..93c5f8a64b9 100644 --- a/src/appengine/handlers/external_update.py +++ b/src/appengine/handlers/external_update.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """External reproduction updates.""" + import json from clusterfuzz._internal.crash_analysis import crash_analyzer @@ -19,16 +20,17 @@ from clusterfuzz._internal.crash_analysis.stack_parsing import stack_analyzer from clusterfuzz._internal.datastore import data_handler from clusterfuzz._internal.datastore import data_types +from clusterfuzz._internal.google_cloud_utils import pubsub from clusterfuzz._internal.metrics import logs from handlers import base_handler from libs import handler from libs import helpers -OLD_PROTOCOL = '1' # Old message format: Data is a stacktrace. -NEW_PROTOCOL = '2' # New message format: Data is a JSON array of stracktraces. +OLD_PROTOCOL: str = '1' # Old message format: Data is a stacktrace. +NEW_PROTOCOL: str = '2' # New format: Data is a JSON array of stacktraces. -def _mark_as_fixed(testcase, revision): +def _mark_as_fixed(testcase: data_types.Testcase, revision: int) -> None: """Mark bug as fixed.""" testcase.open = False # Bisection not available for external reproduction infrastructure. Assume @@ -38,7 +40,8 @@ def _mark_as_fixed(testcase, revision): testcase, revision, message=f'fixed in r{revision}') -def _mark_errored(testcase, revision, error): +def _mark_errored(testcase: data_types.Testcase, revision: int, + error: str) -> None: """Mark testcase as errored out.""" message = 'Received error from external infra, marking testcase as NA.' logs.warning(message, error=error, testcase_id=testcase.key.id()) @@ -49,10 +52,12 @@ def _mark_errored(testcase, revision, error): testcase, revision, message=message) -def handle_update(testcase, revision, stacktraces, error, protocol_version): +def handle_update(testcase: data_types.Testcase, revision: int, + stacktraces: list[str], error: str | None, + protocol_version: str) -> None: """Handle update.""" - def is_still_crashing(st_index, stacktrace): + def is_still_crashing(st_index: int, stacktrace: str) -> bool: """Check if the the given stackstrace indicates the testcase is still crashing""" state = stack_analyzer.get_crash_data( @@ -90,7 +95,7 @@ def is_still_crashing(st_index, stacktrace): last_tested_revision = ( testcase.get_metadata('last_tested_revision') or testcase.crash_revision) - if revision < last_tested_revision: + if last_tested_revision is not None and revision < int(last_tested_revision): logs.warning(f'Revision {revision} less than previously tested ' f'revision {last_tested_revision}.') return @@ -110,14 +115,10 @@ def is_still_crashing(st_index, stacktrace): fuzz_target = testcase.get_fuzz_target() if fuzz_target: fuzz_target_name = fuzz_target.binary + data_handler.record_fuzz_target(fuzz_target.engine, fuzz_target.binary, + testcase.job_type) else: fuzz_target_name = None - - # Record use of fuzz target to avoid garbage collection (since fuzz_task does - # not run). - data_handler.record_fuzz_target(fuzz_target.engine, fuzz_target.binary, - testcase.job_type) - for st_index, stacktrace in enumerate(stacktraces): if is_still_crashing(st_index, stacktrace): logs.info(f'stacktrace {st_index} of {testcase_id} still crashes.') @@ -136,17 +137,18 @@ class Handler(base_handler.Handler): """External reproduction update.""" @handler.pubsub_push - def post(self, message): + def post(self, message: pubsub.Message) -> str: """Handle a post request.""" - testcase_id = message.attributes.get('testcaseId') + attributes = message.attributes or {} + testcase_id = attributes.get('testcaseId') if not testcase_id: raise helpers.EarlyExitError('Missing testcaseId.', 400) - revision = message.attributes.get('revision') + revision = attributes.get('revision') if not revision or not revision.isdigit(): raise helpers.EarlyExitError('Missing revision.', 400) - revision = int(revision) + revision_int = int(revision) testcase = data_handler.get_testcase_by_id(testcase_id) job = data_types.Job.query(data_types.Job.name == testcase.job_type).get() if not job or not job.is_external(): @@ -158,7 +160,7 @@ def post(self, message): logs.info(f'No stacktrace provided (testcase_id={testcase_id}).') stacktrace = '' - protocol_version = message.attributes.get('protocolVersion', OLD_PROTOCOL) + protocol_version = attributes.get('protocolVersion', OLD_PROTOCOL) if protocol_version == OLD_PROTOCOL: # Old: stacktrace is a str. stacktraces = [stacktrace] @@ -173,6 +175,6 @@ def post(self, message): # Invalid: stacktrace is presumably ill-formed. stacktraces = [] - error = message.attributes.get('error') - handle_update(testcase, revision, stacktraces, error, protocol_version) + error = attributes.get('error') + handle_update(testcase, revision_int, stacktraces, error, protocol_version) return 'OK' diff --git a/src/appengine/handlers/fuzzer_stats.py b/src/appengine/handlers/fuzzer_stats.py index 6d619d60072..d02e0c0887f 100644 --- a/src/appengine/handlers/fuzzer_stats.py +++ b/src/appengine/handlers/fuzzer_stats.py @@ -16,11 +16,22 @@ import datetime import html import re +from typing import Any +from typing import Callable +from typing import cast +from typing import Dict +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Type +from typing import Union import urllib.parse from flask import request from googleapiclient.errors import HttpError import yaml +import yaml.parser from clusterfuzz._internal.base import external_users from clusterfuzz._internal.base import memoize @@ -38,17 +49,18 @@ # Old fuzzer stats don't change, we could cache forever and it would never go # stale. Since stats can get pretty large and probably aren't used much the day # after first accessed, use a TTL reflecting this. -MEMCACHE_OLD_TTL_IN_SECONDS = 24 * 60 * 60 +MEMCACHE_OLD_TTL_IN_SECONDS: int = 24 * 60 * 60 # New fuzzer stats change, and aren't as likely to be reaccessed, don't cache # for very long. -MEMCACHE_TODAY_TTL_IN_SECONDS = 30 * 60 +MEMCACHE_TODAY_TTL_IN_SECONDS: int = 30 * 60 class QueryField: """Wrapped fuzzer_stats.QueryField with extra metadata.""" - def __init__(self, field, results_index, field_type, bigquery_type): + def __init__(self, field: fuzzer_stats.QueryField, results_index: int, + field_type: str, bigquery_type: str) -> None: self.field = field self.results_index = results_index self.field_type = field_type @@ -58,12 +70,13 @@ def __init__(self, field, results_index, field_type, bigquery_type): class BuiltinField: """Wrapped fuzzer_stats.BuiltinField with extra metadata.""" - def __init__(self, spec, field): + def __init__(self, spec: fuzzer_stats.BuiltinFieldSpecifier, + field: fuzzer_stats.BuiltinField) -> None: self.spec = spec self.field = field -def _bigquery_type_to_charts_type(typename): +def _bigquery_type_to_charts_type(typename: str) -> str: """Convert bigquery type to charts type.""" typename = typename.lower() if typename in ('integer', 'float'): @@ -75,7 +88,7 @@ def _bigquery_type_to_charts_type(typename): return 'string' -def _python_type_to_charts_type(type_value): +def _python_type_to_charts_type(type_value: Optional[Type[Any]]) -> str: """Convert bigquery type to charts type.""" if type_value in (int, float): return 'number' @@ -86,7 +99,7 @@ def _python_type_to_charts_type(type_value): return 'string' -def _parse_date(date_str): +def _parse_date(date_str: Optional[str]) -> Optional[datetime.date]: """Parse YYYY-MM-DD.""" if not date_str: return None @@ -100,16 +113,21 @@ def _parse_date(date_str): return datetime.date(year, month, day) -def _parse_stats_column_fields(results, stats_columns, group_by, fuzzer, jobs): +def _parse_stats_column_fields( + results: Dict[str, Any], stats_columns: str, group_by: Optional[int], + fuzzer: str, + jobs: Optional[List[str]]) -> List[Union[QueryField, BuiltinField]]: """Parse stats columns.""" - result = [] + result: List[Union[QueryField, BuiltinField]] = [] columns = fuzzer_stats.parse_stats_column_fields(stats_columns) # Insert first column (group by) group_by_field_name = fuzzer_stats.group_by_to_field_name(group_by) - columns.insert(0, fuzzer_stats.QueryField('j', group_by_field_name, None)) + columns.insert( + 0, fuzzer_stats.QueryField('j', cast(str, group_by_field_name), None)) - contexts = {} + contexts: Dict[Type[fuzzer_stats.BuiltinFieldContext], + fuzzer_stats.BuiltinFieldContext] = {} for column in columns: if isinstance(column, fuzzer_stats.QueryField): @@ -132,14 +150,14 @@ def _parse_stats_column_fields(results, stats_columns, group_by, fuzzer, jobs): if not field_class: continue - context_class = field_class.CONTEXT_CLASS + context_class = cast(Any, field_class).CONTEXT_CLASS context = contexts.setdefault(context_class, context_class(fuzzer, jobs)) - result.append(BuiltinField(column, column.create(context))) + result.append(BuiltinField(column, cast(Any, column.create(context)))) return result -def _parse_group_by(group_by): +def _parse_group_by(group_by: Optional[str]) -> Optional[int]: """Parse group_by value.""" if group_by == 'by-day': return fuzzer_stats.QueryGroupBy.GROUP_BY_DAY @@ -155,16 +173,16 @@ def _parse_group_by(group_by): return None -def _get_fuzzer_or_engine(name): +def _get_fuzzer_or_engine(name: str) -> Optional[data_types.Fuzzer]: """Return fuzzer entity, or engine this target is part of.""" fuzz_target = data_handler.get_fuzz_target(name) if fuzz_target: - name = fuzz_target.engine + name = cast(str, fuzz_target.engine) return data_types.Fuzzer.query(data_types.Fuzzer.name == name).get() -def _do_bigquery_query(query): +def _do_bigquery_query(query: str) -> Dict[str, Any]: """Return results from BigQuery.""" logs.info(query) client = big_query.Client() @@ -180,13 +198,14 @@ def _do_bigquery_query(query): return results -def _parse_stats_column_descriptions(stats_column_descriptions): +def _parse_stats_column_descriptions( + stats_column_descriptions: Optional[str]) -> Dict[str, str]: """Parse stats column descriptions.""" if not stats_column_descriptions: return {} try: - result = yaml.safe_load(stats_column_descriptions) + result = cast(dict[str, Any], yaml.safe_load(stats_column_descriptions)) for key, value in result.items(): result[key] = html.escape(value) @@ -196,7 +215,8 @@ def _parse_stats_column_descriptions(stats_column_descriptions): return {} -def _build_columns(result, columns): +def _build_columns(result: Dict[str, Any], + columns: Sequence[Union[QueryField, BuiltinField]]) -> None: """Build columns.""" for column in columns: if isinstance(column, QueryField): @@ -211,7 +231,8 @@ def _build_columns(result, columns): }) -def _try_cast(cell, value_str, cast_function, default_value): +def _try_cast(cell: Dict[str, Any], value_str: Any, + cast_function: Callable[[Any], Any], default_value: Any) -> None: """Try casting the value_str into cast_function.""" try: cell['v'] = cast_function(value_str) @@ -221,13 +242,15 @@ def _try_cast(cell, value_str, cast_function, default_value): # FIXME: Break logic in this function into simpler helper functions. -def _build_rows(result, columns, rows, group_by): +def _build_rows(result: Dict[str, Any], + columns: Sequence[Union[QueryField, BuiltinField]], + rows: List[Dict[str, Any]], group_by: Optional[int]) -> None: """Build rows.""" for row in rows: - row_data = [] - first_column_value = None + row_data: List[Dict[str, Any]] = [] + first_column_value: Any = None for column in columns: - cell = {} + cell: Dict[str, Any] = {} if isinstance(column, QueryField): value = row['f'][column.results_index]['v'] @@ -253,7 +276,7 @@ def _build_rows(result, columns, rows, group_by): first_column_value = first_column_value or cell['v'] elif isinstance(column, BuiltinField): - data = column.field.get(group_by, first_column_value) + data = column.field.get(cast(int, group_by), first_column_value) if data: formatted_value = data.value if data.link: @@ -278,17 +301,18 @@ def _build_rows(result, columns, rows, group_by): result['rows'].append({'c': row_data}) -def _get_cloud_storage_link(bucket_path): +def _get_cloud_storage_link(bucket_path: str) -> str: """Return a clickable link to a cloud storage file given the bucket path.""" return '/gcs-redirect?' + urllib.parse.urlencode({'path': bucket_path}) -def _get_filter_from_job(job): +def _get_filter_from_job(job: Optional[str]) -> Optional[List[str]]: """Creates a job filter from |job|.""" return [str(job)] if job else None -def build_results(fuzzer, jobs, group_by, date_start, date_end): +def build_results(fuzzer: str, jobs: Optional[List[str]], group_by: str, + date_start: str, date_end: str) -> Dict[str, Any]: """Wrapper around the caching wrappers for _build_results. Decides which of those wrappers to call based on how long query should be cached for.""" datetime_end = _parse_date(date_end) @@ -308,25 +332,28 @@ def build_results(fuzzer, jobs, group_by, date_start, date_end): @memoize.wrap(memoize.Memcache(MEMCACHE_TODAY_TTL_IN_SECONDS)) -def _build_todays_results(fuzzer, jobs, group_by, date_start, date_end): +def _build_todays_results(fuzzer: str, jobs: Optional[List[str]], group_by: str, + date_start: str, date_end: str) -> Dict[str, Any]: """Wrapper around _build_results that is intended for use by queries where date_end is today. Caches results for 15 minutes.""" return _build_results(fuzzer, jobs, group_by, date_start, date_end) @memoize.wrap(memoize.Memcache(MEMCACHE_OLD_TTL_IN_SECONDS)) -def _build_old_results(fuzzer, jobs, group_by, date_start, date_end): +def _build_old_results(fuzzer: str, jobs: Optional[List[str]], group_by: str, + date_start: str, date_end: str) -> Dict[str, Any]: """Wrapper around _build_results that is intended for use by queries where date_end is before today. Caches results for 24 hours.""" return _build_results(fuzzer, jobs, group_by, date_start, date_end) -def _build_results(fuzzer, jobs, group_by, date_start, date_end): +def _build_results(fuzzer: str, jobs: Optional[List[str]], group_by: str, + date_start: str, date_end: str) -> Dict[str, Any]: """Build results.""" - date_start = _parse_date(date_start) - date_end = _parse_date(date_end) + date_start_parsed = _parse_date(date_start) + date_end_parsed = _parse_date(date_end) - if not fuzzer or not group_by or not date_start or not date_end: + if not fuzzer or not group_by or not date_start_parsed or not date_end_parsed: raise helpers.EarlyExitError('Missing params.', 400) fuzzer_entity = _get_fuzzer_or_engine(fuzzer) @@ -338,16 +365,17 @@ def _build_results(fuzzer, jobs, group_by, date_start, date_end): else: stats_columns = fuzzer_stats.JobQuery.DEFAULT_FIELDS - group_by = _parse_group_by(group_by) - if group_by is None: + group_by_parsed = _parse_group_by(group_by) + if group_by_parsed is None: raise helpers.EarlyExitError('Invalid grouping.', 400) - table_query = fuzzer_stats.TableQuery(fuzzer, jobs, stats_columns, group_by, - date_start, date_end) + table_query = fuzzer_stats.TableQuery(fuzzer, jobs, stats_columns, + group_by_parsed, date_start_parsed, + date_end_parsed) results = _do_bigquery_query(table_query.build()) - is_timeseries = group_by == fuzzer_stats.QueryGroupBy.GROUP_BY_TIME - result = { + is_timeseries = group_by_parsed == fuzzer_stats.QueryGroupBy.GROUP_BY_TIME + result: Dict[str, Any] = { 'cols': [], 'rows': [], 'column_descriptions': @@ -357,19 +385,20 @@ def _build_results(fuzzer, jobs, group_by, date_start, date_end): is_timeseries } - columns = _parse_stats_column_fields(results, stats_columns, group_by, fuzzer, - jobs) + columns: Sequence[Union[QueryField, BuiltinField]] = ( + _parse_stats_column_fields(results, stats_columns, group_by_parsed, + fuzzer, jobs)) # If we are grouping by time and plotting graphs, skip builtin columns. if is_timeseries: columns = [c for c in columns if not isinstance(c, BuiltinField)] _build_columns(result, columns) - _build_rows(result, columns, results['rows'], group_by) + _build_rows(result, columns, results['rows'], group_by_parsed) return result -def _get_date(date_value, days_ago): +def _get_date(date_value: Optional[str], days_ago: int) -> str: """Returns |date_value| if it is not empty otherwise returns the date |days_ago| number of days ago.""" if date_value: @@ -386,7 +415,7 @@ class Handler(base_handler.Handler): @handler.unsupported_on_local_server @handler.get(handler.HTML) @handler.oauth - def get(self, extra=None): + def get(self, extra: Any = None) -> base_handler.Response: """Handle a GET request.""" if not access.has_access(): # User is an external user of ClusterFuzz (eg: non-Chrome dev who @@ -406,9 +435,10 @@ class LoadFiltersHandler(base_handler.Handler): @handler.unsupported_on_local_server @handler.get(handler.HTML) - def get(self): + def get(self) -> base_handler.Response: """Handle a GET request.""" - project = request.get('project') + request_any = cast(Any, request) + project = request_any.get('project') if access.has_access(): # User is an internal user of ClusterFuzz (eg: ClusterFuzz developer). @@ -450,7 +480,8 @@ def get(self): class LoadHandler(base_handler.Handler): """Load handler.""" - def _check_user_access_and_get_job_filter(self, fuzzer, job): + def _check_user_access_and_get_job_filter( + self, fuzzer: str, job: Optional[str]) -> Optional[List[str]]: """Check whether the current user has access to stats for the fuzzer or job. Returns a job filter that should be applied to the query.""" access_by_fuzzer_or_job = access.has_access( @@ -471,18 +502,19 @@ def _check_user_access_and_get_job_filter(self, fuzzer, job): raise helpers.AccessDeniedError() @handler.post(handler.JSON, handler.JSON) - def post(self): + def post(self) -> base_handler.Response: """Handle a POST request.""" - fuzzer = request.get('fuzzer') - job = request.get('job') - group_by = request.get('group_by') + request_any = cast(Any, request) + fuzzer = request_any.get('fuzzer') + job = request_any.get('job') + group_by = request_any.get('group_by') # If date_start is "": because the front end defaults to using a # start_date 7 days ago, do the same. - date_start = _get_date(request.get('date_start'), 7) + date_start = _get_date(request_any.get('date_start'), 7) # If date_end is "": don't get today's stats as they may not be # available, use yesterdays (as the front end does by default). - date_end = _get_date(request.get('date_end'), 1) + date_end = _get_date(request_any.get('date_end'), 1) job_filter = self._check_user_access_and_get_job_filter(fuzzer, job) return self.render_json( @@ -493,14 +525,13 @@ class PreloadHandler(base_handler.Handler): """Handler for the infrequent task of loading results for expensive stats queries that are commonly accessed into the cache.""" - def _get_fuzzer_job_filters(self): + def _get_fuzzer_job_filters(self) -> List[Tuple[str, Optional[List[str]]]]: """Return list of fuzzer-job filter tuples.""" fuzzer_job_filters = [] for fuzzer_name in data_types.BUILTIN_FUZZERS: fuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == fuzzer_name).get() - - for job in fuzzer.jobs: + for job in cast(Any, fuzzer).jobs: fuzzer_job_filters.append((fuzzer_name, _get_filter_from_job(job))) # None job is explicitly added for fuzzer query across all jobs. @@ -509,7 +540,7 @@ def _get_fuzzer_job_filters(self): return fuzzer_job_filters @handler.cron() - def get(self): + def get(self) -> None: """Handle a GET request.""" date_start = _get_date(None, 7) date_end = _get_date(None, 1) @@ -538,7 +569,7 @@ class RefreshCacheHandler(base_handler.Handler): """Refresh cache.""" @handler.cron() - def get(self): + def get(self) -> None: """Handle a GET request.""" fuzzer_logs_context = fuzzer_stats.FuzzerRunLogsContext() fuzz_targets = data_handler.get_fuzz_targets() @@ -546,5 +577,5 @@ def get(self): # Cache child fuzzer -> logs bucket mappings. for fuzz_target in fuzz_targets: # pylint: disable=protected-access,unexpected-keyword-arg - fuzzer_logs_context._get_logs_bucket_from_fuzzer( + cast(Any, fuzzer_logs_context)._get_logs_bucket_from_fuzzer( fuzz_target.fully_qualified_name(), __memoize_force__=True) diff --git a/src/appengine/handlers/fuzzers.py b/src/appengine/handlers/fuzzers.py index 68b4ed4e8d5..460c3abe851 100644 --- a/src/appengine/handlers/fuzzers.py +++ b/src/appengine/handlers/fuzzers.py @@ -15,8 +15,12 @@ import datetime import io +from typing import Any +from typing import cast +from typing import Optional from flask import request +from flask import Response from google.cloud import ndb from clusterfuzz._internal.base import utils @@ -33,8 +37,8 @@ from libs import handler from libs import helpers -ARCHIVE_READ_SIZE_LIMIT = 16 * 1024 * 1024 -FUZZER_FIELDS_EXCLUDED_FROM_LOG = [ +ARCHIVE_READ_SIZE_LIMIT: int = 16 * 1024 * 1024 +FUZZER_FIELDS_EXCLUDED_FROM_LOG: list[str] = [ 'result', 'result_timestamp', 'console_output', 'return_code', 'sample_testcase', 'stats_columns', 'stats_column_descriptions' ] @@ -47,7 +51,7 @@ class Handler(base_handler.Handler): @handler.oauth @handler.check_admin_access_if_oss_fuzz @handler.check_user_access(need_privileged_access=False) - def get(self): + def get(self) -> Response: """Handle a get request.""" fuzzer_logs_bucket = fuzzer_logs.get_bucket() fuzzers = list(data_types.Fuzzer.query().order(data_types.Fuzzer.name)) @@ -81,7 +85,7 @@ def get(self): class BaseEditHandler(base_handler.GcsUploadHandler): """Base edit handler.""" - def _read_to_bytesio(self, gcs_path): + def _read_to_bytesio(self, gcs_path: str) -> io.BytesIO: """Return a bytesio representing a GCS object.""" data = storage.read_data(gcs_path) if not data: @@ -89,9 +93,10 @@ def _read_to_bytesio(self, gcs_path): return io.BytesIO(data) - def _get_executable_path(self, upload_info): + def _get_executable_path( + self, upload_info: Optional[storage.GcsBlobInfo]) -> Optional[str]: """Get executable path.""" - executable_path = request.get('executable_path') + executable_path = cast(Any, request).get('executable_path') if not upload_info: return executable_path @@ -105,9 +110,10 @@ def _get_executable_path(self, upload_info): with archive.open(upload_info.filename, reader) as archive_reader: return archive_reader.get_first_file_matching(executable_path) - def _get_launcher_script(self, upload_info): + def _get_launcher_script( + self, upload_info: Optional[storage.GcsBlobInfo]) -> Optional[str]: """Get launcher script path.""" - launcher_script = request.get('launcher_script') + launcher_script = cast(Any, request).get('launcher_script') if not upload_info: return launcher_script @@ -126,9 +132,9 @@ def _get_launcher_script(self, upload_info): return launcher_script - def _get_integer_value(self, key): + def _get_integer_value(self, key: str) -> Optional[int]: """Check a numeric input value.""" - value = request.get(key) + value = cast(Any, request).get(key) if value is None: return None @@ -147,7 +153,9 @@ def _get_fuzzer_state_str(self, fuzzer: data_types.Fuzzer) -> str: fuzzer_dict = fuzzer.to_dict(exclude=FUZZER_FIELDS_EXCLUDED_FROM_LOG) return '\n'.join(f"{key}: {val}" for key, val in fuzzer_dict.items()) - def apply_fuzzer_changes(self, fuzzer, upload_info): + def apply_fuzzer_changes( + self, fuzzer: data_types.Fuzzer, + upload_info: Optional[storage.GcsBlobInfo]) -> Response: """Apply changes to a fuzzer.""" if upload_info and not archive.is_archive(upload_info.filename): raise helpers.EarlyExitError( @@ -170,15 +178,16 @@ def apply_fuzzer_changes(self, fuzzer, upload_info): existing_fuzzer_info = self._get_fuzzer_state_str(fuzzer) - jobs = request.get('jobs', []) + jobs = cast(Any, request).get('jobs', []) timeout = self._get_integer_value('timeout') max_testcases = self._get_integer_value('max_testcases') - external_contribution = bool(request.get('external_contribution', False)) - trusted = bool(request.get('trusted', False)) - differential = bool(request.get('differential', False)) - environment_string = request.get('additional_environment_string') - data_bundle_name = request.get('data_bundle_name') - primary_owner = request.get('primary_owner') + external_contribution = bool( + cast(Any, request).get('external_contribution', False)) + trusted = bool(cast(Any, request).get('trusted', False)) + differential = bool(cast(Any, request).get('differential', False)) + environment_string = cast(Any, request).get('additional_environment_string') + data_bundle_name = cast(Any, request).get('data_bundle_name') + primary_owner = cast(Any, request).get('primary_owner') # Save the fuzzer file metadata. if upload_info: @@ -187,7 +196,7 @@ def apply_fuzzer_changes(self, fuzzer, upload_info): fuzzer.file_size = utils.get_size_string(upload_info.size) fuzzer.jobs = jobs - fuzzer.revision = fuzzer.revision + 1 + fuzzer.revision = cast(int, fuzzer.revision) + 1 fuzzer.last_edited_by = helpers.get_user_email() fuzzer.timeout = timeout fuzzer.max_testcases = max_testcases @@ -240,9 +249,9 @@ class CreateHandler(BaseEditHandler): @handler.post(handler.JSON, handler.JSON) @handler.check_user_access(need_privileged_access=True) @handler.require_csrf_token - def post(self): + def post(self) -> Response: """Handle a post request.""" - name = request.get('name') + name = cast(Any, request).get('name') if not name: raise helpers.EarlyExitError('Please give the fuzzer a name!', 400) @@ -276,7 +285,7 @@ class EditHandler(BaseEditHandler): @handler.post(handler.JSON, handler.JSON) @handler.check_user_access(need_privileged_access=True) @handler.require_csrf_token - def post(self): + def post(self) -> Response: """Handle a post request.""" key = helpers.get_integer_key(request) @@ -294,7 +303,7 @@ class DeleteHandler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.check_user_access(need_privileged_access=True) @handler.require_csrf_token - def post(self): + def post(self) -> Response: """Handle a post request.""" key = helpers.get_integer_key(request) @@ -313,7 +322,7 @@ class LogHandler(base_handler.Handler): """Show the console output from a fuzzer run.""" @handler.check_user_access(need_privileged_access=False) - def get(self, fuzzer_name): + def get(self, fuzzer_name: str) -> Response: """Handle a get request.""" helpers.log('LogHandler', fuzzer_name) fuzzer = data_types.Fuzzer.query( @@ -321,7 +330,8 @@ def get(self, fuzzer_name): if not fuzzer: raise helpers.EarlyExitError('Fuzzer not found.', 400) - return self.render('viewer.html', { - 'title': 'Output for ' + fuzzer.name, - 'content': fuzzer.console_output, - }) + return self.render( + 'viewer.html', { + 'title': 'Output for ' + cast(str, fuzzer.name), + 'content': fuzzer.console_output, + }) diff --git a/src/appengine/handlers/gcs_redirector.py b/src/appengine/handlers/gcs_redirector.py index 7a02c625fa3..c4897a87cc6 100644 --- a/src/appengine/handlers/gcs_redirector.py +++ b/src/appengine/handlers/gcs_redirector.py @@ -25,7 +25,7 @@ class Handler(base_handler.Handler): """Gcs redirector.""" @handler.get(handler.HTML) - def get(self): + def get(self) -> base_handler.Response: """Handle a get request.""" gcs_path = request.args.get('path', '') if not gcs_path: diff --git a/src/appengine/handlers/help_redirector.py b/src/appengine/handlers/help_redirector.py index bc3de634dff..cd9b7104501 100644 --- a/src/appengine/handlers/help_redirector.py +++ b/src/appengine/handlers/help_redirector.py @@ -16,14 +16,14 @@ from clusterfuzz._internal.config import db_config from handlers import base_handler -DEFAULT_DOCUMENTATION_URL = 'https://google.github.io/clusterfuzz/' -DEFAULT_BUG_REPORT_URL = 'https://github.com/google/clusterfuzz/issues' +DEFAULT_DOCUMENTATION_URL: str = 'https://google.github.io/clusterfuzz/' +DEFAULT_BUG_REPORT_URL: str = 'https://github.com/google/clusterfuzz/issues' class DocumentationHandler(base_handler.Handler): """Redirect to documentation.""" - def get(self): + def get(self) -> base_handler.Response: """Get the HTML page.""" documentation_url = db_config.get_value('documentation_url') if not documentation_url: @@ -35,7 +35,7 @@ def get(self): class ReportBugHandler(base_handler.Handler): """Redirect to issue tracker for reporting bug.""" - def get(self): + def get(self) -> base_handler.Response: """Get the HTML page.""" bug_report_url = db_config.get_value('bug_report_url') if not bug_report_url: diff --git a/src/appengine/handlers/home.py b/src/appengine/handlers/home.py index b7166dd86a5..8e6219e29dd 100644 --- a/src/appengine/handlers/home.py +++ b/src/appengine/handlers/home.py @@ -14,6 +14,9 @@ """Home page handler.""" import json +from typing import Any +from typing import cast +from typing import Sequence from clusterfuzz._internal.base import external_users from clusterfuzz._internal.base import memoize @@ -33,12 +36,12 @@ MEMCACHE_TTL_IN_SECONDS = 30 * 60 -def _sort_by_name(item): +def _sort_by_name(item: dict[str, Any]) -> Any: """Sort key function.""" return item['name'] -def _get_engine_names(job_name): +def _get_engine_names(job_name: str) -> tuple[str, str]: """Return the (engine display name, engine name) for the job.""" if job_name.startswith('afl_'): return 'AFL', 'afl' @@ -50,7 +53,8 @@ def _get_engine_names(job_name): @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def get_single_fuzz_target_or_none(project, engine_name): +def get_single_fuzz_target_or_none(project: str, + engine_name: str) -> str | None: """Return the name of the single child fuzzer for the project, or None.""" fuzz_targets = data_handler.get_fuzz_targets( engine=engine_name, project=project) @@ -66,28 +70,31 @@ def get_single_fuzz_target_or_none(project, engine_name): return fuzz_target_name -def get_introspector_index(): +def get_introspector_index() -> dict[str, Any]: """Return introspector projects status""" if storage.exists(INTROSPECTOR_INDEX_JSON_URL): - introspector_index = json.loads( - storage.read_data(INTROSPECTOR_INDEX_JSON_URL)) + introspector_data = cast(bytes, + storage.read_data(INTROSPECTOR_INDEX_JSON_URL)) + introspector_index = json.loads(introspector_data) else: introspector_index = {} logs.info('Loaded introspector status: %d' % len(introspector_index)) return introspector_index -def _get_project_results_for_jobs(jobs): +def _get_project_results_for_jobs( + jobs: Sequence[data_types.Job]) -> list[dict[str, Any]]: """Return projects for jobs.""" - projects = {} + projects: dict[str, dict[str, Any]] = {} introspector_index = get_introspector_index() - for job in sorted(jobs, key=lambda j: j.name): - project_name = job.get_environment().get('PROJECT_NAME', job.name) + for job in sorted(jobs, key=lambda j: cast(str, j.name)): + job_name = cast(str, job.name) + project_name = job.get_environment().get('PROJECT_NAME', job_name) if project_name not in projects: projects[project_name] = {'name': project_name, 'jobs': []} if utils.string_is_true(job.get_environment().get('CORPUS_PRUNE')): - projects[project_name]['coverage_job'] = job.name + projects[project_name]['coverage_job'] = job_name projects[project_name]['has_introspector'] = False if project_name in introspector_index: @@ -95,45 +102,46 @@ def _get_project_results_for_jobs(jobs): projects[project_name]['introspector_report'] = introspector_index[ project_name] - engine_display_name, engine_name = _get_engine_names(job.name) + engine_display_name, engine_name = _get_engine_names(job_name) projects[project_name]['jobs'].append({ 'engine_display_name': engine_display_name, 'engine_name': engine_name, 'sanitizer_string': - environment.get_memory_tool_display_string(job.name), + environment.get_memory_tool_display_string(job_name), 'name': - job.name, + job_name, 'single_target': get_single_fuzz_target_or_none(project_name, engine_name), 'has_stats': True }) - projects = list(projects.values()) - projects.sort(key=_sort_by_name) - for project in projects: + project_list = list(projects.values()) + project_list.sort(key=_sort_by_name) + for project in project_list: project['jobs'].sort(key=_sort_by_name) - return projects + return project_list @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def _get_all_project_results(): +def _get_all_project_results() -> list[dict[str, Any]]: """Return all results.""" jobs = list(data_types.Job.query()) return _get_project_results_for_jobs(jobs) -def _get_project_results_for_external_user(external_jobs): +def _get_project_results_for_external_user( + external_jobs: Sequence[str]) -> list[dict[str, Any]]: """Return results for external user.""" jobs = list(data_types.Job.query()) jobs = [job for job in jobs if job.name in external_jobs] return _get_project_results_for_jobs(jobs) -def get_results(): +def get_results() -> dict[str, Any]: """Return results.""" is_user = access.has_access() user_email = helpers.get_user_email() @@ -162,7 +170,7 @@ class Handler(base_handler.Handler): @handler.get(handler.HTML) @handler.oauth - def get(self): + def get(self) -> base_handler.Response: """GET handler.""" return self.render('oss-fuzz-home.html', get_results()) @@ -171,7 +179,7 @@ class RefreshCacheHandler(base_handler.Handler): """Home page handler.""" @handler.cron() - def get(self): + def get(self) -> None: """GET handler.""" # pylint: disable=unexpected-keyword-arg - _get_all_project_results(__memoize_force__=True) + cast(Any, _get_all_project_results)(__memoize_force__=True) diff --git a/src/appengine/handlers/issue_redirector.py b/src/appengine/handlers/issue_redirector.py index f66adc80fcb..c8201549d53 100644 --- a/src/appengine/handlers/issue_redirector.py +++ b/src/appengine/handlers/issue_redirector.py @@ -14,6 +14,8 @@ """Handler for redirecting to the issue url (given a testcase). See crbug.com/665652 on why we need it.""" +from typing import cast + from clusterfuzz._internal.issue_management import issue_tracker_utils from handlers import base_handler from libs import helpers @@ -22,9 +24,9 @@ class Handler(base_handler.Handler): """Handler that redirects user to the issue URL.""" - def get(self, testcase_id=None): + def get(self, testcase_id: str | int | None = None) -> base_handler.Response: """Redirect user to the correct URL.""" - testcase = helpers.get_testcase(testcase_id) + testcase = helpers.get_testcase(cast(int | str, testcase_id)) issue_url = helpers.get_or_exit( lambda: issue_tracker_utils.get_issue_url(testcase), 'Issue tracker for testcase (id=%s) is not found.' % testcase_id, diff --git a/src/appengine/handlers/jobs.py b/src/appengine/handlers/jobs.py index cdd957a532f..40239562fbf 100644 --- a/src/appengine/handlers/jobs.py +++ b/src/appengine/handlers/jobs.py @@ -13,7 +13,11 @@ # limitations under the License. """Manage job types.""" +from typing import Any +from typing import cast + from flask import request +from flask import Response from google.cloud import ndb from clusterfuzz._internal.base import tasks @@ -32,17 +36,17 @@ from libs import helpers from libs.query import datastore_query -PAGE_SIZE = 10 -MORE_LIMIT = 50 - PAGE_SIZE # exactly 5 pages +PAGE_SIZE: int = 10 +MORE_LIMIT: int = 50 - PAGE_SIZE # exactly 5 pages -FILTERS = [ +FILTERS: list[filters.Filter] = [ filters.Keyword([], 'keywords', 'q'), ] -def get_queues(): +def get_queues() -> list[dict[str, str]]: """Return list of task queues.""" - queues = [] + queues: list[dict[str, str]] = [] for name, display_name in tasks.TASK_QUEUE_DISPLAY_NAMES.items(): queue = { 'name': name, @@ -54,28 +58,31 @@ def get_queues(): return queues -def _job_to_dict(job): +def _job_to_dict(job: data_types.Job) -> dict[str, Any]: """Return a dict of job items along with associated fuzzers.""" result = job.to_dict() result['id'] = job.key.id() # Adding all associated fuzzers with each job. fuzzers = data_types.Fuzzer.query() result['fuzzers'] = [ - fuzzer.name for fuzzer in fuzzers if job.name in fuzzer.jobs + fuzzer.name + for fuzzer in fuzzers + if job.name and job.name in cast(Any, fuzzer.jobs) ] return result -def get_results(): +def get_results() -> tuple[dict[str, Any], dict[str, Any]]: """Get results for the jobs page.""" # Return jobs sorted alphabetically by name query = datastore_query.Query(data_types.Job) query.order('name', is_desc=False) - params = dict(request.iterparams()) + request_any = cast(Any, request) + params = dict(request_any.iterparams()) filters.add(query, params, FILTERS) - page = helpers.cast(request.get('page', 1), int, "'page' is not an int.") + page = helpers.cast(request_any.get('page', 1), int, "'page' is not an int.") items, total_pages, total_items, has_more = query.fetch_page( page=page, page_size=PAGE_SIZE, projection=None, more_limit=MORE_LIMIT) helpers.log('Jobs', helpers.VIEW_OPERATION) @@ -97,7 +104,7 @@ class Handler(base_handler.Handler): @handler.get(handler.HTML) @handler.oauth @handler.check_user_access(need_privileged_access=True) - def get(self): + def get(self) -> Response: """Handle a get request.""" templates = list(data_types.JobTemplate.query().order( data_types.JobTemplate.name)) @@ -136,7 +143,7 @@ class UpdateJob(base_handler.GcsUploadHandler): @handler.post(handler.FORM, handler.HTML) @handler.check_user_access(need_privileged_access=True) @handler.require_csrf_token - def post(self): + def post(self) -> Response: """Handle a post request.""" name = request.form.get('name') if not name: @@ -182,7 +189,7 @@ def post(self): job.platform = new_platform job.description = description job.environment_string = environment_string - job.templates = templates + job.templates = cast(Any, templates) blob_info = self.get_upload() if blob_info: @@ -200,7 +207,7 @@ def post(self): fuzzer_selection.update_platform_for_job(name, new_platform) # pylint: disable=unexpected-keyword-arg - _ = data_handler.get_all_job_type_names(__memoize_force__=True) + _ = cast(Any, data_handler.get_all_job_type_names)(__memoize_force__=True) helpers.log('Job created %s' % name, helpers.MODIFY_OPERATION) template_values = { @@ -220,7 +227,7 @@ class UpdateJobTemplate(base_handler.Handler): @handler.post(handler.FORM, handler.HTML) @handler.check_user_access(need_privileged_access=True) @handler.require_csrf_token - def post(self): + def post(self) -> Response: """Handle a post request.""" name = request.form.get('name') if not name: @@ -264,7 +271,7 @@ class DeleteJobHandler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.check_user_access(need_privileged_access=True) @handler.require_csrf_token - def post(self): + def post(self) -> Response: """Handle a post request.""" key = helpers.get_integer_key(request) job = ndb.Key(data_types.Job, key).get() @@ -273,8 +280,8 @@ def post(self): # Delete from fuzzers' jobs' list. for fuzzer in ndb_utils.get_all_from_model(data_types.Fuzzer): - if job.name in fuzzer.jobs: - fuzzer.jobs.remove(job.name) + if job.name in cast(Any, fuzzer).jobs: + cast(Any, fuzzer).jobs.remove(job.name) fuzzer.put() # Delete associated fuzzer-job mapping(s). @@ -295,7 +302,7 @@ class JsonHandler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.check_user_access(need_privileged_access=True) - def post(self): + def post(self) -> Response: """Get and render the jobs in JSON.""" result, _ = get_results() return self.render_json(result) @@ -306,7 +313,7 @@ class GetEnvironmentHandler(base_handler.Handler): @handler.get(handler.JSON) @handler.check_user_access(need_privileged_access=True) - def get(self): + def get(self) -> Response: """Get and render the computed environment in JSON.""" name = request.args.get('name') if not name: diff --git a/src/appengine/handlers/login.py b/src/appengine/handlers/login.py index ea4251de4f1..27f24f48a6a 100644 --- a/src/appengine/handlers/login.py +++ b/src/appengine/handlers/login.py @@ -14,8 +14,11 @@ """Login page.""" import datetime +from typing import Any +from typing import cast from flask import request +from flask import Response from clusterfuzz._internal.config import local_config from clusterfuzz._internal.metrics import logs @@ -24,8 +27,8 @@ from libs import handler from libs import helpers -DEFAULT_REDIRECT = '/' -SESSION_EXPIRY_DAYS = 14 +DEFAULT_REDIRECT: str = '/' +SESSION_EXPIRY_DAYS: int = 14 class Handler(base_handler.Handler): @@ -33,9 +36,9 @@ class Handler(base_handler.Handler): @handler.get(handler.HTML) @handler.unsupported_on_local_server - def get(self): + def get(self) -> Response: """Handle a get request.""" - dest = request.get('dest', DEFAULT_REDIRECT) + dest = cast(str, cast(Any, request).get('dest', DEFAULT_REDIRECT)) base_handler.check_redirect_url(dest) return self.render( @@ -50,9 +53,9 @@ class SessionLoginHandler(base_handler.Handler): """Session login handler.""" @handler.post(handler.JSON, handler.JSON) - def post(self): + def post(self) -> Response: """Handle a post request.""" - id_token = request.get('idToken') + id_token = cast(str, cast(Any, request).get('idToken')) expires_in = datetime.timedelta(days=SESSION_EXPIRY_DAYS) try: session_cookie = auth.create_session_cookie(id_token, expires_in) @@ -72,14 +75,16 @@ class LogoutHandler(base_handler.Handler): @handler.get(handler.HTML) @handler.unsupported_on_local_server @handler.require_csrf_token - def get(self): + def get(self) -> Response: """Handle a get request.""" try: - auth.revoke_session_cookie(auth.get_session_cookie()) + auth.revoke_session_cookie(cast(str, auth.get_session_cookie())) except auth.AuthError: # Even if the revoke failed, remove the cookie. logs.error('Failed to revoke session cookie.') - response = self.redirect(request.get('dest', DEFAULT_REDIRECT)) + response = self.redirect( + cast(str, + cast(Any, request).get('dest', DEFAULT_REDIRECT))) response.delete_cookie('session') return response diff --git a/src/appengine/handlers/report_csp_failure.py b/src/appengine/handlers/report_csp_failure.py index a1ebd978dfb..067fa2331c2 100644 --- a/src/appengine/handlers/report_csp_failure.py +++ b/src/appengine/handlers/report_csp_failure.py @@ -13,6 +13,9 @@ # limitations under the License. """Log incoming reports of CSP violations.""" +from typing import Any +from typing import cast + from flask import request from clusterfuzz._internal.metrics import logs @@ -26,9 +29,9 @@ class ReportCspFailureHandler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.check_user_access(need_privileged_access=False) - def post(self): + def post(self) -> str: """Handle a POST request.""" - report = request.get('csp-report') + report = cast(Any, request).get('csp-report') if not report: raise helpers.EarlyExitError('No CSP report.', 400) diff --git a/src/appengine/handlers/revisions_info.py b/src/appengine/handlers/revisions_info.py index dbb8255640f..575f3218ce6 100644 --- a/src/appengine/handlers/revisions_info.py +++ b/src/appengine/handlers/revisions_info.py @@ -13,6 +13,9 @@ # limitations under the License. """Display information for revision ranges.""" +from typing import Any +from typing import cast + from flask import request from clusterfuzz._internal.build_management import revisions @@ -26,11 +29,11 @@ class Handler(base_handler.Handler): @handler.get(handler.HTML) @handler.oauth - def get(self): + def get(self) -> base_handler.Response: """GET handler.""" - job_type = request.get('job') - revision = request.get('revision') - revision_range = request.get('range') + job_type = cast(Any, request).get('job') + revision = cast(Any, request).get('revision') + revision_range = cast(Any, request).get('range') if revision: if not revision.isdigit(): diff --git a/src/appengine/handlers/testcase_detail/crash_stats.py b/src/appengine/handlers/testcase_detail/crash_stats.py index e42238a1098..c7754a9f907 100644 --- a/src/appengine/handlers/testcase_detail/crash_stats.py +++ b/src/appengine/handlers/testcase_detail/crash_stats.py @@ -13,15 +13,21 @@ # limitations under the License. """Handler for showing crash stats inside the testcase detail page.""" +from typing import Any +from typing import cast + from flask import request +from flask import Response +from clusterfuzz._internal.datastore import data_types from handlers import base_handler from libs import crash_stats from libs import handler from libs import helpers -def get_result(testcase, end, block, days, group_by): +def get_result(testcase: data_types.Testcase, end: int, block: str, days: int, + group_by: str) -> dict[str, Any]: """Get slots for crash stats.""" query = crash_stats.Query() query.group_by = group_by @@ -45,11 +51,13 @@ class Handler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.check_testcase_access - def post(self, testcase): + def post(self, testcase: data_types.Testcase) -> Response: """Server crash stats.""" - end = helpers.cast(request.get('end'), int, "'end' is not an int.") - days = helpers.cast(request.get('days'), int, "'days' is not an int.") + request_any = cast(Any, request) + end = helpers.cast(request_any.get('end'), int, "'end' is not an int.") + days = helpers.cast(request_any.get('days'), int, "'days' is not an int.") group_by = helpers.cast( - request.get('groupBy'), str, "'groupBy' is not a string.") - block = helpers.cast(request.get('block'), str, "'block' is not a string.") + request_any.get('groupBy'), str, "'groupBy' is not a string.") + block = helpers.cast( + request_any.get('block'), str, "'block' is not a string.") return self.render_json(get_result(testcase, end, block, days, group_by)) diff --git a/src/appengine/handlers/testcase_detail/create_issue.py b/src/appengine/handlers/testcase_detail/create_issue.py index e7abf9f853f..27582966b00 100644 --- a/src/appengine/handlers/testcase_detail/create_issue.py +++ b/src/appengine/handlers/testcase_detail/create_issue.py @@ -13,8 +13,13 @@ # limitations under the License. """Handler for creating issue.""" +from typing import Any +from typing import cast + from flask import request +from flask import Response +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.issue_management import issue_filer from clusterfuzz._internal.metrics import events from handlers import base_handler @@ -27,7 +32,8 @@ class Handler(base_handler.Handler): """Handler that creates an issue.""" @staticmethod - def create_issue(testcase, severity, cc_me): + def create_issue(testcase: data_types.Testcase, severity: Any, + cc_me: Any) -> None: """Create an issue.""" issue_tracker = helpers.get_issue_tracker_for_testcase(testcase) user_email = helpers.get_user_email() @@ -61,10 +67,11 @@ def create_issue(testcase, severity, cc_me): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token @handler.check_testcase_access - def post(self, testcase): + def post(self, testcase: data_types.Testcase) -> Response: """Create an issue.""" - cc_me = request.get('ccMe') - severity = request.get('severity') + request_any = cast(Any, request) + cc_me = request_any.get('ccMe') + severity = request_any.get('severity') self.create_issue(testcase, severity, cc_me) return self.render_json(show.get_testcase_detail(testcase)) diff --git a/src/appengine/handlers/testcase_detail/delete.py b/src/appengine/handlers/testcase_detail/delete.py index bfc967390fc..fb4a80ce373 100644 --- a/src/appengine/handlers/testcase_detail/delete.py +++ b/src/appengine/handlers/testcase_detail/delete.py @@ -13,7 +13,11 @@ # limitations under the License. """Handler for creating issue.""" +from typing import Any +from typing import cast + from flask import request +from flask import Response from handlers import base_handler from libs import handler @@ -24,7 +28,7 @@ class Handler(base_handler.Handler): """Handler that creates an issue.""" @staticmethod - def delete_testcase(testcase_id): + def delete_testcase(testcase_id: int | str) -> None: """Delete a testcase.""" testcase = helpers.get_testcase(testcase_id) @@ -40,7 +44,7 @@ def delete_testcase(testcase_id): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token @handler.check_admin_access - def post(self): + def post(self) -> Response: """Delete a testcase.""" - testcase_id = request.get('testcaseId') + testcase_id = cast(Any, request).get('testcaseId') return self.render_json({'testcaseId': self.delete_testcase(testcase_id)}) diff --git a/src/appengine/handlers/testcase_detail/download_testcase.py b/src/appengine/handlers/testcase_detail/download_testcase.py index 44135f26dd5..3034b4e7b7c 100644 --- a/src/appengine/handlers/testcase_detail/download_testcase.py +++ b/src/appengine/handlers/testcase_detail/download_testcase.py @@ -13,11 +13,16 @@ # limitations under the License. """Handler that serves the testcase file.""" +from typing import Any +from typing import cast import urllib.parse from flask import request +from flask import Response +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.google_cloud_utils import blobs +from clusterfuzz._internal.google_cloud_utils import storage from handlers import base_handler from libs import access from libs import gcs @@ -26,10 +31,11 @@ # Blob's filename can be very long. We only preview the last N characters. # A too long file name can cause an issue with `curl` and `wget`. -PREVIEW_BLOB_FILENAME_LENTGH = 20 +PREVIEW_BLOB_FILENAME_LENTGH: int = 20 -def get_testcase_blob_info(testcase): +def get_testcase_blob_info( + testcase: data_types.Testcase) -> tuple[storage.GcsBlobInfo, bool]: """Get testcase file in the binary form.""" blob_key = testcase.minimized_keys using_minimized_keys = True @@ -45,12 +51,12 @@ def get_testcase_blob_info(testcase): blob_key = str(urllib.parse.unquote(blob_key)) blob_info = blobs.get_blob_info(blob_key) - return blob_info, using_minimized_keys + return cast(storage.GcsBlobInfo, blob_info), using_minimized_keys -def get(self): +def get(self: gcs.SignedGcsHandler) -> Response: """Get testcase file and write it to the handler.""" - testcase_id = request.get('id') + testcase_id = cast(Any, request).get('id') testcase = access.check_access_and_get_testcase(testcase_id) blob_info, _ = get_testcase_blob_info(testcase) @@ -68,6 +74,6 @@ class Handler(base_handler.Handler, gcs.SignedGcsHandler): @handler.get(handler.HTML) @handler.oauth - def get(self): + def get(self) -> Response: """Serve the testcase file.""" return get(self) diff --git a/src/appengine/handlers/testcase_detail/find_similar_issues.py b/src/appengine/handlers/testcase_detail/find_similar_issues.py index bca4c76da0b..98b5337d5eb 100644 --- a/src/appengine/handlers/testcase_detail/find_similar_issues.py +++ b/src/appengine/handlers/testcase_detail/find_similar_issues.py @@ -13,9 +13,15 @@ # limitations under the License. """Handler for finding similar issues.""" +from typing import Any +from typing import cast + from flask import request +from flask import Response +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.issue_management import issue_tracker_utils +from clusterfuzz._internal.issue_management.issue_tracker import IssueTracker from handlers import base_handler from libs import handler from libs import helpers @@ -25,13 +31,14 @@ class Handler(base_handler.Handler): """Handler that finds similar issues.""" @staticmethod - def get_issues(issue_tracker, testcase, only_open): + def get_issues(issue_tracker: IssueTracker, testcase: data_types.Testcase, + only_open: bool) -> list[dict[str, Any]]: """Get similar issues. It is used by self.process() and handler.testcase_detail.FindSimilarIssuesHandler.get()""" issues = issue_tracker_utils.get_similar_issues( issue_tracker, testcase, only_open=only_open) - items = [] + items: list[dict[str, Any]] = [] for entry in issues: items.append({ 'owner': entry.assignee, @@ -50,9 +57,9 @@ def get_issues(issue_tracker, testcase, only_open): @handler.get(handler.JSON) @handler.check_admin_access_if_oss_fuzz @handler.check_testcase_access - def get(self, testcase): + def get(self, testcase: data_types.Testcase) -> Response: """Find similar issues.""" - filter_type = request.get('filterType') + filter_type = cast(Any, request).get('filterType') only_open = filter_type == 'open' issue_tracker = helpers.get_issue_tracker_for_testcase(testcase) diff --git a/src/appengine/handlers/testcase_detail/mark_fixed.py b/src/appengine/handlers/testcase_detail/mark_fixed.py index 68b5af800d2..40c0a48198a 100644 --- a/src/appengine/handlers/testcase_detail/mark_fixed.py +++ b/src/appengine/handlers/testcase_detail/mark_fixed.py @@ -13,15 +13,19 @@ # limitations under the License. """Handler for marking a testcase as fixed.""" +from typing import Any +from typing import cast + from flask import request +from clusterfuzz._internal.datastore import data_types from handlers import base_handler from handlers.testcase_detail import show from libs import handler from libs import helpers -def mark(testcase): +def mark(testcase: data_types.Testcase) -> data_types.Testcase: """Mark the testcase as fixed.""" testcase.fixed = 'Yes' testcase.open = False @@ -38,9 +42,9 @@ class Handler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token @handler.check_admin_access - def post(self): + def post(self) -> base_handler.Response: """Mark the testcase as fixed.""" - testcase_id = request.get('testcaseId') + testcase_id = cast(Any, request).get('testcaseId') testcase = helpers.get_testcase(testcase_id) mark(testcase) return self.render_json(show.get_testcase_detail(testcase)) diff --git a/src/appengine/handlers/testcase_detail/mark_security.py b/src/appengine/handlers/testcase_detail/mark_security.py index 7be092546d3..feaf3e0fb82 100644 --- a/src/appengine/handlers/testcase_detail/mark_security.py +++ b/src/appengine/handlers/testcase_detail/mark_security.py @@ -13,6 +13,9 @@ # limitations under the License. """Handler for marking a testcase as security-related.""" +from typing import Any +from typing import cast + from flask import request from clusterfuzz._internal.base import bisection @@ -51,11 +54,11 @@ class Handler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token - def post(self): + def post(self) -> base_handler.Response: """Mark the testcase as security-related.""" - testcase_id = request.get('testcaseId') - security = request.get('security') - severity = request.get('severity') + testcase_id = cast(Any, request).get('testcaseId') + security = cast(Any, request).get('security') + severity = cast(Any, request).get('severity') testcase = helpers.get_testcase(testcase_id) if not access.has_access( diff --git a/src/appengine/handlers/testcase_detail/mark_unconfirmed.py b/src/appengine/handlers/testcase_detail/mark_unconfirmed.py index bd08bb94327..1c748c23cfd 100644 --- a/src/appengine/handlers/testcase_detail/mark_unconfirmed.py +++ b/src/appengine/handlers/testcase_detail/mark_unconfirmed.py @@ -13,15 +13,19 @@ # limitations under the License. """Handler for marking a testcase as unconfirmed.""" +from typing import Any +from typing import cast + from flask import request +from clusterfuzz._internal.datastore import data_types from handlers import base_handler from handlers.testcase_detail import show from libs import handler from libs import helpers -def mark(testcase): +def mark(testcase: data_types.Testcase) -> None: """Mark the testcase as unconfirmed.""" testcase.one_time_crasher_flag = True if not testcase.fixed: @@ -42,9 +46,9 @@ class Handler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token @handler.check_admin_access - def post(self): + def post(self) -> base_handler.Response: """Mark the testcase as unconfirmed.""" - testcase_id = request.get('testcaseId') + testcase_id = cast(Any, request).get('testcaseId') testcase = helpers.get_testcase(testcase_id) mark(testcase) return self.render_json(show.get_testcase_detail(testcase)) diff --git a/src/appengine/handlers/testcase_detail/redo.py b/src/appengine/handlers/testcase_detail/redo.py index 161f33c2933..870d9bfa000 100644 --- a/src/appengine/handlers/testcase_detail/redo.py +++ b/src/appengine/handlers/testcase_detail/redo.py @@ -13,9 +13,13 @@ # limitations under the License. """Handler for redoing.""" +from typing import Any +from typing import cast + from flask import request from clusterfuzz._internal.base import tasks +from clusterfuzz._internal.datastore.data_types import Testcase from handlers import base_handler from handlers.testcase_detail import show from libs import handler @@ -26,7 +30,7 @@ class Handler(base_handler.Handler): """Handler that redo tasks.""" @staticmethod - def redo(testcase, testcase_tasks, user_email): + def redo(testcase: Testcase, testcase_tasks: Any, user_email: str) -> None: """Redo tasks.""" try: tasks.redo_testcase(testcase, testcase_tasks, user_email) @@ -39,9 +43,9 @@ def redo(testcase, testcase_tasks, user_email): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token @handler.check_testcase_access - def post(self, testcase): + def post(self, testcase: Testcase) -> base_handler.Response: """Queue redo tasks.""" - testcase_tasks = request.get('tasks') + testcase_tasks = cast(Any, request).get('tasks') user_email = helpers.get_user_email() self.redo(testcase, testcase_tasks, user_email) diff --git a/src/appengine/handlers/testcase_detail/remove_duplicate.py b/src/appengine/handlers/testcase_detail/remove_duplicate.py index 278b1d171ee..69cc40ee187 100644 --- a/src/appengine/handlers/testcase_detail/remove_duplicate.py +++ b/src/appengine/handlers/testcase_detail/remove_duplicate.py @@ -13,15 +13,19 @@ # limitations under the License. """Handler for removing duplicate_of of a testcase.""" +from typing import Any +from typing import cast + from flask import request +from clusterfuzz._internal.datastore.data_types import Testcase from handlers import base_handler from handlers.testcase_detail import show from libs import handler from libs import helpers -def remove(testcase): +def remove(testcase: Testcase) -> None: """Remove duplicate status from a test case.""" testcase.status = 'Processed' testcase.duplicate_of = None @@ -37,9 +41,9 @@ class Handler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token @handler.check_admin_access - def post(self): + def post(self) -> base_handler.Response: """Remove duplicate status from a test case.""" - testcase_id = request.get('testcaseId') + testcase_id = cast(Any, request).get('testcaseId') testcase = helpers.get_testcase(testcase_id) remove(testcase) return self.render_json(show.get_testcase_detail(testcase)) diff --git a/src/appengine/handlers/testcase_detail/remove_group.py b/src/appengine/handlers/testcase_detail/remove_group.py index 72276483be1..7745f08eb1b 100644 --- a/src/appengine/handlers/testcase_detail/remove_group.py +++ b/src/appengine/handlers/testcase_detail/remove_group.py @@ -13,16 +13,20 @@ # limitations under the License. """Handler for removing a testcase from a group.""" +from typing import Any +from typing import cast + from flask import request from clusterfuzz._internal.datastore import data_handler +from clusterfuzz._internal.datastore import data_types from handlers import base_handler from handlers.testcase_detail import show from libs import handler from libs import helpers -def remove_group(testcase_id): +def remove_group(testcase_id: int | str) -> data_types.Testcase: """Remove the testcase from a group.""" testcase = helpers.get_testcase(testcase_id) group_id = testcase.group_id @@ -42,9 +46,10 @@ class Handler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token @handler.check_admin_access - def post(self): + def post(self) -> base_handler.Response: """Remove the issue from the testcase.""" - testcase_id = request.get('testcaseId') + request_any = cast(Any, request) + testcase_id = request_any.get('testcaseId') updated_testcase = remove_group(testcase_id) return self.render_json(show.get_testcase_detail(updated_testcase)) diff --git a/src/appengine/handlers/testcase_detail/remove_issue.py b/src/appengine/handlers/testcase_detail/remove_issue.py index e54042271bf..60a3c8b1996 100644 --- a/src/appengine/handlers/testcase_detail/remove_issue.py +++ b/src/appengine/handlers/testcase_detail/remove_issue.py @@ -13,8 +13,12 @@ # limitations under the License. """Handler for removing issue from a testcase.""" +from typing import Any +from typing import cast + from flask import request +from clusterfuzz._internal.datastore import data_types from handlers import base_handler from handlers.testcase_detail import show from libs import handler @@ -25,7 +29,7 @@ class Handler(base_handler.Handler): """Handler that removes an issue from a testcase.""" @staticmethod - def remove_issue(testcase_id): + def remove_issue(testcase_id: int | str) -> data_types.Testcase: """Remove the issue from the testcase.""" testcase = helpers.get_testcase(testcase_id) issue_id = testcase.bug_information @@ -43,9 +47,10 @@ def remove_issue(testcase_id): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token @handler.check_admin_access - def post(self): + def post(self) -> base_handler.Response: """Remove the issue from the testcase.""" - testcase_id = request.get('testcaseId') + request_any = cast(Any, request) + testcase_id = request_any.get('testcaseId') updated_testcase = self.remove_issue(testcase_id) return self.render_json(show.get_testcase_detail(updated_testcase)) diff --git a/src/appengine/handlers/testcase_detail/show.py b/src/appengine/handlers/testcase_detail/show.py index 4e49c52f29a..0c30d22a0eb 100644 --- a/src/appengine/handlers/testcase_detail/show.py +++ b/src/appengine/handlers/testcase_detail/show.py @@ -16,6 +16,10 @@ import datetime import html import re +from typing import Any +from typing import cast +from typing import Optional +from typing import Sequence import flask import jinja2 @@ -41,7 +45,7 @@ from libs import handler from libs import helpers -FIND_SIMILAR_ISSUES_OPTIONS = [{ +FIND_SIMILAR_ISSUES_OPTIONS: list[dict[str, str]] = [{ 'type': 'open', 'label': 'Open' }, { @@ -49,25 +53,25 @@ 'label': 'All' }] -CRASH_STATE_REGEXES = [ +CRASH_STATE_REGEXES: list[str] = [ r'bad-cast to (.+) from.*', r'bad-cast to (.+)', r'(.+) +in +.+' ] -COMPILED_CRASH_STATE_REGEXES = [ +COMPILED_CRASH_STATE_REGEXES: list[re.Pattern] = [ re.compile(r, flags=re.IGNORECASE) for r in CRASH_STATE_REGEXES ] # [] dump_backtrace+0x0/0x34c # http://go/pakernel/msm-google/+/40e9b2ff3a280a8775cfcd5841e530ce78f94355/ # arch/arm64/kernel/traps.c#96;msm-google/arch/arm64/kernel/traps.c -KERNEL_LINK_REGEX = re.compile( +KERNEL_LINK_REGEX: re.Pattern = re.compile( r'(.+)(http:\/\/go\/pakernel\/[^;]+);([^;]+);(.*)') -KERNEL_LINK_FORMAT = r'%s%s%s' +KERNEL_LINK_FORMAT: str = r'%s%s%s' -STACKTRACE_MAX_LENGTH = 500000 +STACKTRACE_MAX_LENGTH: int = 500000 -def _truncate_stacktrace(stacktrace): +def _truncate_stacktrace(stacktrace: str) -> str: """Truncate stacktrace if necessary.""" if len(stacktrace) > STACKTRACE_MAX_LENGTH: # Read first and last |STACKTRACE_MAX_LENGTH/2| bytes. @@ -79,7 +83,8 @@ def _truncate_stacktrace(stacktrace): return stacktrace -def _parse_suspected_cls(predator_result): +def _parse_suspected_cls( + predator_result: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: """Parse raw suspected_cls into dict.""" if not predator_result: return None @@ -87,24 +92,24 @@ def _parse_suspected_cls(predator_result): # The raw result contains some additional information that we don't need here. # Everything we're concerned with is a part of the "result" object included # with the response. - predator_result = predator_result['result'] + predator_result_dict = cast(dict[str, Any], predator_result['result']) return { - 'found': predator_result.get('found'), - 'suspected_project': predator_result.get('suspected_project'), - 'suspected_components': predator_result.get('suspected_components'), - 'changelists': predator_result.get('suspected_cls'), - 'feedback_url': predator_result.get('feedback_url'), - 'error_message': predator_result.get('error_message'), + 'found': predator_result_dict.get('found'), + 'suspected_project': predator_result_dict.get('suspected_project'), + 'suspected_components': predator_result_dict.get('suspected_components'), + 'changelists': predator_result_dict.get('suspected_cls'), + 'feedback_url': predator_result_dict.get('feedback_url'), + 'error_message': predator_result_dict.get('error_message'), } -def highlight_common_stack_frames(crash_stacktrace): +def highlight_common_stack_frames(crash_stacktrace: str) -> str: """Highlights common stack frames between first two stacks.""" - crash_stacks = [[]] - highlighted_crash_stacktrace_lines = [] - old_frame_no = 0 - stack_index = 0 - stack_trace_line_format = '^ *#([0-9]+) *0x[0-9a-f]+ (.*)' + crash_stacks: list[list[str]] = [[]] + highlighted_crash_stacktrace_lines: list[str] = [] + old_frame_no: int = 0 + stack_index: int = 0 + stack_trace_line_format = r'^ *#([0-9]+) *0x[0-9a-f]+ (.*)' for line in crash_stacktrace.splitlines(): # Stacktrace separator prefix. @@ -129,7 +134,7 @@ def highlight_common_stack_frames(crash_stacktrace): return crash_stacktrace # Compare stack frames between first two stacks. - match_index = -1 + match_index: Any = -1 start_index_crash_stack_1 = len(crash_stacks[0]) - 1 start_index_crash_stack_2 = len(crash_stacks[1]) - 1 while True: @@ -176,7 +181,7 @@ def highlight_common_stack_frames(crash_stacktrace): return '\n'.join(highlighted_crash_stacktrace_lines) -def _linkify_android_kernel_stack_frame_if_needed(line): +def _linkify_android_kernel_stack_frame_if_needed(line: str) -> str: """Linkify links to android kernel source.""" match = KERNEL_LINK_REGEX.match(line) if match: @@ -186,8 +191,10 @@ def _linkify_android_kernel_stack_frame_if_needed(line): return line -def filter_stacktrace(crash_stacktrace, crash_type, revisions_dict, platform, - job_type): +def filter_stacktrace(crash_stacktrace: Optional[str], + crash_type: Optional[str], + revisions_dict: Optional[dict[str, Any]], + platform: Optional[str], job_type: Optional[str]) -> str: """Clean up and format a stack trace for display.""" if not crash_stacktrace: return '' @@ -195,8 +202,8 @@ def filter_stacktrace(crash_stacktrace, crash_type, revisions_dict, platform, # Truncate stacktrace if it's too big. crash_stacktrace = _truncate_stacktrace(crash_stacktrace) - filtered_crash_lines = [] - linkifier = source_mapper.StackFrameLinkifier(revisions_dict) + filtered_crash_lines: list[str] = [] + linkifier = source_mapper.StackFrameLinkifier(cast(Any, revisions_dict)) for line in crash_stacktrace.splitlines(): # Html escape line content to prevent XSS. line = html.escape(line, quote=True) @@ -219,22 +226,26 @@ def filter_stacktrace(crash_stacktrace, crash_type, revisions_dict, platform, class Line: """Represent a stacktrace line.""" - def __init__(self, line_number, content, important): + line_number: int + content: str + important: bool + + def __init__(self, line_number: int, content: str, important: bool) -> None: self.line_number = line_number self.content = content self.important = important - def __str__(self): + def __str__(self) -> str: return 'Line(%d, "%s", %s)' % (self.line_number, self.content, self.important) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return hash(self) == hash(other) - def __hash__(self): + def __hash__(self) -> int: return hash(self.__str__()) - def to_dict(self): + def to_dict(self) -> dict[str, Any]: return { 'lineNumber': self.line_number, 'content': self.content, @@ -243,7 +254,7 @@ def to_dict(self): } -def _is_line_important(line_content, frames): +def _is_line_important(line_content: str, frames: Sequence[str]) -> bool: """Check if the line contains a frame; it means the line is important.""" for frame in frames: @@ -252,12 +263,12 @@ def _is_line_important(line_content, frames): return False -def get_stack_frames(crash_state_lines): +def get_stack_frames(crash_state_lines: Sequence[str]) -> list[str]: """Get the stack frames from the crash state. Sometimes the crash state contains a type of crash, e.g. 'Bad-cast to content::RenderWidget from content::RenderWidgetHostViewAura'. The stack frame is 'content::RenderWidget'.""" - frames = [] + frames: list[str] = [] for line in crash_state_lines: added = False for regex in COMPILED_CRASH_STATE_REGEXES: @@ -272,7 +283,9 @@ def get_stack_frames(crash_state_lines): return frames -def convert_to_lines(raw_stacktrace, crash_state_lines, crash_type): +def convert_to_lines(raw_stacktrace: Optional[str], + crash_state_lines: Sequence[str], + crash_type: str) -> list[Line]: """Convert an array of string to an array of Line.""" if not raw_stacktrace or not raw_stacktrace.strip(): return [] @@ -281,30 +294,32 @@ def convert_to_lines(raw_stacktrace, crash_state_lines, crash_type): frames = get_stack_frames(crash_state_lines) escaped_frames = [markupsafe.escape(f) for f in frames] - combined_frames = frames + escaped_frames + combined_frames = frames + cast(list[str], escaped_frames) # Certain crash types have their own customized frames that are not related to # the stacktrace. Therefore, we make our best effort to preview stacktrace - # in a reasonable way; we preview around the the top of the stacktrace. + # in a reasonable way; we preview around the top of the stacktrace. for unique_type in data_types.CRASH_TYPES_WITH_UNIQUE_STATE: if crash_type.startswith(unique_type): combined_frames = ['ERROR'] break - lines = [] + lines: list[Line] = [] for index, content in enumerate(raw_lines): important = _is_line_important(content, combined_frames) lines.append(Line(index + 1, content, important)) return lines -def get_testcase_detail_by_id(testcase_id): +def get_testcase_detail_by_id(testcase_id: Any) -> dict[str, Any]: """Get testcase detail for rendering the testcase detail page.""" testcase = access.check_access_and_get_testcase(testcase_id) return get_testcase_detail(testcase) -def _get_revision_range_html_from_string(job_type, platform_id, revision_range): +def _get_revision_range_html_from_string(job_type: Optional[str], + platform_id: Optional[str], + revision_range: str) -> str: """Return revision range html for a revision range and job type given a range string.""" try: @@ -316,10 +331,10 @@ def _get_revision_range_html_from_string(job_type, platform_id, revision_range): end_revision) -def _get_revision_range_html(job_type, - platform_id, - start_revision, - end_revision=None): +def _get_revision_range_html(job_type: Optional[str], + platform_id: Optional[str], + start_revision: Any, + end_revision: Any = None) -> str: """Return revision range html for a revision range and job type.""" if end_revision is None: end_revision = start_revision @@ -333,7 +348,7 @@ def _get_revision_range_html(job_type, return revisions.format_revision_list(component_rev_list) -def _get_blob_size_string(blob_key): +def _get_blob_size_string(blob_key: Optional[str]) -> Optional[str]: """Return blob size string.""" blob_size = blobs.get_blob_size(blob_key) if blob_size is None: @@ -342,7 +357,7 @@ def _get_blob_size_string(blob_key): return utils.get_size_string(blob_size) -def _format_reproduction_help(reproduction_help): +def _format_reproduction_help(reproduction_help: Optional[str]) -> str: """Format a reproduction help string as HTML (linkified with break tags).""" if not reproduction_help: return '' @@ -350,12 +365,12 @@ def _format_reproduction_help(reproduction_help): return jinja2.utils.urlize(reproduction_help).replace('\n', '
') -def get_testcase_detail(testcase): +def get_testcase_detail(testcase: data_types.Testcase) -> dict[str, Any]: """Get testcase detail for rendering the testcase detail page.""" config = db_config.get() crash_address = testcase.crash_address crash_state = testcase.crash_state - crash_state_lines = crash_state.strip().splitlines() + crash_state_lines = cast(str, crash_state).strip().splitlines() crash_type = data_handler.get_crash_type_string(testcase) external_user = not access.has_access(job_type=testcase.job_type) issue_url = issue_tracker_utils.get_issue_url(testcase) @@ -408,11 +423,11 @@ def get_testcase_detail(testcase): crash_revisions_dict = revisions.get_component_revisions_dict( crash_revision, testcase.job_type, platform_id=testcase.platform_id) crash_stacktrace = data_handler.get_stacktrace(testcase) - crash_stacktrace = filter_stacktrace(crash_stacktrace, testcase.crash_type, - crash_revisions_dict, testcase.platform, - testcase.job_type) - crash_stacktrace = convert_to_lines(crash_stacktrace, crash_state_lines, - crash_type) + filtered_crash_stacktrace = filter_stacktrace( + crash_stacktrace, testcase.crash_type, crash_revisions_dict, + testcase.platform, testcase.job_type) + crash_stacktrace_lines = convert_to_lines(filtered_crash_stacktrace, + crash_state_lines, crash_type) last_tested_crash_revision = metadata.get('last_tested_crash_revision') last_tested_crash_revisions_dict = revisions.get_component_revisions_dict( @@ -421,11 +436,11 @@ def get_testcase_detail(testcase): platform_id=testcase.platform_id) last_tested_crash_stacktrace = data_handler.get_stacktrace( testcase, stack_attribute='last_tested_crash_stacktrace') - last_tested_crash_stacktrace = filter_stacktrace( + filtered_last_tested_stacktrace = filter_stacktrace( last_tested_crash_stacktrace, testcase.crash_type, last_tested_crash_revisions_dict, testcase.platform, testcase.job_type) - last_tested_crash_stacktrace = convert_to_lines(last_tested_crash_stacktrace, - crash_state_lines, crash_type) + last_tested_crash_stacktrace_lines = convert_to_lines( + filtered_last_tested_stacktrace, crash_state_lines, crash_type) privileged_user = access.has_access(need_privileged_access=True) @@ -469,14 +484,14 @@ def get_testcase_detail(testcase): # no associated bug. if not testcase.bug_information: auto_delete_timestamp = utils.utc_datetime_to_timestamp( - last_crash_time + datetime.timedelta( + cast(Any, last_crash_time) + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE)) # Set auto-close timestamp for unreproducible testcases with # an associated bug. if testcase.open and testcase.bug_information: auto_close_timestamp = utils.utc_datetime_to_timestamp( - last_crash_time + datetime.timedelta( + cast(Any, last_crash_time) + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)) memory_tool_display_string = environment.get_memory_tool_display_string( @@ -529,7 +544,7 @@ def get_testcase_detail(testcase): regression, 'crash_stacktrace': { 'lines': - crash_stacktrace, + crash_stacktrace_lines, 'revision': revisions.get_real_revision( crash_revision, @@ -539,7 +554,7 @@ def get_testcase_detail(testcase): }, 'last_tested_crash_stacktrace': { 'lines': - last_tested_crash_stacktrace, + last_tested_crash_stacktrace_lines, 'revision': revisions.get_real_revision( last_tested_crash_revision, @@ -592,11 +607,11 @@ def get_testcase_detail(testcase): 'is_local_development': environment.is_running_on_app_engine_development(), 'fuzzer_display': - fuzzer_display._asdict(), + cast(Any, fuzzer_display)._asdict(), } -def is_admin_or_not_oss_fuzz(): +def is_admin_or_not_oss_fuzz() -> bool: """Return True if the current user is an admin or if this is not OSS-Fuzz.""" return not utils.is_oss_fuzz() or auth.is_current_user_admin() @@ -605,7 +620,7 @@ class Handler(base_handler.Handler): """Handler that shows a testcase in detail.""" @handler.get(handler.HTML) - def get(self, testcase_id=None): + def get(self, testcase_id: Any = None) -> base_handler.Response: """Serve the testcase detail HTML page.""" values = {'info': get_testcase_detail_by_id(testcase_id)} return self.render('testcase-detail.html', values) @@ -614,7 +629,7 @@ def get(self, testcase_id=None): class DeprecatedHandler(base_handler.Handler): """Deprecated handler to show old style testcase link with key.""" - def get(self): + def get(self) -> base_handler.Response: """Serve the redirect to the current test case detail page.""" testcase_id = flask.request.args.get('key') if not testcase_id: @@ -628,9 +643,9 @@ class RefreshHandler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.oauth - def post(self): + def post(self) -> base_handler.Response: """Serve the testcase detail JSON.""" - testcase_id = flask.request.get('testcaseId') + testcase_id = cast(Any, flask.request).get('testcaseId') return self.render_json(get_testcase_detail_by_id(testcase_id)) @@ -638,10 +653,10 @@ class TaskLogHandler(base_handler.Handler): """Handler for downloading a task's log.""" @handler.get(handler.TEXT) - def get(self): + def get(self) -> base_handler.Response: """Serve the task log.""" testcase = access.check_access_and_get_testcase( - flask.request.args.get('testcase_id')) + cast(Any, flask.request.args.get('testcase_id'))) task_id = flask.request.args.get('task_id') if not task_id: diff --git a/src/appengine/handlers/testcase_detail/testcase_status_events.py b/src/appengine/handlers/testcase_detail/testcase_status_events.py index ac140715039..ee302957816 100644 --- a/src/appengine/handlers/testcase_detail/testcase_status_events.py +++ b/src/appengine/handlers/testcase_detail/testcase_status_events.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for getting testcase status information from events.""" +from collections.abc import Callable +from collections.abc import Iterator +from collections.abc import Mapping from dataclasses import asdict import datetime import json -from typing import Iterator -from typing import Mapping +from typing import Any +from typing import cast from typing import TypeAlias import urllib.parse @@ -44,10 +47,10 @@ def _format_timestamp(timestamp: datetime.datetime) -> str: class TestcaseStatusInfo: """Methods to retrieve and format testcase events information.""" - TASK_EVENTS_NAMES = ('analyze', 'minimize', 'progression', 'regression', - 'variant') - CHROME_TASK_EVENTS_NAMES = ('blame', 'impact') - LIFECYCLE_EVENTS_TYPES = ( + TASK_EVENTS_NAMES: tuple[str, ...] = ('analyze', 'minimize', 'progression', + 'regression', 'variant') + CHROME_TASK_EVENTS_NAMES: tuple[str, ...] = ('blame', 'impact') + LIFECYCLE_EVENTS_TYPES: tuple[str, ...] = ( events.EventTypes.ISSUE_CLOSING, events.EventTypes.ISSUE_FILING, events.EventTypes.TESTCASE_CREATION, @@ -56,10 +59,10 @@ class TestcaseStatusInfo: events.EventTypes.TESTCASE_REJECTION, ) - def __init__(self, testcase_id: int): + def __init__(self, testcase_id: int) -> None: """Initializes the TestcaseStatusInfo with a testcase ID.""" self._testcase_id = testcase_id - self._formatters = { + self._formatters: dict[str, Callable[[Any], EventInfo]] = { events.EventTypes.TASK_EXECUTION: self._format_task_execution_event, events.EventTypes.TESTCASE_CREATION: @@ -79,7 +82,7 @@ def __init__(self, testcase_id: int): self._format_testcase_grouping_event, } - def _get_event_selectors(self): + def _get_event_selectors(self) -> tuple[tuple[str, ...], tuple[str, ...]]: """Returns the selectors for task and lifecycle events.""" task_events_names = self.TASK_EVENTS_NAMES lifecycle_events_types = self.LIFECYCLE_EVENTS_TYPES @@ -204,8 +207,8 @@ def get_info(self) -> Mapping[str, list[EventInfo]]: # String sorting works here because timestamps are in the # `strftime('%Y-%m-%d %H:%M:%S.%f UTC')` format. - task_events_info.sort(key=lambda x: x.get('timestamp', '')) - lifecycle_events_info.sort(key=lambda x: x.get('timestamp', '')) + task_events_info.sort(key=lambda x: cast(str, x.get('timestamp', ''))) + lifecycle_events_info.sort(key=lambda x: cast(str, x.get('timestamp', ''))) return { 'task_events_info': task_events_info, @@ -216,7 +219,7 @@ def get_info(self) -> Mapping[str, list[EventInfo]]: class TestcaseEventHistory: """Methods to retrieve the testcase events history.""" - def __init__(self, testcase_id: int): + def __init__(self, testcase_id: int) -> None: self._testcase_id = testcase_id def _get_time_range_filter(self, days: int) -> str: @@ -235,7 +238,8 @@ def _get_task_log_query_filter(self, task_id: str, task_name: str) -> str: query += f' AND {self._get_time_range_filter(days=31)}' return query - def _enrich_event_info_with_gcp_log_url(self, event_info: EventInfo) -> None: + def _enrich_event_info_with_gcp_log_url(self, + event_info: dict[str, Any]) -> None: """Adds the GCP log URL to the event info.""" required_info = { 'project_id': utils.get_logging_cloud_project_id(), @@ -244,7 +248,8 @@ def _enrich_event_info_with_gcp_log_url(self, event_info: EventInfo) -> None: } if all(required_info.values()): project_id, task_id, task_name = required_info.values() - query = self._get_task_log_query_filter(task_id, task_name) + query = self._get_task_log_query_filter( + cast(str, task_id), cast(str, task_name)) encoded_query = urllib.parse.quote(query) event_info['gcp_log_url'] = ( _BASE_LOGS_URL + f'?project={project_id}&query={encoded_query}') @@ -253,13 +258,15 @@ def _enrich_event_info_with_gcp_log_url(self, event_info: EventInfo) -> None: logs.error('Unable to generate GCP log URL due to missing info. ' f'Missing info: {missing}') - def _format_event_for_history(self, event: events.Event) -> EventInfo: + def _format_event_for_history(self, event: events.Event) -> dict[str, Any]: """Formats an event for display in the event history table.""" - event_info = {k: v for k, v in asdict(event).items() if v is not None} + event_info: dict[str, Any] = { + k: v for k, v in asdict(event).items() if v is not None + } event_info['timestamp'] = _format_timestamp(event.timestamp) return event_info - def get_history(self) -> Iterator[Mapping]: + def get_history(self) -> Iterator[dict[str, Any]]: """Get all testcase events information in reverse chronological order.""" event_history = events.get_events_from_testcase(self._testcase_id) for event in event_history: @@ -285,7 +292,7 @@ def get_testcase_status_info(testcase_id: int) -> Mapping[str, list[EventInfo]]: return TestcaseStatusInfo(testcase_id).get_info() -def get_testcase_event_history(testcase_id: int) -> list[Mapping]: +def get_testcase_event_history(testcase_id: int) -> list[dict[str, Any]]: """Public function to get event history (reverse chronological order).""" return list(TestcaseEventHistory(testcase_id).get_history()) diff --git a/src/appengine/handlers/testcase_detail/testcase_variants.py b/src/appengine/handlers/testcase_detail/testcase_variants.py index c49946a5fca..c1c2e3f2110 100644 --- a/src/appengine/handlers/testcase_detail/testcase_variants.py +++ b/src/appengine/handlers/testcase_detail/testcase_variants.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. """Handler for getting testcase variants.""" + +from typing import Any + from clusterfuzz._internal.datastore import data_types from handlers import base_handler from libs import handler @@ -20,10 +23,10 @@ class Handler(base_handler.Handler): """Handler that return testcase variants.""" - def get_variants(self, testcase): + def get_variants(self, testcase: data_types.Testcase) -> list[dict[str, Any]]: """Get testcase variants""" - def _display_status(status): + def _display_status(status: int | None) -> str: """Return status for display.""" if status == data_types.TestcaseVariantStatus.PENDING: return 'Pending' @@ -36,7 +39,7 @@ def _display_status(status): return 'Unknown' - items = [] + items: list[dict[str, Any]] = [] variants = data_types.TestcaseVariant.query( data_types.TestcaseVariant.testcase_id == testcase.key.id()).order( data_types.TestcaseVariant.job_type) @@ -47,7 +50,7 @@ def _display_status(status): continue is_pending = variant.status == data_types.TestcaseVariantStatus.PENDING - item = { + item: dict[str, Any] = { 'isPending': is_pending, 'status': _display_status(variant.status), 'job': variant.job_type, @@ -72,10 +75,10 @@ def _display_status(status): @handler.get(handler.JSON) @handler.check_testcase_access - def get(self, testcase): + def get(self, testcase: data_types.Testcase) -> base_handler.Response: """Return testcase variants.""" - items = [] - message = None + items: list[dict[str, Any]] = [] + message: str | None = None if testcase.one_time_crasher_flag: message = 'Not run for unreproducible testcases.' elif not testcase.minimized_keys: diff --git a/src/appengine/handlers/testcase_detail/update_from_trunk.py b/src/appengine/handlers/testcase_detail/update_from_trunk.py index 04524369393..12105c2cce7 100644 --- a/src/appengine/handlers/testcase_detail/update_from_trunk.py +++ b/src/appengine/handlers/testcase_detail/update_from_trunk.py @@ -12,14 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. """Handler for updating from trunk. In other words, updating the stacktrace.""" + +from flask import Response + from clusterfuzz._internal.base import tasks +from clusterfuzz._internal.datastore import data_types from handlers import base_handler from handlers.testcase_detail import show from libs import handler from libs import helpers -def update(testcase): +def update(testcase: data_types.Testcase) -> None: """Update from trunk.""" testcase.last_tested_crash_stacktrace = 'Pending' testcase.put() @@ -41,7 +45,7 @@ class Handler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.require_csrf_token @handler.check_testcase_access - def post(self, testcase): + def post(self, testcase: data_types.Testcase) -> Response: """Update from trunk.""" update(testcase) return self.render_json(show.get_testcase_detail(testcase)) diff --git a/src/appengine/handlers/testcase_detail/update_issue.py b/src/appengine/handlers/testcase_detail/update_issue.py index ac003a7c441..adcd81ffbed 100644 --- a/src/appengine/handlers/testcase_detail/update_issue.py +++ b/src/appengine/handlers/testcase_detail/update_issue.py @@ -13,9 +13,14 @@ # limitations under the License. """Handler for updating issue.""" +from typing import Any +from typing import cast + from flask import request +from flask import Response from clusterfuzz._internal.datastore import data_handler +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.issue_management import issue_filer from clusterfuzz._internal.issue_management import issue_tracker_policy from handlers import base_handler @@ -28,7 +33,8 @@ class Handler(base_handler.Handler): """Handler that updates an issue.""" @staticmethod - def update_issue(testcase, issue_id, needs_summary_update): + def update_issue(testcase: data_types.Testcase, issue_id: Any, + needs_summary_update: Any) -> None: """Associate (or update) an existing issue with the testcase.""" issue_id = helpers.cast(issue_id, int, 'Issue ID (%s) is not a number!' % issue_id) @@ -73,10 +79,11 @@ def update_issue(testcase, issue_id, needs_summary_update): @handler.require_csrf_token @handler.check_admin_access_if_oss_fuzz @handler.check_testcase_access - def post(self, testcase): + def post(self, testcase: data_types.Testcase) -> Response: """Update an issue.""" - issue_id = request.get('issueId') - needs_summary_update = request.get('needsSummaryUpdate') + request_any = cast(Any, request) + issue_id = request_any.get('issueId') + needs_summary_update = request_any.get('needsSummaryUpdate') self.update_issue(testcase, issue_id, needs_summary_update) return self.render_json(show.get_testcase_detail(testcase)) diff --git a/src/appengine/handlers/testcase_list.py b/src/appengine/handlers/testcase_list.py index 43b3a50d9d0..71c18e9dcc5 100644 --- a/src/appengine/handlers/testcase_list.py +++ b/src/appengine/handlers/testcase_list.py @@ -13,6 +13,10 @@ # limitations under the License. """Handler that gets the testcase list page.""" +from typing import Any +from typing import cast +from typing import Mapping + from flask import request from clusterfuzz._internal.base import errors @@ -27,9 +31,9 @@ from libs import helpers from libs.query import datastore_query -PAGE_SIZE = 20 -MORE_LIMIT = 100 - PAGE_SIZE # exactly 5 pages -FIELDS = [ +PAGE_SIZE: int = 20 +MORE_LIMIT: int = 100 - PAGE_SIZE # exactly 5 pages +FIELDS: list[str] = [ 'crash_type', 'crash_state', 'job_type', @@ -54,10 +58,13 @@ class GroupFilter(filters.Filter): """Filter for group.""" - def __init__(self): + param_key: str + + def __init__(self) -> None: self.param_key = 'group' - def add(self, query, params): + def add(self, query: datastore_query.Query, + params: Mapping[str, Any]) -> None: """Add group filter.""" value = params.get(self.param_key, '') if filters.is_empty(value): @@ -66,7 +73,7 @@ def add(self, query, params): query.filter('group_id', helpers.cast(value, int, "'group' must be int.")) -KEYWORD_FILTERS = [ +KEYWORD_FILTERS: list[filters.Filter] = [ GroupFilter(), filters.String('bug_indices', 'issue'), filters.String('platform', 'platform'), @@ -78,7 +85,7 @@ def add(self, query, params): filters.String('job_type', 'job'), ] -FILTERS = [ +FILTERS: list[filters.Filter] = [ filters.String('impact_version_indices', 'impact'), filters.Boolean('has_bug_flag', 'issue'), filters.Boolean('open', 'open'), @@ -88,11 +95,11 @@ def add(self, query, params): filters.String('job_type', 'job'), filters.String('fuzzer_name_indices', 'fuzzer'), filters.String('project_name', 'project'), - filters.Int('crash_revision', 'revision_greater_than', operator='>') + filters.Int('crash_revision', 'revision_greater_than', operator='>'), ] -def add_filters(query, params): +def add_filters(query: datastore_query.Query, params: dict[str, Any]) -> None: """Add filters based on params.""" if not filters.has_params(params, FILTERS) and not params.get('showall'): params['open'] = 'yes' @@ -110,10 +117,12 @@ def add_filters(query, params): filters.add(query, params, FILTERS) -def get_result(): +def get_result() -> tuple[dict[str, Any], dict[str, Any]]: """Get the result for the testcase list page.""" - params = dict(request.iterparams()) - page = helpers.cast(request.get('page') or 1, int, "'page' is not an int.") + request_any = cast(Any, request) + params = dict(request_any.iterparams()) + page = helpers.cast( + request_any.get('page') or 1, int, "'page' is not an int.") query = datastore_query.Query(data_types.Testcase) crash_access.add_scope(query, params, 'security_flag', 'job_type', @@ -181,7 +190,7 @@ class Handler(base_handler.Handler): @handler.get(handler.HTML) @handler.oauth - def get(self): + def get(self) -> base_handler.Response: """Get and render the testcase list in HTML.""" result, params = get_result() field_values = { @@ -199,18 +208,18 @@ class CacheHandler(base_handler.Handler): """Handler for exercising cache.""" @handler.cron() - def get(self): + def get(self) -> None: """Handle a GET request.""" # pylint: disable=unexpected-keyword-arg # Memoize all project and job names. - _ = data_handler.get_all_project_names(__memoize_force__=True) - _ = data_handler.get_all_job_type_names(__memoize_force__=True) + _ = cast(Any, data_handler.get_all_project_names)(__memoize_force__=True) + _ = cast(Any, data_handler.get_all_job_type_names)(__memoize_force__=True) # Memoize both variants of get_all_fuzzer_names_including_children. - _ = data_handler.get_all_fuzzer_names_including_children( + _ = cast(Any, data_handler.get_all_fuzzer_names_including_children)( include_parents=True, __memoize_force__=True) - _ = data_handler.get_all_fuzzer_names_including_children( + _ = cast(Any, data_handler.get_all_fuzzer_names_including_children)( __memoize_force__=True) # Memoize expensive testcase attribute calls. @@ -230,7 +239,7 @@ class JsonHandler(base_handler.Handler): @handler.post(handler.JSON, handler.JSON) @handler.oauth - def post(self): + def post(self) -> base_handler.Response: """Get and render the testcase list in JSON.""" result, _ = get_result() return self.render_json(result) diff --git a/src/appengine/handlers/upload_testcase.py b/src/appengine/handlers/upload_testcase.py index 691e9b5711a..5371a506b29 100644 --- a/src/appengine/handlers/upload_testcase.py +++ b/src/appengine/handlers/upload_testcase.py @@ -19,8 +19,13 @@ import io import json import os +from typing import Any +from typing import cast +from typing import Optional +from typing import Union from flask import request +from flask import Response from google.cloud import ndb from clusterfuzz._internal import fuzzing @@ -47,59 +52,69 @@ from libs import helpers from libs.query import datastore_query -MAX_RETRIES = 50 -RUN_FILE_PATTERNS = ['run', 'fuzz-', 'index.', 'crash.'] -PAGE_SIZE = 20 -MORE_LIMIT = 100 - PAGE_SIZE -UPLOAD_URL = '/upload-testcase/upload-oauth' -MEMCACHE_TTL_IN_SECONDS = 60 * 60 # 1 hour. +MAX_RETRIES: int = 50 +RUN_FILE_PATTERNS: list[str] = ["run", "fuzz-", "index.", "crash."] +PAGE_SIZE: int = 20 +MORE_LIMIT: int = 100 - PAGE_SIZE +UPLOAD_URL: str = "/upload-testcase/upload-oauth" +MEMCACHE_TTL_IN_SECONDS: int = 60 * 60 # 1 hour. -TRUSTED_AGREEMENT_TEXT = 'This testcase is safe to run' +TRUSTED_AGREEMENT_TEXT: str = "This testcase is safe to run" -def _is_uploader_allowed(email): +def _is_uploader_allowed(email: str) -> bool: """Return bool on whether user is allowed to upload to any job or fuzzer.""" return external_users.is_upload_allowed_for_user(email) -def _is_trusted_uploader_allowed(email): +def _is_trusted_uploader_allowed(email: str) -> bool: """Return whether or not uploader is allowed and trusted.""" return access.has_access( need_privileged_access=True) and _is_uploader_allowed(email) -def attach_testcases(rows): +def attach_testcases(rows: list[dict[str, Any]]) -> None: """Attach testcase to each crash.""" - testcases = {} + testcases: dict[int, Any] = {} for index, row in enumerate(rows): - testcases[index] = query_testcase(row['testcaseId']) + testcases[index] = query_testcase(row["testcaseId"]) for index, row in enumerate(rows): - testcase = (list(testcases[index]) or [None])[0] - if testcase: + testcase_obj = (list(testcases[index]) or [None])[0] + testcase: Optional[dict[str, Any]] = None + if testcase_obj: testcase = { - 'crashType': testcase.crash_type, - 'crashStateLines': (testcase.crash_state or '').strip().splitlines(), - 'isSecurity': testcase.security_flag, - 'issueNumber': testcase.bug_information, - 'job': testcase.job_type, - 'fuzzerName': testcase.actual_fuzzer_name(), - 'projectName': testcase.project_name, + "crashType": + testcase_obj.crash_type, + "crashStateLines": + cast(str, testcase_obj.crash_state).strip().splitlines(), + "isSecurity": + testcase_obj.security_flag, + "issueNumber": + testcase_obj.bug_information, + "job": + testcase_obj.job_type, + "fuzzerName": + testcase_obj.actual_fuzzer_name(), + "projectName": + testcase_obj.project_name, } - row['testcase'] = testcase + row["testcase"] = testcase -def get_result(): +def get_result() -> tuple[dict[str, Any], dict[str, Any]]: """Get the result.""" - params = dict(request.iterparams()) - page = helpers.cast(request.get('page') or 1, int, "'page' is not an int.") + request_any = cast(Any, request) + params = dict(request_any.iterparams()) + page = helpers.cast( + request_any.get("page") or 1, int, "'page' is not an int.") query = datastore_query.Query(data_types.TestcaseUploadMetadata) - query.order('timestamp', is_desc=True) + query.order("timestamp", is_desc=True) if not access.has_access(need_privileged_access=True): - query.filter('uploader_email', helpers.get_user_email()) - params['permission'] = {'uploaderEmail': helpers.get_user_email()} + query.filter("uploader_email", helpers.get_user_email()) + params["permission"] = {"uploaderEmail": helpers.get_user_email()} entities, total_pages, total_items, has_more = query.fetch_page( page=page, page_size=PAGE_SIZE, projection=None, more_limit=MORE_LIMIT) @@ -107,38 +122,39 @@ def get_result(): items = [] for entity in entities: items.append({ - 'timestamp': utils.utc_datetime_to_timestamp(entity.timestamp), - 'testcaseId': entity.testcase_id, - 'uploaderEmail': entity.uploader_email, - 'filename': entity.filename, - 'bundled': entity.bundled, - 'pathInArchive': entity.path_in_archive, - 'status': entity.status + "timestamp": utils.utc_datetime_to_timestamp(entity.timestamp), + "testcaseId": entity.testcase_id, + "uploaderEmail": entity.uploader_email, + "filename": entity.filename, + "bundled": entity.bundled, + "pathInArchive": entity.path_in_archive, + "status": entity.status }) attach_testcases(items) result = { - 'hasMore': has_more, - 'items': items, - 'page': page, - 'pageSize': PAGE_SIZE, - 'totalItems': total_items, - 'totalPages': total_pages, + "hasMore": has_more, + "items": items, + "page": page, + "pageSize": PAGE_SIZE, + "totalItems": total_items, + "totalPages": total_pages, } return result, params -def _read_to_bytesio(gcs_path): +def _read_to_bytesio(gcs_path: str) -> io.BytesIO: """Return a bytesio representing a GCS object.""" data = storage.read_data(gcs_path) if not data: - raise helpers.EarlyExitError('Failed to read uploaded archive.', 500) + raise helpers.EarlyExitError("Failed to read uploaded archive.", 500) return io.BytesIO(data) -def guess_input_file(uploaded_file, filename): +def guess_input_file(uploaded_file: storage.GcsBlobInfo, + filename: str) -> Optional[str]: """Guess the main test case file from an archive.""" for file_pattern in RUN_FILE_PATTERNS: blob_reader = _read_to_bytesio(uploaded_file.gcs_path) @@ -150,7 +166,7 @@ def guess_input_file(uploaded_file, filename): return None -def query_testcase(testcase_id): +def query_testcase(testcase_id: Optional[Union[str, int]]) -> Any: """Start a query for an associated testcase.""" if not testcase_id: return [] @@ -159,34 +175,35 @@ def query_testcase(testcase_id): data_types.Testcase, testcase_id)).iter( limit=1, projection=[ - 'crash_type', - 'crash_state', - 'security_flag', - 'bug_information', - 'job_type', - 'fuzzer_name', - 'overridden_fuzzer_name', - 'project_name', + "crash_type", + "crash_state", + "security_flag", + "bug_information", + "job_type", + "fuzzer_name", + "overridden_fuzzer_name", + "project_name", ]) -def filter_target_names(targets, engine): +def filter_target_names(targets: list[str], engine: str) -> list[str]: """Filter target names for a fuzzer and remove parent fuzzer prefixes.""" - prefix = engine + '_' + prefix = engine + "_" return [t[len(prefix):] for t in targets if t.startswith(prefix)] -def filter_blackbox_fuzzers(fuzzers): +def filter_blackbox_fuzzers(fuzzers: list[str]) -> list[str]: """Filter out fuzzers such that only blackbox fuzzers are included.""" - def is_engine_fuzzer(name): + def is_engine_fuzzer(name: str) -> bool: return any(name.startswith(engine) for engine in fuzzing.ENGINES) return [f for f in fuzzers if not is_engine_fuzzer(f)] @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def find_fuzz_target(engine, target_name, job_name): +def find_fuzz_target(engine: str, target_name: str, + job_name: str) -> tuple[str, str]: """Return fuzz target values given the engine, target name (which may or may not be prefixed with project), and job.""" project_name = data_handler.get_project_name(job_name) @@ -195,12 +212,12 @@ def find_fuzz_target(engine, target_name, job_name): target = data_handler.get_fuzz_target(candidate_name) if not target: - raise helpers.EarlyExitError('Fuzz target does not exist.', 400) + raise helpers.EarlyExitError("Fuzz target does not exist.", 400) - return target.fully_qualified_name(), target.binary + return target.fully_qualified_name(), cast(str, target.binary) -def _allow_unprivileged_metadata(testcase_metadata): +def _allow_unprivileged_metadata(testcase_metadata: dict[str, Any]) -> bool: """Returns whether or not the provided testcase metadata can be set by an unprivileged user.""" if utils.is_oss_fuzz(): @@ -210,7 +227,7 @@ def _allow_unprivileged_metadata(testcase_metadata): # Allow *only* issue labels and skip_minimization to be set. # issue_labels must be present if any metadata is provided. - if 'issue_labels' not in testcase_metadata: + if "issue_labels" not in testcase_metadata: return False if len(testcase_metadata) == 1: @@ -218,7 +235,7 @@ def _allow_unprivileged_metadata(testcase_metadata): return True return len( - testcase_metadata) == 2 and 'skip_minimization' in testcase_metadata + testcase_metadata) == 2 and "skip_minimization" in testcase_metadata class Handler(base_handler.Handler): @@ -226,7 +243,7 @@ class Handler(base_handler.Handler): @handler.get(handler.HTML) @handler.oauth - def get(self): + def get(self) -> Response: """Handles get request.""" email = helpers.get_user_email() if not email: @@ -252,22 +269,22 @@ def get(self): result, params = get_result() return self.render( - 'upload.html', { - 'fieldValues': { - 'blackboxFuzzers': filter_blackbox_fuzzers(allowed_fuzzers), - 'jobs': allowed_jobs, - 'targets': { + "upload.html", { + "fieldValues": { + "blackboxFuzzers": filter_blackbox_fuzzers(allowed_fuzzers), + "jobs": allowed_jobs, + "targets": { engine: filter_target_names(allowed_fuzzers, engine) for engine in fuzzing.ENGINES }, - 'isChromium': utils.is_chromium(), - 'csrfToken': form.generate_csrf_token(), - 'isExternalUser': not is_privileged_or_domain_user, - 'uploadInfo': gcs.prepare_blob_upload()._asdict(), - 'hasIssueTracker': has_issue_tracker, + "isChromium": utils.is_chromium(), + "csrfToken": form.generate_csrf_token(), + "isExternalUser": not is_privileged_or_domain_user, + "uploadInfo": gcs.prepare_blob_upload()._asdict(), + "hasIssueTracker": has_issue_tracker, }, - 'params': params, - 'result': result + "params": params, + "result": result }) @@ -275,9 +292,9 @@ class PrepareUploadHandler(base_handler.Handler): """Handler that creates an upload URL.""" @handler.check_user_access(need_privileged_access=False) - def post(self): + def post(self) -> Response: """Serves the url.""" - return self.render_json({'uploadInfo': gcs.prepare_blob_upload()._asdict()}) + return self.render_json({"uploadInfo": gcs.prepare_blob_upload()._asdict()}) class UploadUrlHandlerOAuth(base_handler.Handler): @@ -285,10 +302,10 @@ class UploadUrlHandlerOAuth(base_handler.Handler): @handler.oauth @handler.check_user_access(need_privileged_access=False) - def post(self): + def post(self) -> Response: """Serves the url.""" return self.render_json({ - 'uploadUrl': request.host_url + UPLOAD_URL, + "uploadUrl": request.host_url + UPLOAD_URL, }) @@ -296,7 +313,7 @@ class JsonHandler(base_handler.Handler): """JSON handler for past testcase uploads.""" @handler.post(handler.JSON, handler.JSON) - def post(self): + def post(self) -> Response: """Handles a post request.""" if not helpers.get_user_email(): raise helpers.AccessDeniedError() @@ -308,61 +325,59 @@ def post(self): class UploadHandlerCommon: """Handler that uploads the testcase file.""" - def get_upload(self): + def get_upload(self) -> Optional[storage.GcsBlobInfo]: """Get the upload.""" raise NotImplementedError def _handle_upload(self, - uploaded_file, - job_type, - fuzzer_name, - target_name, - additional_arguments, - app_launch_command, - gestures, - platform_id, - http_flag, - high_end_job=None, - bug_information=None, - crash_revision=None, - timeout=None, - retries=None, - bug_summary_update_flag=None, - quiet_flag=None, - issue_labels=None, - stacktrace=None, - multiple_testcases=None, - trusted_agreement_signed=False, - testcase_id=None, - testcase_metadata=None, - skip_minimization=False) -> str: + uploaded_file: Optional[storage.GcsBlobInfo], + job_type: Optional[str], + fuzzer_name: Optional[str], + target_name: Optional[str], + additional_arguments: Optional[str], + app_launch_command: Optional[str], + gestures: Any, + platform_id: Optional[str], + http_flag: Any = None, + high_end_job: Any = None, + bug_information: Optional[str] = None, + crash_revision: Optional[Union[str, int]] = None, + timeout: Optional[Union[str, int]] = None, + retries: Optional[Union[str, int]] = None, + bug_summary_update_flag: Any = None, + quiet_flag: Any = None, + issue_labels: Optional[str] = None, + stacktrace: Optional[str] = None, + multiple_testcases: Optional[bool] = None, + trusted_agreement_signed: bool = False, + testcase_id: Optional[Union[str, int]] = None, + testcase_metadata: Any = None, + skip_minimization: bool = False) -> Response: """Holds the logic for actually performing a testcase upload.""" + self_any = cast(Any, self) if testcase_id and not uploaded_file: testcase = helpers.get_testcase(testcase_id) if not access.can_user_access_testcase(testcase): raise helpers.AccessDeniedError() - # Use minimized testcase for upload (if available). - if not access.can_user_access_testcase(testcase): - raise helpers.AccessDeniedError() - # Use minimized testcase for upload (if available). key = ( testcase.minimized_keys if testcase.minimized_keys and - testcase.minimized_keys != 'NA' else testcase.fuzzed_keys) + testcase.minimized_keys != "NA" else testcase.fuzzed_keys) - uploaded_file = blobs.get_blob_info(key) + uploaded_file = cast(storage.GcsBlobInfo, + blobs.get_blob_info(cast(str, key))) # Extract filename part from blob. uploaded_file.filename = os.path.basename( - uploaded_file.filename.replace('\\', os.sep)) + uploaded_file.filename.replace("\\", os.sep)) if not job_type: - raise helpers.EarlyExitError('Missing job name.', 400) + raise helpers.EarlyExitError("Missing job name.", 400) job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: - raise helpers.EarlyExitError('Invalid job name.', 400) + raise helpers.EarlyExitError("Invalid job name.", 400) job_type_lowercase = job_type.lower() @@ -374,19 +389,19 @@ def _handle_upload(self, if not is_engine_job and target_name: raise helpers.EarlyExitError( - 'Target name is not applicable to non-engine jobs (AFL, libFuzzer).', + "Target name is not applicable to non-engine jobs (AFL, libFuzzer).", 400) if is_engine_job and not target_name: raise helpers.EarlyExitError( - 'Missing target name for engine job (AFL, libFuzzer).', 400) + "Missing target name for engine job (AFL, libFuzzer).", 400) if (target_name and not data_types.Fuzzer.VALID_NAME_REGEX.match(target_name)): - raise helpers.EarlyExitError('Invalid target name.', 400) + raise helpers.EarlyExitError("Invalid target name.", 400) email = helpers.get_user_email() - fully_qualified_fuzzer_name = '' + fully_qualified_fuzzer_name = "" if is_engine_job and target_name: if _is_trusted_uploader_allowed(email) or job.is_external(): # External jobs don't run and set FuzzTarget entities as part of @@ -394,19 +409,21 @@ def _handle_upload(self, # Additionally, record fuzz target here for trusted uploaders # to avoid race conditions with newly added fuzz targets. fuzz_target = ( - data_handler.record_fuzz_target(fuzzer_name, target_name, job_type)) - fully_qualified_fuzzer_name = fuzz_target.fully_qualified_name() - target_name = fuzz_target.binary + data_handler.record_fuzz_target( + cast(str, fuzzer_name), target_name, job_type)) + fully_qualified_fuzzer_name = cast(Any, + fuzz_target).fully_qualified_name() + target_name = cast(Any, fuzz_target).binary else: fully_qualified_fuzzer_name, target_name = find_fuzz_target( - fuzzer_name, target_name, job_type) + cast(str, fuzzer_name), target_name, cast(str, job_type)) if (not access.has_access( need_privileged_access=False, job_type=job_type, fuzzer_name=(fully_qualified_fuzzer_name or fuzzer_name)) and not _is_uploader_allowed(email)): - helpers.log(f'User {email} does not have access', helpers.VIEW_OPERATION) + helpers.log(f"User {email} does not have access", helpers.VIEW_OPERATION) raise helpers.AccessDeniedError() # Ensure the job supports untrusted workloads if the uploader hasn't signed @@ -421,11 +438,11 @@ def _handle_upload(self, crash_data = None if job.is_external(): if not stacktrace: - raise helpers.EarlyExitError('Stacktrace required for external jobs.', + raise helpers.EarlyExitError("Stacktrace required for external jobs.", 400) if not crash_revision: - raise helpers.EarlyExitError('Revision required for external jobs.', + raise helpers.EarlyExitError("Revision required for external jobs.", 400) crash_data = stack_analyzer.get_crash_data( @@ -436,32 +453,32 @@ def _handle_upload(self, detect_ooms_and_hangs=True) elif stacktrace: raise helpers.EarlyExitError( - 'Should not specify stacktrace for non-external jobs.', 400) + "Should not specify stacktrace for non-external jobs.", 400) if testcase_metadata: try: testcase_metadata = json.loads(testcase_metadata) except Exception as e: - raise helpers.EarlyExitError('Invalid metadata JSON.', 400) from e + raise helpers.EarlyExitError("Invalid metadata JSON.", 400) from e if not isinstance(testcase_metadata, dict): - raise helpers.EarlyExitError('Metadata is not a JSON object.', 400) + raise helpers.EarlyExitError("Metadata is not a JSON object.", 400) if not testcase_metadata: testcase_metadata = {} if skip_minimization: - testcase_metadata['skip_minimization'] = True + testcase_metadata["skip_minimization"] = True if issue_labels: - testcase_metadata['issue_labels'] = issue_labels + testcase_metadata["issue_labels"] = issue_labels try: gestures = ast.literal_eval(gestures) except Exception as e: - raise helpers.EarlyExitError('Failed to parse gestures.', 400) from e + raise helpers.EarlyExitError("Failed to parse gestures.", 400) from e archive_state = 0 bundled = False - file_path_input = '' + file_path_input = "" # Certain modifications such as app launch command, issue updates are only # allowed for privileged users. @@ -469,67 +486,65 @@ def _handle_upload(self, if not privileged_user: if bug_information or bug_summary_update_flag: raise helpers.EarlyExitError( - 'You are not privileged to update existing issues.', 400) + "You are not privileged to update existing issues.", 400) need_privileged_access = utils.string_is_true( data_handler.get_value_from_job_definition(job_type, - 'PRIVILEGED_ACCESS')) + "PRIVILEGED_ACCESS")) if need_privileged_access: raise helpers.EarlyExitError( - 'You are not privileged to run this job type.', 400) + "You are not privileged to run this job type.", 400) if app_launch_command: raise helpers.EarlyExitError( - 'You are not privileged to run arbitrary launch commands.', 400) + "You are not privileged to run arbitrary launch commands.", 400) if (testcase_metadata and not _allow_unprivileged_metadata(testcase_metadata)): raise helpers.EarlyExitError( - 'You are not privileged to set testcase metadata.', 400) + "You are not privileged to set testcase metadata.", 400) if gestures: raise helpers.EarlyExitError( - 'You are not privileged to run arbitrary gestures.', 400) + "You are not privileged to run arbitrary gestures.", 400) - if crash_revision and crash_revision.isdigit(): + if crash_revision and cast(Any, crash_revision).isdigit(): crash_revision = int(crash_revision) else: crash_revision = 0 - if bug_information == '0': # Auto-recover from this bad input. + if bug_information == "0": # Auto-recover from this bad input. bug_information = None - if bug_information and not bug_information.isdigit(): - raise helpers.EarlyExitError('Bug is not a number.', 400) + if bug_information and not cast(Any, bug_information).isdigit(): + raise helpers.EarlyExitError("Bug is not a number.", 400) if not timeout: timeout = 0 - elif not timeout.isdigit() or timeout == '0': + elif not cast(Any, timeout).isdigit() or timeout == "0": raise helpers.EarlyExitError( - 'Testcase timeout must be a number greater than 0.', 400) + "Testcase timeout must be a number greater than 0.", 400) else: timeout = int(timeout) if timeout > 120: raise helpers.EarlyExitError( - 'Testcase timeout may not be greater than 120 seconds.', 400) + "Testcase timeout may not be greater than 120 seconds.", 400) if retries: - if retries.isdigit(): + if cast(Any, retries).isdigit(): retries = int(retries) else: retries = None if retries is None or retries > MAX_RETRIES: raise helpers.EarlyExitError( - 'Testcase retries must be a number less than %d.' % MAX_RETRIES, + "Testcase retries must be a number less than %d." % MAX_RETRIES, 400) - else: - retries = None job_queue = tasks.queue_for_job(job_type, is_high_end=high_end_job) - if uploaded_file is not None: - filename = ''.join([ - x for x in uploaded_file.filename if x not in ' ;/?:@&=+$,{}|<>()\\' + if uploaded_file: + filename = "".join([ + x for x in uploaded_file.filename if x not in " ;/?:@&=+$,{}|<>()\\" ]) key = str(uploaded_file.key()) if archive.is_archive(filename): @@ -559,7 +574,7 @@ def _handle_upload(self, # Use wait_time=0 to execute the task ASAP, since it is # user-facing. tasks.add_task( - 'unpack', + "unpack", str(metadata.key.id()), job_type, queue=tasks.queue_for_job(job_type), @@ -571,10 +586,10 @@ def _handle_upload(self, upload_metadata.filename = filename upload_metadata.blobstore_key = key upload_metadata.original_blobstore_key = key - upload_metadata.status = 'Pending' + upload_metadata.status = "Pending" upload_metadata.bundled = True upload_metadata.uploader_email = email - upload_metadata.retries = retries + upload_metadata.retries = cast(int | None, retries) upload_metadata.bug_summary_update_flag = bug_summary_update_flag upload_metadata.quiet_flag = quiet_flag upload_metadata.additional_metadata_string = json.dumps( @@ -582,17 +597,18 @@ def _handle_upload(self, upload_metadata.bug_information = bug_information upload_metadata.put() - helpers.log('Uploaded multiple testcases.', helpers.VIEW_OPERATION) - return self.render_json({'multiple': True}) + helpers.log("Uploaded multiple testcases.", helpers.VIEW_OPERATION) + return self_any.render_json({"multiple": True}) - file_path_input = guess_input_file(uploaded_file, filename) - if not file_path_input: + guessed_input = guess_input_file(uploaded_file, filename) + if not guessed_input: raise helpers.EarlyExitError( - ("Unable to detect which file to launch. The main file\'s name " - 'must contain either of %s.' % str(RUN_FILE_PATTERNS)), 400) + ("Unable to detect which file to launch. The main file's name " + "must contain either of %s." % str(RUN_FILE_PATTERNS)), 400) + file_path_input = guessed_input else: - raise helpers.EarlyExitError('Please select a file to upload.', 400) + raise helpers.EarlyExitError("Please select a file to upload.", 400) testcase_id = data_handler.create_user_uploaded_testcase( key, @@ -611,11 +627,11 @@ def _handle_upload(self, email, platform_id, app_launch_command, - fuzzer_name, + cast(str, fuzzer_name), fully_qualified_fuzzer_name, target_name, bundled, - retries, + cast(Optional[int], retries), bug_summary_update_flag, quiet_flag, additional_metadata=testcase_metadata, @@ -634,42 +650,43 @@ def _handle_upload(self, report_url = data_handler.TESTCASE_REPORT_URL.format( domain=data_handler.get_domain(), testcase_id=testcase_id) - comment = ('ClusterFuzz is analyzing your testcase. ' - 'Developers can follow the progress at %s.' % report_url) + comment = ("ClusterFuzz is analyzing your testcase. " + "Developers can follow the progress at %s." % report_url) issue.save(new_comment=comment) - helpers.log(f'Uploaded testcase {testcase_id}', helpers.VIEW_OPERATION) - return self.render_json({'id': str(testcase_id)}) # pylint: disable=no-member + helpers.log(f"Uploaded testcase {testcase_id}", helpers.VIEW_OPERATION) + return self_any.render_json({"id": str(testcase_id)}) # pylint: disable=no-member - def do_post(self): + def do_post(self) -> Response: """Upload a testcase.""" # Set artifical task id env to be used by tracing. - environment.set_task_id_vars(task_name='upload_testcase') - testcase_id = request.get('testcaseId') + environment.set_task_id_vars(task_name="upload_testcase") + request_any = cast(Any, request) + testcase_id = request_any.get("testcaseId") uploaded_file = self.get_upload() - job_type = request.get('job') - fuzzer_name = request.get('fuzzer') - target_name = request.get('target') - multiple_testcases = bool(request.get('multiple')) - http_flag = bool(request.get('http')) - high_end_job = bool(request.get('highEnd')) - bug_information = request.get('issue') - crash_revision = request.get('revision') - timeout = request.get('timeout') - retries = request.get('retries') - bug_summary_update_flag = bool(request.get('updateIssue')) - quiet_flag = bool(request.get('quiet')) - additional_arguments = request.get('args') - app_launch_command = request.get('cmd') - platform_id = request.get('platform') - issue_labels = request.get('issue_labels') - gestures = request.get('gestures') or '[]' - stacktrace = request.get('stacktrace') - trusted_agreement_signed = request.get( - 'trustedAgreement') == TRUSTED_AGREEMENT_TEXT.strip() - testcase_metadata = request.get('metadata') + job_type = request_any.get("job") + fuzzer_name = request_any.get("fuzzer") + target_name = request_any.get("target") + multiple_testcases = bool(request_any.get("multiple")) + http_flag = bool(request_any.get("http")) + high_end_job = bool(request_any.get("highEnd")) + bug_information = request_any.get("issue") + crash_revision = request_any.get("revision") + timeout = request_any.get("timeout") + retries = request_any.get("retries") + bug_summary_update_flag = bool(request_any.get("updateIssue")) + quiet_flag = bool(request_any.get("quiet")) + additional_arguments = request_any.get("args") + app_launch_command = request_any.get("cmd") + platform_id = request_any.get("platform") + issue_labels = request_any.get("issue_labels") + gestures = request_any.get("gestures") or "[]" + stacktrace = request_any.get("stacktrace") + trusted_agreement_signed = request_any.get( + "trustedAgreement") == TRUSTED_AGREEMENT_TEXT.strip() + testcase_metadata = request_any.get("metadata") # TODO: Consider using utils.is_string to handle "false" properly. - skip_minimization = bool(request.get('skip_minimization')) + skip_minimization = bool(request_any.get("skip_minimization")) return self._handle_upload( uploaded_file=uploaded_file, @@ -702,23 +719,25 @@ class UploadHandler(UploadHandlerCommon, base_handler.GcsUploadHandler): """Handler that uploads the testcase file.""" # pylint: disable=unused-argument - def before_render_json(self, values, status): + def before_render_json(self, values: dict[str, Any], status: int) -> None: """Add upload info when the request fails.""" - values['uploadInfo'] = gcs.prepare_blob_upload()._asdict() + values["uploadInfo"] = gcs.prepare_blob_upload()._asdict() - def get_upload(self): + def get_upload(self) -> Optional[storage.GcsBlobInfo]: return base_handler.GcsUploadHandler.get_upload(self) @handler.post(handler.FORM, handler.JSON) @handler.require_csrf_token - def post(self): + def post(self) -> Response: return self.do_post() class NamedBytesIO(io.BytesIO): """Named bytesio.""" - def __init__(self, name, value): + name: str + + def __init__(self, name: str, value: bytes) -> None: self.name = name io.BytesIO.__init__(self, value) @@ -727,47 +746,49 @@ class UploadHandlerOAuth(base_handler.Handler, UploadHandlerCommon): """Handler that uploads the testcase file (OAuth).""" # pylint: disable=unused-argument - def before_render_json(self, values, status): + def before_render_json(self, values: dict[str, Any], status: int) -> None: """Add upload info when the request fails.""" - values['uploadUrl'] = request.host_url + UPLOAD_URL + values["uploadUrl"] = request.host_url + UPLOAD_URL - def get_upload(self): + def get_upload(self) -> Optional[storage.GcsBlobInfo]: """Get the upload.""" - uploaded_file = request.files.get('file') + request_any = cast(Any, request) + uploaded_file = request_any.files.get("file") if not uploaded_file: - raise helpers.EarlyExitError('File upload not found.', 400) + raise helpers.EarlyExitError("File upload not found.", 400) - bytes_io = NamedBytesIO(uploaded_file.filename, uploaded_file.stream.read()) + bytes_io = NamedBytesIO( + cast(str, uploaded_file.filename), uploaded_file.stream.read()) key = blobs.write_blob(bytes_io) return blobs.get_blob_info(key) @handler.post(handler.FORM, handler.JSON) @handler.oauth - def post(self, *args): + def post(self, *args: Any) -> Response: return self.do_post() class CrashReplicationUploadHandler(base_handler.Handler, UploadHandlerCommon): """Handler that picks up the pubsub notification.""" - def get_upload(self): + def get_upload(self) -> None: pass @handler.pubsub_push - def post(self, message): + def post(self, message: Any) -> Response: """Uploads a crash sampled from fuzz task.""" with monitor.wrap_with_monitoring(): - message_data = json.loads(message.data.decode('utf-8')) - helpers.log(f'Message: {type(message)} {message}', helpers.VIEW_OPERATION) - - job = message_data.get('job', None) - fuzzer = message_data.get('fuzzer', None) - fuzz_target = message_data.get('target_name', None) - original_task_id = message_data.get('original_task_id', None) - helpers.log(f'Uploading testcase from fuzz task id {original_task_id}', + message_data = json.loads(message.data.decode("utf-8")) + helpers.log(f"Message: {type(message)} {message}", helpers.VIEW_OPERATION) + + job = message_data.get("job", None) + fuzzer = message_data.get("fuzzer", None) + fuzz_target = message_data.get("target_name", None) + original_task_id = message_data.get("original_task_id", None) + helpers.log(f"Uploading testcase from fuzz task id {original_task_id}", helpers.VIEW_OPERATION) helpers.log(message.data.decode(), helpers.VIEW_OPERATION) - uploaded_file_key = message_data['fuzzed_key'] + uploaded_file_key = message_data["fuzzed_key"] uploaded_file = blobs.get_blob_info(uploaded_file_key) try: response = self._handle_upload( @@ -775,26 +796,26 @@ def post(self, message): job_type=job, fuzzer_name=fuzzer, target_name=fuzz_target, - additional_arguments=message_data.get('arguments', None), - app_launch_command=message_data.get('application_command_line', + additional_arguments=message_data.get("arguments", None), + app_launch_command=message_data.get("application_command_line", None), - gestures=message_data.get('gestures', '[]'), - http_flag=message_data.get('http_flag', None), - platform_id='Linux', + gestures=message_data.get("gestures", "[]"), + http_flag=message_data.get("http_flag", None), + platform_id="Linux", trusted_agreement_signed=True, ) monitoring_metrics.UPLOAD_TESTCASE_COUNT.increment({ - 'fuzzer': fuzzer, - 'job': job, - 'fuzz_target': fuzz_target, - 'success': True, + "fuzzer": fuzzer, + "job": job, + "fuzz_target": fuzz_target, + "success": True, }) return response except Exception as e: monitoring_metrics.UPLOAD_TESTCASE_COUNT.increment({ - 'fuzzer': fuzzer, - 'job': job, - 'fuzz_target': fuzz_target, - 'success': False, + "fuzzer": fuzzer, + "job": job, + "fuzz_target": fuzz_target, + "success": False, }) raise e diff --git a/src/appengine/handlers/viewer.py b/src/appengine/handlers/viewer.py index 20fecfd3aef..1dd8cc23008 100644 --- a/src/appengine/handlers/viewer.py +++ b/src/appengine/handlers/viewer.py @@ -13,6 +13,9 @@ # limitations under the License. """Content viewer.""" +from typing import Any +from typing import cast + from flask import request from clusterfuzz._internal.base import utils @@ -22,20 +25,20 @@ from libs import handler from libs import helpers -MAX_ALLOWED_CONTENT_SIZE = 10 * 1024 * 1024 +MAX_ALLOWED_CONTENT_SIZE: int = 10 * 1024 * 1024 class Handler(base_handler.Handler): """Content Viewer.""" @handler.get(handler.HTML) - def get(self): + def get(self) -> base_handler.Response: """Get the HTML page.""" - key = request.get('key') + key = cast(Any, request).get('key') if not key: raise helpers.EarlyExitError('No key provided.', 400) - testcase_id = request.get('testcase_id') + testcase_id = cast(Any, request).get('testcase_id') if testcase_id: testcase = helpers.get_testcase(testcase_id) if not access.can_user_access_testcase(testcase): @@ -56,7 +59,8 @@ def get(self): # once it is fixed properly upstream: # https://github.com/googleapis/google-cloud-python/issues/6572 try: - content = blobs.read_key(key).decode('utf-8', errors='replace') + content = cast(bytes, blobs.read_key(key)).decode( + 'utf-8', errors='replace') except Exception: raise helpers.EarlyExitError('Failed to read content.', 400) else: diff --git a/src/appengine/libs/access.py b/src/appengine/libs/access.py index 8cf8fa534cb..c84a2151591 100644 --- a/src/appengine/libs/access.py +++ b/src/appengine/libs/access.py @@ -19,6 +19,7 @@ from clusterfuzz._internal.config import db_config from clusterfuzz._internal.config import local_config from clusterfuzz._internal.datastore import data_handler +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.google_cloud_utils import google_groups from clusterfuzz._internal.issue_management import issue_tracker_utils from clusterfuzz._internal.metrics import logs @@ -26,11 +27,11 @@ from libs import helpers from libs import request_cache -PRIVILEGED_USER_CACHE_CAPACITY = 10 +PRIVILEGED_USER_CACHE_CAPACITY: int = 10 @request_cache.wrap(PRIVILEGED_USER_CACHE_CAPACITY) -def _is_privileged_user(email): +def _is_privileged_user(email: str) -> bool: """Check if an email is in the privileged users list.""" if local_config.AuthConfig().get('all_users_privileged'): return True @@ -66,7 +67,7 @@ def _is_privileged_user(email): return False -def _is_blacklisted_user(email): +def _is_blacklisted_user(email: str) -> bool: """Check if an email is in the privileged users list.""" blacklisted_user_emails = (db_config.get_value('blacklisted_users') or '').splitlines() @@ -75,7 +76,7 @@ def _is_blacklisted_user(email): for blacklisted_user_email in blacklisted_user_emails) -def get_user_job_type(): +def get_user_job_type() -> str | None: """Return the job_type that is assigned to the current user. None means one can access any job type. You might want to invoke get_access(..) with the job type afterward.""" @@ -92,7 +93,7 @@ def get_user_job_type(): return None -def _is_domain_allowed(email): +def _is_domain_allowed(email: str) -> bool: """Check if the email's domain is allowed.""" domains = local_config.AuthConfig().get('whitelisted_domains', default=[]) for domain in domains: @@ -102,7 +103,7 @@ def _is_domain_allowed(email): return False -def _is_trusted_domain_user(user): +def _is_trusted_domain_user(user: auth.User | None) -> bool: """Check if a user should be trusted based on their email domain. Domain-allowlist trust requires a verified email: a self-assertable @@ -112,10 +113,16 @@ def _is_trusted_domain_user(user): class UserAccess: - Allowed, Denied, Redirected = list(range(3)) # pylint: disable=invalid-name + # pylint: disable=invalid-name + Allowed: int + Denied: int + Redirected: int + Allowed, Denied, Redirected = list(range(3)) -def has_access(need_privileged_access=False, job_type=None, fuzzer_name=None): +def has_access(need_privileged_access: bool = False, + job_type: str | None = None, + fuzzer_name: str | None = None) -> bool: """Check if the user has access.""" result = get_access( need_privileged_access=need_privileged_access, @@ -125,7 +132,9 @@ def has_access(need_privileged_access=False, job_type=None, fuzzer_name=None): return result == UserAccess.Allowed -def get_access(need_privileged_access=False, job_type=None, fuzzer_name=None): +def get_access(need_privileged_access: bool = False, + job_type: str | None = None, + fuzzer_name: str | None = None) -> int: """Return 'allowed', 'redirected', or 'failed'.""" if auth.is_current_user_admin(): return UserAccess.Allowed @@ -154,11 +163,12 @@ def get_access(need_privileged_access=False, job_type=None, fuzzer_name=None): return UserAccess.Denied -def can_user_access_testcase(testcase): +def can_user_access_testcase(testcase: data_types.Testcase) -> bool: """Checks if the current user can access the testcase.""" config = db_config.get() - need_privileged_access = ( - testcase.security_flag and not config.relax_security_bug_restrictions) + need_privileged_access = bool( + testcase.security_flag and + not (config and config.relax_security_bug_restrictions)) if has_access( fuzzer_name=testcase.actual_fuzzer_name(), @@ -176,6 +186,8 @@ def can_user_access_testcase(testcase): return False issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(testcase) + if not issue_tracker: + return False associated_issue = issue_tracker.get_issue(issue_id) if not associated_issue: return False @@ -188,9 +200,8 @@ def can_user_access_testcase(testcase): if original_issue: issues_to_check.append(original_issue) - relaxed_restrictions = ( - config.relax_testcase_restrictions or - _is_trusted_domain_user(auth.get_current_user())) + relaxed_restrictions = bool((config and config.relax_testcase_restrictions) or + _is_trusted_domain_user(auth.get_current_user())) for issue in issues_to_check: if relaxed_restrictions: if (any(utils.emails_equal(user_email, cc) for cc in issue.ccs) or @@ -204,7 +215,8 @@ def can_user_access_testcase(testcase): return False -def check_access_and_get_testcase(testcase_id): +def check_access_and_get_testcase( + testcase_id: int | str) -> data_types.Testcase: """Check the failed attempt count and get the testcase.""" if not helpers.get_user_email(): raise helpers.UnauthorizedError() diff --git a/src/appengine/libs/auth.py b/src/appengine/libs/auth.py index 0ee33b2a075..0aa70dde843 100644 --- a/src/appengine/libs/auth.py +++ b/src/appengine/libs/auth.py @@ -13,9 +13,15 @@ # limitations under the License. """Authentication helpers.""" -import collections +import datetime +from typing import Any +from typing import cast +from typing import NamedTuple +from typing import Optional +from typing import Union from firebase_admin import auth +from firebase_admin import exceptions from google.auth.transport import requests as google_requests from google.cloud import ndb from google.oauth2 import id_token @@ -32,8 +38,12 @@ from libs import helpers from libs import request_cache -User = collections.namedtuple( - 'User', ['email', 'email_verified'], defaults=(True,)) + +class User(NamedTuple): + email: str + email_verified: bool = True + + BEARER_PREFIX = 'Bearer ' @@ -41,7 +51,7 @@ class AuthError(Exception): """Auth error.""" -def auth_domain(): +def auth_domain() -> str: """Get the auth domain.""" domain = local_config.ProjectConfig().get('firebase.auth_domain') if not domain: @@ -50,16 +60,17 @@ def auth_domain(): return domain -def real_auth_domain(): +def real_auth_domain() -> str: """Get the real auth domain""" real_domain = local_config.ProjectConfig().get('firebase.real_auth_domain') if real_domain: return real_domain - return utils.get_application_id() + '.firebaseapp.com' + app_id = utils.get_application_id() + return cast(str, app_id) + '.firebaseapp.com' -def is_current_user_admin(): +def is_current_user_admin() -> bool: """Returns whether or not the current logged in user is an admin.""" if environment.is_local_development(): return True @@ -73,7 +84,7 @@ def is_current_user_admin(): @memoize.wrap(memoize.FifoInMemory(1)) -def _project_number_from_id(project_id): +def _project_number_from_id(project_id: str) -> str: """Get the project number from project ID.""" resource_manager = discovery.build('cloudresourcemanager', 'v1') # pylint: disable=no-member @@ -85,7 +96,7 @@ def _project_number_from_id(project_id): @memoize.wrap(memoize.FifoInMemory(1)) -def _get_iap_key(key_id): +def _get_iap_key(key_id: str) -> str: """Retrieves a public key from the list published by Identity-Aware Proxy, re-fetching the key file if necessary. """ @@ -103,9 +114,9 @@ def _get_iap_key(key_id): return key -def _validate_iap_jwt(iap_jwt): +def _validate_iap_jwt(iap_jwt: str) -> str: """Validate JWT assertion.""" - project_id = utils.get_application_id() + project_id = cast(str, utils.get_application_id()) expected_audience = f'/projects/{_project_number_from_id(project_id)}' \ f'/apps/{project_id}' @@ -127,7 +138,7 @@ def _validate_iap_jwt(iap_jwt): raise AuthError('JWT assertion decode error: ' + str(e)) -def get_iap_email(current_request): +def get_iap_email(current_request: Any) -> Optional[str]: """Get Cloud IAP email.""" jwt_assertion = current_request.headers.get('X-Goog-IAP-JWT-Assertion') if not jwt_assertion: @@ -136,7 +147,7 @@ def get_iap_email(current_request): return _validate_iap_jwt(jwt_assertion) -def get_email_from_bearer_token(request): +def get_email_from_bearer_token(request: Any) -> Optional[str]: """Gets the email corresponding to a bearer token in an http request.""" bearer_token = request.headers.get('Authorization', '') if not bearer_token.startswith(BEARER_PREFIX): @@ -147,16 +158,17 @@ def get_email_from_bearer_token(request): claim = id_token.verify_oauth2_token(token, google_requests.Request()) except ValueError: raise helpers.UnauthorizedError('Malformed bearer token') - if (not claim.get('email_verified') or - claim.get('email') != utils.service_account_email()): + claim_dict = cast(dict[str, Any], claim) + if (not claim_dict.get('email_verified') or + claim_dict.get('email') != utils.service_account_email()): raise helpers.UnauthorizedError('Invalid ID token.') - if not claim.get('email_verified'): + if not claim_dict.get('email_verified'): return None - return claim.get('email') + return claim_dict.get('email') -def get_current_user(): +def get_current_user() -> Optional[User]: """Get the current logged in user, or None.""" if environment.is_local_development(): return User('user@localhost') @@ -194,7 +206,7 @@ def get_current_user(): return None try: - decoded_claims = decode_claims(get_session_cookie()) + decoded_claims = decode_claims(session_cookie) except AuthError: logs.warning('Invalid session cookie.') return None @@ -231,28 +243,29 @@ def get_current_user(): return User(email, email_verified) -def create_session_cookie(token, expires_in): +def create_session_cookie(token: str, + expires_in: Union[int, datetime.timedelta]) -> str: """Create a new session cookie.""" try: return auth.create_session_cookie(token, expires_in=expires_in) - except auth.AuthError: + except exceptions.FirebaseError: raise AuthError('Failed to create session cookie.') -def get_session_cookie(): +def get_session_cookie() -> Optional[str]: """Get the current session cookie.""" return request_cache.get_current_request().cookies.get('session') -def revoke_session_cookie(session_cookie): +def revoke_session_cookie(session_cookie: str) -> None: """Revoke a session cookie.""" decoded_claims = decode_claims(session_cookie) auth.revoke_refresh_tokens(decoded_claims['sub']) -def decode_claims(session_cookie): +def decode_claims(session_cookie: str) -> dict[str, Any]: """Decode the claims for the current session cookie.""" try: return auth.verify_session_cookie(session_cookie, check_revoked=True) - except (ValueError, auth.AuthError): + except (ValueError, exceptions.FirebaseError): raise AuthError('Invalid session cookie.') diff --git a/src/appengine/libs/crash_access.py b/src/appengine/libs/crash_access.py index 705af5c77aa..bcaebadf5f0 100644 --- a/src/appengine/libs/crash_access.py +++ b/src/appengine/libs/crash_access.py @@ -19,20 +19,25 @@ Crash access control is used by the testcase list page and the crash stats page.""" -import collections +from typing import Any +from typing import NamedTuple from clusterfuzz._internal.base import external_users from clusterfuzz._internal.datastore import data_types from libs import access from libs import helpers +from libs.query import base -Scope = collections.namedtuple('Scope', [ - 'everything', 'is_privileged', 'job_types', 'fuzzer_names', - 'allowed_job_type' -]) +class Scope(NamedTuple): + everything: bool + is_privileged: bool + job_types: list[str] + fuzzer_names: list[str] + allowed_job_type: str | None -def get_permission_names(entity_kind): + +def get_permission_names(entity_kind: int) -> list[str]: """Get scoped fuzzer names.""" # pylint: disable=protected-access permissions = external_users._get_permissions_query_for_user( @@ -45,7 +50,7 @@ def get_permission_names(entity_kind): return names -def get_scope(): +def get_scope() -> Scope: """Get the scope object for the user.""" user_email = helpers.get_user_email() is_privileged = access.has_access(need_privileged_access=True) @@ -67,7 +72,7 @@ def get_scope(): allowed_job_type) -def add_permissions_to_params(scope, params): +def add_permissions_to_params(scope: Scope, params: dict[str, Any]) -> None: """Add permissions to params.""" params['permissions'] = { 'everything': scope.everything, @@ -80,7 +85,8 @@ def add_permissions_to_params(scope, params): params['permissions']['jobs'].append(scope.allowed_job_type) -def add_scope(query, params, security_field, job_type_field, fuzzer_name_field): +def add_scope(query: base.Query, params: dict[str, Any], security_field: str, + job_type_field: str, fuzzer_name_field: str) -> None: """Add scope to the query according to permissions and modify params.""" scope = get_scope() add_permissions_to_params(scope, params) diff --git a/src/appengine/libs/crash_stats.py b/src/appengine/libs/crash_stats.py index 6c0d3599c1b..175139a3bdb 100644 --- a/src/appengine/libs/crash_stats.py +++ b/src/appengine/libs/crash_stats.py @@ -15,26 +15,34 @@ testcase detail page. This can't be with common.crash_stats because it imports specific libraries on appengine.""" +from typing import Any +from typing import cast + from clusterfuzz._internal.metrics import crash_stats from libs import helpers from libs.query import big_query_query # We don't allow showing more than 3 days when viewing by hours because it'd # break the UI. -MAX_DAYS_FOR_BY_HOURS = 3 +MAX_DAYS_FOR_BY_HOURS: int = 3 class Query(big_query_query.Query): """A query class for crash stats. It contains two additional fields.""" - def __init__(self): + end: int | None + days: int | None + block: str | None + group_by: str | None + + def __init__(self) -> None: self.end = None self.days = None self.block = None self.group_by = None super().__init__() - def set_time_params(self, end, days, block): + def set_time_params(self, end: int, days: int, block: str) -> None: """Set time-related params.""" if block == 'hour' and days > MAX_DAYS_FOR_BY_HOURS: raise helpers.EarlyExitError( @@ -48,15 +56,20 @@ def set_time_params(self, end, days, block): self.block = block -def get(query, group_query, offset, limit): +def get( + query: Query, + group_query: big_query_query.Query, + offset: int, + limit: int | None = None, +) -> tuple[int, list[dict[str, Any]]]: """Query from BigQuery given the query object.""" return crash_stats.get( - end=query.end, - days=query.days, - block=query.block, - group_by=query.group_by, + end=cast(int, query.end), + days=cast(int, query.days), + block=cast(str, query.block), + group_by=cast(str, query.group_by), where_clause=query.get_where_clause(), group_having_clause=group_query.get_where_clause(), - sort_by=query.sort_by, + sort_by=cast(str, query.sort_by), offset=offset, limit=limit) diff --git a/src/appengine/libs/csp.py b/src/appengine/libs/csp.py index a9ea63dcec5..43306cdc4de 100644 --- a/src/appengine/libs/csp.py +++ b/src/appengine/libs/csp.py @@ -20,10 +20,10 @@ class CSPBuilder: """Helper to build a Content Security Policy string.""" - def __init__(self): - self.directives = collections.defaultdict(list) + def __init__(self) -> None: + self.directives: dict[str, list[str]] = collections.defaultdict(list) - def add(self, directive, source, quote=False): + def add(self, directive: str, source: str, quote: bool = False) -> None: """Add a source for a given directive.""" # Some values for sources are expected to be quoted. No escaping is done # since these are specific literal values that don't require it. @@ -34,13 +34,13 @@ def add(self, directive, source, quote=False): f'Duplicate source "{source}" for directive "{directive}"') self.directives[directive].append(source) - def add_sourceless(self, directive): + def add_sourceless(self, directive: str) -> None: assert directive not in self.directives, ( f'Sourceless directive "{directive}" already exists.') self.directives[directive] = [] - def remove(self, directive, source, quote=False): + def remove(self, directive: str, source: str, quote: bool = False) -> None: """Remove a source for a given directive.""" if quote: source = f'\'{source}\'' @@ -49,7 +49,7 @@ def remove(self, directive, source, quote=False): f'Removing nonexistent "{source}" for directive "{directive}"') self.directives[directive].remove(source) - def __str__(self): + def __str__(self) -> str: """Convert to a string to send with a Content-Security-Policy header.""" parts = [] @@ -61,7 +61,7 @@ def __str__(self): return ' '.join(parts) -def get_default_builder(): +def get_default_builder() -> CSPBuilder: """Get a CSPBuilder object for the default policy. Can be modified for specific pages if needed.""" @@ -141,6 +141,6 @@ def get_default_builder(): return builder -def get_default(): +def get_default() -> str: """Get the default Content Security Policy as a string.""" return str(get_default_builder()) diff --git a/src/appengine/libs/filters.py b/src/appengine/libs/filters.py index 41373e409f1..23c1b0574d5 100644 --- a/src/appengine/libs/filters.py +++ b/src/appengine/libs/filters.py @@ -18,25 +18,31 @@ import re import sys +from typing import Any +from typing import Callable +from typing import Mapping +from typing import Optional +from typing import Sequence from clusterfuzz._internal.datastore import search_tokenizer from libs import helpers -KEYWORD_FIELD_REGEX = ( +KEYWORD_FIELD_REGEX: str = ( '(?: +|^)%s:((?:"[^"]*")|(?:\'[^\']*\')|(?:[^ ]*))(?: +|$)') -def is_empty(value): +def is_empty(value: Any) -> bool: """Determine if the param's value is considered as empty.""" return not value -def has_params(params, filters): +def has_params(params: Mapping[str, Any], filters: Sequence['Filter']) -> bool: """Check if there's any param.""" return any(params.get(fltr.param_key) for fltr in filters) -def extract_keyword_field(keyword, field): +def extract_keyword_field(keyword: str, + field: str) -> tuple[str, Optional[str]]: """Extract the value from the keyword given the field and return the new keyword.""" regex = re.compile(KEYWORD_FIELD_REGEX % field, flags=re.IGNORECASE) @@ -52,7 +58,7 @@ def extract_keyword_field(keyword, field): return keyword, None -def get_boolean(value): +def get_boolean(value: Any) -> bool: """Convert yes/no to boolean or raise Exception.""" if value == 'yes': return True @@ -61,15 +67,16 @@ def get_boolean(value): raise ValueError("The value must be 'yes' or 'no'.") -def get_string(value): +def get_string(value: str) -> str: """Get sanitized string.""" return value.strip() class Filter: """Base filter.""" + param_key: str = '' - def add(self, query, params): + def add(self, query: Any, params: Mapping[str, Any]) -> None: """Set query according to params.""" raise NotImplementedError @@ -77,12 +84,18 @@ def add(self, query, params): class SimpleFilter(Filter): """A simple filter that reads value from only one key.""" + field: str + param_key: str + transformers: Sequence[Callable[[Any], Any]] + required: bool + extras: dict[str, Any] + def __init__(self, - field, - param_key, - transformers=None, - required=False, - operator=None): + field: str, + param_key: str, + transformers: Optional[Sequence[Callable[[Any], Any]]] = None, + required: bool = False, + operator: Optional[str] = None): self.field = field self.param_key = param_key self.transformers = transformers or [] @@ -91,7 +104,7 @@ def __init__(self, if operator: self.extras['operator'] = operator - def add(self, query, params): + def add(self, query: Any, params: Mapping[str, Any]) -> None: """Set query according to params.""" value = params.get(self.param_key) if is_empty(value): @@ -109,19 +122,20 @@ def add(self, query, params): query.filter(self.field, value, **self.extras) -def String(field, param_key, required=False): +def String(field: str, param_key: str, required: bool = False) -> SimpleFilter: """Return a string filter.""" return SimpleFilter( field, param_key, transformers=[get_string], required=required) -def Boolean(field, param_key, required=False): +def Boolean(field: str, param_key: str, required: bool = False) -> SimpleFilter: """Return a boolean filter that converts yes/no to True/False.""" return SimpleFilter( field, param_key, transformers=[get_boolean], required=required) -def NegativeBoolean(field, param_key, required=False): +def NegativeBoolean(field: str, param_key: str, + required: bool = False) -> SimpleFilter: """Return a boolean filter that converts yes/no to False/True.""" return SimpleFilter( field, @@ -130,7 +144,10 @@ def NegativeBoolean(field, param_key, required=False): required=required) -def Int(field, param_key, required=False, operator=None): +def Int(field: str, + param_key: str, + required: bool = False, + operator: Optional[str] = None) -> SimpleFilter: """return an int filter.""" return SimpleFilter( field, @@ -143,11 +160,14 @@ def Int(field, param_key, required=False, operator=None): class Keyword(SimpleFilter): """Extract keyword fields and filter by the rest.""" - def __init__(self, keyword_filters, field, param_key): + keyword_filters: Sequence[Filter] + + def __init__(self, keyword_filters: Sequence[Filter], field: str, + param_key: str): self.keyword_filters = keyword_filters super().__init__(field, param_key, required=False) - def add(self, query, params): + def add(self, query: Any, params: Mapping[str, Any]) -> None: """Add filter.""" value = params.get(self.param_key, '') for fltr in self.keyword_filters: @@ -161,7 +181,8 @@ def add(self, query, params): query.filter(self.field, keyword) -def add(query, params, filters): +def add(query: Any, params: Mapping[str, Any], + filters: Sequence[Filter]) -> None: """Add filters to query, given the param.""" for fltr in filters: fltr.add(query, params) diff --git a/src/appengine/libs/form.py b/src/appengine/libs/form.py index ef28721d76a..69b5b214134 100644 --- a/src/appengine/libs/form.py +++ b/src/appengine/libs/form.py @@ -15,6 +15,7 @@ import base64 import datetime import os +from typing import cast from clusterfuzz._internal.base import utils from clusterfuzz._internal.datastore import data_types @@ -22,7 +23,9 @@ from libs import helpers -def generate_csrf_token(length=64, valid_seconds=3600, html=False): +def generate_csrf_token(length: int = 64, + valid_seconds: int = 3600, + html: bool = False) -> str: """Generate a CSRF token.""" now = utils.utcnow() valid_token = None @@ -32,7 +35,7 @@ def generate_csrf_token(length=64, valid_seconds=3600, html=False): data_types.CSRFToken.user_email == helpers.get_user_email()) tokens_to_delete = [] for token in tokens: - if token.expiration_time > now: + if token.expiration_time and token.expiration_time > now: valid_token = token continue tokens_to_delete.append(token.key) @@ -41,13 +44,13 @@ def generate_csrf_token(length=64, valid_seconds=3600, html=False): # Generate a new token. if not valid_token: valid_token = data_types.CSRFToken() - valid_token.value = base64.b64encode(os.urandom(length)) + valid_token.value = base64.b64encode(os.urandom(length)).decode('utf-8') valid_token.expiration_time = ( now + datetime.timedelta(seconds=valid_seconds)) valid_token.user_email = helpers.get_user_email() valid_token.put() - value = valid_token.value + value = cast(data_types.CSRFToken, valid_token).value if html: return '' % value return value diff --git a/src/appengine/libs/gcs.py b/src/appengine/libs/gcs.py index d4f5755f36a..5cae9da83d9 100644 --- a/src/appengine/libs/gcs.py +++ b/src/appengine/libs/gcs.py @@ -14,13 +14,15 @@ """App Engine GCS helpers.""" import base64 -import collections import datetime import json import time +from typing import Any +from typing import cast +from typing import NamedTuple import urllib.parse -import googleapiclient +import googleapiclient.discovery from clusterfuzz._internal.base import utils from clusterfuzz._internal.google_cloud_utils import blobs @@ -31,20 +33,25 @@ DEFAULT_URL_VALID_SECONDS = 30 * 60 # 30 minutes. MAX_UPLOAD_SIZE = 15 * 1024 * 1024 * 1024 # 15 GB. -GcsUpload = collections.namedtuple( - 'GcsUpload', - ['url', 'bucket', 'key', 'google_access_id', 'policy', 'signature']) + +class GcsUpload(NamedTuple): + url: str + bucket: str + key: str + google_access_id: str + policy: bytes + signature: bytes class GcsError(Exception): """Base class for exceptions in this module.""" -def sign_data(data): +def sign_data(data: bytes) -> bytes: """Sign data with the default App Engine service account.""" iam = googleapiclient.discovery.build('iamcredentials', 'v1') - service_account = 'projects/-/serviceAccounts/' + utils.service_account_email( - ) + service_account = 'projects/-/serviceAccounts/' + cast( + str, utils.service_account_email()) response = iam.projects().serviceAccounts().signBlob( # pylint: disable=no-member name=service_account, @@ -62,7 +69,10 @@ def sign_data(data): class SignedGcsHandler: """Handler for signing and redirecting to a GCS object.""" - def serve_gcs_object(self, bucket, object_path, content_disposition=None): + def serve_gcs_object(self, + bucket: str, + object_path: str, + content_disposition: str | None = None) -> Any: """Serve a GCS object.""" url = get_signed_url(bucket, object_path) @@ -73,18 +83,18 @@ def serve_gcs_object(self, bucket, object_path, content_disposition=None): url += '&' + urllib.parse.urlencode(content_disposition_params) - return self.redirect(url) # pylint: disable=no-member + return self.redirect(url) # type: ignore # pylint: disable=no-member -def _get_expiration_time(expiry_seconds): +def _get_expiration_time(expiry_seconds: int) -> int: """Return a timestamp |expiry_seconds| from now.""" return int(time.time() + expiry_seconds) -def get_signed_url(bucket_name, - path, - method='GET', - expiry=DEFAULT_URL_VALID_SECONDS): +def get_signed_url(bucket_name: str, + path: str, + method: str = 'GET', + expiry: int = DEFAULT_URL_VALID_SECONDS) -> str: """Return a signed url.""" timestamp = _get_expiration_time(expiry) blob = '%s\n\n\n%d\n/%s/%s' % (method, timestamp, bucket_name, path) @@ -97,7 +107,7 @@ def get_signed_url(bucket_name, else: url = STORAGE_URL % bucket_name signed_blob = sign_data(blob.encode('utf-8')) - service_account_name = utils.service_account_email() + service_account_name = cast(str, utils.service_account_email()) params = { 'GoogleAccessId': service_account_name, @@ -108,12 +118,14 @@ def get_signed_url(bucket_name, return str(url + '/' + path + '?' + urllib.parse.urlencode(params)) -def prepare_upload(bucket_name, path, expiry=DEFAULT_URL_VALID_SECONDS): +def prepare_upload(bucket_name: str, + path: str, + expiry: int = DEFAULT_URL_VALID_SECONDS) -> GcsUpload: """Prepare a signed GCS upload.""" expiration_time = ( datetime.datetime.utcnow() + datetime.timedelta(seconds=expiry)) - conditions = [ + conditions: list[Any] = [ { 'key': path }, @@ -138,12 +150,12 @@ def prepare_upload(bucket_name, path, expiry=DEFAULT_URL_VALID_SECONDS): else: url = STORAGE_URL % bucket_name signature = base64.b64encode(sign_data(policy)) - service_account_name = utils.service_account_email() + service_account_name = cast(str, utils.service_account_email()) return GcsUpload(url, bucket_name, path, service_account_name, policy, signature) -def prepare_blob_upload(): +def prepare_blob_upload() -> GcsUpload: """Prepare a signed GCS blob upload.""" return prepare_upload(storage.blobs_bucket(), blobs.generate_new_blob_name()) diff --git a/src/appengine/libs/handler.py b/src/appengine/libs/handler.py index 8c981635fda..66e42dfc5b6 100644 --- a/src/appengine/libs/handler.py +++ b/src/appengine/libs/handler.py @@ -13,15 +13,21 @@ # limitations under the License. """handler.py provides decorators for POST and GET handlers.""" +from collections.abc import Callable +from collections.abc import Iterator +from collections.abc import Mapping import datetime import functools import json import re +from typing import Any +from typing import cast from flask import g from flask import make_response from flask import request import google.auth +import google.auth.exceptions import requests from clusterfuzz._internal.base import utils @@ -46,10 +52,10 @@ CLUSTERFUZZ_AUTHORIZATION_IDENTITY = 'x-clusterfuzz-identity' BEARER_PREFIX = 'Bearer ' -_auth_config_obj = None +_auth_config_obj: local_config.AuthConfig | None = None -def _auth_config(): +def _auth_config() -> local_config.AuthConfig: """Return a config with auth root.""" global _auth_config_obj if not _auth_config_obj: @@ -58,13 +64,13 @@ def _auth_config(): return _auth_config_obj -def extend_request(req, params): +def extend_request(req: Any, params: Mapping[Any, Any]) -> None: """Extends a request.""" - def _iterparams(): + def _iterparams() -> Iterator[tuple[Any, Any]]: yield from params.items() - def _get(key, default_value=None): + def _get(key: Any, default_value: Any = None) -> Any: """Return the value of the key or the default value.""" return params.get(key, default_value) @@ -72,7 +78,7 @@ def _get(key, default_value=None): req.iterparams = _iterparams -def extend_json_request(req): +def extend_json_request(req: Any) -> None: """Extends a request to support JSON.""" try: params = json.loads(req.data) @@ -83,14 +89,14 @@ def extend_json_request(req): extend_request(req, params) -def cron(): +def cron() -> Callable[[Callable[..., Any]], Callable[..., Any]]: """Wrap a handler with cron.""" - def decorator(func): + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: """Decorator.""" @functools.wraps(func) - def wrapper(self): + def wrapper(self: Any) -> Any: """Wrapper.""" if not self.is_cron(): raise helpers.AccessDeniedError('You are not a cron.') @@ -110,14 +116,14 @@ def wrapper(self): return decorator -def check_admin_access(func): +def check_admin_access(func: Callable[..., Any]) -> Callable[..., Any]: """Wrap a handler with admin checking. This decorator must be below post(..) and get(..) when used. """ @functools.wraps(func) - def wrapper(self): + def wrapper(self: Any) -> Any: """Wrapper.""" if not auth.is_current_user_admin(): raise helpers.AccessDeniedError('Admin access is required.') @@ -127,14 +133,15 @@ def wrapper(self): return wrapper -def check_admin_access_if_oss_fuzz(func): +def check_admin_access_if_oss_fuzz( + func: Callable[..., Any]) -> Callable[..., Any]: """Wrap a handler with an admin check if this is OSS-Fuzz. This decorator must be below post(..) and get(..) when used. """ @functools.wraps(func) - def wrapper(self): + def wrapper(self: Any) -> Any: """Wrapper.""" if utils.is_oss_fuzz(): return check_admin_access(func)(self) @@ -144,7 +151,7 @@ def wrapper(self): return wrapper -def unsupported_on_local_server(func): +def unsupported_on_local_server(func: Callable[..., Any]) -> Callable[..., Any]: """Wrap a handler to raise error when running in local App Engine development environment. @@ -152,7 +159,7 @@ def unsupported_on_local_server(func): """ @functools.wraps(func) - def wrapper(self, *args, **kwargs): + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: """Wrapper.""" if environment.is_running_on_app_engine_development(): raise helpers.EarlyExitError( @@ -164,7 +171,7 @@ def wrapper(self, *args, **kwargs): return wrapper -def validate_id_token(access_token): +def validate_id_token(access_token: str) -> requests.Response | None: """Validates a JWT as an id token.""" response_id_token = requests.get( 'https://www.googleapis.com/oauth2/v3/tokeninfo', @@ -177,7 +184,7 @@ def validate_id_token(access_token): return None -def validate_access_token(access_token): +def validate_access_token(access_token: str) -> requests.Response | None: """Validates a JWT as an access token.""" response_access_token = requests.get( 'https://www.googleapis.com/oauth2/v3/tokeninfo', @@ -190,7 +197,7 @@ def validate_access_token(access_token): return None -def validate_token(authorization): +def validate_token(authorization: str) -> requests.Response: """Validates a JWT as either an access or id token, or raises.""" access_token = authorization.split(' ')[1] id_token_response = validate_id_token(access_token) @@ -206,7 +213,7 @@ def validate_token(authorization): 'is neither a valid id or access token.') -def get_email_and_access_token(authorization): +def get_email_and_access_token(authorization: str) -> tuple[str, str]: """Get user email from the request. See: https://developers.google.com/identity/protocols/OAuth2InstalledApp @@ -245,13 +252,13 @@ def get_email_and_access_token(authorization): 'Parsing the JSON response body failed: %s' % response.text, 500) from e -def oauth(func): +def oauth(func: Callable[..., Any]) -> Callable[..., Any]: """Wrap a handler with OAuth authentication by reading the Authorization header and getting user email. """ @functools.wraps(func) - def wrapper(self, *args, **kwargs): + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: """Wrapper.""" auth_header = request.headers.get('Authorization') if auth_header: @@ -269,11 +276,11 @@ def wrapper(self, *args, **kwargs): return wrapper -def pubsub_push(func): +def pubsub_push(func: Callable[..., Any]) -> Callable[..., Any]: """Wrap a handler with pubsub push authentication.""" @functools.wraps(func) - def wrapper(self): + def wrapper(self: Any) -> Any: """Wrapper.""" try: email = auth.get_email_from_bearer_token(request) @@ -289,17 +296,18 @@ def wrapper(self): return wrapper -def check_user_access(need_privileged_access): +def check_user_access(need_privileged_access: bool + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: """Wrap a handler with check_user_access. This decorator must be below post(..) and get(..) when used. """ - def decorator(func): + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: """Decorator.""" @functools.wraps(func) - def wrapper(self, *args, **kwargs): + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: """Wrapper.""" if not access.has_access(need_privileged_access=need_privileged_access): raise helpers.AccessDeniedError() @@ -311,7 +319,7 @@ def wrapper(self, *args, **kwargs): return decorator -def check_testcase_access(func): +def check_testcase_access(func: Callable[..., Any]) -> Callable[..., Any]: """Wrap a handler with check_testcase_access. It expects the param @@ -321,10 +329,10 @@ def check_testcase_access(func): """ @functools.wraps(func) - def wrapper(self): + def wrapper(self: Any) -> Any: """Wrapper.""" testcase_id = helpers.cast( - request.get('testcaseId'), int, + cast(Any, request).get('testcaseId'), int, "The param 'testcaseId' is not a number.") testcase = access.check_access_and_get_testcase(testcase_id) @@ -333,12 +341,12 @@ def wrapper(self): return wrapper -def allowed_cors(func): +def allowed_cors(func: Callable[..., Any]) -> Callable[..., Any]: """Wrap a handler with 'Access-Control-Allow-Origin to allow cross-domain AJAX calls.""" @functools.wraps(func) - def wrapper(self): + def wrapper(self: Any) -> Any: """Wrapper.""" origin = request.headers.get('Origin') whitelisted_cors_urls = _auth_config().get('whitelisted_cors_urls') @@ -362,14 +370,15 @@ def wrapper(self): return wrapper -def post(request_content_type, response_content_type): +def post(request_content_type: str, response_content_type: str + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: """Wrap a POST handler, parse request, and set response's content type.""" - def decorator(func): + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: """Decorator.""" @functools.wraps(func) - def wrapper(self): + def wrapper(self: Any) -> Any: """Wrapper.""" if response_content_type == JSON: self.is_json = True @@ -398,14 +407,15 @@ def wrapper(self): return decorator -def get(response_content_type): +def get(response_content_type: str + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: """Wrap a GET handler and set response's content type.""" - def decorator(func): + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: """Decorator.""" @functools.wraps(func) - def wrapper(self, *args, **kwargs): + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: """Wrapper.""" if response_content_type == JSON: self.is_json = True @@ -428,12 +438,12 @@ def wrapper(self, *args, **kwargs): return decorator -def require_csrf_token(func): +def require_csrf_token(func: Callable[..., Any]) -> Callable[..., Any]: """Wrap a handler to require a valid CSRF token.""" - def wrapper(self, *args, **kwargs): + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: """Check to see if this handler has a valid CSRF token provided to it.""" - token_value = request.get('csrf_token') + token_value = cast(Any, request).get('csrf_token') user = auth.get_current_user() if not user: raise helpers.AccessDeniedError('Not logged in.') @@ -446,7 +456,8 @@ def wrapper(self, *args, **kwargs): raise helpers.AccessDeniedError('Invalid CSRF token.') # Make sure that the token is not expired. - if token.expiration_time < datetime.datetime.utcnow(): + if cast(datetime.datetime, + token.expiration_time) < datetime.datetime.utcnow(): token.key.delete() raise helpers.AccessDeniedError('Expired CSRF token.') diff --git a/src/appengine/libs/helpers.py b/src/appengine/libs/helpers.py index 94575b49719..db6fe2b816c 100644 --- a/src/appengine/libs/helpers.py +++ b/src/appengine/libs/helpers.py @@ -18,14 +18,22 @@ import logging import sys import traceback +import typing +from typing import Any +from typing import Callable +from typing import TypeVar from clusterfuzz._internal.base import errors from clusterfuzz._internal.datastore import data_handler +from clusterfuzz._internal.datastore import data_types +from clusterfuzz._internal.issue_management import issue_tracker from clusterfuzz._internal.issue_management import issue_tracker_utils from libs import auth -VIEW_OPERATION = 'View' -MODIFY_OPERATION = 'Modify' +VIEW_OPERATION: str = 'View' +MODIFY_OPERATION: str = 'Modify' + +T = TypeVar('T') class _DoNotCatchException(Exception): @@ -35,7 +43,11 @@ class _DoNotCatchException(Exception): class EarlyExitError(Exception): """Serve as an exception for exiting a handler's method early.""" - def __init__(self, message, status, trace_dump=None): + status: int + trace_dump: str | None + + def __init__(self, message: str, status: int, + trace_dump: str | None = None) -> None: super().__init__(message) self.status = status self.trace_dump = trace_dump @@ -45,7 +57,7 @@ def __init__(self, message, status, trace_dump=None): else: self.trace_dump = ''.join(traceback.format_stack()) - def to_dict(self): + def to_dict(self) -> dict[str, Any]: """Build dict that is used for JSON serialisation.""" return { 'traceDump': self.trace_dump, @@ -59,18 +71,18 @@ def to_dict(self): class AccessDeniedError(EarlyExitError): """Serve as an exception for exiting a handler's method with 403.""" - def __init__(self, message=''): + def __init__(self, message: str = '') -> None: super().__init__(message, 403, '') class UnauthorizedError(EarlyExitError): """Serve as an exception for exiting a handler's method with 401.""" - def __init__(self, message=''): + def __init__(self, message: str = '') -> None: super().__init__(message, 401, '') -def get_testcase(testcase_id): +def get_testcase(testcase_id: int | str) -> data_types.Testcase: """Get a valid testcase or raise EarlyExitError.""" testcase = None try: @@ -83,16 +95,17 @@ def get_testcase(testcase_id): return testcase -def get_issue_tracker_for_testcase(testcase): +def get_issue_tracker_for_testcase( + testcase: data_types.Testcase) -> issue_tracker.IssueTracker: """Get an IssueTracker or raise EarlyExitError.""" - issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(testcase) - if not issue_tracker: + tracker = issue_tracker_utils.get_issue_tracker_for_testcase(testcase) + if not tracker: raise EarlyExitError( "The testcase doesn't have a corresponding issue tracker", 404) - return issue_tracker + return tracker -def cast(value, fn, error_message): +def cast(value: Any, fn: Callable[[Any], T], error_message: str) -> T: """Return `fn(value)` or raise an EarlyExitError with 400.""" try: return fn(value) @@ -100,12 +113,12 @@ def cast(value, fn, error_message): raise EarlyExitError(error_message, 400) -def should_render_json(accepts, content_type): +def should_render_json(accepts: str, content_type: str) -> bool: """Check accepts and content_type to see if we should render JSON.""" return 'application/json' in accepts or content_type == 'application/json' -def _is_not_empty(value): +def _is_not_empty(value: Any) -> bool: """Check if value is empty value or a tuple of empty values.""" if isinstance(value, tuple): return any(bool(elem) for elem in value) @@ -113,11 +126,12 @@ def _is_not_empty(value): return bool(value) -def get_or_exit(fn, - not_found_message, - error_message, - not_found_exception=_DoNotCatchException, - non_empty_fn=_is_not_empty): +def get_or_exit(fn: Callable[[], T | None], + not_found_message: str, + error_message: str, + not_found_exception: type[Exception] + | tuple[type[Exception], ...] = _DoNotCatchException, + non_empty_fn: Callable[[Any], bool] = _is_not_empty) -> T: """Get an entity using `fn`. If the returning entity is nothing (e.g. None or a tuple on Nones), it raises 404. @@ -141,19 +155,20 @@ def get_or_exit(fn, sys.exc_info()[1])), 500) if non_empty_fn(result): - return result + return typing.cast(T, result) raise EarlyExitError(not_found_message, 404) -def get_user_email(): +def get_user_email() -> str: """Returns currently logged-in user's email.""" try: - return auth.get_current_user().email + user = auth.get_current_user() + return user.email if user else '' except Exception: return '' -def get_integer_key(request): +def get_integer_key(request: Any) -> int: """Convenience function for getting an integer datastore key ID.""" key = request.get('key') try: @@ -162,7 +177,7 @@ def get_integer_key(request): raise EarlyExitError('Invalid key format.', 400) -def log(message, operation_type): +def log(message: str, operation_type: Any) -> None: """Logs operation being carried by current logged-in user.""" logging.info('ClusterFuzz: %s (%s): %s.', operation_type, get_user_email(), message) diff --git a/src/appengine/libs/query/base.py b/src/appengine/libs/query/base.py index ba2021dbf99..130b275b1d7 100644 --- a/src/appengine/libs/query/base.py +++ b/src/appengine/libs/query/base.py @@ -17,22 +17,26 @@ This must be compatible with libs.filters and libs.crash_access.""" +from collections.abc import Sequence +from typing import Any +from typing import Self + class Query: """Represent the interface for Query.""" - def filter(self, field, value, operator='='): + def filter(self, field: str, value: Any, operator: str = '=') -> None: """Filter by a single value.""" raise NotImplementedError - def filter_in(self, field, values): + def filter_in(self, field: str, values: Sequence[Any]) -> None: """Filter by multiple values.""" raise NotImplementedError - def union(self, *queries): + def union(self, *queries: Self) -> None: """Union all queries with OR conditions.""" raise NotImplementedError - def new_subquery(self): + def new_subquery(self) -> Self: """Instantiate a query that is compatible with the current query.""" raise NotImplementedError diff --git a/src/appengine/libs/query/big_query_query.py b/src/appengine/libs/query/big_query_query.py index 2865a1b94a9..787cff878d0 100644 --- a/src/appengine/libs/query/big_query_query.py +++ b/src/appengine/libs/query/big_query_query.py @@ -13,7 +13,9 @@ # limitations under the License. """Query handles constructing BigQuery's SQL.""" +from collections.abc import Sequence import json +from typing import Any from libs.query import base @@ -21,35 +23,35 @@ class Query(base.Query): """Represent a query for BigQuery's query. Named parameter mode is used.""" - def __init__(self): - self.conditions = [] - self.or_groups = [] - self.sort_by = None + def __init__(self) -> None: + self.conditions: list[str] = [] + self.or_groups: list[Sequence[Query]] = [] + self.sort_by: str | None = None - def raw_filter(self, cond): + def raw_filter(self, cond: str) -> None: """Add raw filter directly.""" self.conditions.append(cond) - def filter(self, field, value, operator='='): + def filter(self, field: str, value: Any, operator: str = '=') -> None: """Filter by a single value.""" # json.dumps converts Python literals to BigQuery literals perfectly well. # See tests. self.conditions.append('%s %s %s' % (field, operator, json.dumps(value))) - def filter_in(self, field, values): + def filter_in(self, field: str, values: Sequence[Any]) -> None: """Filter by multiple values.""" literals = [json.dumps(v) for v in values] self.conditions.append('%s IN (%s)' % (field, ', '.join(literals))) - def union(self, *queries): + def union(self, *queries: Any) -> None: """Combine queries with OR conditions.""" self.or_groups.append(queries) - def new_subquery(self): + def new_subquery(self) -> 'Query': """Generate a new subquery.""" return Query() - def get_where_clause(self): + def get_where_clause(self) -> str: """Get the where clause.""" subquery_wheres = [] for or_queries in self.or_groups: diff --git a/src/appengine/libs/query/datastore_query.py b/src/appengine/libs/query/datastore_query.py index 48be9bee53a..849c9976049 100644 --- a/src/appengine/libs/query/datastore_query.py +++ b/src/appengine/libs/query/datastore_query.py @@ -13,21 +13,28 @@ # limitations under the License. """Query handles pagination and OR conditions with its best effort.""" +from collections.abc import Callable +from collections.abc import Sequence +from typing import Any +from typing import cast + +from google.cloud import ndb from google.cloud.ndb import exceptions from libs.query import base -def _get_key_fn(attribute_name): +def _get_key_fn(attribute_name: str | None) -> Callable[[Any], Any]: """Return the function to get attr of an item. This is used in sorting.""" - def get_key(item): - return getattr(item, attribute_name) + def get_key(item: Any) -> Any: + return getattr(item, cast(str, attribute_name)) return get_key -def compute_projection(projection, order_property): +def compute_projection(projection: Sequence[str] | None, + order_property: str) -> list[str] | None: """Set projection.""" if projection is None: return None @@ -37,7 +44,7 @@ def compute_projection(projection, order_property): return list(combined_projection) -def _combine(q1, q2): +def _combine(q1: '_KeyQuery', q2: '_KeyQuery') -> '_KeyQuery': """Combine KeyQuery q1 and q2. We ignore or_filters because we assume q1 and q2 are flat. In other words, they are results of _KeyQuery.flatten(..).""" assert not q1.or_filters @@ -55,7 +62,10 @@ def _combine(q1, q2): class _Run: """Encapsulate a query and its run.""" - def __init__(self, query, **kwargs): + query: ndb.Query + result: Any + + def __init__(self, query: ndb.Query, **kwargs: Any) -> None: self.query = query self.result = query.iter(**kwargs) @@ -63,18 +73,24 @@ def __init__(self, query, **kwargs): class _KeyQuery: """Query only keys. It supports an OR condition.""" - def __init__(self, model): + model: type[ndb.Model] + or_filters: list[tuple['_KeyQuery', ...]] + filters: list[tuple[str, str, Any]] + order_property: str | None + order_desc: bool + + def __init__(self, model: type[ndb.Model]) -> None: self.model = model self.or_filters = [] self.filters = [] self.order_property = None self.order_desc = False - def union(self, *queries): + def union(self, *queries: '_KeyQuery') -> None: """Specify the OR condition.""" self.or_filters.append(queries) - def filter(self, operator, prop, value): + def filter(self, operator: str, prop: str, value: Any) -> None: """Specify the filter.""" if operator == 'IN': subqueries = [] @@ -86,11 +102,11 @@ def filter(self, operator, prop, value): else: self.filters.append((operator, prop, value)) - def order(self, prop, is_desc): + def order(self, prop: str | None, is_desc: bool) -> None: """Specify the order.""" self.order_property, self.order_desc = prop, is_desc - def flatten(self): + def flatten(self) -> list['_KeyQuery']: """Flatten self into multiple queries if or_filters is not empty.""" if not self.or_filters: return [self] @@ -99,13 +115,13 @@ def flatten(self): for q in qs: q.order(self.order_property, self.order_desc) - queries = [] + queries: list['_KeyQuery'] = [] for query in self.or_filters[0]: for q in query.flatten(): queries.append(q) for or_queries in self.or_filters[1:]: - new_queries = [] + new_queries: list['_KeyQuery'] = [] for oq in or_queries: for fq in oq.flatten(): for q in queries: @@ -118,13 +134,14 @@ def flatten(self): return queries - def to_datastore_query(self): + def to_datastore_query(self) -> ndb.Query: """Return the corresponding datastore query.""" assert not self.or_filters query = self.model.query() - properties = self.model._properties # pylint: disable=protected-access + properties = cast(Any, self.model)._properties # pylint: disable=protected-access for (prop_op, prop, value) in self.filters: + filter_func: Any = None if prop_op == '=': filter_func = properties[prop].__eq__ elif prop_op == '!=': @@ -138,7 +155,8 @@ def to_datastore_query(self): elif prop_op == '>=': filter_func = properties[prop].__ge__ - query = query.filter(filter_func(value)) + if filter_func: + query = query.filter(filter_func(value)) if self.order_property: order_property = properties[self.order_property] @@ -149,7 +167,7 @@ def to_datastore_query(self): return query - def _build_runs(self, total): + def _build_runs(self, total: int) -> list[_Run]: """Construct queries and run them.""" queries = self.flatten() @@ -166,7 +184,9 @@ def _build_runs(self, total): limit=total)) return runs - def _get_total_count(self, runs, offset, limit, items, more_limit): + def _get_total_count(self, runs: Sequence[_Run], offset: int, limit: int, + items: Sequence[Any], + more_limit: int) -> tuple[int, bool]: """Get total count by querying more items.""" max_total_count = offset + limit + more_limit current_count = len(items) @@ -201,7 +221,8 @@ def _get_total_count(self, runs, offset, limit, items, more_limit): return total_count, has_more - def fetch(self, offset, limit, more_limit): + def fetch(self, offset: int, limit: int, + more_limit: int) -> tuple[list[Any], int, bool]: """Construct multiple queries based on the or_filters, query them, combined the results, return items and total_count.""" runs = self._build_runs(offset + limit) @@ -227,35 +248,41 @@ class Query(base.Query): """Query that returns items with a smart count. A smart count indicates a lowerbound of the total count (e.g. 500+).""" - def __init__(self, model): + model: type[ndb.Model] + key_query: _KeyQuery + order_property: str | None + order_desc: bool + + def __init__(self, model: type[ndb.Model]) -> None: self.model = model self.key_query = _KeyQuery(model) self.order_property = None self.order_desc = False - def filter(self, field, value, operator='='): + def filter(self, field: str, value: Any, operator: str = '=') -> None: """Specify the filter.""" self.key_query.filter(operator, field, value) - def filter_in(self, field, values): + def filter_in(self, field: str, values: Sequence[Any]) -> None: """Specify the filter IN.""" self.key_query.filter('IN', field, values) - def union(self, *queries): + def union(self, *queries: Any) -> None: """Specify the OR condition.""" self.key_query.union(*[q.key_query for q in queries]) - def new_subquery(self): + def new_subquery(self) -> 'Query': """Generate a new subquery with the same model.""" return Query(self.model) - def order(self, field, is_desc): + def order(self, field: str, is_desc: bool) -> None: """Specify the order.""" self.order_property = field self.order_desc = is_desc self.key_query.order(field, is_desc) - def fetch(self, offset, limit, projection, more_limit): + def fetch(self, offset: int, limit: int, projection: Sequence[str] | None, + more_limit: int) -> tuple[list[Any], int, bool]: """Return the items, the total count, and if there are more number of items than the total count.""" assert self.order_property @@ -265,7 +292,7 @@ def fetch(self, offset, limit, projection, more_limit): if keys: item_query = self.key_query.model.query( - self.key_query.model.key.IN([key.key for key in keys])) + cast(Any, self.key_query.model).key.IN([key.key for key in keys])) items = item_query.fetch( limit=limit, @@ -277,7 +304,9 @@ def fetch(self, offset, limit, projection, more_limit): return items, total_count, has_more - def fetch_page(self, page, page_size, projection, more_limit): + def fetch_page(self, page: int, page_size: int, + projection: Sequence[str] | None, + more_limit: int) -> tuple[list[Any], int, int, bool]: """Return the items, total_pages, total_items, and has_more.""" # Validation check to convert all negative page numbers to 1. diff --git a/src/appengine/libs/request_cache.py b/src/appengine/libs/request_cache.py index b7c0c6100f5..e43f019abcb 100644 --- a/src/appengine/libs/request_cache.py +++ b/src/appengine/libs/request_cache.py @@ -14,19 +14,27 @@ """Request specific caching..""" import collections +from typing import Any +from typing import Callable +from typing import Optional +from typing import ParamSpec +from typing import TypeVar import flask from clusterfuzz._internal.base import memoize from clusterfuzz._internal.metrics import logs +_P = ParamSpec('_P') +_R = TypeVar('_R') -def get_current_request(): + +def get_current_request() -> Any: """Get the current request.""" return flask.request -def get_cache_backing(): +def get_cache_backing() -> Any: """Get the cache backing for saving current context data.""" return flask.g @@ -34,12 +42,12 @@ def get_cache_backing(): class _FifoRequestCache(memoize.FifoInMemory): """In memory caching engine scoped to a request.""" - def __init__(self, cache_key, capacity): + def __init__(self, cache_key: Any, capacity: int) -> None: super().__init__(capacity) self._cache_key = str(cache_key) @property - def cache(self): + def cache(self) -> Optional[collections.OrderedDict[Any, Any]]: """Get the cache backing.""" cache_backing = get_cache_backing() if not cache_backing: @@ -57,10 +65,10 @@ def cache(self): return backing -def wrap(capacity): +def wrap(capacity: int) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: """Wraps a function to use the per request cache.""" - def decorator(func): + def decorator(func: Callable[_P, _R]) -> Callable[_P, _R]: """Decorator function.""" engine = _FifoRequestCache(id(func), capacity) return memoize.wrap(engine)(func) diff --git a/src/appengine/main.py b/src/appengine/main.py index 5a8a69f3f47..169e25e8e9c 100644 --- a/src/appengine/main.py +++ b/src/appengine/main.py @@ -19,11 +19,11 @@ # Add necessary directories to path. sys.path.append('third_party') -config_modules_path = os.path.join('config', 'modules') +config_modules_path: str = os.path.join('config', 'modules') if os.path.exists(config_modules_path): sys.path.append(config_modules_path) -gae_env = os.environ.get('GAE_ENV') +gae_env: str | None = os.environ.get('GAE_ENV') if gae_env: import pkg_resources importlib.reload(pkg_resources) @@ -34,7 +34,7 @@ try: # Run module initialization code for internal issue tracker - import module_init + import module_init # type: ignore module_init.init() except ImportError: pass diff --git a/src/appengine/server.py b/src/appengine/server.py index b2b4f3f132f..7bf8dc55bab 100644 --- a/src/appengine/server.py +++ b/src/appengine/server.py @@ -13,6 +13,10 @@ # limitations under the License. """server.py initialises the appengine server for ClusterFuzz.""" +from typing import Any +from typing import Callable +from typing import Sequence + from flask import Flask from flask import redirect from flask import request @@ -67,20 +71,22 @@ from handlers.testcase_detail import update_from_trunk from handlers.testcase_detail import update_issue -ndb_client = ndb.Client() +ndb_client: ndb.Client = ndb.Client() -def ndb_wsgi_middleware(wsgi_app): +def ndb_wsgi_middleware(wsgi_app: Callable[..., Any]) -> Any: """WSGI middleware for ndb_datastore context allocation to the app.""" - def middleware(environ, start_response): + def middleware(environ: dict[str, Any], + start_response: Callable[..., Any]) -> Any: with ndb_client.context(): return wsgi_app(environ, start_response) return middleware -def register_routes(flask_app, routes): +def register_routes(flask_app: Flask, + routes: Sequence[tuple[str, Any]]) -> None: """Utility function to register all routes to the flask app.""" for route, handler in routes: flask_app.add_url_rule(route, view_func=handler.as_view(route)) @@ -92,8 +98,8 @@ def register_routes(flask_app, routes): base_handler.add_menu('Crash Statistics', '/crash-stats') base_handler.add_menu('Upload Testcase', '/upload-testcase') -_is_chromium = utils.is_chromium() -_is_oss_fuzz = utils.is_oss_fuzz() +_is_chromium: bool = utils.is_chromium() +_is_oss_fuzz: bool = utils.is_oss_fuzz() if _is_chromium: base_handler.add_menu('Crashes by range', '/commit-range') @@ -109,7 +115,7 @@ def register_routes(flask_app, routes): base_handler.add_menu('Documentation', '/docs') # We need to separate routes for cron to avoid redirection. -cron_routes = [ +cron_routes: list[tuple[str, Any]] = [ ('/cleanup', cleanup.Handler), ('/external-update', external_update.Handler), ('/fuzzer-stats/cache', fuzzer_stats.RefreshCacheHandler), @@ -120,7 +126,7 @@ def register_routes(flask_app, routes): ('/triage', triage.Handler), ] -handlers = [ +handlers: list[tuple[str, Any]] = [ ('/', home.Handler if _is_oss_fuzz else testcase_list.Handler), ('/__/auth/', auth.Handler), ('/add-external-user-permission', configuration.AddExternalUserPermission), @@ -205,12 +211,12 @@ def register_routes(flask_app, routes): ] logs.configure('appengine') -config = local_config.GAEConfig() -main_domain = config.get('domains.main') -redirect_domains = config.get('domains.redirects') +config: local_config.GAEConfig = local_config.GAEConfig() +main_domain: Any = config.get('domains.main') +redirect_domains: Any = config.get('domains.redirects') -def redirect_handler(): +def redirect_handler() -> Any: """Redirection handler.""" if not redirect_domains: return None @@ -221,7 +227,7 @@ def redirect_handler(): return None -app = Flask(__name__) +app: Flask = Flask(__name__) # To also process trailing slash urls. app.url_map.strict_slashes = False app.before_request(redirect_handler) diff --git a/src/clusterfuzz/_internal/base/bisection.py b/src/clusterfuzz/_internal/base/bisection.py index 47ebdafd47c..2b99e354717 100644 --- a/src/clusterfuzz/_internal/base/bisection.py +++ b/src/clusterfuzz/_internal/base/bisection.py @@ -18,17 +18,18 @@ from clusterfuzz._internal.config import local_config from clusterfuzz._internal.crash_analysis import severity_analyzer from clusterfuzz._internal.datastore import data_handler +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.google_cloud_utils import blobs from clusterfuzz._internal.google_cloud_utils import pubsub from clusterfuzz._internal.system import environment -def _get_topic(): +def _get_topic() -> str | None: """Get the Pub/Sub topic for publishing tasks.""" return local_config.ProjectConfig().get('bisect_service.pubsub_topic') -def notify_bisection_invalid(testcase): +def notify_bisection_invalid(testcase: data_types.Testcase) -> None: """Notify the bisection infrastructure of a testcase getting into invalid state.""" pubsub_topic = _get_topic() @@ -44,7 +45,7 @@ def notify_bisection_invalid(testcase): ]) -def request_bisection(testcase): +def request_bisection(testcase: data_types.Testcase) -> None: """Request precise bisection.""" pubsub_topic = _get_topic() if not pubsub_topic: @@ -80,7 +81,9 @@ def request_bisection(testcase): testcase.set_metadata('requested_fixed_bisect', True) -def _check_commits(testcase, bisect_type, old_commit, new_commit): +def _check_commits(testcase: data_types.Testcase, bisect_type: str, + old_commit: str | None, + new_commit: str | None) -> tuple[str | None, str | None]: """Check old and new commit validity.""" if old_commit != new_commit or build_manager.is_custom_binary(): return old_commit, new_commit @@ -103,12 +106,14 @@ def _check_commits(testcase, bisect_type, old_commit, new_commit): if bisect_type == 'fixed': # Narrowest range: last crashing revision up to the latest build. + assert revision_list is not None return _get_commits( str(known_crash_revision) + ':' + str(revision_list[-1]), testcase.job_type) if bisect_type == 'regressed': # Narrowest range: first build to the first crashing revision. + assert revision_list is not None return _get_commits( str(revision_list[0]) + ':' + str(testcase.crash_revision), testcase.job_type) @@ -116,7 +121,9 @@ def _check_commits(testcase, bisect_type, old_commit, new_commit): raise ValueError('Invalid bisection type: ' + bisect_type) -def _make_bisection_request(pubsub_topic, testcase, target, bisect_type): +def _make_bisection_request(pubsub_topic: str, testcase: data_types.Testcase, + target: data_types.FuzzTarget, + bisect_type: str) -> bool: """Make a bisection request to the external bisection service. Returns whether or not a request was actually made.""" if bisect_type == 'fixed': @@ -134,6 +141,14 @@ def _make_bisection_request(pubsub_topic, testcase, target, bisect_type): old_commit, new_commit = _check_commits(testcase, bisect_type, old_commit, new_commit) + assert old_commit is not None + assert new_commit is not None + assert testcase.job_type is not None + assert testcase.timestamp is not None + memory_tool_name = environment.get_memory_tool_name(testcase.job_type) + assert memory_tool_name is not None + sanitizer = environment.SANITIZER_NAME_MAP[memory_tool_name] + repo_url = data_handler.get_main_repo(testcase.job_type) or '' reproducer = blobs.read_key(_get_keys(testcase)) pubsub_client = pubsub.PubSubClient() @@ -145,9 +160,7 @@ def _make_bisection_request(pubsub_topic, testcase, target, bisect_type): 'project_name': target.project, 'sanitizer': - environment.SANITIZER_NAME_MAP[ - environment.get_memory_tool_name(testcase.job_type) - ], + sanitizer, 'fuzz_target': target.binary, 'old_commit': @@ -176,7 +189,8 @@ def _make_bisection_request(pubsub_topic, testcase, target, bisect_type): return True -def _get_commits(commit_range, job_type): +def _get_commits(commit_range: str | None, + job_type: str | None) -> tuple[str | None, str | None]: """Get commits from range.""" if not commit_range or commit_range == 'NA': return None, None @@ -198,7 +212,7 @@ def _get_commits(commit_range, job_type): return old_commit, new_commit -def _get_keys(testcase): +def _get_keys(testcase: data_types.Testcase) -> str | None: if testcase.minimized_keys and testcase.minimized_keys != 'NA': return testcase.minimized_keys return testcase.fuzzed_keys diff --git a/src/clusterfuzz/_internal/base/concurrency.py b/src/clusterfuzz/_internal/base/concurrency.py index 9934011153f..de8b5f72036 100644 --- a/src/clusterfuzz/_internal/base/concurrency.py +++ b/src/clusterfuzz/_internal/base/concurrency.py @@ -12,17 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. """Tools for concurrency/parallelism.""" +from collections.abc import Iterator from concurrent import futures import contextlib import multiprocessing from clusterfuzz._internal.system import environment -POOL_SIZE = multiprocessing.cpu_count() +POOL_SIZE: int = multiprocessing.cpu_count() @contextlib.contextmanager -def make_pool(pool_size=POOL_SIZE, max_pool_size=None, use_threads=False): +def make_pool( + pool_size: int = POOL_SIZE, + max_pool_size: int | None = None, + use_threads: bool = False, +) -> Iterator[futures.ThreadPoolExecutor | futures.ProcessPoolExecutor]: """Returns a pool that can (usually) execute tasks concurrently.""" if max_pool_size is not None: pool_size = min(pool_size, max_pool_size) diff --git a/src/clusterfuzz/_internal/base/dates.py b/src/clusterfuzz/_internal/base/dates.py index 9c88b73b3d3..477f5e66941 100644 --- a/src/clusterfuzz/_internal/base/dates.py +++ b/src/clusterfuzz/_internal/base/dates.py @@ -13,13 +13,14 @@ # limitations under the License. """Date or time helper functions.""" +import datetime import time from clusterfuzz._internal.base import utils from clusterfuzz._internal.system import environment -def initialize_timezone_from_environment(): +def initialize_timezone_from_environment() -> None: """Initializes timezone for date functions based on environment.""" plt = environment.platform() if plt == 'WINDOWS': @@ -29,13 +30,16 @@ def initialize_timezone_from_environment(): time.tzset() -def time_has_expired(timestamp, - compare_to=None, - days=0, - hours=0, - minutes=0, - seconds=0): +def time_has_expired(timestamp: datetime.datetime | None, + compare_to: datetime.datetime | None = None, + days: int = 0, + hours: int = 0, + minutes: int = 0, + seconds: int = 0) -> bool: """Checks to see if a timestamp is older than another by a certain amount.""" + if timestamp is None: + return False + if compare_to is None: compare_to = utils.utcnow() diff --git a/src/clusterfuzz/_internal/base/errors.py b/src/clusterfuzz/_internal/base/errors.py index e5e4b86330a..27fc0985579 100644 --- a/src/clusterfuzz/_internal/base/errors.py +++ b/src/clusterfuzz/_internal/base/errors.py @@ -13,14 +13,17 @@ # limitations under the License. """Functions for errors management.""" +from typing import Any +from typing import Sequence + # Phrases to look in an error stacktrace to determine if something bad happened # which is non recoverable and we just want to hang in that situation without # proceeding further. This also helps to later come up and debug the problem. -BOT_ERROR_HANG_LIST = ['no space left'] +BOT_ERROR_HANG_LIST: list[str] = ['no space left'] # Phrases to look in an error stacktrace to determine whether the bot should # terminate or not. -BOT_ERROR_TERMINATION_LIST = [ +BOT_ERROR_TERMINATION_LIST: list[str] = [ 'can\'t start new thread', 'cannot allocate memory', 'hostexception', @@ -31,13 +34,13 @@ ] -def error_in_list(error_stacktrace, error_list): +def error_in_list(error_stacktrace: str, error_list: Sequence[str]) -> bool: """Returns if the error exists in the error list.""" # Change all strings to lowercase for comparison. error_stacktrace = error_stacktrace.lower() - error_list = [error.lower() for error in error_list] + lowered_error_list = [error.lower() for error in error_list] - for error in error_list: + for error in lowered_error_list: if error in error_stacktrace: return True @@ -51,41 +54,40 @@ class Error(Exception): class InvalidTestcaseError(Error): """Error thrown when there is an attempt to access an invalid test case.""" - def __init__(self, testcase_id): + def __init__(self, testcase_id: Any) -> None: super().__init__(f'Invalid test case {testcase_id!r}.') class InvalidFuzzerError(Error): """Error thrown when there is an attempt to set up a nonexistent fuzzer.""" - def __init__(self): + def __init__(self) -> None: super().__init__('Invalid fuzzer.') class BadStateError(Error): """We are in an unexpected state that we cannot recover from.""" - def __init__(self, message=None): + def __init__(self, message: str | None = None) -> None: super().__init__('Entered a bad state.' or message) class BuildNotFoundError(Error): """Exception type for build not found failures.""" - def __init__(self, revision, job_type): + def __init__(self, revision: int | str | None, job_type: str | None) -> None: self.revision = revision self.job_type = job_type super().__init__() - def __str__(self): - return 'Build not found (revision %d, job %s).' % (self.revision, - self.job_type) + def __str__(self) -> str: + return f'Build not found (revision {self.revision}, job {self.job_type}).' class BadConfigError(Error): """Error thrown when configuration is bad.""" - def __init__(self, config_dir): + def __init__(self, config_dir: str) -> None: super().__init__( 'Bad configuration at: {config_dir}'.format(config_dir=config_dir)) @@ -93,27 +95,27 @@ def __init__(self, config_dir): class ConfigParseError(Error): """Error thrown when we failed to parse a config yaml file.""" - def __init__(self, file_path): + def __init__(self, file_path: str) -> None: self.file_path = file_path super().__init__() - def __str__(self): + def __str__(self) -> str: return 'Failed to parse config file %s.' % self.file_path class InvalidConfigKey(Error): """Error thrown when we failed to parse a config yaml file.""" - def __init__(self, key_name): + def __init__(self, key_name: str) -> None: self.key_name = key_name super().__init__() - def __str__(self): + def __str__(self) -> str: return 'Invalid config key %s.' % self.key_name class QueueLimitReachedError(Error): """Error thrown when the queue limit is reached.""" - def __init__(self, size, queue): + def __init__(self, size: int, queue: str) -> None: super().__init__(f'Queue {queue} has reached the limit of {size}.') diff --git a/src/clusterfuzz/_internal/base/external_tasks.py b/src/clusterfuzz/_internal/base/external_tasks.py index 879afe73d43..b933efa4a52 100644 --- a/src/clusterfuzz/_internal/base/external_tasks.py +++ b/src/clusterfuzz/_internal/base/external_tasks.py @@ -14,16 +14,18 @@ """External tasks.""" from clusterfuzz._internal.datastore import data_handler +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.google_cloud_utils import blobs from clusterfuzz._internal.google_cloud_utils import pubsub from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment # Retry every task for 3 times by default to capture non-deterministic crashes. -_NUM_TRIALS = 3 +_NUM_TRIALS: int = 3 -def add_external_task(command, testcase_id, job): +def add_external_task(command: str, testcase_id: int | str, + job: data_types.Job) -> None: """Add external task.""" if command != 'progression': # Only progression is supported. @@ -35,6 +37,7 @@ def add_external_task(command, testcase_id, job): testcase = data_handler.get_testcase_by_id(testcase_id) fuzz_target = testcase.get_fuzz_target() + assert fuzz_target is not None memory_tool_name = environment.get_memory_tool_name(job.name) sanitizer = environment.SANITIZER_NAME_MAP.get(memory_tool_name) diff --git a/src/clusterfuzz/_internal/base/external_users.py b/src/clusterfuzz/_internal/base/external_users.py index 7c84809bb27..9a624888ada 100644 --- a/src/clusterfuzz/_internal/base/external_users.py +++ b/src/clusterfuzz/_internal/base/external_users.py @@ -13,6 +13,10 @@ # limitations under the License. """External user permission utilities.""" +from collections.abc import Sequence + +from google.cloud import ndb + from clusterfuzz._internal.base import memoize from clusterfuzz._internal.base import utils from clusterfuzz._internal.datastore import data_handler @@ -23,7 +27,7 @@ MEMCACHE_TTL_IN_SECONDS = 15 * 60 -def _fuzzers_for_job(job_type, include_parents): +def _fuzzers_for_job(job_type: str, include_parents: bool) -> list[str]: """Return all fuzzers that have the job associated. Args: @@ -33,7 +37,7 @@ def _fuzzers_for_job(job_type, include_parents): Returns: A list of fuzzer names. """ - fuzzers = [] + fuzzers: list[str] = [] engine_fuzzers = data_handler.get_fuzzing_engines() for fuzzer in data_types.Fuzzer.query(): @@ -43,15 +47,17 @@ def _fuzzers_for_job(job_type, include_parents): # Add this if we're including all parents or this is not an engine fuzzer # with fuzz targets. if include_parents or fuzzer.name not in engine_fuzzers: + assert fuzzer.name is not None fuzzers.append(fuzzer.name) for target_job in fuzz_target_utils.get_fuzz_target_jobs(job=job_type): + assert target_job.fuzz_target_name is not None fuzzers.append(target_job.fuzz_target_name) return sorted(fuzzers) -def _expand_prefix(all_names, prefix): +def _expand_prefix(all_names: Sequence[str], prefix: str) -> list[str]: """Expand the given prefix into real entity names. Args: @@ -64,7 +70,8 @@ def _expand_prefix(all_names, prefix): return [name for name in all_names if name.startswith(prefix)] -def _get_permissions_query_for_user(user_email, entity_kind=None): +def _get_permissions_query_for_user( + user_email: str, entity_kind: int | None = None) -> ndb.Query: """Get a permissions query for a given user. Args: @@ -86,7 +93,8 @@ def _get_permissions_query_for_user(user_email, entity_kind=None): return permissions_for_user -def _allowed_entities_for_user(user_email, entity_kind): +def _allowed_entities_for_user(user_email: str | None, + entity_kind: int) -> list[str]: """Return the entity names that the given user can access. Args: @@ -99,7 +107,7 @@ def _allowed_entities_for_user(user_email, entity_kind): if not user_email: return [] - allowed = [] + allowed: list[str] = [] permissions = _get_permissions_query_for_user(user_email, entity_kind) if entity_kind == data_types.PermissionEntityKind.FUZZER: @@ -108,6 +116,7 @@ def _allowed_entities_for_user(user_email, entity_kind): all_names = data_handler.get_all_job_type_names() for permission in permissions: + assert permission.entity_name is not None if permission.is_prefix: allowed.extend(_expand_prefix(all_names, permission.entity_name)) elif permission.entity_name in all_names: @@ -116,7 +125,8 @@ def _allowed_entities_for_user(user_email, entity_kind): return sorted(allowed) -def _is_entity_allowed_for_user(user_email, name, entity_kind): +def _is_entity_allowed_for_user(user_email: str | None, name: str | None, + entity_kind: int) -> bool: """Return whether if the given user has access to the entity. Args: @@ -133,6 +143,7 @@ def _is_entity_allowed_for_user(user_email, name, entity_kind): permissions = _get_permissions_query_for_user(user_email, entity_kind) for permission in permissions: + assert permission.entity_name is not None if permission.is_prefix: if name.startswith(permission.entity_name): return True @@ -142,7 +153,9 @@ def _is_entity_allowed_for_user(user_email, name, entity_kind): return False -def _allowed_users_for_entity(name, entity_kind, auto_cc=None): +def _allowed_users_for_entity(name: str | None, + entity_kind: int, + auto_cc: int | None = None) -> list[str]: """Return a list of users that have permissions for the given entity. Args: @@ -166,7 +179,11 @@ def _allowed_users_for_entity(name, entity_kind, auto_cc=None): direct_match_permissions = direct_match_permissions.filter( data_types.ExternalUserPermission.auto_cc == auto_cc) - allowed_users = [permission.email for permission in direct_match_permissions] + allowed_users = [ + permission.email + for permission in direct_match_permissions + if permission.email + ] # Find all permissions where the prefix matches the fuzzer_name. # Unfortunately, Datastore doesn't give us an easy way of doing so. To iterate @@ -193,12 +210,14 @@ def _allowed_users_for_entity(name, entity_kind, auto_cc=None): continue if name.startswith(permission.entity_name): + assert permission.email is not None allowed_users.append(permission.email) return sorted(allowed_users) -def _cc_users_for_entity(name, entity_type, security_flag): +def _cc_users_for_entity(name: str | None, entity_type: int, + security_flag: bool) -> list[str]: """Return CC users for entity.""" users = _allowed_users_for_entity(name, entity_type, data_types.AutoCCType.ALL) @@ -212,9 +231,9 @@ def _cc_users_for_entity(name, entity_type, security_flag): @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def allowed_fuzzers_for_user(user_email, - include_from_jobs=False, - include_parents=False): +def allowed_fuzzers_for_user(user_email: str | None, + include_from_jobs: bool = False, + include_parents: bool = False) -> list[str]: """Return allowed fuzzers for the given user. Args: @@ -243,7 +262,7 @@ def allowed_fuzzers_for_user(user_email, @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def allowed_jobs_for_user(user_email): +def allowed_jobs_for_user(user_email: str | None) -> list[str]: """Return allowed jobs for the given user. Args: @@ -257,7 +276,7 @@ def allowed_jobs_for_user(user_email): data_types.PermissionEntityKind.JOB) -def allowed_users_for_fuzzer(fuzzer_name): +def allowed_users_for_fuzzer(fuzzer_name: str | None) -> list[str]: """Return allowed external users for the given fuzzer. Args: @@ -272,7 +291,8 @@ def allowed_users_for_fuzzer(fuzzer_name): data_types.PermissionEntityKind.FUZZER) -def cc_users_for_fuzzer(fuzzer_name, security_flag): +def cc_users_for_fuzzer(fuzzer_name: str | None, + security_flag: bool) -> list[str]: """Return external users that should be CC'ed according to the given rule. Args: @@ -286,8 +306,9 @@ def cc_users_for_fuzzer(fuzzer_name, security_flag): fuzzer_name, data_types.PermissionEntityKind.FUZZER, security_flag) -def is_fuzzer_allowed_for_user(user_email, fuzzer_name, - include_from_jobs=False): +def is_fuzzer_allowed_for_user(user_email: str | None, + fuzzer_name: str | None, + include_from_jobs: bool = False) -> bool: """Return whether if the given user has access to the fuzzer. Args: @@ -308,7 +329,8 @@ def is_fuzzer_allowed_for_user(user_email, fuzzer_name, return is_allowed -def is_job_allowed_for_user(user_email, job_type): +def is_job_allowed_for_user(user_email: str | None, + job_type: str | None) -> bool: """Return whether if the given user has access to the job. Args: @@ -322,7 +344,7 @@ def is_job_allowed_for_user(user_email, job_type): data_types.PermissionEntityKind.JOB) -def is_upload_allowed_for_user(user_email): +def is_upload_allowed_for_user(user_email: str) -> bool: """Return whether if the given user has upload permissions. Args: @@ -336,7 +358,7 @@ def is_upload_allowed_for_user(user_email): return bool(permissions.get()) -def cc_users_for_job(job_type, security_flag): +def cc_users_for_job(job_type: str | None, security_flag: bool) -> list[str]: """Return external users that should be CC'ed according to the given rule. Args: diff --git a/src/clusterfuzz/_internal/base/feature_flags.py b/src/clusterfuzz/_internal/base/feature_flags.py index 9660745990c..2c5c53433a8 100644 --- a/src/clusterfuzz/_internal/base/feature_flags.py +++ b/src/clusterfuzz/_internal/base/feature_flags.py @@ -47,7 +47,7 @@ class FeatureFlags(Enum): CALL_ANDROID_API = 'call_android_api' @property - def flag(self): + def flag(self) -> data_types.FeatureFlag | None: """Get the feature flag.""" flag = ndb.Key(data_types.FeatureFlag, self.value).get() if not flag: @@ -55,15 +55,15 @@ def flag(self): return flag @property - def enabled(self): + def enabled(self) -> bool: """Check if a feature flag is enabled.""" flag = self.flag if not flag: return False - return flag.enabled + return bool(flag.enabled) @property - def content(self): + def content(self) -> float | None: """Get the feature flag content.""" flag = self.flag if not flag or flag.value is None: @@ -71,7 +71,7 @@ def content(self): return flag.value @property - def description(self): + def description(self) -> str: """Get the feature flag description.""" flag = self.flag if not flag or flag.description is None: @@ -79,7 +79,7 @@ def description(self): return flag.description @property - def string_value(self): + def string_value(self) -> str: """Get the feature flag string value.""" flag = self.flag if not flag or flag.string_value is None: diff --git a/src/clusterfuzz/_internal/base/json_utils.py b/src/clusterfuzz/_internal/base/json_utils.py index 3fce50dddcb..579e5d52e07 100644 --- a/src/clusterfuzz/_internal/base/json_utils.py +++ b/src/clusterfuzz/_internal/base/json_utils.py @@ -15,13 +15,14 @@ import datetime import json +from typing import Any class JSONEncoder(json.JSONEncoder): """Custom version of JSON encoder with support for additional object types (e.g. datetime).""" - def default(self, o): # pylint: disable=method-hidden + def default(self, o: Any) -> Any: # pylint: disable=method-hidden if isinstance(o, datetime.datetime): return { '__type__': 'datetime', @@ -47,12 +48,15 @@ def default(self, o): # pylint: disable=method-hidden class JSONDecoder(json.JSONDecoder): """Custom version of JSON decoder with support for additional object types (e.g. datetime).""" - _TYPES = {'datetime': datetime.datetime, 'date': datetime.date} + _TYPES: dict[str, type[datetime.date]] = { + 'datetime': datetime.datetime, + 'date': datetime.date, + } - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(object_hook=self.dict_to_object, *args, **kwargs) - def dict_to_object(self, d): + def dict_to_object(self, d: dict[str, Any]) -> Any: if '__type__' not in d: return d @@ -64,14 +68,14 @@ def dict_to_object(self, d): return d -def dumps(obj, *args, **kwargs): +def dumps(obj: Any, *args: Any, **kwargs: Any) -> str: """Custom json.dumps using custom encoder JSONEncoder defined in this file.""" kwargs['cls'] = JSONEncoder kwargs['sort_keys'] = True return json.dumps(obj, *args, **kwargs) -def loads(obj, *args, **kwargs): +def loads(obj: str | bytes | bytearray, *args: Any, **kwargs: Any) -> Any: """Custom json.loads using custom encoder JSONDecoder defined in this file.""" kwargs['cls'] = JSONDecoder return json.loads(obj, *args, **kwargs) diff --git a/src/clusterfuzz/_internal/base/memoize.py b/src/clusterfuzz/_internal/base/memoize.py index 8b214f9f91a..c62e94d3fcc 100644 --- a/src/clusterfuzz/_internal/base/memoize.py +++ b/src/clusterfuzz/_internal/base/memoize.py @@ -18,6 +18,14 @@ import json import threading import time +from typing import Any +from typing import Callable +from typing import cast +from typing import Mapping +from typing import Optional +from typing import ParamSpec +from typing import Sequence +from typing import TypeVar from clusterfuzz._internal.base import persistent_cache from clusterfuzz._internal.metrics import logs @@ -32,8 +40,11 @@ _DEFAULT_REDIS_HOST = 'localhost' _DEFAULT_REDIS_PORT = 6379 +_P = ParamSpec('_P') +_R = TypeVar('_R') -def _redis_client(): + +def _redis_client() -> Any: """Get the redis client.""" import redis @@ -41,7 +52,7 @@ def _redis_client(): return _local.redis host = environment.get_value('REDIS_HOST', _DEFAULT_REDIS_HOST) - port = environment.get_value('REDIS_PORT', _DEFAULT_REDIS_PORT) + port = cast(int, environment.get_value('REDIS_PORT', _DEFAULT_REDIS_PORT)) _local.redis = redis.Redis(host=host, port=port) return _local.redis @@ -50,20 +61,20 @@ def _redis_client(): class FifoInMemory: """In-memory caching engine.""" - def __init__(self, capacity): + def __init__(self, capacity: int) -> None: self.capacity = capacity self.lock = threading.Lock() - self._cache = None + self._cache: Optional[collections.OrderedDict[Any, Any]] = None @property - def cache(self): + def cache(self) -> Optional[collections.OrderedDict[Any, Any]]: """Get the cache backing. None may be returned.""" if self._cache is None: self._cache = collections.OrderedDict() return self._cache - def put(self, key, value): + def put(self, key: Any, value: Any) -> None: """Put (key, value) into cache.""" if self.cache is None: return @@ -78,14 +89,15 @@ def put(self, key, value): self.lock.release() - def get(self, key): + def get(self, key: Any) -> Any: """Get the value from cache.""" if self.cache is None: return None return self.cache.get(key) - def get_key(self, func, args, kwargs): + def get_key(self, func: Callable[..., Any], args: Sequence[Any], + kwargs: Mapping[str, Any]) -> str: """Get a key name based on function, arguments and keyword arguments.""" return _default_key(func, args, kwargs) @@ -93,15 +105,15 @@ def get_key(self, func, args, kwargs): class InMemory(FifoInMemory): """In-memory caching engine with TTL.""" - def __init__(self, ttl_in_seconds, capacity=1000): + def __init__(self, ttl_in_seconds: float, capacity: int = 1000) -> None: super().__init__(capacity) self.ttl_in_seconds = ttl_in_seconds - def put(self, key, value): + def put(self, key: Any, value: Any) -> None: """Put (key, value) into cache.""" super().put(key, (value, time.time() + self.ttl_in_seconds)) - def get(self, key): + def get(self, key: Any) -> Any: """Get the value from cache.""" entry = super().get(key) if entry is None: @@ -117,13 +129,13 @@ def get(self, key): class FifoOnDisk: """On-disk caching engine.""" - def __init__(self, capacity): + def __init__(self, capacity: int) -> None: self.capacity = capacity - self.keys = [] + self.keys: list[str] = [] self.lock = threading.Lock() @appengine_noop - def put(self, key, value): + def put(self, key: str, value: Any) -> None: """Put (key, value) into cache.""" # Lock to avoid race condition in pop. self.lock.acquire() @@ -138,11 +150,12 @@ def put(self, key, value): self.lock.release() @appengine_noop - def get(self, key): + def get(self, key: str) -> Any: """Get the value from cache.""" return persistent_cache.get_value(key) - def get_key(self, func, args, kwargs): + def get_key(self, func: Callable[..., Any], args: Sequence[Any], + kwargs: Mapping[str, Any]) -> str: """Get a key name based on function, arguments and keyword arguments.""" return _default_key(func, args, kwargs) @@ -150,13 +163,15 @@ def get_key(self, func, args, kwargs): class Memcache: """Memcache caching engine.""" - def __init__(self, ttl_in_seconds, key_fn=None): + def __init__(self, + ttl_in_seconds: int, + key_fn: Optional[Callable[..., str]] = None) -> None: self.ttl_in_seconds = ttl_in_seconds self.key_fn = key_fn or _default_key @local_noop @if_redis_available - def put(self, key, value): + def put(self, key: Any, value: Any) -> None: """Put (key, value) into cache.""" import redis try: @@ -167,7 +182,7 @@ def put(self, key, value): @local_noop @if_redis_available - def get(self, key): + def get(self, key: Any) -> Any: """Get the value from cache.""" import redis try: @@ -181,34 +196,37 @@ def get(self, key): return json.loads(value_raw) - def get_key(self, func, args, kwargs): + def get_key(self, func: Callable[..., Any], args: Sequence[Any], + kwargs: Mapping[str, Any]) -> str: return self.key_fn(func, args, kwargs) -def _default_key(func, args, kwargs): +def _default_key(func: Any, args: Sequence[Any], + kwargs: Mapping[str, Any]) -> str: """Get a key name based on function, arguments and keyword arguments.""" # Use unicode instead of str where possible. This makes it less likely to # have false misses. - args = tuple(arg if not isinstance(arg, str) else str(arg) for arg in args) + args_tuple = tuple( + arg if not isinstance(arg, str) else str(arg) for arg in args) - kwargs = { + kwargs_dict = { key: value if not isinstance(value, str) else str(value) for key, value in kwargs.items() } - return 'memoize:%s' % [func.__name__, args, sorted(kwargs.items())] + return 'memoize:%s' % [func.__name__, args_tuple, sorted(kwargs_dict.items())] -def wrap(engine): +def wrap(engine: Any) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: """Decorator for caching the result of method calls. Arguments must be hashable. None is not cached because we don't tell the difference between having None and not having a key.""" - def decorator(func): + def decorator(func: Callable[_P, _R]) -> Callable[_P, _R]: """Decorator function.""" @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: """Wrapper function.""" force_update = kwargs.pop('__memoize_force__', False) diff --git a/src/clusterfuzz/_internal/base/modules.py b/src/clusterfuzz/_internal/base/modules.py index 8ef0161bdb1..29eb8fa7120 100644 --- a/src/clusterfuzz/_internal/base/modules.py +++ b/src/clusterfuzz/_internal/base/modules.py @@ -19,7 +19,7 @@ import sys -def _config_modules_directory(root_directory): +def _config_modules_directory(root_directory: str) -> str: """Get the config modules directory.""" config_dir = os.getenv('CONFIG_DIR_OVERRIDE') if not config_dir: @@ -28,7 +28,7 @@ def _config_modules_directory(root_directory): return os.path.join(config_dir, 'modules') -def _patch_appengine_modules_for_bots(): +def _patch_appengine_modules_for_bots() -> None: """Patch out App Engine reliant behaviour from bots.""" if os.getenv('SERVER_SOFTWARE'): # Not applicable on App Engine. @@ -44,7 +44,7 @@ def _patch_appengine_modules_for_bots(): pass -def fix_module_search_paths(): +def fix_module_search_paths() -> None: """Add directories that we must be able to import from to path.""" root_directory = os.environ['ROOT_DIR'] source_directory = os.path.join(root_directory, 'src') diff --git a/src/clusterfuzz/_internal/base/persistent_cache.py b/src/clusterfuzz/_internal/base/persistent_cache.py index 658a3f11d08..8bdc4fdc454 100644 --- a/src/clusterfuzz/_internal/base/persistent_cache.py +++ b/src/clusterfuzz/_internal/base/persistent_cache.py @@ -16,6 +16,10 @@ import hashlib import os +from typing import Any +from typing import Callable +from typing import cast +from typing import Optional from clusterfuzz._internal.base import json_utils from clusterfuzz._internal.metrics import logs @@ -23,22 +27,22 @@ # For any given value file, if a file with the same name with this added # extension exists, it is not cleared during initialization. -PERSIST_FILE_EXTENSION = '.persist' +PERSIST_FILE_EXTENSION: str = '.persist' -def initialize(): +def initialize() -> None: """Initialize the persistent cache, creating the directory used to store the values.""" - cache_directory_path = environment.get_value('CACHE_DIR') + cache_directory_path = cast(str, environment.get_value('CACHE_DIR')) if os.path.exists(cache_directory_path): clear_values() else: os.makedirs(cache_directory_path) -def clear_values(clear_all=False): +def clear_values(clear_all: bool = False) -> None: """Remove all values.""" - cache_directory_path = environment.get_value('CACHE_DIR') + cache_directory_path = cast(str, environment.get_value('CACHE_DIR')) if not os.path.exists(cache_directory_path): return @@ -55,14 +59,16 @@ def clear_values(clear_all=False): os.remove(file_path) -def delete_value(key): +def delete_value(key: Any) -> None: """Removes the value for a key.""" value_path = get_value_file_path(key) if os.path.exists(value_path): os.remove(value_path) -def get_value(key, default_value=None, constructor=None): +def get_value(key: Any, + default_value: Any = None, + constructor: Optional[Callable[[Any], Any]] = None) -> Any: """Get the value for a key.""" value_path = get_value_file_path(key) @@ -95,16 +101,17 @@ def get_value(key, default_value=None, constructor=None): return value -def get_value_file_path(key): +def get_value_file_path(key: Any) -> str: """Return the full path to the value file for the given key.""" # Not using utils.string_hash here to avoid a circular dependency. # TODO(mbarbella): Avoid this once utils.py is broken into multiple files. key_filename = 'cache-%s.json' % hashlib.sha1(str(key).encode()).hexdigest() - cache_directory_path = environment.get_value('CACHE_DIR') + cache_directory_path = cast(str, environment.get_value('CACHE_DIR')) return os.path.join(cache_directory_path, key_filename) -def set_value(key, value, persist_across_reboots=False): +def set_value(key: Any, value: Any, + persist_across_reboots: bool = False) -> None: """Set the value for a key. If |persist_across_restarts| is set, then the key won't be deleted even run.py is restarted. """ value_path = get_value_file_path(key) diff --git a/src/clusterfuzz/_internal/base/retry.py b/src/clusterfuzz/_internal/base/retry.py index f0179de7f75..1dafd31bb1c 100644 --- a/src/clusterfuzz/_internal/base/retry.py +++ b/src/clusterfuzz/_internal/base/retry.py @@ -19,26 +19,39 @@ import inspect import sys import time +from typing import Any +from typing import Callable +from typing import cast +from typing import Optional +from typing import ParamSpec +from typing import Sequence +from typing import TypeVar from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment +_P = ParamSpec('_P') +_R = TypeVar('_R') -def sleep(seconds): + +def sleep(seconds: float | int) -> None: """Invoke time.sleep. This is to avoid the flakiness of time.sleep. See: crbug.com/770375""" time.sleep(seconds) -def _should_ignore_delay_for_testing(): - return (environment.get_value('PY_UNITTESTS') or - environment.get_value('INTEGRATION') or - environment.get_value('UNTRUSTED_RUNNER_TESTS') or - environment.get_value('LOCAL_DEVELOPMENT') or - environment.get_value('UTASK_TESTS')) +def _should_ignore_delay_for_testing() -> bool: + return cast( + bool, + environment.get_value('PY_UNITTESTS') or + environment.get_value('INTEGRATION') or + environment.get_value('UNTRUSTED_RUNNER_TESTS') or + environment.get_value('LOCAL_DEVELOPMENT') or + environment.get_value('UTASK_TESTS')) -def get_delay(num_try, delay, backoff): +def get_delay(num_try: int, delay: float | int, + backoff: float | int) -> float | int: """Compute backoff delay.""" delay = delay * (backoff**(num_try - 1)) if _should_ignore_delay_for_testing(): @@ -48,12 +61,12 @@ def get_delay(num_try, delay, backoff): return delay -def wrap(retries, - delay, - function, - backoff=2, - exception_types=None, - retry_on_false=False): +def wrap(retries: int, + delay: float | int, + function: str, + backoff: float | int = 2, + exception_types: Optional[Sequence[type[BaseException]]] = None, + retry_on_false: bool = False) -> Callable[[Any], Any]: """Retry decorator for a function.""" assert delay > 0 @@ -63,12 +76,12 @@ def wrap(retries, if exception_types is None: exception_types = [Exception] - def is_exception_type(exception): + def is_exception_type(exception: BaseException) -> bool: return any( isinstance(exception, exception_type) for exception_type in exception_types) - def decorator(func): + def decorator(func: Any) -> Any: """Decorator for the given function.""" tries = retries + 1 is_generator = inspect.isgeneratorfunction(func) @@ -76,7 +89,8 @@ def decorator(func): if is_generator: function_with_type += ' (generator)' - def handle_retry(num_try, exception=None): + def handle_retry(num_try: int, + exception: Optional[BaseException] = None) -> bool: """Handle retry.""" from clusterfuzz._internal.metrics import monitoring_metrics @@ -101,7 +115,7 @@ def handle_retry(num_try, exception=None): return False @functools.wraps(func) - def _wrapper(*args, **kwargs): + def _wrapper(*args: Any, **kwargs: Any) -> Any: """Regular function wrapper.""" from clusterfuzz._internal.metrics import monitoring_metrics @@ -126,7 +140,7 @@ def _wrapper(*args, **kwargs): return None @functools.wraps(func) - def _generator_wrapper(*args, **kwargs): + def _generator_wrapper(*args: Any, **kwargs: Any) -> Any: """Generator function wrapper.""" # This argument is not applicable for generator functions. assert not retry_on_false diff --git a/src/clusterfuzz/_internal/base/tasks/__init__.py b/src/clusterfuzz/_internal/base/tasks/__init__.py index 98709dcbb21..ddd6bf33b13 100644 --- a/src/clusterfuzz/_internal/base/tasks/__init__.py +++ b/src/clusterfuzz/_internal/base/tasks/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. """Task queue functions.""" +from collections.abc import Generator import contextlib import datetime import functools @@ -20,6 +21,7 @@ import random import threading import time +from typing import Any from typing import List from typing import Optional @@ -340,7 +342,7 @@ def get_postprocess_task(): # Postprocess is platform-agnostic, so we run all such tasks on our # most generic and plentiful bots only. In other words, we avoid # wasting our precious non-linux bots on generic postprocess tasks. - if not environment.platform().lower() == 'linux': + if environment.platform().lower() != 'linux': return None queue_name = POSTPROCESS_QUEUE @@ -387,7 +389,7 @@ def get_preprocess_task(): return task -def tworker_get_task(override_queue: str = None): +def tworker_get_task(override_queue: str | None = None): """Gets a task for a tworker to do.""" assert environment.is_tworker() # TODO(metzman): Pulling tasks is relatively expensive compared to @@ -503,8 +505,8 @@ def __init__(self, def __repr__(self): return f'Task: {self.command} {self.argument} {self.job} {self.queue}' - def attribute(self, _): - return None + def attribute(self, key): + del key def payload(self): """Get the payload.""" @@ -529,7 +531,7 @@ def to_pubsub_message(self): return pubsub.Message(attributes=attributes) @contextlib.contextmanager - def lease(self): + def lease(self) -> Generator[Any, None, None]: """Maintain a lease for the task. Track only start and end by default.""" # Assume default time for non-pubsub tasks. track_task_start(self, TASK_LEASE_SECONDS) @@ -789,7 +791,7 @@ def handle_multiple_utask_main_messages(messages, queue) -> List[PubSubTask]: return tasks -def initialize_task(message, task_cls=None) -> PubSubTask: +def initialize_task(message, task_cls=None) -> PubSubTask | None: """Creates a task from |messages|.""" if task_cls is None: task_cls = PubSubTask @@ -818,17 +820,18 @@ def initialize_task(message, task_cls=None) -> PubSubTask: class PostprocessPubSubTask(PubSubTask): """A postprocess task received over pub/sub.""" - def __init__(self, - output_url_argument, - pubsub_message, - is_command_override=False): + def __init__( # pylint: disable=super-init-not-called + self, + output_url_argument, + pubsub_message, + is_command_override=False): command = 'postprocess' job_type = 'none' eta = None high_end = False - grandparent_class = super(PubSubTask, self) - grandparent_class.__init__(command, output_url_argument, job_type, eta, - is_command_override, high_end) + Task.__init__( # pylint: disable=non-parent-init-called + self, command, output_url_argument, job_type, eta, is_command_override, + high_end) self._pubsub_message = pubsub_message def ack(self): diff --git a/src/clusterfuzz/_internal/base/tasks/pub_sub_task_queue.py b/src/clusterfuzz/_internal/base/tasks/pub_sub_task_queue.py index e6860b82be5..98bc687599b 100644 --- a/src/clusterfuzz/_internal/base/tasks/pub_sub_task_queue.py +++ b/src/clusterfuzz/_internal/base/tasks/pub_sub_task_queue.py @@ -44,16 +44,16 @@ def get_max_target_size(self) -> int: # Default target size for the preprocess queue. -PREPROCESS_TARGET_SIZE_DEFAULT = 10000 +PREPROCESS_TARGET_SIZE_DEFAULT: int = 10000 # Default limit for the utask main queue. -UTASK_MAIN_QUEUE_LIMIT_DEFAULT = 10000 +UTASK_MAIN_QUEUE_LIMIT_DEFAULT: int = 10000 # Default target size for the swarming preprocess queue. -SWARMING_PREPROCESS_TARGET_SIZE_DEFAULT = 10 +SWARMING_PREPROCESS_TARGET_SIZE_DEFAULT: int = 10 # Default limit for the swarming utask main queue. -SWARMING_UTASK_MAIN_QUEUE_LIMIT_DEFAULT = 25 +SWARMING_UTASK_MAIN_QUEUE_LIMIT_DEFAULT: int = 25 PREPROCESS_QUEUE = PubSubTaskQueue( name=tasks.PREPROCESS_QUEUE, diff --git a/src/clusterfuzz/_internal/base/tasks/task_rate_limiting.py b/src/clusterfuzz/_internal/base/tasks/task_rate_limiting.py index af22f51d01b..8fb614e994e 100644 --- a/src/clusterfuzz/_internal/base/tasks/task_rate_limiting.py +++ b/src/clusterfuzz/_internal/base/tasks/task_rate_limiting.py @@ -26,35 +26,35 @@ # an error rather than except. This needs to be handled in postprocess. -def _get_datetime_now(): +def _get_datetime_now() -> datetime.datetime: return datetime.datetime.now() # Things that are sometimes run as tasks by commands.py but are really portions # of actual tasks. -_UTASK_PSEUDO_TASKS = {'uworker_main', 'postprocess', 'preprocess'} +_UTASK_PSEUDO_TASKS: set[str] = {'uworker_main', 'postprocess', 'preprocess'} @memoize.wrap(memoize.FifoInMemory(1)) -def optin_to_task_rate_limiting(): +def optin_to_task_rate_limiting() -> bool: enabled = local_config.ProjectConfig().get('rate_limit_tasks.enabled', False) - return enabled + return bool(enabled) class TaskRateLimiter: """Rate limiter for tasks. This limits tasks to 100 erroneous runs or 2000 succesful runs in 6 hours. It keeps track of task completion when record_task is called at the end of every task.""" - TASK_RATE_LIMIT_MAX_ERRORS = 100 + TASK_RATE_LIMIT_MAX_ERRORS: int = 100 # TODO(metzman): Reevaluate this number, it's probably too high. - TASK_RATE_LIMIT_MAX_COMPLETIONS = 2000 + TASK_RATE_LIMIT_MAX_COMPLETIONS: int = 2000 - def __init__(self, task_name, task_argument, job_name): + def __init__(self, task_name: str, task_argument: str, job_name: str) -> None: self.task_name = task_name self.task_argument = task_argument self.job_name = job_name - def __str__(self): + def __str__(self) -> str: return ' '.join([self.task_name, self.task_argument, self.job_name]) def record_task(self, success: bool) -> None: diff --git a/src/clusterfuzz/_internal/base/tasks/task_utils.py b/src/clusterfuzz/_internal/base/tasks/task_utils.py index b2fe0803920..64dab180146 100644 --- a/src/clusterfuzz/_internal/base/tasks/task_utils.py +++ b/src/clusterfuzz/_internal/base/tasks/task_utils.py @@ -16,11 +16,13 @@ appengine.""" import os +from typing import Any +from typing import cast from clusterfuzz._internal.protos import uworker_msg_pb2 from clusterfuzz._internal.system import environment -_TESTCASE_BASED_TASKS = { +_TESTCASE_BASED_TASKS: set[str] = { 'analyze', 'blame', 'impact', @@ -31,7 +33,7 @@ 'variant', } -_FUZZER_BASED_TASKS = { +_FUZZER_BASED_TASKS: set[str] = { 'corpus_pruning', 'fuzz', } @@ -48,28 +50,30 @@ def is_remotely_executing_utasks() -> bool: """Returns True if the utask_main portions of utasks are being remotely executed on Google cloud batch.""" # TODO(metzman): REMOTE_UTASK_EXECUTION should be a config not an env var. - return (environment.is_production() and - environment.get_value('REMOTE_UTASK_EXECUTION')) + return cast( + bool, + environment.is_production() and + environment.get_value('REMOTE_UTASK_EXECUTION')) class UworkerMsgParseError(RuntimeError): """Error for parsing UworkerMsgs.""" -def is_testcase_based_task(task_name: str): +def is_testcase_based_task(task_name: str) -> bool: return task_name in _TESTCASE_BASED_TASKS -def is_fuzzer_based_task(task_name: str): +def is_fuzzer_based_task(task_name: str) -> bool: return task_name in _FUZZER_BASED_TASKS def get_task_execution_event_data( task_command: str, task_argument: str | uworker_msg_pb2.Input | None, # pylint: disable=no-member - job_type: str | None = None) -> dict: + job_type: str | None = None) -> dict[str, Any]: """Returns a formatted dict with task execution event data.""" - event_data = {} + event_data: dict[str, Any] = {} event_data['task_job'] = job_type if is_testcase_based_task(task_command): testcase_id = ( diff --git a/src/clusterfuzz/_internal/base/untrusted.py b/src/clusterfuzz/_internal/base/untrusted.py index 5b3fb815934..b695dce9a64 100644 --- a/src/clusterfuzz/_internal/base/untrusted.py +++ b/src/clusterfuzz/_internal/base/untrusted.py @@ -13,26 +13,47 @@ # limitations under the License. """Helpers for untrusted runner.""" +from collections.abc import Callable import functools +from typing import Any +from typing import overload +from typing import ParamSpec +from typing import TypeVar from clusterfuzz._internal.base import tasks from clusterfuzz._internal.base import utils from clusterfuzz._internal.system import environment +_P = ParamSpec('_P') +_R = TypeVar('_R') +_T = TypeVar('_T') + class HostError(SystemExit): """Unrecoverable Exception.""" -def untrusted_noop(return_value=None): +@overload +def untrusted_noop() -> Callable[[Callable[_P, _T]], Callable[_P, _T | None]]: + ... + + +@overload +def untrusted_noop(return_value: _R, + ) -> Callable[[Callable[_P, _T]], Callable[_P, _T | _R]]: + ... + + +def untrusted_noop(return_value: Any = None, + ) -> Callable[[Callable[_P, Any]], Callable[_P, Any]]: """Return a decorator that turns functions into no-ops if the bot is untrusted.""" - def decorator(func): + def decorator(func: Callable[_P, Any]) -> Callable[_P, Any]: """Decorator function.""" @functools.wraps(func) - def wrapped(*args, **kwargs): + def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> Any: if environment.is_untrusted_worker(): return return_value @@ -43,16 +64,16 @@ def wrapped(*args, **kwargs): return decorator -def internal_network_domain(): +def internal_network_domain() -> str: """Return the internal network domain.""" return '.c.%s.internal' % utils.get_application_id() -def platform_name(project, platform): +def platform_name(project: str, platform: str) -> str: """"Get the untrusted platform name.""" return project.upper() + '_' + platform.upper() -def queue_name(project, platform): +def queue_name(project: str, platform: str) -> str: """Get the untrusted queue name for the project and platform.""" return tasks.queue_for_platform(platform_name(project, platform)) diff --git a/src/clusterfuzz/_internal/base/utils.py b/src/clusterfuzz/_internal/base/utils.py index 55b208d7fba..2603cfee3ed 100644 --- a/src/clusterfuzz/_internal/base/utils.py +++ b/src/clusterfuzz/_internal/base/utils.py @@ -24,7 +24,17 @@ import re import sys import tempfile +import threading import time +from typing import Any +from typing import Callable +from typing import cast +from typing import Iterable +from typing import Iterator +from typing import overload +from typing import ParamSpec +from typing import Sequence +from typing import TypeVar import urllib.parse import urllib.request import weakref @@ -39,13 +49,18 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import shell +psutil: Any = None try: import psutil except ImportError: - psutil = None + pass + +_T = TypeVar('_T') +_P = ParamSpec('_P') +_R = TypeVar('_R') # FIXME: Binary extensions list is still very basic. -BINARY_EXTENSIONS = [ +BINARY_EXTENSIONS: list[str] = [ # Media formats. '.mp3', '.ogg', @@ -59,46 +74,58 @@ '.pdf', '.swf', ] -FUZZ_PREFIX = 'fuzz-' -TEXT_EXTENSIONS = [ +FUZZ_PREFIX: str = 'fuzz-' +TEXT_EXTENSIONS: list[str] = [ '.css', '.js', '.htm', '.html', '.svg', '.xhtml', '.xht', '.xml', '.xsl' ] -URL_REQUEST_RETRIES = 5 -URL_REQUEST_FAIL_WAIT = 1 -WINDOWS_PREFIX_PATH = '\\\\?\\' +URL_REQUEST_RETRIES: int = 5 +URL_REQUEST_FAIL_WAIT: int = 1 +WINDOWS_PREFIX_PATH: str = '\\\\?\\' # Thread pool for use in function timeouts. -THREAD_POOL = None +THREAD_POOL: Any = None -LOCAL_SOURCE_MANIFEST = os.path.join('src', 'appengine', 'resources', - 'clusterfuzz-source.manifest') +LOCAL_SOURCE_MANIFEST: str = os.path.join('src', 'appengine', 'resources', + 'clusterfuzz-source.manifest') # The length we're reading from the file-object in search_bytes_in_file. -DEFAULT_SEARCH_BUFFER_LENGTH = 10 * 1024 * 1024 # 10MB +DEFAULT_SEARCH_BUFFER_LENGTH: int = 10 * 1024 * 1024 # 10MB -def utcnow(): +def utcnow() -> datetime.datetime: """Return datetime.datetime.utcnow(). We need this method because we can't mock built-in methods.""" return datetime.datetime.utcnow() # pragma: no cover. -def current_date_time(): +def current_date_time() -> str: """Returns current date and time.""" return datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC') -def utc_date_to_timestamp(date): +def utc_date_to_timestamp(date: datetime.date) -> float: """Converts a (UTC) datetime.date to a UNIX timestamp.""" return (date - datetime.date(1970, 1, 1)).total_seconds() -def utc_datetime_to_timestamp(dt): +def utc_datetime_to_timestamp(dt: datetime.datetime | None) -> float | None: """Converts a (UTC) datetime.date to a UNIX timestamp.""" + if dt is None: + return None return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() -def decode_to_unicode(obj): +@overload +def decode_to_unicode(obj: bytes | bytearray | str) -> str: + ... + + +@overload +def decode_to_unicode(obj: _T) -> _T: + ... + + +def decode_to_unicode(obj: Any) -> Any: """Decode object to unicode encoding.""" if not hasattr(obj, 'decode'): return obj @@ -106,7 +133,17 @@ def decode_to_unicode(obj): return obj.decode('utf-8', errors='ignore') -def encode_as_unicode(obj): +@overload +def encode_as_unicode(obj: str) -> bytes: + ... + + +@overload +def encode_as_unicode(obj: _T) -> _T: + ... + + +def encode_as_unicode(obj: Any) -> Any: """Encode a string as unicode, or leave bytes as they are.""" if not hasattr(obj, 'encode'): return obj @@ -119,8 +156,8 @@ def encode_as_unicode(obj): delay=URL_REQUEST_FAIL_WAIT, function='base.utils.fetch_url') def fetch_url(url: str, - params: dict | None = None, - headers: dict | None = None, + params: dict[str, Any] | None = None, + headers: dict[str, Any] | None = None, request_timeout: int | float | None = None, raise_for_not_found: bool = False, stream: bool = False) -> str | requests.Response | None: @@ -144,7 +181,9 @@ def fetch_url(url: str, returns an error status. """ if request_timeout is None: - request_timeout = environment.get_value('URL_BLOCKING_OPERATIONS_TIMEOUT') + request_timeout = cast( + float | int | None, + environment.get_value('URL_BLOCKING_OPERATIONS_TIMEOUT')) logs.info(f'Request for {url}', params=params, headers=headers, stream=stream) response = requests.get( @@ -182,10 +221,10 @@ def post_url(url: str, data: str, headers: dict) -> str: return response.text -def fields_match(string_1, - string_2, - field_separator=':', - allow_empty_fields=True): +def fields_match(string_1: str | None, + string_2: str | None, + field_separator: str = ':', + allow_empty_fields: bool = True) -> bool: """Match fields of two strings, separated by a |field_separator|. Empty fields can be ignored via |allow_empty_fields| flag.""" if string_1 is None or string_2 is None: @@ -207,7 +246,7 @@ def fields_match(string_1, return True -def file_path_to_file_url(path): +def file_path_to_file_url(path: str | None) -> str: """Return a path as a file scheme url.""" if not path: return '' @@ -216,10 +255,10 @@ def file_path_to_file_url(path): return urllib.parse.urljoin('file:', urllib.request.pathname2url(path)) -def filter_file_list(file_list): +def filter_file_list(file_list: Sequence[str]) -> list[str]: """Filters file list by removing duplicates, non-existent files and directories.""" - filtered_file_list = [] + filtered_file_list: list[str] = [] for file_path in file_list: if not os.path.exists(file_path): continue @@ -241,7 +280,8 @@ def filter_file_list(file_list): return filtered_file_list -def find_binary_path(app_directory, binary_file_subpath): +def find_binary_path(app_directory: str, + binary_file_subpath: str) -> str | None: """Find the path to a binary given the app directory and the file name. This is necessary as cov files are created in the root app directory, and we @@ -272,7 +312,7 @@ def find_binary_path(app_directory, binary_file_subpath): return None -def get_application_id(): +def get_application_id() -> str | None: """Return application id. Code simplified based off original implementation in AppEngine SDK get_identity.get_application_id.""" app_id = environment.get_value('APPLICATION_ID') @@ -286,16 +326,16 @@ def get_application_id(): return app_id -def get_logging_cloud_project_id(): +def get_logging_cloud_project_id() -> str | None: """Return logging cloud project id.""" return environment.get_value('LOGGING_CLOUD_PROJECT_ID') -def get_clusterfuzz_release(): +def get_clusterfuzz_release() -> str: return os.getenv('CLUSTERFUZZ_RELEASE', 'prod') -def _get_manifest_release_suffix(release): +def _get_manifest_release_suffix(release: str) -> str: suffix = '' if sys.version_info.major == 3: suffix += '.3' @@ -305,7 +345,7 @@ def _get_manifest_release_suffix(release): return suffix -def _get_deployment_zip_release_suffix(release): +def _get_deployment_zip_release_suffix(release: str) -> str: suffix = '' if sys.version_info.major == 3: suffix += '-3' @@ -315,7 +355,7 @@ def _get_deployment_zip_release_suffix(release): return suffix -def get_platform_deployment_filename(platform, release): +def get_platform_deployment_filename(platform: str, release: str) -> str: """Return the platform deployment filename.""" # Expects linux, macos or windows. base_filename = platform @@ -323,14 +363,14 @@ def get_platform_deployment_filename(platform, release): return f'{base_filename}{release_filename_suffix}.zip' -def get_remote_manifest_filename(release): +def get_remote_manifest_filename(release: str) -> str: return f'clusterfuzz-source.manifest{_get_manifest_release_suffix(release)}' -def service_account_email(): +def service_account_email() -> str | None: """Get the service account name.""" # TODO(ochang): Detect GCE and return the GCE service account instead. - email_id = get_application_id() + email_id = cast(str, get_application_id()) if ':' in email_id: domain, application_id = email_id.split(':') email_id = application_id + '.' + domain @@ -338,7 +378,7 @@ def service_account_email(): return email_id + '@appspot.gserviceaccount.com' -def get_bot_testcases_file_path(input_directory): +def get_bot_testcases_file_path(input_directory: str) -> str: """Returns path to bot-specific fuzzed testcases.""" # Using |FUZZ_INPUTS| prevents putting high load on nfs servers for cases # when |input_directory| is a cloud storage data bundle. We can't rely @@ -357,14 +397,14 @@ def get_bot_testcases_file_path(input_directory): return bot_testcases_file_path -def get_crash_stacktrace_output(application_command_line, - symbolized_stacktrace, - unsymbolized_stacktrace=None, - build_type=None): +def get_crash_stacktrace_output(application_command_line: str, + symbolized_stacktrace: str, + unsymbolized_stacktrace: str | None = None, + build_type: str | None = None) -> str: """Return output string with symbolized and unsymbolized stacktraces combined.""" - def _guess_build_type(application_command_line): + def _guess_build_type(application_command_line: str) -> str: if 'stable' in application_command_line: return 'stable' if 'beta' in application_command_line: @@ -398,9 +438,9 @@ def _guess_build_type(application_command_line): return crash_stacktraces_output -def get_directory_hash_for_path(file_path): +def get_directory_hash_for_path(file_path: str) -> str: """Return the directory hash for a file path (excludes file name).""" - root_directory = environment.get_value('ROOT_DIR') + root_directory = cast(str, environment.get_value('ROOT_DIR')) directory_path = os.path.dirname(file_path) normalized_directory_path = remove_prefix(directory_path, @@ -409,7 +449,7 @@ def get_directory_hash_for_path(file_path): return string_hash(normalized_directory_path) -def get_file_contents_with_fatal_error_on_failure(path): +def get_file_contents_with_fatal_error_on_failure(path: str) -> bytes: """Return the contents of the specified file, or None on error.""" try: with open(path, 'rb') as file_handle: @@ -421,14 +461,14 @@ def get_file_contents_with_fatal_error_on_failure(path): raise errors.BadStateError -def get_line_seperator(label=''): +def get_line_seperator(label: str = '') -> str: """Return a line separator with an optional label.""" separator = '-' * 40 result = '\n\n%s%s%s\n\n' % (separator, label, separator) return result -def get_normalized_relative_path(file_path, directory_path): +def get_normalized_relative_path(file_path: str, directory_path: str) -> str: """Return normalized relative path for file w.r.t to a directory.""" normalized_relative_file_path = remove_prefix(file_path, directory_path + os.sep) @@ -437,12 +477,12 @@ def get_normalized_relative_path(file_path, directory_path): return normalized_relative_file_path -def get_path_without_ext(path): +def get_path_without_ext(path: str) -> str: """Return a path excluding the extension.""" return os.path.splitext(path)[0] -def get_process_ids(process_id, recursive=True): +def get_process_ids(process_id: int, recursive: bool = True) -> list[int]: """Return list of pids for a process and its descendants.""" # Try to find the running process. if not psutil.pid_exists(process_id): @@ -466,7 +506,7 @@ def get_process_ids(process_id, recursive=True): return pids -def get_line_count_string(line_count): +def get_line_count_string(line_count: int) -> str: """Return string representation for size.""" if line_count == 0: return 'empty' @@ -475,7 +515,7 @@ def get_line_count_string(line_count): return '%d lines' % line_count -def get_size_string(size): +def get_size_string(size: int) -> str: """Return string representation for size.""" if size < 1 << 10: return '%d B' % size @@ -486,8 +526,8 @@ def get_size_string(size): return '%d GB' % (size >> 30) -def get_unique_lines_in_unsymbolized_stack(symbolized_stacktrace, - unsymbolized_stacktrace): +def get_unique_lines_in_unsymbolized_stack(symbolized_stacktrace: str, + unsymbolized_stacktrace: str) -> str: """Return unique lines in unsymbolized stacktrace that are not in the symbolized stacktrace.""" if symbolized_stacktrace == unsymbolized_stacktrace: @@ -525,7 +565,7 @@ def get_unique_lines_in_unsymbolized_stack(symbolized_stacktrace, return result -def indent_string(string, chars): +def indent_string(string: str, chars: int) -> str: """Indents a string by x number of characters.""" indented_string = '' @@ -536,7 +576,7 @@ def indent_string(string, chars): return indented_string[0:-1] -def is_binary_file(file_path, bytes_to_read=1024): +def is_binary_file(file_path: str, bytes_to_read: int = 1024) -> bool | None: """Return true if the file looks like a binary file.""" file_extension = os.path.splitext(file_path)[1].lower() if file_extension in BINARY_EXTENSIONS: @@ -557,7 +597,7 @@ def is_binary_file(file_path, bytes_to_read=1024): return len(binary_data) > len(data) * 0.1 -def is_recursive_call(): +def is_recursive_call() -> bool: """Returns true if the caller function is called recursively.""" try: stack_frames = inspect.stack() @@ -571,10 +611,11 @@ def is_recursive_call(): return False -def is_valid_testcase_file(file_path, - check_if_exists=True, - size_limit=None, - allowed_extensions=None): +def is_valid_testcase_file( + file_path: str, + check_if_exists: bool = True, + size_limit: int | None = None, + allowed_extensions: Sequence[str] | set[str] | None = None) -> bool: """Return true if the file looks like a testcase file.""" filename = os.path.basename(file_path) if filename.startswith('.') or filename.startswith(FUZZ_PREFIX): @@ -600,27 +641,28 @@ def is_valid_testcase_file(file_path, return True -def maximum_parallel_processes_allowed(): +def maximum_parallel_processes_allowed() -> int: """Return maximum number of parallel processes allowed. Adjust it based on thread multiplier.""" if environment.is_trusted_host(): # gRPC only supports 1 thread/process. return 1 - max_parallel_process_count = environment.get_value('MAX_FUZZ_THREADS', 1) - thread_multiplier = environment.get_value('THREAD_MULTIPLIER', 1) + max_parallel_process_count = cast( + int, environment.get_value('MAX_FUZZ_THREADS', 1)) + thread_multiplier = cast(int, environment.get_value('THREAD_MULTIPLIER', 1)) max_parallel_process_count *= thread_multiplier return int(max_parallel_process_count) -def normalize_path(path): +def normalize_path(path: str) -> str: """Normalize path. This is needed on windows because windows' paths are case-insensitive.""" return os.path.normcase(os.path.normpath(path)) -def python_gc(): +def python_gc() -> None: """Call python's garbage collector.""" # gc_collect isn't perfectly synchronous, because it may # break reference cycles that then take time to fully @@ -629,22 +671,23 @@ def python_gc(): gc.collect() -def random_element_from_list(element_list): +def random_element_from_list(element_list: Sequence[_T]) -> _T: """Returns a random element from list.""" return element_list[random.SystemRandom().randint(0, len(element_list) - 1)] -def random_number(start, end): +def random_number(start: int, end: int) -> int: """Returns a random number between start and end.""" return random.SystemRandom().randint(start, end) # pylint: disable=inconsistent-return-statements -def random_weighted_choice(element_list, weight_attribute='weight'): +def random_weighted_choice(element_list: Sequence[_T], + weight_attribute: str = 'weight') -> _T: """Returns a random element from list taking its weight into account.""" total = sum(getattr(e, weight_attribute) for e in element_list) random_pick = random.SystemRandom().uniform(0, total) - temp = 0 + temp = 0.0 for element in element_list: element_weight = getattr(element, weight_attribute) if element_weight == 0: @@ -656,14 +699,16 @@ def random_weighted_choice(element_list, weight_attribute='weight'): assert False, 'Failed to make a random weighted choice.' -def read_data_from_file(file_path, eval_data=True, default=None): +def read_data_from_file(file_path: str, + eval_data: bool = True, + default: Any = None) -> Any: """Returns eval-ed data from file.""" if not os.path.exists(file_path): return default - failure_wait_interval = environment.get_value('FAIL_WAIT', 0) + failure_wait_interval = cast(float, environment.get_value('FAIL_WAIT', 0)) file_content = None - retry_limit = environment.get_value('FAIL_RETRIES', 1) + retry_limit = cast(int, environment.get_value('FAIL_RETRIES', 1)) for _ in range(retry_limit): try: with open(file_path, 'rb') as file_handle: @@ -690,7 +735,9 @@ def read_data_from_file(file_path, eval_data=True, default=None): return None -def read_data_from_file_and_remove(file_path, eval_data=False, default=None): +def read_data_from_file_and_remove(file_path: str | None, + eval_data: bool = False, + default: Any = None) -> Any: """Reads file content and removes the file after reading. Args: @@ -711,7 +758,7 @@ def read_data_from_file_and_remove(file_path, eval_data=False, default=None): shell.remove_file(file_path) -def remove_prefix(string, prefix): +def remove_prefix(string: str, prefix: str) -> str: """Strips the prefix from a string.""" if string.startswith(prefix): return string[len(prefix):] @@ -719,7 +766,7 @@ def remove_prefix(string, prefix): return string -def remove_sub_strings(string, substrings): +def remove_sub_strings(string: str, substrings: Sequence[str]) -> str: """Strips substrings from a given string.""" result = string for substring in substrings: @@ -728,7 +775,7 @@ def remove_sub_strings(string, substrings): return result -def restart_machine(): +def restart_machine() -> None: """Restart machine.""" if environment.platform() == 'WINDOWS': os.system('shutdown /f /r /t 0') @@ -737,7 +784,7 @@ def restart_machine(): os.system('sudo shutdown -r now') -def search_bytes_in_file(search_bytes, file_handle) -> bool: +def search_bytes_in_file(search_bytes: bytes, file_handle: Any) -> bool: """Searches `search_bytes` in the file represented by the file-object `file_handle`. @@ -774,12 +821,12 @@ def search_bytes_in_file(search_bytes, file_handle) -> bool: return False -def string_hash(obj): +def string_hash(obj: Any) -> str: """Returns a SHA-1 hash of the object. Not used for security purposes.""" return hashlib.sha1(str(obj).encode('utf-8')).hexdigest() -def entity_hash(obj): +def entity_hash(obj: Any) -> str: """Returns a deterministic hash of a ndb entity. If an entity has been recently modified, put() must be called on it before @@ -793,26 +840,27 @@ def entity_hash(obj): return hasher.hexdigest() -def string_is_true(value): +def string_is_true(value: Any) -> bool: """Check to see if a string has a value that should be treated as True.""" - return value and value != 'false' and value != 'False' and value != '0' + return cast(bool, value and value != 'false' and value != 'False' and + value != '0') -def strip_from_left(string, prefix): +def strip_from_left(string: str, prefix: str) -> str: """Strip a prefix from start from string.""" if not string.startswith(prefix): return string return string[len(prefix):] -def strip_from_right(string, suffix): +def strip_from_right(string: str, suffix: str) -> str: """Strip a suffix from end of string.""" if not string.endswith(suffix): return string return string[:len(string) - len(suffix)] -def sub_string_exists_in(substring_list, string): +def sub_string_exists_in(substring_list: Sequence[str], string: str) -> bool: """Return true if one of the substring in the list is found in |string|.""" for substring in substring_list: if substring in string: @@ -821,7 +869,7 @@ def sub_string_exists_in(substring_list, string): return False -def time_difference_string(timestamp): +def time_difference_string(timestamp: datetime.datetime | None) -> str: """Return time difference as a string.""" if not timestamp: return '' @@ -848,27 +896,27 @@ def time_difference_string(timestamp): return 'in the future' -def timeout(duration): +def timeout( + duration: float | int) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: """Timeout decorator for functions.""" - def decorator(func): + def decorator(func: Callable[_P, _R]) -> Callable[_P, _R]: """Decorates the given function.""" if environment.is_running_on_app_engine(): # multiprocessing doesn't work on App Engine. return func @functools.wraps(func) - def _wrapper(*args, **kwargs): + def _wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: """Wrapper.""" # FIXME: Weird exceptions in imports, might be something relating to our # reload module. Needs further investigation, try this as a temporary fix. import multiprocessing.pool - import threading # Fix for Python < 2.7.2. if not hasattr(threading.current_thread(), '_children'): - # pylint: disable=protected-access - threading.current_thread()._children = weakref.WeakKeyDictionary() + setattr(threading.current_thread(), '_children', + weakref.WeakKeyDictionary()) global THREAD_POOL if THREAD_POOL is None: @@ -889,13 +937,15 @@ def _wrapper(*args, **kwargs): logs.log_fatal_and_exit( f'Exception occurred in function {func}: args: {args}, kwargs: ' f'{kwargs} exception: {sys.exc_info()[1]}') + assert False, 'Unreachable after log_fatal_and_exit.' return _wrapper return decorator -def wait_until_timeout(threads, thread_timeout): +def wait_until_timeout(threads: Sequence[threading.Thread], + thread_timeout: float | int) -> bool: """Wait for all threads to finish unless the given timeout is reached. If no thread is alive, it waits much shorter than the given timeout. @@ -910,7 +960,7 @@ def wait_until_timeout(threads, thread_timeout): wait_timeout = time.time() + thread_timeout while time.time() < wait_timeout: - time.sleep(thread_alive_check_interval) + time.sleep(cast(float, thread_alive_check_interval)) thread_alive = False for thread in threads: @@ -924,11 +974,12 @@ def wait_until_timeout(threads, thread_timeout): return True -def write_data_to_file(content, file_path, append=False): +def write_data_to_file(content: Any, file_path: str, + append: bool = False) -> None: """Writes data to file.""" - failure_wait_interval = environment.get_value('FAIL_WAIT') + failure_wait_interval = cast(float, environment.get_value('FAIL_WAIT')) file_mode = 'ab' if append else 'wb' - retry_limit = environment.get_value('FAIL_RETRIES') + retry_limit = cast(int, environment.get_value('FAIL_RETRIES')) # TODO(mbarbella): One extra iteration is allowed for the type conversion hack # included here. Once this function is converted to only accept bytes-like @@ -955,22 +1006,22 @@ def write_data_to_file(content, file_path, append=False): @memoize.wrap(memoize.FifoInMemory(1)) -def default_backup_bucket(): +def default_backup_bucket() -> str | None: """Return the default backup bucket for this instance of ClusterFuzz.""" # Do not use |BACKUP_BUCKET| environment variable as that is the overridden # backup bucket from job type and is not the default backup bucket. - return local_config.ProjectConfig().get('env.BACKUP_BUCKET') + return cast(str | None, local_config.ProjectConfig().get('env.BACKUP_BUCKET')) @memoize.wrap(memoize.FifoInMemory(1)) -def default_project_name(): +def default_project_name() -> str | None: """Return the default project name for this instance of ClusterFuzz.""" # Do not use |PROJECT_NAME| environment variable as that is the overridden # project name from job type and is not the default project name. - return local_config.ProjectConfig().get('env.PROJECT_NAME') + return cast(str | None, local_config.ProjectConfig().get('env.PROJECT_NAME')) -def current_project(): +def current_project() -> str | None: """Return the project for the current job, or the default project.""" return environment.get_value('PROJECT_NAME', default_project_name()) @@ -1022,7 +1073,7 @@ def get_instance_name() -> str: return environment.get_value('BOT_NAME', '') -def read_from_handle_truncated(file_handle, max_len): +def read_from_handle_truncated(file_handle: Any, max_len: int) -> bytes: """Read from file handle, limiting output to |max_len| by removing output in the middle.""" file_handle.seek(0, os.SEEK_END) @@ -1043,14 +1094,14 @@ def read_from_handle_truncated(file_handle, max_len): return start + truncated_marker + end -def normalize_email(email): +def normalize_email(email: str) -> str: """Normalize an email address.""" # TODO(ochang): Investigate whether if it makes sense to replace # @googlemail.com with @gmail.com. return email.lower() -def emails_equal(first, second): +def emails_equal(first: str | None, second: str | None) -> bool: """Return whether or not the 2 emails are equal after being normalized.""" if not first or not second: return False @@ -1066,8 +1117,10 @@ def is_service_account(email: str) -> bool: return sa_email.endswith('gserviceaccount.com') -def parse_delimited(value_or_handle, delimiter, strip=False, - remove_empty=False): +def parse_delimited(value_or_handle: Any, + delimiter: str, + strip: bool = False, + remove_empty: bool = False) -> list[str]: """Parse a delimter separated value.""" if hasattr(value_or_handle, 'read'): results = value_or_handle.read().split(delimiter) @@ -1090,19 +1143,19 @@ def parse_delimited(value_or_handle, delimiter, strip=False, return processed_results -def is_oss_fuzz(): +def is_oss_fuzz() -> bool: """If this is an instance of OSS-Fuzz.""" return default_project_name() == 'oss-fuzz' -def is_chromium(): +def is_chromium() -> bool: """If this is an instance of chromium fuzzing.""" return default_project_name() in ('chromium', 'chromium-testing', 'clusterfuzz-staging', 'clusterfuzz-development') -def file_hash(file_path): +def file_hash(file_path: str) -> str: """Returns the SHA-1 hash of |file_path| contents.""" chunk_size = 51200 # Read in 50 KB chunks. digest = hashlib.sha1() @@ -1115,16 +1168,17 @@ def file_hash(file_path): return digest.hexdigest() -def cpu_count(): +def cpu_count() -> int: """Get the CPU count.""" # Does not import on App Engine. import multiprocessing - return environment.get_value('CPU_COUNT_OVERRIDE', - multiprocessing.cpu_count()) + return cast( + int, + environment.get_value('CPU_COUNT_OVERRIDE', multiprocessing.cpu_count())) -def batched(iterator, batch_size): +def batched(iterator: Iterable[_T], batch_size: int) -> Iterator[list[_T]]: """Implementation of itertools.py's batched that was added after Python3.11.""" # TODO(metzman): Replace this with itertools.batched. diff --git a/src/clusterfuzz/_internal/batch/service.py b/src/clusterfuzz/_internal/batch/service.py index 32fd12f6683..e7fb7c897ff 100644 --- a/src/clusterfuzz/_internal/batch/service.py +++ b/src/clusterfuzz/_internal/batch/service.py @@ -21,8 +21,12 @@ import json import random import threading +from typing import Any +from typing import cast from typing import Dict from typing import List +from typing import NamedTuple +from typing import Optional from typing import Tuple import urllib.request import uuid @@ -43,59 +47,60 @@ from clusterfuzz._internal.remote_task import remote_task_types from clusterfuzz._internal.system import environment + # A named tuple that defines the execution environment for a batch workload. # This includes details about the machine, disk, network, and container image, # as well as ClusterFuzz-specific settings. -BatchWorkloadSpec = collections.namedtuple( - 'BatchWorkloadSpec', [ - 'clusterfuzz_release', - 'disk_size_gb', - 'disk_type', - 'docker_image', - 'user_data', - 'service_account_email', - 'subnetwork', - 'preemptible', - 'project', - 'machine_type', - 'network', - 'gce_region', - 'priority', - 'max_run_duration', - 'retry', - 'project_number', - ], - defaults=(None,)) - -WeightedSubconfig = collections.namedtuple('WeightedSubconfig', - ['name', 'weight']) +class BatchWorkloadSpec(NamedTuple): + clusterfuzz_release: str + disk_size_gb: int + disk_type: str + docker_image: str + user_data: str + service_account_email: str + subnetwork: str + preemptible: bool + project: str + machine_type: str + network: str + gce_region: str + priority: int + max_run_duration: str + retry: bool + project_number: Optional[str] = None + + +class WeightedSubconfig(NamedTuple): + name: str + weight: float + # See https://cloud.google.com/batch/quotas#job_limits -MAX_CONCURRENT_VMS_PER_JOB = 1000 +MAX_CONCURRENT_VMS_PER_JOB: int = 1000 -MAX_QUEUE_SIZE = 100 +MAX_QUEUE_SIZE: int = 100 class AllRegionsOverloadedError(Exception): """Raised when all batch regions are overloaded.""" -_local = threading.local() +_local: threading.local = threading.local() -DEFAULT_RETRY_COUNT = 0 +DEFAULT_RETRY_COUNT: int = 0 # Controls how many containers (ClusterFuzz tasks) can run on a single VM. # THIS SHOULD BE 1 OR THERE WILL BE SECURITY PROBLEMS. -TASK_COUNT_PER_NODE = 1 +TASK_COUNT_PER_NODE: int = 1 -def _create_batch_client_new(): +def _create_batch_client_new() -> batch.BatchServiceClient: """Creates a batch client.""" creds, _ = credentials.get_default() return batch.BatchServiceClient(credentials=creds) -def _batch_client(): +def _batch_client() -> batch.BatchServiceClient: """Gets the batch client, creating it if it does not exist.""" if hasattr(_local, 'client'): return _local.client @@ -104,11 +109,11 @@ def _batch_client(): return _local.client -def get_job_name(): +def get_job_name() -> str: return 'j-' + str(uuid.uuid4()).lower() -def _get_task_spec(batch_workload_spec): +def _get_task_spec(batch_workload_spec: BatchWorkloadSpec) -> batch.TaskSpec: """Gets the task spec based on the batch workload spec.""" runnable = batch.Runnable() runnable.container = batch.Runnable.Container() @@ -131,11 +136,12 @@ def _get_task_spec(batch_workload_spec): task_spec.max_retry_count = 4 else: task_spec.max_retry_count = DEFAULT_RETRY_COUNT - task_spec.max_run_duration = batch_workload_spec.max_run_duration + task_spec.max_run_duration = cast(Any, batch_workload_spec.max_run_duration) return task_spec -def _set_preemptible(instance_policy, batch_workload_spec) -> None: +def _set_preemptible(instance_policy: batch.AllocationPolicy.InstancePolicy, + batch_workload_spec: BatchWorkloadSpec) -> None: if batch_workload_spec.preemptible: instance_policy.provisioning_model = ( batch.AllocationPolicy.ProvisioningModel.PREEMPTIBLE) @@ -144,7 +150,7 @@ def _set_preemptible(instance_policy, batch_workload_spec) -> None: batch.AllocationPolicy.ProvisioningModel.STANDARD) -def _get_allocation_policy(spec): +def _get_allocation_policy(spec: BatchWorkloadSpec) -> batch.AllocationPolicy: """Returns the allocation policy for a BatchWorkloadSpec.""" disk = batch.AllocationPolicy.Disk() disk.image = 'batch-cos' @@ -179,7 +185,8 @@ def _get_allocation_policy(spec): retries=3, delay=2, function='google_cloud_utils.batch._send_create_job_request') -def _send_create_job_request(create_request): +def _send_create_job_request( + create_request: batch.CreateJobRequest) -> batch.Job: return _batch_client().create_job(create_request) @@ -188,7 +195,7 @@ def count_queued_or_scheduled_tasks(project: str, """Counts the number of queued and scheduled tasks.""" region = f'projects/{project}/locations/{region}' jobs_filter = 'Status.State="SCHEDULED" OR Status.State="QUEUED"' - req = batch.types.ListJobsRequest(parent=region, filter=jobs_filter) + req = batch.ListJobsRequest(parent=region, filter=jobs_filter) queued = 0 scheduled = 0 for job in _batch_client().list_jobs(request=req): @@ -231,9 +238,9 @@ def get_region_load(project: str, region: str) -> int: # We'll assume the structure is standard for Google APIs. job_counts = data.get('jobCounts', {}) for state, count in job_counts.items(): - count = int(count) + count_int = int(count) if state == 'QUEUED': - total += count + total += count_int else: logs.error(f'Unknown state: {state}') @@ -243,7 +250,7 @@ def get_region_load(project: str, region: str) -> int: return 0 -def _get_batch_config(): +def _get_batch_config() -> local_config.BatchConfig: """Returns the batch config. This function was made to make mocking easier.""" return local_config.BatchConfig() @@ -263,7 +270,9 @@ def is_remote_task(command: str, job_name: str) -> bool: return False -def _get_config_names(batch_tasks: List[remote_task_types.RemoteTask]): +def _get_config_names( + batch_tasks: List[remote_task_types.RemoteTask] +) -> Dict[Tuple[str, str], Tuple[str, Optional[int], Optional[str]]]: """Gets the name of the configs for each batch_task. Returns a dict that is indexed by command and job_type for efficient lookup.""" job_names = {task.job_type for task in batch_tasks} @@ -301,7 +310,9 @@ def _get_config_names(batch_tasks: List[remote_task_types.RemoteTask]): return config_map -def _get_subconfig(batch_config, instance_spec, should_check_regions=True): +def _get_subconfig(batch_config: local_config.BatchConfig, + instance_spec: Dict[str, Any], + should_check_regions: bool = True) -> Dict[str, Any]: all_subconfigs = batch_config.get('subconfigs', {}) instance_subconfigs = instance_spec['subconfigs'] @@ -345,7 +356,8 @@ def _get_subconfig(batch_config, instance_spec, should_check_regions=True): def _get_specs_from_config(batch_tasks: List[remote_task_types.RemoteTask], - should_check_regions: bool = True) -> Dict: + should_check_regions: bool = True + ) -> Dict[Tuple[str, str], BatchWorkloadSpec]: """Gets the configured specifications for a batch workload.""" if not batch_tasks: return {} @@ -430,9 +442,10 @@ class GcpBatchService(remote_task_types.RemoteTaskInterface): provides a way to check if a task is configured to run remotely. """ - def create_job(self, spec: BatchWorkloadSpec, input_urls: List[str]): + def create_job(self, spec: BatchWorkloadSpec, + input_urls: List[str]) -> batch.Job: """Creates and starts a batch job from |spec| that executes all tasks. - + This method creates a new GCP Batch job with a single task group. The task group is configured to run a containerized task for each of the input URLs. The tasks are run in parallel, with each task having its @@ -468,8 +481,9 @@ def create_job(self, spec: BatchWorkloadSpec, input_urls: List[str]): logs.info(f'Created batch job id={job_name}.', spec=spec) return job_result - def create_utask_main_job(self, module: str, job_type: str, - input_download_url: str): + def create_utask_main_job( + self, module: str, job_type: str, + input_download_url: str) -> Optional[remote_task_types.RemoteTask]: """Creates a single batch job for a uworker main task.""" command = task_utils.get_command_from_module(module) batch_tasks = [ @@ -481,7 +495,8 @@ def create_utask_main_job(self, module: str, job_type: str, return result[0] def create_utask_main_jobs(self, - remote_tasks: List[remote_task_types.RemoteTask]): + remote_tasks: List[remote_task_types.RemoteTask] + ) -> List[remote_task_types.RemoteTask]: """Creates a batch job for a list of uworker main tasks. This method groups the tasks by their workload specification and creates a diff --git a/src/clusterfuzz/_internal/batch/use_batch.py b/src/clusterfuzz/_internal/batch/use_batch.py index 12585a7fb41..e10da355193 100644 --- a/src/clusterfuzz/_internal/batch/use_batch.py +++ b/src/clusterfuzz/_internal/batch/use_batch.py @@ -15,32 +15,39 @@ import multiprocessing.pool import os +from typing import Any from unittest import mock -from clusterfuzz._internal.google_cloud_utils import batch +from clusterfuzz._internal.batch import service +from clusterfuzz._internal.remote_task import remote_task_types -def _create_many(): +def _create_many() -> None: """Creates many jobs.""" - many = [None for _ in range(2000)] + many: list[None] = [None for _ in range(2000)] with multiprocessing.pool.Pool(120) as pool: pool.map(_send_test_job, many) @mock.patch( - 'clusterfuzz._internal.google_cloud_utils.batch._get_job', - return_value=mock.Mock(platform='LINUX')) + 'clusterfuzz._internal.batch.service._get_specs_from_config', + return_value={}) @mock.patch( 'clusterfuzz._internal.system.environment.get_config_directory', - return_value=os.environ['BATCH_TEST_CONFIG_PATH']) -def _send_test_job(_=None, get_config_directory=None, get_job=None): + return_value=os.environ.get('BATCH_TEST_CONFIG_PATH', '')) +def _send_test_job( + _: Any = None, + get_config_directory: Any = None, + get_specs_from_config: Any = None, +) -> None: """Creates a test batch job for local manual testing to ensure job creation actually works.""" del _ del get_config_directory - del get_job + del get_specs_from_config tasks = [ - batch.BatchTask('variant', 'libfuzzer_chrome_asan', 'https://fake/') - for _ in range(10) + remote_task_types.RemoteTask('variant', 'libfuzzer_chrome_asan', + 'https://fake/') for _ in range(10) ] - batch.create_utask_main_jobs(tasks) + batch_service = service.GcpBatchService() + batch_service.create_utask_main_jobs(tasks) diff --git a/src/clusterfuzz/_internal/bot/fuzzers/afl/constants.py b/src/clusterfuzz/_internal/bot/fuzzers/afl/constants.py index c4f6f50d643..fa69afbda73 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/afl/constants.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/afl/constants.py @@ -16,109 +16,109 @@ """ # AFL++ flags. -DICT_FLAG = '-x' +DICT_FLAG: str = '-x' -INPUT_FLAG = '-i' +INPUT_FLAG: str = '-i' -MEMORY_LIMIT_FLAG = '-m' +MEMORY_LIMIT_FLAG: str = '-m' -OUTPUT_FLAG = '-o' +OUTPUT_FLAG: str = '-o' -TIMEOUT_FLAG = '-t' +TIMEOUT_FLAG: str = '-t' -INSTANCE_ID_FLAG = '-S' +INSTANCE_ID_FLAG: str = '-S' -MOPT_FLAG = '-L' +MOPT_FLAG: str = '-L' -CMPLOG_LEVEL_FLAG = '-l' +CMPLOG_LEVEL_FLAG: str = '-l' -QUEUE_OLD_STRATEGY_FLAG = '-Z' +QUEUE_OLD_STRATEGY_FLAG: str = '-Z' -SCHEDULER_FLAG = '-p' +SCHEDULER_FLAG: str = '-p' -CMPLOG_FLAG = '-c' +CMPLOG_FLAG: str = '-c' -FUZZING_TIMEOUT_FLAG = '-V' +FUZZING_TIMEOUT_FLAG: str = '-V' -MUTATION_STATE_FLAG = '-P' +MUTATION_STATE_FLAG: str = '-P' -INPUT_TYPE_FLAG = '-a' +INPUT_TYPE_FLAG: str = '-a' # AFL MUTATION suboptions. -MUTATION_EXPLORE = 'explore' +MUTATION_EXPLORE: str = 'explore' -MUTATION_EXPLOIT = 'exploit' +MUTATION_EXPLOIT: str = 'exploit' # AFL INPUT suboptions. -INPUT_ASCII = 'ascii' +INPUT_ASCII: str = 'ascii' -INPUT_BINARY = 'binary' +INPUT_BINARY: str = 'binary' # AFL CMPLOG suboptions. -CMPLOG_ARITH = 'A' +CMPLOG_ARITH: str = 'A' -CMPLOG_TRANS = 'T' +CMPLOG_TRANS: str = 'T' -CMPLOG_XTREME = 'X' +CMPLOG_XTREME: str = 'X' -CMPLOG_RAND = 'R' +CMPLOG_RAND: str = 'R' # AFL environment variables. -IGNORE_UNKNOWN_ENVS_ENV_VAR = 'AFL_IGNORE_UNKNOWN_ENVS' +IGNORE_UNKNOWN_ENVS_ENV_VAR: str = 'AFL_IGNORE_UNKNOWN_ENVS' -SKIP_CRASHES_ENV_VAR = 'AFL_SKIP_CRASHES' +SKIP_CRASHES_ENV_VAR: str = 'AFL_SKIP_CRASHES' -SKIP_CPUFREQ_ENV_VAR = 'AFL_SKIP_CPUFREQ' +SKIP_CPUFREQ_ENV_VAR: str = 'AFL_SKIP_CPUFREQ' -BENCH_UNTIL_CRASH_ENV_VAR = 'AFL_BENCH_UNTIL_CRASH' +BENCH_UNTIL_CRASH_ENV_VAR: str = 'AFL_BENCH_UNTIL_CRASH' -DONT_DEFER_ENV_VAR = 'AFL_DRIVER_DONT_DEFER' +DONT_DEFER_ENV_VAR: str = 'AFL_DRIVER_DONT_DEFER' -IGNORE_SEED_PROBLEMS = 'AFL_IGNORE_SEED_PROBLEMS' +IGNORE_SEED_PROBLEMS: str = 'AFL_IGNORE_SEED_PROBLEMS' -FAST_CAL_ENV_VAR = 'AFL_FAST_CAL' +FAST_CAL_ENV_VAR: str = 'AFL_FAST_CAL' -FORKSRV_INIT_TMOUT_ENV_VAR = 'AFL_FORKSRV_INIT_TMOUT' +FORKSRV_INIT_TMOUT_ENV_VAR: str = 'AFL_FORKSRV_INIT_TMOUT' -KEEP_TIMEOUTS_ENV_VAR = 'AFL_KEEP_TIMEOUTS' +KEEP_TIMEOUTS_ENV_VAR: str = 'AFL_KEEP_TIMEOUTS' -NO_AFFINITY_ENV_VAR = 'AFL_NO_AFFINITY' +NO_AFFINITY_ENV_VAR: str = 'AFL_NO_AFFINITY' -STDERR_FILENAME_ENV_VAR = 'AFL_DRIVER_STDERR_DUPLICATE_FILENAME' +STDERR_FILENAME_ENV_VAR: str = 'AFL_DRIVER_STDERR_DUPLICATE_FILENAME' -CLOSE_FD_MASK_ENV_VAR = 'AFL_DRIVER_CLOSE_FD_MASK' +CLOSE_FD_MASK_ENV_VAR: str = 'AFL_DRIVER_CLOSE_FD_MASK' -EXPAND_HAVOC_NOW_VAR = 'AFL_EXPAND_HAVOC_NOW' +EXPAND_HAVOC_NOW_VAR: str = 'AFL_EXPAND_HAVOC_NOW' -DEBUG_VAR = 'AFL_DEBUG' +DEBUG_VAR: str = 'AFL_DEBUG' -CMPLOG_ONLY_NEW_ENV_VAR = 'AFL_CMPLOG_ONLY_NEW' +CMPLOG_ONLY_NEW_ENV_VAR: str = 'AFL_CMPLOG_ONLY_NEW' -DISABLE_TRIM_ENV_VAR = 'AFL_DISABLE_TRIM' +DISABLE_TRIM_ENV_VAR: str = 'AFL_DISABLE_TRIM' -IGNORE_PROBLEMS_ENV_VAR = 'AFL_IGNORE_PROBLEMS' +IGNORE_PROBLEMS_ENV_VAR: str = 'AFL_IGNORE_PROBLEMS' -IGNORE_TIMEOUTS_ENV_VAR = 'AFL_IGNORE_TIMEOUTS' +IGNORE_TIMEOUTS_ENV_VAR: str = 'AFL_IGNORE_TIMEOUTS' # Other settings. -MAX_FILE_BYTES = 2**20 # 1 MB +MAX_FILE_BYTES: int = 2**20 # 1 MB -FORKSERVER_TIMEOUT = 60000 # milliseconds +FORKSERVER_TIMEOUT: int = 60000 # milliseconds # This should be as high as possible, otherwise AFL will restart the binary too # often to be competitive with pure libFuzzer. 2147483647 is the maximum signed # integer. afl_driver accepts one argument which it converts to a signed int # using atoi hence this is the largest value we can pick. -MAX_PERSISTENT_EXECUTIONS = '2147483647' +MAX_PERSISTENT_EXECUTIONS: str = '2147483647' # Resume is used by passing -i- to AFL. See https://goo.gl/rZi455 -RESUME_INPUT = '-' +RESUME_INPUT: str = '-' # Don't let afl set a memory limit. Otherwise we will not be able to use # sanitizers. -MAX_MEMORY_LIMIT = 'none' +MAX_MEMORY_LIMIT: str = 'none' -CORE_PATTERN_FILE_PATH = '/proc/sys/kernel/core_pattern' +CORE_PATTERN_FILE_PATH: str = '/proc/sys/kernel/core_pattern' -DEFAULT_INSTANCE_ID = 'default' +DEFAULT_INSTANCE_ID: str = 'default' diff --git a/src/clusterfuzz/_internal/bot/fuzzers/afl/engine.py b/src/clusterfuzz/_internal/bot/fuzzers/afl/engine.py index f64f096d14a..241d72a243b 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/afl/engine.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/afl/engine.py @@ -17,10 +17,13 @@ from clusterfuzz._internal.bot.fuzzers.afl import launcher from clusterfuzz._internal.bot.fuzzers.afl import stats +from clusterfuzz._internal.system import new_process from clusterfuzz.fuzz import engine -def _run_single_testcase(fuzzer_runner, testcase_file_path): +def _run_single_testcase( + fuzzer_runner: launcher.AflRunner | launcher.AflAndroidRunner, + testcase_file_path: str) -> new_process.ProcessResult: """Loads a crash testcase if it exists.""" # To ensure that we can run the fuzzer. os.chmod(fuzzer_runner.executable_path, stat.S_IRWXU | stat.S_IRGRP @@ -33,10 +36,11 @@ class Engine(engine.Engine): """AFL engine implementation.""" @property - def name(self): + def name(self) -> str: return 'afl' - def prepare(self, corpus_dir, target_path, build_dir): # pylint: disable=unused-argument + def prepare(self, corpus_dir: str, target_path: str, + build_dir: str) -> engine.FuzzOptions: # pylint: disable=unused-argument """Prepare for a fuzzing session, by generating options. Returns a FuzzOptions object. @@ -60,7 +64,8 @@ def prepare(self, corpus_dir, target_path, build_dir): # pylint: disable=unused return engine.FuzzOptions(corpus_dir, arguments, strategies) - def fuzz(self, target_path, options, reproducers_dir, max_time): + def fuzz(self, target_path: str, options: engine.FuzzOptions, + reproducers_dir: str, max_time: int) -> engine.FuzzResult: """Run a fuzz session. Args: @@ -86,6 +91,8 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): strategy_dict=options.strategies) fuzz_result = runner.fuzz() + assert fuzz_result is not None + assert fuzz_result.time_executed is not None command = fuzz_result.command time_executed = fuzz_result.time_executed @@ -103,7 +110,7 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): new_units_added, corpus_size, runner.strategies, runner.fuzzer_stderr, fuzz_result.output) - crashes = [] + crashes: list[engine.Crash] = [] if os.path.exists(testcase_file_path): crash = engine.Crash(testcase_file_path, runner.fuzzer_stderr, [], fuzz_result.time_executed) @@ -112,7 +119,8 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): return engine.FuzzResult(fuzzing_logs, command, crashes, stats_getter.stats, time_executed) - def reproduce(self, target_path, input_path, arguments, max_time): + def reproduce(self, target_path: str, input_path: str, arguments: list[str], + max_time: int) -> engine.ReproduceResult: """Reproduce a crash given an input. Args: @@ -140,8 +148,10 @@ def reproduce(self, target_path, input_path, arguments, max_time): return engine.ReproduceResult(command, return_code, time_executed, output) - def minimize_corpus(self, target_path, arguments, input_dirs, output_dir, - reproducers_dir, max_time): + def minimize_corpus(self, target_path: str, arguments: list[str], + input_dirs: list[str], output_dir: str, + reproducers_dir: str | None, + max_time: int) -> engine.FuzzResult: """Optional (but recommended): run corpus minimization. Args: diff --git a/src/clusterfuzz/_internal/bot/fuzzers/afl/fuzzer.py b/src/clusterfuzz/_internal/bot/fuzzers/afl/fuzzer.py index 6c869482b4a..72c5d0bfd2f 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/afl/fuzzer.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/afl/fuzzer.py @@ -25,7 +25,7 @@ AFL_DUMMY_INPUT = 'in1' -def write_dummy_file(input_dir): +def write_dummy_file(input_dir: str) -> None: """Afl will refuse to run if the corpus directory is empty or contains empty files. So write the bare minimum to get afl to run if there is no corpus yet.""" @@ -40,12 +40,14 @@ def write_dummy_file(input_dir): class Afl(builtin.EngineFuzzer): """Builtin AFL fuzzer.""" - def generate_arguments(self, fuzzer_path): # pylint: disable=unused-argument + def generate_arguments(self, fuzzer_path: str) -> str: # pylint: disable=unused-argument """Generate arguments for fuzzer using .options file or default values.""" return '' - def run(self, input_directory, output_directory, no_of_files): + def run(self, input_directory: str, output_directory: str, + no_of_files: int) -> builtin.BuiltinFuzzerResult: result = super().run(input_directory, output_directory, no_of_files) + assert result.corpus_directory is not None write_dummy_file(result.corpus_directory) return result diff --git a/src/clusterfuzz/_internal/bot/fuzzers/afl/launcher.py b/src/clusterfuzz/_internal/bot/fuzzers/afl/launcher.py index 10ced5a2c18..1f5bd82ea45 100755 --- a/src/clusterfuzz/_internal/bot/fuzzers/afl/launcher.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/afl/launcher.py @@ -22,7 +22,6 @@ pass import atexit -import collections import enum import os import re @@ -31,6 +30,10 @@ import stat import subprocess import sys +from typing import Any +from typing import Callable +from typing import cast +from typing import NamedTuple from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot.fuzzers import dictionary_manager @@ -50,20 +53,20 @@ # Allow 30 minutes to merge the testcases back into the corpus. This matches # libFuzzer's merge timeout. -DEFAULT_MERGE_TIMEOUT = 30 * 60 +DEFAULT_MERGE_TIMEOUT: int = 30 * 60 -BOT_NAME = environment.get_value('BOT_NAME', '') +BOT_NAME: str = environment.get_value('BOT_NAME', '') -STDERR_FILENAME = 'stderr.out' +STDERR_FILENAME: str = 'stderr.out' -MAX_OUTPUT_LEN = 1 * 1024 * 1024 # 1 MB +MAX_OUTPUT_LEN: int = 1 * 1024 * 1024 # 1 MB # .options file option for the number of persistent executions. -PERSISTENT_EXECUTIONS_OPTION = 'n' +PERSISTENT_EXECUTIONS_OPTION: str = 'n' # Grace period for the launcher to complete any processing before it's killed. # This will no longer be needed when we migrate to the engine interface. -POSTPROCESSING_TIMEOUT = 30 +POSTPROCESSING_TIMEOUT: int = 30 class AflOptionType(enum.Enum): @@ -73,7 +76,9 @@ class AflOptionType(enum.Enum): # Afl options have names and can either be commandline arguments or environment # variables. -AflOption = collections.namedtuple('AflOption', ['name', 'type']) +class AflOption(NamedTuple): + name: str + type: AflOptionType class AflConfig: @@ -83,24 +88,25 @@ class AflConfig: Determines these mainly by parsing the .options file for the target.""" # Mapping of libfuzzer option names to AflOption objects. - LIBFUZZER_TO_AFL_OPTIONS = { + LIBFUZZER_TO_AFL_OPTIONS: dict[str, AflOption] = { 'dict': AflOption(constants.DICT_FLAG, AflOptionType.ARG), 'close_fd_mask': AflOption(constants.CLOSE_FD_MASK_ENV_VAR, AflOptionType.ENV_VAR), } - def __init__(self): + def __init__(self) -> None: """Sets the configs to sane defaults. Use from_target_path if you want to use the .options file and possibly .dict to set the configs.""" - self.additional_afl_arguments = [] - self.additional_env_vars = {} - self.num_persistent_executions = constants.MAX_PERSISTENT_EXECUTIONS - self.dict_path = None - self._executable_path = None + self.additional_afl_arguments: list[str] = [] + self.additional_env_vars: dict[str, str] = {} + self.num_persistent_executions: str | int = ( + constants.MAX_PERSISTENT_EXECUTIONS) + self.dict_path: str | None = None + self._executable_path: str | None = None @classmethod - def from_target_path(cls, target_path): + def from_target_path(cls, target_path: str) -> 'AflConfig': """Instantiates and returns an AFLConfig object. The object is configured based on |target_path|.""" config = cls() @@ -113,7 +119,7 @@ def from_target_path(cls, target_path): return config - def parse_options(self, target_path): + def parse_options(self, target_path: str) -> None: """Parses a target's .options file (determined using |target_path|) if it exists and sets configs based on it.""" fuzzer_options = options.get_fuzz_target_options(target_path) @@ -140,7 +146,7 @@ def parse_options(self, target_path): self.num_persistent_executions = afl_options.get( PERSISTENT_EXECUTIONS_OPTION, constants.MAX_PERSISTENT_EXECUTIONS) - def use_default_dict(self, target_path): + def use_default_dict(self, target_path: str) -> None: """Set the dictionary argument in |self.additional_afl_arguments| to %target_binary_name%.dict if no dictionary argument is already specified. Also update |self.dict_path|.""" @@ -167,39 +173,39 @@ class AflFuzzOutputDirectory: # AFL usually copies over old units from the corpus to the queue and adds the # string 'orig' to the new filename. Therefore we know that testcases # containing 'orig' are copied. - COPIED_FILE_STRING = 'orig' + COPIED_FILE_STRING: str = 'orig' - TESTCASE_REGEX = re.compile(r'id:\d{6},.+') + TESTCASE_REGEX: re.Pattern[str] = re.compile(r'id:\d{6},.+') - def __init__(self): - self.output_directory = os.path.join(fuzzer_utils.get_temp_dir(), - 'afl_output_dir') + def __init__(self) -> None: + self.output_directory: str = os.path.join(fuzzer_utils.get_temp_dir(), + 'afl_output_dir') engine_common.recreate_directory(self.output_directory) @classmethod - def is_testcase(cls, path): + def is_testcase(cls, path: str) -> bool: """Is the path an AFL testcase file or something else.""" return (os.path.isfile(path) and bool(re.match(cls.TESTCASE_REGEX, os.path.basename(path)))) @property - def instance_directory(self): + def instance_directory(self) -> str: """Returns afl-fuzz's instance directory.""" return os.path.join(self.output_directory, constants.DEFAULT_INSTANCE_ID) @property - def queue(self): + def queue(self) -> str: """Returns afl-fuzz's queue directory.""" return os.path.join(self.instance_directory, 'queue') - def is_new_testcase(self, path): + def is_new_testcase(self, path: str) -> bool: """Determine if |path| is a new unit.""" # Clearly non-testcases can't be new testcases. return (self.is_testcase(path) and self.COPIED_FILE_STRING not in os.path.basename(path)) - def count_new_units(self, corpus_path): + def count_new_units(self, corpus_path: str) -> int: """Count the number of new units (testcases) in |corpus_path|.""" corpus_files = os.listdir(corpus_path) num_new_units = 0 @@ -210,7 +216,7 @@ def count_new_units(self, corpus_path): return num_new_units - def copy_crash_if_needed(self, testcase_path): + def copy_crash_if_needed(self, testcase_path: str) -> None: """Copy the first crash found by AFL to |testcase_path| (the input file created by run.py). """ @@ -223,7 +229,7 @@ def copy_crash_if_needed(self, testcase_path): shutil.copyfile(crash_path, testcase_path) break - def remove_hang_in_queue(self, hang_filename): + def remove_hang_in_queue(self, hang_filename: str) -> None: """Removes the hanging testcase from queue.""" # AFL copies all inputs to the queue and renames them in the format # "id:NUMBER,orig:$hang_filename". So remove that file from the queue. @@ -238,7 +244,7 @@ def remove_hang_in_queue(self, hang_filename): remove_path(hang_queue_path) @property - def stats_path(self): + def stats_path(self) -> str: """Returns the path of AFL's stats file: "fuzzer_stats".""" return os.path.join(self.instance_directory, 'fuzzer_stats') @@ -248,7 +254,7 @@ class AflAndroidFuzzOutputDirectory(AflFuzzOutputDirectory): and help copy contents from device to local. """ - def remove_hang_in_queue(self, hang_filename): + def remove_hang_in_queue(self, hang_filename: str) -> None: """Removes the hanging testcase from queue.""" queue_paths = list_full_file_paths_device(self.queue) @@ -260,7 +266,7 @@ def remove_hang_in_queue(self, hang_filename): android.adb.remove_file(hang_queue_path_device) - def copy_crash_if_needed(self, testcase_path): + def copy_crash_if_needed(self, testcase_path: str) -> None: """Copy the first crash found by AFL. Before calling super method copy the crashes directory from device to local. """ @@ -282,55 +288,55 @@ class FuzzingStrategies: and to record the decision for StatsGetter to use later.""" # Probability for the level of CMPLOG to set. (-l X) - CMPLOG_LEVEL_PROBS = [ + CMPLOG_LEVEL_PROBS: list[tuple[str, float]] = [ ('1', 0.7), # Level 1 ('2', 0.25), # Level 2 ('3', 0.05), # Level 3 ] # Probabilty for arithmetic CMPLOG calculations. - CMPLOG_ARITH_PROB = 0.4 + CMPLOG_ARITH_PROB: float = 0.4 # Probability for transforming CMPLOG solving. - CMPLOG_TRANS_PROB = 0.1 + CMPLOG_TRANS_PROB: float = 0.1 # Probability for extreme CMPLOG solving. - CMPLOG_XTREME_PROB = 0.05 + CMPLOG_XTREME_PROB: float = 0.05 # Probability for randomized coloring. - CMPLOG_RAND_PROB = 0.1 + CMPLOG_RAND_PROB: float = 0.1 # Probability to disable trimming. (AFL_DISABLE_TRIM=1) - DISABLE_TRIM_PROB = 0.70 + DISABLE_TRIM_PROB: float = 0.70 # Probability to keep long running finds. (AFL_KEEP_TIMEOUTS=1) - KEEP_TIMEOUTS_PROB = 0.7 + KEEP_TIMEOUTS_PROB: float = 0.7 # Probability for increased havoc intensity. (AFL_EXPAND_HAVOC_NOW=1) - EXPAND_HAVOC_PROB = 0.5 + EXPAND_HAVOC_PROB: float = 0.5 # Probability for a fixed mutation type. (-P) - MUTATION_PROB = 0.5 - MUTATION_EXPLORE_PROB = 0.75 + MUTATION_PROB: float = 0.5 + MUTATION_EXPLORE_PROB: float = 0.75 # PROBABILITY for input type. (-a) - INPUT_PROB = 0.4 - INPUT_ASCII_PROB = 0.3 + INPUT_PROB: float = 0.4 + INPUT_ASCII_PROB: float = 0.3 # Probability to use the MOpt mutator. (-L0) - MOPT_PROB = 0.4 + MOPT_PROB: float = 0.4 # Probability to use the original afl queue walking mechanism. (-Z) - QUEUE_OLD_STRATEGY_PROB = 0.2 + QUEUE_OLD_STRATEGY_PROB: float = 0.2 # Probability to ignore long running inputs. (AFL_IGNORE_TIMEOUTS=1) - IGNORE_TIMEOUTS_PROB = 0.7 + IGNORE_TIMEOUTS_PROB: float = 0.7 # Probability to enable the schedule cycler. (AFL_CYCLE_SCHEDULES=1) - SCHEDULER_CYCLE_PROB = 0.1 + SCHEDULER_CYCLE_PROB: float = 0.1 # Propability which scheduler to select. (-p SCHEDULER) - SCHEDULER_PROBS = [ + SCHEDULER_PROBS: list[tuple[str, float]] = [ ('fast', .3), ('explore', .3), ('exploit', .2), @@ -342,9 +348,11 @@ class FuzzingStrategies: # to_strategy_dict function should be removed when everything is fully # converted to the engine pipeline. For now this allows the code to be shared # between both cases while still adhering to the new pipeline's API. - def __init__(self, target_path, strategy_dict=None): - self.corpus_subset_size = None - self.candidate_generator = engine_common.Generator.NONE + def __init__(self, + target_path: str | None, + strategy_dict: dict[str, Any] | None = None) -> None: + self.corpus_subset_size: int | None = None + self.candidate_generator: int = engine_common.Generator.NONE # If we have already generated a strategy dict, use that in favor of # creating a new pool and picking randomly. @@ -363,7 +371,7 @@ def __init__(self, target_path, strategy_dict=None): # Select a generator to attempt to use for existing testcase mutations. self.candidate_generator = engine_common.select_generator( - strategy_pool, target_path) + strategy_pool, cast(str, target_path)) self.use_corpus_subset = strategy_pool.do_strategy( strategy.CORPUS_SUBSET_STRATEGY) @@ -372,18 +380,18 @@ def __init__(self, target_path, strategy_dict=None): self.corpus_subset_size = engine_common.random_choice( engine_common.CORPUS_SUBSET_NUM_TESTCASES) - self.is_mutations_run = ( + self.is_mutations_run: bool = ( self.candidate_generator != engine_common.Generator.NONE) # Generator that is actually used. Initialize to none, change if new # testcase mutations are properly generated by the candidate generator. - self.generator_strategy = engine_common.Generator.NONE + self.generator_strategy: int = engine_common.Generator.NONE - def to_strategy_dict(self): + def to_strategy_dict(self) -> dict[str, Any]: """Convert to a strategy dict in the format used by the engine pipeline.""" # The decision on whether or not fast cal should be used is made during # fuzzing. This function is expected to be called whe preparing for fuzzing. - strategies_dict = {} + strategies_dict: dict[str, Any] = {} if self.generator_strategy == engine_common.Generator.RADAMSA: strategies_dict[strategy.CORPUS_MUTATION_RADAMSA_STRATEGY.name] = 1 @@ -399,7 +407,8 @@ class AflFuzzInputDirectory: afl-fuzz as the -i argument. """ - def __init__(self, input_directory, target_path, fuzzing_strategies): + def __init__(self, input_directory: str, target_path: str, + fuzzing_strategies: FuzzingStrategies) -> None: """Inits AflFuzzInputDirectory. Args: @@ -408,12 +417,12 @@ def __init__(self, input_directory, target_path, fuzzing_strategies): fuzzing_strategies: fuzzing strategies to use. """ - self.input_directory = input_directory - self.strategies = fuzzing_strategies + self.input_directory: str = input_directory + self.strategies: FuzzingStrategies = fuzzing_strategies # We only need to use this when a temporary input directory is made. # (ie: when there is an oversized testcase in the input). - self.original_input_directory = None + self.original_input_directory: str | None = None engine_common.unpack_seed_corpus_if_needed( target_path, self.input_directory, max_bytes=constants.MAX_FILE_BYTES) @@ -424,7 +433,7 @@ def __init__(self, input_directory, target_path, fuzzing_strategies): if not list_full_file_paths(self.input_directory): write_dummy_file(self.input_directory) - def restore_if_needed(self): + def restore_if_needed(self) -> None: """Restore the original input directory if self.original_input_directory is set. Used to by merge() to get rid of the temporary input directory if it exists and merge new units into the original input directory. @@ -445,55 +454,62 @@ class AflRunnerCommon: """Afl runner common routines.""" # Window of time for afl to exit gracefully before we kill it. - AFL_CLEAN_EXIT_TIME = 10.0 + AFL_CLEAN_EXIT_TIME: float = 10.0 # Time to wait for SIGTERM handler. - SIGTERM_WAIT_TIME = 10.0 + SIGTERM_WAIT_TIME: float = 10.0 # Maximum number of times we will retry fuzzing after fixing an issue. - MAX_FUZZ_RETRIES = 40 + MAX_FUZZ_RETRIES: int = 40 # Maximum number of times we will retry fuzzing with a strict autocalibrated # timeout. After this number of fuzzing retries, if we see a hang we will set # the timeout to to '-t' + str(self.MANUAL_TIMEOUT_MILLISECONDS) + '+' to tell # AFL to skip testcases that take longer. - MAX_FUZZ_RETRIES_WITH_STRICT_TIMEOUT = 20 + MAX_FUZZ_RETRIES_WITH_STRICT_TIMEOUT: int = 20 # The number of times we will retry fuzzing with the deferred fork server # after the first testcase hangs. Afterwards we will use # AFL_DRIVER_DONT_DEFER, since this is a common symptom of fuzzers that # cant use the deferred forkserver. - MAX_FIRST_HANGS_WITH_DEFERRED_FORKSERVER = 5 + MAX_FIRST_HANGS_WITH_DEFERRED_FORKSERVER: int = 5 # The timeout we will use if autocalibrating results in too many hangs. This # is the maximum autocalibrated timeout afl-fuzz can set. - MANUAL_TIMEOUT_MILLISECONDS = 5000 + MANUAL_TIMEOUT_MILLISECONDS: int = 5000 # Regexes used to determine which file caused AFL to quit. - CRASH_REGEX = re.compile( + CRASH_REGEX: re.Pattern[str] = re.compile( r'Test case \'id\:\d+,orig:(?P.*)\' results in a' ' crash') - HANG_REGEX = re.compile( + HANG_REGEX: re.Pattern[str] = re.compile( r'Test case \'(?P.*)\' results in a (hang|timeout)') - CPU_BIND_ERROR_REGEX = re.compile('PROGRAM ABORT :.*No more free CPU cores') + CPU_BIND_ERROR_REGEX: re.Pattern[str] = re.compile( + 'PROGRAM ABORT :.*No more free CPU cores') # Log messages we format and log as error when afl-fuzz stops running. - CRASH_LOG_MESSAGE = 'Testcase {0} in corpus causes a crash' - HANG_LOG_MESSAGE = 'Testcase {0} in corpus causes a hang, retrying without it' + CRASH_LOG_MESSAGE: str = 'Testcase {0} in corpus causes a crash' + HANG_LOG_MESSAGE: str = ( + 'Testcase {0} in corpus causes a hang, retrying without it') + + SHOWMAP_FILENAME: str = 'afl_showmap_output' + SHOWMAP_REGEX: re.Pattern[bytes] = re.compile( + br'(?P\d{6}):(?P\d+)\n') - SHOWMAP_FILENAME = 'afl_showmap_output' - SHOWMAP_REGEX = re.compile(br'(?P\d{6}):(?P\d+)\n') + _default_args: list[str] + _executable_path: str + run_and_wait: Callable[..., new_process.ProcessResult] def __init__(self, - target_path, - config, - testcase_file_path, - input_directory, - timeout=None, - afl_tools_path=None, - strategy_dict=None): + target_path: str, + config: AflConfig, + testcase_file_path: str, + input_directory: str | None = None, + timeout: float | int | None = None, + afl_tools_path: str | None = None, + strategy_dict: dict[str, Any] | None = None) -> None: """Inits the AflRunner. Args: @@ -504,48 +520,54 @@ def __init__(self, afl_tools_path: Path that is used to locate afl-* tools. """ - self.target_path = target_path - self.config = config - self.testcase_file_path = testcase_file_path - self._input_directory = input_directory - self.timeout = timeout + self.target_path: str = target_path + self.config: AflConfig = config + self.testcase_file_path: str = testcase_file_path + self._input_directory: str | None = input_directory + self.timeout: float | int | None = timeout if afl_tools_path is None: afl_tools_path = os.path.dirname(target_path) # Set paths to afl tools. - self.afl_fuzz_path = os.path.join(afl_tools_path, 'afl-fuzz') - self.afl_showmap_path = os.path.join(afl_tools_path, 'afl-showmap') + self.afl_fuzz_path: str = os.path.join(afl_tools_path, 'afl-fuzz') + self.afl_showmap_path: str = os.path.join(afl_tools_path, 'afl-showmap') - self._afl_input = None - self._afl_output = None + self._afl_input: AflFuzzInputDirectory | None = None + self._afl_output: AflFuzzOutputDirectory | None = None - self.strategies = FuzzingStrategies( + self.strategies: FuzzingStrategies = FuzzingStrategies( target_path, strategy_dict=strategy_dict) # Set this to None so we can tell if it has never been set or if it's just # empty. - self._fuzzer_stderr = None + self._fuzzer_stderr: str | None = None - self.initial_max_total_time = 0 + self.initial_max_total_time: float = 0 for env_var, value in config.additional_env_vars.items(): environment.set_value(env_var, value) - self.showmap_output_path = os.path.join(fuzzer_utils.get_temp_dir(), - self.SHOWMAP_FILENAME) - self.merge_timeout = engine_common.get_merge_timeout(DEFAULT_MERGE_TIMEOUT) - self.showmap_no_output_logged = False - self._fuzz_args = [] - self._executable_path = None + self.showmap_output_path: str = os.path.join(fuzzer_utils.get_temp_dir(), + self.SHOWMAP_FILENAME) + self.merge_timeout: float = float( + engine_common.get_merge_timeout(DEFAULT_MERGE_TIMEOUT)) + self.showmap_no_output_logged: bool = False + self._fuzz_args: list[str] = [] + self._executable_path: str = '' + + @property + def executable_path(self) -> str: + """Returns the executable path.""" + return self._executable_path @property - def stderr_file_path(self): + def stderr_file_path(self) -> str: """Returns the file for afl to output stack traces.""" return os.path.join(fuzzer_utils.get_temp_dir(), STDERR_FILENAME) @property - def fuzzer_stderr(self): + def fuzzer_stderr(self) -> str: """Returns the stderr of the fuzzer. Reads it first if it wasn't already read. Because ClusterFuzz terminates this process after seeing a stacktrace printed, make sure that printing this property is the last code a program @@ -564,7 +586,7 @@ def fuzzer_stderr(self): self._fuzzer_stderr = '' return self._fuzzer_stderr - def set_environment_variables(self): + def set_environment_variables(self) -> None: """Sets environment variables needed by afl.""" # Tell afl_driver to duplicate stderr to STDERR_FILENAME. # Environment variable names and values that must be set before running afl. @@ -583,8 +605,9 @@ def set_environment_variables(self): stderr_file_path = android.util.get_device_path(self.stderr_file_path) environment.set_value(constants.STDERR_FILENAME_ENV_VAR, stderr_file_path) - def get_afl_environment_variables(self): - afl_environment_vars = [] + def get_afl_environment_variables(self) -> list[str]: + """Returns list of AFL environment variables.""" + afl_environment_vars: list[str] = [] for env_var in os.environ: if env_var and env_var.startswith('AFL_'): afl_environment_vars.append(env_var + '=' + @@ -592,7 +615,11 @@ def get_afl_environment_variables(self): return afl_environment_vars - def check_return_code(self, result, additional_return_codes=None): + def check_return_code( + self, + result: new_process.ProcessResult, + additional_return_codes: list[int] | None = None) -> None: + """Checks the return code of the target execution.""" expected_return_codes = [0, 1, -6] if additional_return_codes: expected_return_codes += additional_return_codes @@ -602,7 +629,8 @@ def check_return_code(self, result, additional_return_codes=None): f'AFL target exited with abnormal exit code: {result.return_code}.', output=result.output) - def run_single_testcase(self, testcase_path): + def run_single_testcase(self, + testcase_path: str) -> new_process.ProcessResult: """Runs a single testcase. Args: @@ -618,12 +646,13 @@ def run_single_testcase(self, testcase_path): self.afl_setup() result = self.run_and_wait(additional_args=[testcase_path]) + assert result.command is not None print('Running command:', engine_common.get_command_quoted(result.command)) self.check_return_code(result) return result - def afl_setup(self): + def afl_setup(self) -> None: """Make sure we can run afl. Delete any files that afl_driver needs to create and set any environmnet variables it needs. """ @@ -631,13 +660,14 @@ def afl_setup(self): remove_path(self.stderr_file_path) @staticmethod - def set_resume(afl_args): + def set_resume(afl_args: list[str]) -> list[str]: """Changes afl_args so afl-fuzz will resume fuzzing rather than restarting. """ return AflRunner.set_input_arg(afl_args, constants.RESUME_INPUT) @staticmethod - def get_arg_index(afl_args, flag): + def get_arg_index(afl_args: list[str], flag: str) -> int: + """Gets the index of the flag in afl_args.""" for idx, arg in enumerate(afl_args): if arg.startswith(flag): return idx @@ -645,7 +675,7 @@ def get_arg_index(afl_args, flag): return -1 @classmethod - def set_arg(cls, afl_args, flag, value): + def set_arg(cls, afl_args: list[str], flag: str, value: Any) -> list[str]: """Sets the afl |flag| to |value| in |afl_args|. If |flag| is already in |afl_args|, then the old value is replaced by |value|, otherwise |flag| and |value| are added. @@ -665,20 +695,25 @@ def set_arg(cls, afl_args, flag, value): return afl_args @classmethod - def set_input_arg(cls, afl_args, new_input_value): + def set_input_arg(cls, afl_args: list[str], + new_input_value: str) -> list[str]: """Changes the input argument (-i) in |afl_args| to |new_input_value|.""" return cls.set_arg(afl_args, constants.INPUT_FLAG, new_input_value) @classmethod - def set_timeout_arg(cls, afl_args, timeout_value, skip_hangs=False): - timeout_value = str(int(timeout_value)) + def set_timeout_arg(cls, + afl_args: list[str], + timeout_value: int | float | str, + skip_hangs: bool = False) -> list[str]: + """Sets the timeout argument in afl_args.""" + timeout_str = str(int(timeout_value)) if skip_hangs: - timeout_value += '+' + timeout_str += '+' - cls.set_arg(afl_args, constants.TIMEOUT_FLAG, timeout_value) + cls.set_arg(afl_args, constants.TIMEOUT_FLAG, timeout_str) return afl_args - def do_offline_mutations(self): + def do_offline_mutations(self) -> None: """Mutate the corpus offline using Radamsa.""" if not self.strategies.is_mutations_run: return @@ -698,12 +733,13 @@ def do_offline_mutations(self): if os.path.getsize(input_path) >= constants.MAX_FILE_BYTES: remove_path(input_path) - def generate_afl_args(self, - afl_input=None, - afl_output=None, - target_path=None, - additional_args=None, - mem_limit=constants.MAX_MEMORY_LIMIT): + def generate_afl_args( + self, + afl_input: str | None = None, + afl_output: str | None = None, + target_path: str | None = None, + additional_args: list[str] | None = None, + mem_limit: int | str = constants.MAX_MEMORY_LIMIT) -> list[str]: """Generate arguments to pass to Process.run_and_wait. Args: @@ -750,7 +786,7 @@ def generate_afl_args(self, return afl_args - def should_try_fuzzing(self, max_total_time, num_retries): + def should_try_fuzzing(self, max_total_time: float, num_retries: int) -> bool: """Returns True if we should try fuzzing, based on the number of times we've already tried, |num_retries|, and the amount of time we have left (calculated using |max_total_time|). @@ -770,7 +806,7 @@ def should_try_fuzzing(self, max_total_time, num_retries): return True - def run_afl_fuzz(self, fuzz_args): + def run_afl_fuzz(self, fuzz_args: list[str]) -> new_process.ProcessResult: """Run afl-fuzz and if there is an input that causes afl-fuzz to hang or if it can't bind to a cpu, try fixing the issue and running afl-fuzz again. If there is a crash in the starting corpus then report it. @@ -782,17 +818,19 @@ def run_afl_fuzz(self, fuzz_args): """ # Define here to capture in closures. max_total_time = self.initial_max_total_time - fuzz_result = None + fuzz_result: new_process.ProcessResult | None = None - def get_time_spent_fuzzing(): + def get_time_spent_fuzzing() -> float: """Gets the amount of time spent running afl-fuzz so far.""" return self.initial_max_total_time - max_total_time - def check_error_and_log(error_regex, log_message_format): + def check_error_and_log(error_regex: re.Pattern[str], + log_message_format: str) -> str | None: """See if error_regex can match in fuzz_result.output. If it can, then it uses the match to format and print log_message and return the match. Otherwise returns None. """ + assert fuzz_result is not None matches = re.search(error_regex, fuzz_result.output) if matches: erroring_filename = matches.groups()[0] @@ -816,6 +854,7 @@ def check_error_and_log(error_regex, log_message_format): # If the target was compiled for CMPLOG we need to set this. build_directory = environment.get_value('BUILD_DIR') + assert build_directory is not None cmplog_build_file = os.path.join(build_directory, 'afl_cmplog.txt') if os.path.exists(cmplog_build_file): self.set_arg(fuzz_args, constants.CMPLOG_FLAG, self.target_path) @@ -898,7 +937,7 @@ def check_error_and_log(error_regex, log_message_format): terminate_wait_time=self.SIGTERM_WAIT_TIME, ) else: - android_params = [] + android_params: list[str] = [] android_params = self.get_afl_environment_variables() android_params.append(android.util.get_device_path(self.afl_fuzz_path)) @@ -911,6 +950,7 @@ def check_error_and_log(error_regex, log_message_format): ) # Reduce max_total_time by the amount of time the last attempt took. + assert fuzz_result.time_executed is not None max_total_time -= fuzz_result.time_executed # Break now only if everything went well. Note that if afl finds a crash @@ -984,9 +1024,11 @@ def check_error_and_log(error_regex, log_message_format): break + assert fuzz_result is not None return fuzz_result - def prepare_retry_if_cpu_error(self, fuzz_result): + def prepare_retry_if_cpu_error( + self, fuzz_result: new_process.ProcessResult) -> bool: """AFL will try to bind targets to a particular core for a speed improvement. If this isn't possible, then AFL won't run unless AFL_NO_AFFINITY=1. One way this can happen is if afl-fuzz leaves zombies @@ -1020,17 +1062,18 @@ def prepare_retry_if_cpu_error(self, fuzz_result): return True @property - def afl_input(self): + def afl_input(self) -> AflFuzzInputDirectory: """Don't create the object until we need it, since it isn't used for reproducing testcases.""" if self._afl_input is None: + assert self._input_directory is not None self._afl_input = AflFuzzInputDirectory(self._input_directory, self.target_path, self.strategies) return self._afl_input @property - def afl_output(self): + def afl_output(self) -> AflFuzzOutputDirectory: """Don't create the object until we need it, since it isn't used for reproducing testcases.""" @@ -1042,7 +1085,7 @@ def afl_output(self): return self._afl_output - def fuzz(self): + def fuzz(self) -> new_process.ProcessResult: """Running fuzzing command. Wrapper around run_afl_fuzz that performs one time setup. @@ -1065,15 +1108,18 @@ def fuzz(self): return self.run_afl_fuzz(self._fuzz_args) - def get_feature_tuple(self, showmap_output): - features = set() + def get_feature_tuple(self, + showmap_output: bytes) -> frozenset[tuple[int, int]]: + """Parses showmap output into a set of (guard, hit_count) tuples.""" + features: set[tuple[int, int]] = set() for match in re.finditer(self.SHOWMAP_REGEX, showmap_output): d = match.groupdict() features.add((int(d['guard']), int(d['hit_count']))) return frozenset(features) - def get_file_features(self, input_file_path, showmap_args): + def get_file_features(self, input_file_path: str, showmap_args: list[str] + ) -> tuple[frozenset[tuple[int, int]] | None, bool]: """Get the features (edge hit counts) of |input_file_path| using afl-showmap.""" # TODO(metzman): Figure out if we should worry about CPU affinity errors @@ -1086,6 +1132,7 @@ def get_file_features(self, input_file_path, showmap_args): timeout=self.merge_timeout, extra_env={constants.DEBUG_VAR: '1'}) + assert showmap_result.time_executed is not None self.merge_timeout -= showmap_result.time_executed # TODO(metzman): Figure out why negative values are accepted by @@ -1114,7 +1161,7 @@ def get_file_features(self, input_file_path, showmap_args): showmap_output = engine_common.read_data_from_file(self.showmap_output_path) return self.get_feature_tuple(showmap_output), False - def merge_corpus(self): + def merge_corpus(self) -> int: """Merge new testcases into the input corpus.""" logs.info('Merging corpus.') # Don't tell the fuzz target to write its stderr to the same file written @@ -1131,8 +1178,8 @@ def merge_corpus(self): ] input_dir = self.afl_input.input_directory corpus = Corpus() - input_inodes = set() - input_filenames = set() + input_inodes: set[int] = set() + input_filenames: set[str] = set() for file_path in shell.get_files_list(input_dir): file_features, timed_out = self.get_file_features(file_path, showmap_args) if timed_out: @@ -1184,7 +1231,7 @@ def merge_corpus(self): return new_units_added - def libfuzzerize_corpus(self): + def libfuzzerize_corpus(self) -> tuple[int, int, int]: """Make corpus directories libFuzzer compatible, merge new testcases if needed and return the number of new testcases added to corpus. """ @@ -1210,13 +1257,13 @@ class AflRunner(AflRunnerCommon, new_process.UnicodeProcessRunner): """Afl runner.""" def __init__(self, - target_path, - config, - testcase_file_path, - input_directory, - timeout=None, - afl_tools_path=None, - strategy_dict=None): + target_path: str, + config: AflConfig, + testcase_file_path: str, + input_directory: str | None = None, + timeout: float | int | None = None, + afl_tools_path: str | None = None, + strategy_dict: dict[str, Any] | None = None) -> None: super().__init__(target_path, config, testcase_file_path, input_directory, timeout, afl_tools_path, strategy_dict) @@ -1230,39 +1277,40 @@ class AflAndroidRunner(AflRunnerCommon, new_process.UnicodeProcessRunner): # while fuzzing on device # NOTE: if run_and_wait times out before afl on device process it will return # an error, and clusterfuzz will think there was an error with the fuzzing - DEVICE_FUZZING_CLEAN_EXIT_TIME = 10.0 + DEVICE_FUZZING_CLEAN_EXIT_TIME: float = 10.0 def __init__(self, - target_path, - config, - testcase_file_path, - input_directory, - timeout=None, - afl_tools_path=None, - strategy_dict=None): + target_path: str, + config: AflConfig, + testcase_file_path: str, + input_directory: str | None = None, + timeout: float | int | None = None, + afl_tools_path: str | None = None, + strategy_dict: dict[str, Any] | None = None) -> None: super().__init__(target_path, config, testcase_file_path, input_directory, timeout, afl_tools_path, strategy_dict) new_process.ProcessRunner.__init__( self, executable_path=android.adb.get_adb_path()) - self.showmap_output_path = os.path.join( + self.showmap_output_path: str = os.path.join( android.constants.DEVICE_FUZZING_DIR, self.SHOWMAP_FILENAME) - self._showmap_results_dir = os.path.join(environment.get_root_directory(), - "tmp/showmap_results") + self._showmap_results_dir: str = os.path.join( + environment.get_root_directory(), "tmp/showmap_results") - def _copy_local_directories_to_device(self, local_directories): + def _copy_local_directories_to_device(self, + local_directories: list[str]) -> None: """Copies local directories to device.""" for local_directory in sorted(set(local_directories)): self._copy_local_directory_to_device(local_directory) - def _copy_local_directory_to_device(self, local_directory): + def _copy_local_directory_to_device(self, local_directory: str) -> None: """Copies single local directory to device.""" device_directory = android.util.get_device_path(local_directory) android.adb.remove_directory(device_directory, recreate=True) android.adb.copy_local_directory_to_remote(local_directory, device_directory) - def _copy_directories_from_device(self, local_directories): + def _copy_directories_from_device(self, local_directories: list[str]) -> None: """Copies directories from device to local.""" for local_directory in sorted(set(local_directories)): device_directory = android.util.get_device_path(local_directory) @@ -1271,11 +1319,12 @@ def _copy_directories_from_device(self, local_directories): android.adb.copy_remote_directory_to_local(device_directory, local_directory) - def afl_setup(self): + def afl_setup(self) -> None: android.adb.remove_file(android.util.get_device_path(self.stderr_file_path)) super().afl_setup() - def run_single_testcase(self, testcase_path): + def run_single_testcase(self, + testcase_path: str) -> new_process.ProcessResult: """Runs a single testcase. Args: testcase_path: Path to testcase to be run. @@ -1303,7 +1352,8 @@ def run_single_testcase(self, testcase_path): self.check_return_code(result, [134]) return result - def get_file_features(self, input_file_path, showmap_args): + def get_file_features(self, input_file_path: str, showmap_args: list[str] + ) -> tuple[frozenset[tuple[int, int]] | None, bool]: """Get the features (edge hit counts) of |input_file_path| using afl-showmap.""" # TODO(metzman): Figure out if we should worry about CPU affinity errors @@ -1322,7 +1372,7 @@ def get_file_features(self, input_file_path, showmap_args): intput_file_showmap_results_file) return self.get_feature_tuple(showmap_output), False - def fuzz(self): + def fuzz(self) -> new_process.ProcessResult: self._executable_path = android.adb.get_adb_path() self._default_args = ['shell'] @@ -1331,6 +1381,7 @@ def fuzz(self): self.strategies.is_mutations_run, full_timeout=self.timeout) - self.AFL_CLEAN_EXIT_TIME - self.SIGTERM_WAIT_TIME) + assert self.timeout is not None on_device_fuzzing_timeout = self.timeout - int( self.AFL_CLEAN_EXIT_TIME + self.SIGTERM_WAIT_TIME + self.DEVICE_FUZZING_CLEAN_EXIT_TIME) @@ -1393,23 +1444,24 @@ class UnshareAflRunner(new_process.ModifierProcessRunnerMixin, AflRunner): class CorpusElement: """An element (file) in a corpus.""" - def __init__(self, path): - self.path = path - self.size = os.path.getsize(self.path) + def __init__(self, path: str) -> None: + self.path: str = path + self.size: int = os.path.getsize(self.path) class Corpus: """A minimal set of input files (elements) for a fuzz target.""" - def __init__(self): - self.features_and_elements = {} + def __init__(self) -> None: + self.features_and_elements: dict[tuple[int, int], CorpusElement] = {} @property - def element_paths(self): + def element_paths(self) -> set[str]: """Returns the filepaths of all elements in the corpus.""" return {element.path for element in self.features_and_elements.values()} - def _associate_feature_with_element(self, feature, element): + def _associate_feature_with_element(self, feature: tuple[int, int], + element: CorpusElement) -> None: """Associate a feature with an element if the element is the smallest for the feature.""" if feature not in self.features_and_elements: @@ -1421,7 +1473,9 @@ def _associate_feature_with_element(self, feature, element): if incumbent_element.size > element.size: self.features_and_elements[feature] = element - def associate_features_with_file(self, features, path): + def associate_features_with_file( + self, features: frozenset[tuple[int, int]] | set[tuple[int, int]], + path: str) -> None: """Associate features with a file when the file is the smallest for the features.""" element = CorpusElement(path) @@ -1429,10 +1483,10 @@ def associate_features_with_file(self, features, path): self._associate_feature_with_element(feature, element) -def _verify_system_config(): +def _verify_system_config() -> None: """Verifies system settings required for AFL.""" - def _check_core_pattern_file(): + def _check_core_pattern_file() -> bool: """Verifies that core pattern file content is set to 'core'.""" if not os.path.exists(constants.CORE_PATTERN_FILE_PATH): return False @@ -1453,7 +1507,8 @@ def _check_core_pattern_file(): path=constants.CORE_PATTERN_FILE_PATH)) -def load_testcase_if_exists(fuzzer_runner, testcase_file_path): +def load_testcase_if_exists(fuzzer_runner: AflRunnerCommon, + testcase_file_path: str) -> bool: """Loads a crash testcase if it exists.""" # To ensure that we can run the fuzzer. os.chmod(fuzzer_runner.executable_path, stat.S_IRWXU | stat.S_IRGRP @@ -1464,7 +1519,7 @@ def load_testcase_if_exists(fuzzer_runner, testcase_file_path): return True -def set_additional_sanitizer_options_for_afl_fuzz(): +def set_additional_sanitizer_options_for_afl_fuzz() -> None: """Set *SAN_OPTIONS to afl's liking. If ASAN_OPTIONS or MSAN_OPTION is set, they must contain certain options or @@ -1495,11 +1550,12 @@ def set_additional_sanitizer_options_for_afl_fuzz(): continue options_env_value = environment.get_memory_tool_options(options_env_var) + assert options_env_value is not None options_env_value.update(option_values) environment.set_memory_tool_options(options_env_var, options_env_value) -def remove_path(path): +def remove_path(path: str) -> None: """Remove |path| if it exists. Similar to running rm -rf |path|.""" # Remove links to files and files. if os.path.isfile(path): @@ -1509,7 +1565,7 @@ def remove_path(path): # Else path doesn't exist. Do nothing. -def list_full_file_paths_device(directory): +def list_full_file_paths_device(directory: str) -> list[str]: """List the absolute paths of files in |directory| on Android device.""" directory_absolute_path = os.path.abspath(directory) directory_absolute_path = android.util.get_device_path( @@ -1518,7 +1574,8 @@ def list_full_file_paths_device(directory): dir_contents = android.adb.run_command( ['shell', 'ls', directory_absolute_path]) - paths = [] + assert dir_contents is not None + paths: list[str] = [] for rel_path in dir_contents.split(): full_path = os.path.join(directory_absolute_path, rel_path) if android.adb.file_exists(full_path): @@ -1527,10 +1584,10 @@ def list_full_file_paths_device(directory): return paths -def list_full_file_paths(directory): +def list_full_file_paths(directory: str) -> list[str]: """List the absolute paths of files in |directory|.""" directory_absolute_path = os.path.abspath(directory) - paths = [] + paths: list[str] = [] for relative_path in os.listdir(directory): absolute_path = os.path.join(directory_absolute_path, relative_path) if os.path.isfile(absolute_path): # Only return paths to files. @@ -1538,7 +1595,7 @@ def list_full_file_paths(directory): return paths -def get_first_stacktrace(stderr_data): +def get_first_stacktrace(stderr_data: str) -> str: """If |stderr_data| contains stack traces, only returns the first one. Otherwise returns the entire string.""" @@ -1558,9 +1615,10 @@ def get_first_stacktrace(stderr_data): # TODO(mbarbella): After deleting the non-engine AFL code, remove this # function and replace it with a simple check based on the timeout set in # Engine.fuzz. Mutations should also be moved to fuzz. -def get_fuzz_timeout(is_mutations_run, full_timeout=None): +def get_fuzz_timeout(is_mutations_run: bool, + full_timeout: float | int | None = None) -> float: """Get the maximum amount of time that should be spent fuzzing.""" - fuzz_timeout = full_timeout + fuzz_timeout: float | int | None = full_timeout # TODO(mbarbella): Delete this check once non-engine support is removed. The # engine pipeline always specifies a timeout. @@ -1576,18 +1634,20 @@ def get_fuzz_timeout(is_mutations_run, full_timeout=None): fuzz_timeout -= engine_common.get_merge_timeout(DEFAULT_MERGE_TIMEOUT) assert fuzz_timeout > 0 - return fuzz_timeout + return float(fuzz_timeout) -def prepare_runner(fuzzer_path, - config, - testcase_file_path, - input_directory, - timeout=None, - strategy_dict=None): +def prepare_runner(fuzzer_path: str, + config: AflConfig, + testcase_file_path: str, + input_directory: str | None = None, + timeout: float | int | None = None, + strategy_dict: dict[str, Any] | None = None + ) -> AflRunner | AflAndroidRunner: """Common initialization code shared by the new pipeline and main.""" # Set up temp dir. engine_common.recreate_directory(fuzzer_utils.get_temp_dir()) + runner_class: type[AflRunner | AflAndroidRunner] if environment.get_value('USE_UNSHARE'): runner_class = UnshareAflRunner elif environment.is_android(): @@ -1617,7 +1677,7 @@ def prepare_runner(fuzzer_path, return runner -def rand_schedule(scheduler_probs): +def rand_schedule(scheduler_probs: list[tuple[str, float]]) -> str: """Returns a random queue scheduler.""" schedule = 'fast' rnd = engine_common.get_probability() @@ -1630,7 +1690,7 @@ def rand_schedule(scheduler_probs): return schedule -def rand_cmplog_level(strategies): +def rand_cmplog_level(strategies: FuzzingStrategies) -> str: """Returns a random CMPLOG intensity level.""" cmplog_level = '2' rnd = engine_common.get_probability() @@ -1652,7 +1712,7 @@ def rand_cmplog_level(strategies): # pylint: disable=too-many-function-args -def main(argv): +def main(argv: list[str]) -> None: """Run afl as specified by argv.""" atexit.register(fuzzer_utils.cleanup) @@ -1687,6 +1747,8 @@ def main(argv): fuzz_result = runner.fuzz() # Print info for the fuzzer logs. + assert fuzz_result.command is not None + assert fuzz_result.time_executed is not None print( engine_common.get_log_header(fuzz_result.command, fuzz_result.time_executed)) @@ -1704,12 +1766,13 @@ def main(argv): try: new_units_generated, new_units_added, corpus_size = ( runner.libfuzzerize_corpus()) + assert fuzz_result.time_executed is not None stats_getter.set_stats(fuzz_result.time_executed, new_units_generated, new_units_added, corpus_size, runner.strategies, runner.fuzzer_stderr, fuzz_result.output) engine_common.dump_big_query_data(stats_getter.stats, testcase_file_path, - fuzz_result.command) + cast(str, fuzz_result.command)) finally: print(runner.fuzzer_stderr) diff --git a/src/clusterfuzz/_internal/bot/fuzzers/afl/stats.py b/src/clusterfuzz/_internal/bot/fuzzers/afl/stats.py index f68a01d7aac..37af55c1388 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/afl/stats.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/afl/stats.py @@ -15,6 +15,8 @@ import os import re +from typing import Any +from typing import TYPE_CHECKING from clusterfuzz._internal.bot.fuzzers import dictionary_manager from clusterfuzz._internal.bot.fuzzers import engine_common @@ -22,6 +24,9 @@ from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment +if TYPE_CHECKING: + from clusterfuzz._internal.bot.fuzzers.afl import launcher + SANITIZER_START_REGEX = re.compile(r'.*ERROR: [A-Za-z]+Sanitizer:.*') SANITIZER_SEPERATOR_REGEX = re.compile(r'^=+$') @@ -49,7 +54,7 @@ class StatsGetter: CORPUS_CRASH_REGEX = re.compile( r'program crashed with one of the test cases provided') - def __init__(self, afl_stats_path, dict_path): + def __init__(self, afl_stats_path: str, dict_path: str | None) -> None: """Set important attributes and initialize all stats to 0. Args: @@ -62,11 +67,11 @@ def __init__(self, afl_stats_path, dict_path): self.afl_stats_path = afl_stats_path # The parsed stats in afl_stats_path. - self.afl_stats = {} + self.afl_stats: dict[str, str] = {} self.dict_path = dict_path # Default values. - self.stats = { + self.stats: dict[str, Any] = { 'actual_duration': 0, 'average_exec_per_sec': 0, 'bad_instrumentation': 0, @@ -83,7 +88,7 @@ def __init__(self, afl_stats_path, dict_path): 'timeout_limit': 0, } - def set_afl_stats(self): + def set_afl_stats(self) -> None: """Read statistics from afl-fuzz's "fuzzer_stats" file and save them as self.afl_stats. """ @@ -95,9 +100,10 @@ def set_afl_stats(self): afl_stats_string = engine_common.read_data_from_file( self.afl_stats_path).decode('utf-8') matches_iterator = re.finditer(self.STATS_REGEX, afl_stats_string) - self.afl_stats = dict(match.groups() for match in matches_iterator) + self.afl_stats = dict( + match.groups() for match in matches_iterator) # type: ignore - def get_afl_stat(self, afl_stat_name): + def get_afl_stat(self, afl_stat_name: str) -> int | float: """Try to get |afl_stat_name| from self.afl_stats, otherwise return a default value instead. Note that is imporant that the types are correct since a type change will prevent this data from being loaded into @@ -132,7 +138,7 @@ def get_afl_stat(self, afl_stat_name): except ValueError: return float(afl_stat_value) - def _get_unwanted_log_line_count(self, log_output): + def _get_unwanted_log_line_count(self, log_output: str) -> int: """Return number of unwanted lines in log output.""" count = 0 for line in log_output.splitlines(): @@ -145,13 +151,13 @@ def _get_unwanted_log_line_count(self, log_output): return count def set_stats(self, - actual_duration, - new_units_generated, - new_units_added, - corpus_size, - fuzzing_strategies, - fuzzer_stderr, - afl_fuzz_output=''): + actual_duration: float, + new_units_generated: int, + new_units_added: int, + corpus_size: int, + fuzzing_strategies: 'launcher.FuzzingStrategies', + fuzzer_stderr: str, + afl_fuzz_output: str = '') -> dict[str, Any]: """Create a dict of statistics that can be uploaded to ClusterFuzz and save it in self.stats. @@ -213,7 +219,7 @@ def set_stats(self, self.set_output_stats(afl_fuzz_output) return self.stats - def set_output_stats(self, afl_fuzz_output): + def set_output_stats(self, afl_fuzz_output: str) -> None: """Set stats gotten from the output of afl-fuzz, |afl_fuzz_output|.""" # If there is no instrumentation, note it. if re.search(self.NO_INSTRUMENTATION_REGEX, afl_fuzz_output): @@ -227,7 +233,8 @@ def set_output_stats(self, afl_fuzz_output): if re.search(self.CORPUS_CRASH_REGEX, afl_fuzz_output): self.stats['corpus_crash_count'] = 1 - def set_strategy_stats(self, fuzzing_strategies): + def set_strategy_stats( + self, fuzzing_strategies: 'launcher.FuzzingStrategies') -> None: """Sets strategy related stats for afl-fuzz to correct values based on |strategies|.""" diff --git a/src/clusterfuzz/_internal/bot/fuzzers/builtin.py b/src/clusterfuzz/_internal/bot/fuzzers/builtin.py index 133068ffa13..9769aa74f1e 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/builtin.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/builtin.py @@ -15,6 +15,7 @@ # NOTE: This module is deprecated and will be replaced with # lib.clusterfuzz.fuzz.engine. +from collections.abc import Sequence import os import random import sys @@ -31,7 +32,7 @@ class BuiltinFuzzerResult: """Result of running a builtin fuzzer.""" - def __init__(self, output, corpus_directory=None): + def __init__(self, output: str, corpus_directory: str | None = None) -> None: self.output = output self.corpus_directory = corpus_directory @@ -45,16 +46,19 @@ class BuiltinFuzzerError(Exception): class BuiltinFuzzer: """Builtin fuzzer.""" - def run(self, input_directory, output_directory, no_of_files): + def run(self, input_directory: str, output_directory: str, + no_of_files: int) -> BuiltinFuzzerResult: raise NotImplementedError @property - def fuzzer_directory(self): - return os.path.abspath( - os.path.dirname(sys.modules[self.__module__].__file__)) + def fuzzer_directory(self) -> str: + module_file = sys.modules[self.__module__].__file__ + assert module_file is not None + return os.path.abspath(os.path.dirname(module_file)) -def get_corpus_directory(input_directory, project_qualified_name): +def get_corpus_directory(input_directory: str, + project_qualified_name: str) -> str: """Get the corpus directory given a project qualified fuzz target name.""" corpus_directory = os.path.join(input_directory, project_qualified_name) if environment.is_trusted_host(): @@ -73,11 +77,11 @@ def get_corpus_directory(input_directory, project_qualified_name): class EngineFuzzer(BuiltinFuzzer): """Builtin fuzzer for fuzzing engines such as libFuzzer.""" - def generate_arguments(self, fuzzer_path): + def generate_arguments(self, fuzzer_path: str) -> str: """Generate arguments for the given fuzzer.""" raise NotImplementedError - def _get_fuzzer_binary_name_and_path(self): + def _get_fuzzer_binary_name_and_path(self) -> tuple[str, str]: """Returns the fuzzer binary name and its path.""" # Fuchsia doesn't use file paths to call fuzzers, just the name of the # fuzzer, so we set both from FUZZ_TARGET here. @@ -103,7 +107,8 @@ def _get_fuzzer_binary_name_and_path(self): fuzzer_binary_name = os.path.basename(fuzzer_path) return fuzzer_binary_name, fuzzer_path - def run(self, input_directory, output_directory, no_of_files): + def run(self, input_directory: str, output_directory: str, + no_of_files: int) -> BuiltinFuzzerResult: """Run the fuzzer to generate testcases.""" fuzzer_binary_name, fuzzer_path = self._get_fuzzer_binary_name_and_path() @@ -159,7 +164,7 @@ def run(self, input_directory, output_directory, no_of_files): return BuiltinFuzzerResult(output=output, corpus_directory=corpus_directory) -def _get_fuzzer_path(target_list, fuzzer_name): +def _get_fuzzer_path(target_list: Sequence[str], fuzzer_name: str) -> str: """Return the full fuzzer path and actual binary name of |fuzzer_name|.""" fuzzer_filename = environment.get_executable_filename(fuzzer_name) for path in target_list: diff --git a/src/clusterfuzz/_internal/bot/fuzzers/centipede/constants.py b/src/clusterfuzz/_internal/bot/fuzzers/centipede/constants.py index 223310fe284..a18a0c4dfdb 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/centipede/constants.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/centipede/constants.py @@ -19,38 +19,38 @@ "-flag=value". """ -FORK_SERVER_FLAGNAME = 'fork_server' -FORK_SERVER_DEFAULT = 1 +FORK_SERVER_FLAGNAME: str = 'fork_server' +FORK_SERVER_DEFAULT: int = 1 -RSS_LIMIT_MB_FLAGNAME = 'rss_limit_mb' -RSS_LIMIT_MB_DEFAULT = 4096 +RSS_LIMIT_MB_FLAGNAME: str = 'rss_limit_mb' +RSS_LIMIT_MB_DEFAULT: int = 4096 -TIMEOUT_PER_INPUT_FLAGNAME = 'timeout_per_input' -TIMEOUT_PER_INPUT_DEFAULT = 25 -TIMEOUT_PER_INPUT_REPR_DEFAULT = 60 +TIMEOUT_PER_INPUT_FLAGNAME: str = 'timeout_per_input' +TIMEOUT_PER_INPUT_DEFAULT: int = 25 +TIMEOUT_PER_INPUT_REPR_DEFAULT: int = 60 -ADDRESS_SPACE_LIMIT_FLAGNAME = 'address_space_limit_mb' -ADDRESS_SPACE_LIMIT_DEFAULT = 4096 +ADDRESS_SPACE_LIMIT_FLAGNAME: str = 'address_space_limit_mb' +ADDRESS_SPACE_LIMIT_DEFAULT: int = 4096 -DICTIONARY_FLAGNAME = 'dictionary' -WORKDIR_FLAGNAME = 'workdir' -CORPUS_DIR_FLAGNAME = 'corpus_dir' -BINARY_FLAGNAME = 'binary' -EXTRA_BINARIES_FLAGNAME = 'extra_binaries' -EXIT_ON_CRASH_FLAGNAME = 'exit_on_crash' +DICTIONARY_FLAGNAME: str = 'dictionary' +WORKDIR_FLAGNAME: str = 'workdir' +CORPUS_DIR_FLAGNAME: str = 'corpus_dir' +BINARY_FLAGNAME: str = 'binary' +EXTRA_BINARIES_FLAGNAME: str = 'extra_binaries' +EXIT_ON_CRASH_FLAGNAME: str = 'exit_on_crash' -MAX_LEN_FLAGNAME = 'max_len' -NUM_RUNS_FLAGNAME = 'num_runs' -BATCH_SIZE_FLAGNAME = 'batch_size' -STOP_AFTER_FLAGNAME = 'stop_after' +MAX_LEN_FLAGNAME: str = 'max_len' +NUM_RUNS_FLAGNAME: str = 'num_runs' +BATCH_SIZE_FLAGNAME: str = 'batch_size' +STOP_AFTER_FLAGNAME: str = 'stop_after' -SYMBOLIZER_PATH_FLAGNAME = 'symbolizer_path' -SYMBOLIZER_PATH_DEFAULT = '/dev/null' +SYMBOLIZER_PATH_FLAGNAME: str = 'symbolizer_path' +SYMBOLIZER_PATH_DEFAULT: str = '/dev/null' -NUM_RUNS_PER_MINIMIZATION = 100000 +NUM_RUNS_PER_MINIMIZATION: int = 100000 -def get_default_arguments(): +def get_default_arguments() -> dict[str, int | str]: return { FORK_SERVER_FLAGNAME: FORK_SERVER_DEFAULT, RSS_LIMIT_MB_FLAGNAME: RSS_LIMIT_MB_DEFAULT, diff --git a/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py b/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py index 269f3a614d2..9207a8a734f 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/centipede/engine.py @@ -13,16 +13,13 @@ # limitations under the License. """Centipede engine interface.""" -from collections import namedtuple import csv import os import pathlib import re import shutil -from typing import Dict -from typing import List -from typing import Optional -from typing import Union +from typing import Any +from typing import NamedTuple from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot.fuzzers import dictionary_manager @@ -41,7 +38,11 @@ CRASH_REGEX = re.compile(r'[sS]aving input to:?\s*(.*)') _CRASH_LOG_PREFIX = 'CRASH LOG: ' -TargetBinaries = namedtuple('TargetBinaries', ['unsanitized', 'sanitized']) + + +class TargetBinaries(NamedTuple): + unsanitized: pathlib.Path | None + sanitized: pathlib.Path class CentipedeError(Exception): @@ -51,27 +52,28 @@ class CentipedeError(Exception): class CentipedeOptions(engine.FuzzOptions): """Centipede engine options.""" - def __init__(self, corpus_dir, arguments, strategies, workdir, - new_corpus_dir): + def __init__(self, corpus_dir: str, arguments: list[str], + strategies: dict[str, Any], workdir: str, + new_corpus_dir: str) -> None: super().__init__(corpus_dir, arguments, strategies) # Directory to add new units self.new_corpus_dir = new_corpus_dir self.workdir = workdir -def _get_runner(target_path): +def _get_runner(target_path: str) -> new_process.UnicodeProcessRunner: """Gets the Centipede runner.""" centipede_path = pathlib.Path(target_path).parent / 'centipede' if not centipede_path.exists(): raise CentipedeError('Centipede not found in build') - centipede_path = str(centipede_path) + centipede_path_str = str(centipede_path) if environment.get_value('USE_UNSHARE'): - return new_process.UnicodeModifierRunner(centipede_path) - return new_process.UnicodeProcessRunner(centipede_path) + return new_process.UnicodeModifierRunner(centipede_path_str) + return new_process.UnicodeProcessRunner(centipede_path_str) -def _get_reproducer_path(log, reproducers_dir): +def _get_reproducer_path(log: str, reproducers_dir: str) -> pathlib.Path | None: """Gets the reproducer path, if any.""" crash_match = CRASH_REGEX.search(log) if not crash_match: @@ -82,17 +84,17 @@ def _get_reproducer_path(log, reproducers_dir): return crash_path -def _set_sanitizer_options(fuzzer_path): +def _set_sanitizer_options(fuzzer_path: str) -> None: """Sets sanitizer options based on .options file overrides.""" engine_common.process_sanitizer_options_overrides(fuzzer_path) sanitizer_options_var = environment.get_current_memory_tool_var() + assert sanitizer_options_var is not None sanitizer_options = environment.get_memory_tool_options( sanitizer_options_var, {}) environment.set_memory_tool_options(sanitizer_options_var, sanitizer_options) -def _parse_centipede_stats( - stats_file: str) -> Optional[Dict[str, Union[int, float]]]: +def _parse_centipede_stats(stats_file: str) -> dict[str, int | float] | None: """Parses the Centipede stats file and returns a dictionary with labels and their respective values. @@ -120,7 +122,7 @@ def _parse_centipede_stats( desc = rows[0][:-1] latest_stats = rows[-1][:-1] - def to_number(x: str) -> Union[int, float]: + def to_number(x: str) -> int | float: return int(x) if x.isdigit() else float(x) return {desc[i]: to_number(latest_stats[i]) for i in range(0, len(desc))} @@ -129,7 +131,7 @@ def to_number(x: str) -> Union[int, float]: return None -def _parse_centipede_logs(log_lines: List[str]) -> Dict[str, int]: +def _parse_centipede_logs(log_lines: list[str]) -> dict[str, int]: """Parses Centipede outputs and generates stats for it. Args: @@ -161,10 +163,10 @@ class Engine(engine.Engine): """Centipede engine implementation.""" @property - def name(self): + def name(self) -> str: return 'centipede' - def _get_arguments(self, fuzzer_path): + def _get_arguments(self, fuzzer_path: str) -> fuzzer_options.FuzzerArguments: """Gets the fuzzer arguments. Returns default arguments and arguments specified by the options field. Args: @@ -193,7 +195,8 @@ def _get_arguments(self, fuzzer_path): return arguments - def fuzz_additional_processing_timeout(self, options): + def fuzz_additional_processing_timeout(self, + options: engine.FuzzOptions) -> int: """Return the maximum additional timeout in seconds for additional operations in fuzz() (e.g. merging back new items). @@ -204,10 +207,12 @@ def fuzz_additional_processing_timeout(self, options): An int representing the number of seconds required. """ del options - return engine_common.get_merge_timeout(engine_common.DEFAULT_MERGE_TIMEOUT) + return int( + engine_common.get_merge_timeout(engine_common.DEFAULT_MERGE_TIMEOUT)) # pylint: disable=unused-argument - def prepare(self, corpus_dir, target_path, build_dir): + def prepare(self, corpus_dir: str, target_path: str, + build_dir: str) -> CentipedeOptions: """Prepares for a fuzzing session, by generating options. Args: @@ -249,7 +254,7 @@ def prepare(self, corpus_dir, target_path, build_dir): return CentipedeOptions(corpus_dir, arguments.list(), {}, workdir, new_corpus_dir) - def _get_binary_paths(self, target_path): + def _get_binary_paths(self, target_path: str) -> TargetBinaries: """Gets the paths to the main and auxiliary binaries based on |target_path| Args: target_path: Path to the main target in a string. @@ -283,7 +288,7 @@ def _get_binary_paths(self, target_path): return target_binaries - def _get_auxiliary_target_path(self, target_path): + def _get_auxiliary_target_path(self, target_path: str) -> pathlib.Path: """Gets the auxiliary target path based on the main |target_path|. When exists, it points to the sanitized binary, which is required by fuzzing (as an auxiliary) and crash reproduction. @@ -296,12 +301,14 @@ def _get_auxiliary_target_path(self, target_path): """ # Assuming they will be in child dirs named by fuzzer_utils.EXTRA_BUILD_DIR. build_dir = environment.get_value('BUILD_DIR') + assert build_dir is not None auxiliary_target_name = pathlib.Path(target_path).name auxiliary_target_path = pathlib.Path( build_dir, fuzzer_utils.EXTRA_BUILD_DIR, auxiliary_target_name) return auxiliary_target_path - def fuzz(self, target_path, options, reproducers_dir, max_time): # pylint: disable=unused-argument + def fuzz(self, target_path: str, options: CentipedeOptions, + reproducers_dir: str, max_time: int) -> engine.FuzzResult: # pylint: disable=unused-argument """Runs a fuzz session. Args: @@ -335,6 +342,9 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): # pylint: disa timeout=timeout, terminate_before_kill=True, terminate_wait_time=_CLEAN_EXIT_SECS) + assert fuzz_result.output is not None + assert fuzz_result.command is not None + assert fuzz_result.time_executed is not None log_lines = fuzz_result.output.splitlines() fuzz_result.output = Engine.trim_logs(fuzz_result.output) @@ -342,7 +352,7 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): # pylint: disa workdir = options.workdir reproducer_path = _get_reproducer_path(fuzz_result.output, reproducers_dir) - crashes = [] + crashes: list[engine.Crash] = [] if reproducer_path: # Centipde doesn't remove carshing inputs from the corpus, this workaround # removes the crashing input in case it's present in the corpus directory. @@ -359,9 +369,7 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): # pylint: disa stats_filename = f'fuzzing-stats-{os.path.basename(target_path)}.000000.csv' stats_file = os.path.join(workdir, stats_filename) - stats = _parse_centipede_stats(stats_file) - if not stats: - stats = {} + stats: dict[str, Any] = _parse_centipede_stats(stats_file) or {} actual_duration = int( stats.get('FuzzTimeSec_Avg', fuzz_result.time_executed or 0.0)) fuzzing_time_percent = 100 * actual_duration / float(max_time) @@ -387,8 +395,9 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): # pylint: disa input_dirs=[options.new_corpus_dir], output_dir=options.corpus_dir, reproducers_dir=reproducers_dir, - max_time=engine_common.get_merge_timeout( - engine_common.DEFAULT_MERGE_TIMEOUT), + max_time=int( + engine_common.get_merge_timeout( + engine_common.DEFAULT_MERGE_TIMEOUT)), # Use the same workdir that was used for fuzzing. # This allows us to skip rerunning the fuzzing inputs. workdir=workdir) @@ -409,11 +418,11 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): # pylint: disa stats, fuzz_result.time_executed) @staticmethod - def trim_logs(fuzz_log): + def trim_logs(fuzz_log: str) -> str: """Strips the 'CRASH LOG:' prefix that breaks stacktrace parsing. Args: - fuzz_result: The ProcessResult returned by running fuzzer binary. + fuzz_log: The ProcessResult returned by running fuzzer binary. """ trimmed_log_lines = [ line[len(_CRASH_LOG_PREFIX):] @@ -422,7 +431,8 @@ def trim_logs(fuzz_log): ] return '\n'.join(trimmed_log_lines) - def reproduce(self, target_path, input_path, arguments, max_time): # pylint: disable=unused-argument + def reproduce(self, target_path: str, input_path: str, arguments: list[str], + max_time: int) -> engine.ReproduceResult: # pylint: disable=unused-argument """Reproduces a crash given an input. Args: @@ -471,12 +481,17 @@ def reproduce(self, target_path, input_path, arguments, max_time): # pylint: di os.environ['CENTIPEDE_RUNNER_FLAGS'] = existing_runner_flags else: os.unsetenv('CENTIPEDE_RUNNER_FLAGS') + assert result.output is not None + assert result.command is not None + assert result.return_code is not None + assert result.time_executed is not None result.output = Engine.trim_logs(result.output) return engine.ReproduceResult(result.command, result.return_code, result.time_executed, result.output) - def _strip_fuzzing_arguments(self, arguments): + def _strip_fuzzing_arguments(self, arguments: fuzzer_options.FuzzerArguments + ) -> fuzzer_options.FuzzerArguments: """Remove arguments only needed for fuzzing.""" for argument in [ constants.FORK_SERVER_FLAGNAME, @@ -491,13 +506,13 @@ def _strip_fuzzing_arguments(self, arguments): return arguments def minimize_corpus(self, - target_path, - arguments, - input_dirs, - output_dir, - reproducers_dir, - max_time, - workdir=None): + target_path: str, + arguments: list[str], + input_dirs: list[str], + output_dir: str, + reproducers_dir: str, + max_time: int, + workdir: str | None = None) -> engine.FuzzResult: """Runs corpus minimization. Args: target_path: Path to the target. @@ -536,7 +551,10 @@ def minimize_corpus(self, ] logs.info(f'Running Generate Corpus file for Centipede with args: {args}') result = runner.run_and_wait(additional_args=args, timeout=max_time) - max_time -= result.time_executed + assert result.time_executed is not None + assert result.output is not None + assert result.command is not None + max_time -= int(result.time_executed) if result.timed_out or max_time < 0: logs.warning( @@ -553,7 +571,10 @@ def minimize_corpus(self, ] logs.info(f'Running Corpus Distillation with args: {args}') result = runner.run_and_wait(additional_args=args, timeout=max_time) - max_time -= result.time_executed + assert result.time_executed is not None + assert result.output is not None + assert result.command is not None + max_time -= int(result.time_executed) if result.timed_out or max_time < 0: logs.warning( @@ -581,6 +602,9 @@ def minimize_corpus(self, ] logs.info(f'Converting corpus to files with the following args: {args}') result = runner.run_and_wait(additional_args=args, timeout=max_time) + assert result.output is not None + assert result.command is not None + assert result.time_executed is not None if result.timed_out or max_time < 0: logs.warning( @@ -608,7 +632,7 @@ def minimize_corpus(self, return engine.FuzzResult(result.output, result.command, [], None, result.time_executed, result.timed_out) - def _get_smallest_crasher(self, workdir_path): + def _get_smallest_crasher(self, workdir_path: str) -> str | None: """Returns the path to the smallest crash in Centipede's |workdir_path|.""" if not os.path.isdir(workdir_path): logs.error(f'Work directory does not exist: {workdir_path}') @@ -630,8 +654,9 @@ def _get_smallest_crasher(self, workdir_path): minimum_testcase = min(testcases, key=os.path.getsize) return minimum_testcase - def minimize_testcase(self, target_path, arguments, input_path, output_path, - max_time): + def minimize_testcase(self, target_path: str, arguments: list[str], + input_path: str, output_path: str, + max_time: int) -> engine.ReproduceResult: """Minimizes a testcase. Args: target_path: Path to the target. @@ -669,10 +694,15 @@ def minimize_testcase(self, target_path, arguments, input_path, output_path, shutil.copyfile(minimum_testcase, output_path) else: shutil.copyfile(input_path, output_path) + assert result.command is not None + assert result.return_code is not None + assert result.time_executed is not None + assert result.output is not None return engine.ReproduceResult(result.command, result.return_code, result.time_executed, result.output) - def cleanse(self, target_path, arguments, input_path, output_path, max_time): + def cleanse(self, target_path: str, arguments: list[str], input_path: str, + output_path: str, max_time: int) -> engine.ReproduceResult: """Cleanses a testcase. Args: target_path: Path to the target. diff --git a/src/clusterfuzz/_internal/bot/fuzzers/dictionary_manager.py b/src/clusterfuzz/_internal/bot/fuzzers/dictionary_manager.py index f90c39b0da4..38d4cbf9dcf 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/dictionary_manager.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/dictionary_manager.py @@ -28,7 +28,7 @@ DICTIONARY_PART_PATTERN = re.compile(r'([^"]+\s*=\s*)?(.*)') -def extract_dictionary_element(line): +def extract_dictionary_element(line: str) -> str | None: """Extract a dictionary element from the given string.""" # An element should start and end with a double-quote. start_index = line.find('"') @@ -40,13 +40,13 @@ def extract_dictionary_element(line): return element -def get_default_dictionary_path(fuzz_target_path): +def get_default_dictionary_path(fuzz_target_path: str) -> str: """Return default dictionary path.""" return fuzzer_utils.get_supporting_file(fuzz_target_path, DICTIONARY_FILE_EXTENSION) -def get_dictionary_size(dictionary_content): +def get_dictionary_size(dictionary_content: str) -> int: """Calculate number of dictionary elements in the given string.""" count = 0 for line in dictionary_content.splitlines(): @@ -56,7 +56,7 @@ def get_dictionary_size(dictionary_content): return count -def get_stats_for_dictionary_file(dictionary_path): +def get_stats_for_dictionary_file(dictionary_path: str | None) -> int: """Calculate size of manual section of given dictionary.""" if not dictionary_path or not os.path.exists(dictionary_path): return 0 @@ -71,7 +71,7 @@ def get_stats_for_dictionary_file(dictionary_path): return manual_dictionary_size -def _fix_dictionary_line(line, dict_path): +def _fix_dictionary_line(line: str, dict_path: str) -> str: """Correct a single dictionary line.""" # Ignore blank and comment lines. if not line or line.strip().startswith('#'): @@ -115,7 +115,7 @@ def _fix_dictionary_line(line, dict_path): return name_part + new_entry -def correct_if_needed(dict_path): +def correct_if_needed(dict_path: str | None) -> None: """Corrects obvious errors such as missing quotes in a dictionary.""" if not dict_path or not os.path.exists(dict_path): return diff --git a/src/clusterfuzz/_internal/bot/fuzzers/engine_common.py b/src/clusterfuzz/_internal/bot/fuzzers/engine_common.py index b6881d86e5a..2b5851adbb0 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/engine_common.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/engine_common.py @@ -13,6 +13,8 @@ # limitations under the License. """Common functionality for engine fuzzers (ie: libFuzzer or AFL).""" +from collections.abc import Iterator +from collections.abc import Sequence import contextlib import glob import json @@ -24,6 +26,13 @@ import string import sys import time +import types +from typing import Any +from typing import cast +from typing import IO +from typing import NoReturn +from typing import TYPE_CHECKING +from typing import TypeVar from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot.fuzzers import options @@ -38,53 +47,61 @@ from clusterfuzz._internal.system import new_process from clusterfuzz._internal.system import shell +if TYPE_CHECKING: + from clusterfuzz._internal.bot.fuzzers import strategy_selection + # Maximum allowed size of a corpus file. -CORPUS_INPUT_SIZE_LIMIT = 5 * 1024 * 1024 +CORPUS_INPUT_SIZE_LIMIT: int = 5 * 1024 * 1024 # Number of testcases to use for the corpus subset strategy. # See https://crbug.com/682311 for more information. # Size 100 has a slightly higher chance as it seems to be the best one so far. -CORPUS_SUBSET_NUM_TESTCASES = [10, 20, 50, 75, 75, 100, 100, 100, 125, 125, 150] +CORPUS_SUBSET_NUM_TESTCASES: list[int] = [ + 10, 20, 50, 75, 75, 100, 100, 100, 125, 125, 150 +] # Suffix used for seed corpus archive generated with build. Does not include # file extension. -SEED_CORPUS_ARCHIVE_SUFFIX = '_seed_corpus' +SEED_CORPUS_ARCHIVE_SUFFIX: str = '_seed_corpus' # Maximum number of files in the corpus for which we will unpack the seed # corpus. -MAX_FILES_FOR_UNPACK = 5 +MAX_FILES_FOR_UNPACK: int = 5 # Extension for owners file containing list of people to be notified. -OWNERS_FILE_EXTENSION = '.owners' +OWNERS_FILE_EXTENSION: str = '.owners' # Extension for per-fuzz target labels to be added to issue tracker. -LABELS_FILE_EXTENSION = '.labels' +LABELS_FILE_EXTENSION: str = '.labels' # Extension for per-fuzz target components to be added to issue tracker. -COMPONENTS_FILE_EXTENSION = '.components' +COMPONENTS_FILE_EXTENSION: str = '.components' # Extension for additional metadata to be added to issue descriptions. -METADATA_FILE_EXTENSION = '.issue_metadata' +METADATA_FILE_EXTENSION: str = '.issue_metadata' # Number of radamsa mutations. -RADAMSA_MUTATIONS = 2000 +RADAMSA_MUTATIONS: int = 2000 # Maximum number of seconds to run radamsa for. -RADAMSA_TIMEOUT = 3 +RADAMSA_TIMEOUT: int = 3 -HEXDIGITS_SET = set(string.hexdigits) +HEXDIGITS_SET: set[str] = set(string.hexdigits) # Allow 30 minutes to merge the testcases back into the corpus. -DEFAULT_MERGE_TIMEOUT = 30 * 60 +DEFAULT_MERGE_TIMEOUT: int = 30 * 60 + +_T = TypeVar('_T') class Generator: """Generators we can use.""" - NONE = 0 - RADAMSA = 1 + NONE: int = 0 + RADAMSA: int = 1 -def select_generator(strategy_pool, fuzzer_path): +def select_generator(strategy_pool: 'strategy_selection.StrategyPool', + fuzzer_path: str) -> int: """Pick a generator to generate new testcases before fuzzing or return Generator.NONE if no generator selected.""" if environment.is_lib() or environment.platform() == 'FUCHSIA': @@ -102,8 +119,9 @@ def select_generator(strategy_pool, fuzzer_path): return Generator.NONE -def generate_new_testcase_mutations( - corpus_directory, new_testcase_mutations_directory, candidate_generator): +def generate_new_testcase_mutations(corpus_directory: str, + new_testcase_mutations_directory: str, + candidate_generator: int) -> bool: """Generate new testcase mutations, using existing corpus directory or other methods. @@ -128,12 +146,13 @@ def generate_new_testcase_mutations( # Filename length limit on ext4. -FILENAME_LENGTH_LIMIT = 255 +FILENAME_LENGTH_LIMIT: int = 255 -RADAMSA_FILENAME_REGEX = re.compile(r'radamsa-\d+-(.*)', re.DOTALL) +RADAMSA_FILENAME_REGEX: re.Pattern[str] = re.compile(r'radamsa-\d+-(.*)', + re.DOTALL) -def get_radamsa_output_filename(initial_filename, i): +def get_radamsa_output_filename(initial_filename: str, i: int) -> str: """Get the name of a file mutated by radamsa.""" # Don't add the radamsa prefix to a file that already has it to avoid hitting # filename/path length limits. @@ -150,7 +169,8 @@ def get_radamsa_output_filename(initial_filename, i): def generate_new_testcase_mutations_using_radamsa( - corpus_directory, new_testcase_mutations_directory, generation_timeout): + corpus_directory: str, new_testcase_mutations_directory: str, + generation_timeout: float | int) -> None: """Generate new testcase mutations based on Radamsa.""" radamsa_path = get_radamsa_path() if not radamsa_path: @@ -199,7 +219,7 @@ def generate_new_testcase_mutations_using_radamsa( (new_corpus_size - old_corpus_size)) -def get_radamsa_path(): +def get_radamsa_path() -> str | None: """Return path to radamsa binary for current platform.""" bin_directory_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'bin') @@ -213,17 +233,17 @@ def get_radamsa_path(): return None -def get_new_testcase_mutations_timeout(): +def get_new_testcase_mutations_timeout() -> float: """Get the timeout for new testcase mutations.""" return get_overridable_timeout(10 * 60, 'MUTATIONS_TIMEOUT_OVERRIDE') -def current_timestamp(): +def current_timestamp() -> float: """Returns the current timestamp. Needed for mocking.""" return time.time() -def get_strategy_probability(strategy_name, default): +def get_strategy_probability(strategy_name: str, default: float) -> float: """Returns a strategy weight based on env variable |FUZZING_STRATEGIES|""" fuzzing_strategies = environment.get_value('FUZZING_STRATEGIES') if fuzzing_strategies is None or not isinstance(fuzzing_strategies, dict): @@ -235,17 +255,18 @@ def get_strategy_probability(strategy_name, default): return fuzzing_strategies[strategy_name] -def decide_with_probability(probability): +def decide_with_probability(probability: float) -> bool: """Decide if we want to do something with the given probability.""" return random.SystemRandom().random() < probability -def get_probability(): +def get_probability() -> float: """Obtain a random number.""" return random.SystemRandom().random() -def get_testcase_run(stats, fuzzer_command): +def get_testcase_run(stats: dict[str, Any] | None, + fuzzer_command: str) -> fuzzer_stats.TestcaseRun: """Get testcase run for stats.""" build_revision = fuzzer_utils.get_build_revision() job = environment.get_value('JOB_NAME') @@ -259,13 +280,15 @@ def get_testcase_run(stats, fuzzer_command): return testcase_run -def dump_big_query_data(stats, testcase_file_path, fuzzer_command): +def dump_big_query_data(stats: dict[str, Any] | None, testcase_file_path: str, + fuzzer_command: str) -> None: """Dump BigQuery stats.""" testcase_run = get_testcase_run(stats, fuzzer_command) fuzzer_stats.TestcaseRun.write_to_disk(testcase_run, testcase_file_path) -def find_fuzzer_path(build_directory, fuzzer_name): +def find_fuzzer_path(build_directory: str | None, + fuzzer_name: str) -> str | None: """Find the fuzzer path with the given name.""" if not build_directory: # Grey-box fuzzers might not have the build directory for a particular job @@ -302,23 +325,24 @@ def find_fuzzer_path(build_directory, fuzzer_name): return None -def get_command_quoted(command): +def get_command_quoted(command: Sequence[str]) -> str: """Return shell quoted command string.""" return ' '.join(shlex.quote(part) for part in command) -def get_overridable_timeout(default_timeout, override_env_var): +def get_overridable_timeout(default_timeout: float | int | None, + override_env_var: str) -> float: """Returns a timeout given a |default_timeout| and the environment variable, |override_env_var|, that overrides it. Returns the overriden value if |override_env_var| is set, otherwise returns default_timeout. Throws an assertion error if the return value is negative.""" timeout_override = environment.get_value(override_env_var) - timeout = float(timeout_override or default_timeout) + timeout = float(cast(Any, timeout_override or default_timeout)) assert timeout >= 0, timeout return timeout -def get_hard_timeout(total_timeout=None): +def get_hard_timeout(total_timeout: float | int | None = None) -> float: """Get the hard timeout for fuzzing.""" if total_timeout is None: total_timeout = environment.get_value('FUZZ_TEST_TIMEOUT') @@ -326,13 +350,13 @@ def get_hard_timeout(total_timeout=None): return get_overridable_timeout(total_timeout, 'HARD_TIMEOUT_OVERRIDE') -def get_merge_timeout(default_merge_timeout): +def get_merge_timeout(default_merge_timeout: float | int) -> float: """Get the maximum amount of time that should be spent merging a corpus.""" return get_overridable_timeout(default_merge_timeout, 'MERGE_TIMEOUT_OVERRIDE') -def is_lpm_fuzz_target(fuzzer_path): +def is_lpm_fuzz_target(fuzzer_path: str) -> bool: """Returns True if |fuzzer_path| is a libprotobuf-mutator based fuzz target.""" # TODO(metzman): Use this function to disable running LPM targets with AFL. @@ -340,7 +364,7 @@ def is_lpm_fuzz_target(fuzzer_path): return utils.search_bytes_in_file(b'TestOneProtoInput', file_handle) -def get_issue_owners(fuzz_target_path): +def get_issue_owners(fuzz_target_path: str) -> list[str]: """Return list of owner emails given a fuzz target path. Format of an owners file is described at: @@ -356,7 +380,7 @@ def get_issue_owners(fuzz_target_path): if not os.path.exists(owners_file_path): return [] - owners = [] + owners: list[str] = [] with open(owners_file_path) as owners_file_handle: owners_file_content = owners_file_handle.read() @@ -383,7 +407,7 @@ def get_issue_owners(fuzz_target_path): return owners -def get_issue_metadata(fuzz_target_path, extension): +def get_issue_metadata(fuzz_target_path: str, extension: str) -> list[str]: """Get issue metadata.""" metadata_file_path = fuzzer_utils.get_supporting_file(fuzz_target_path, extension) @@ -400,17 +424,17 @@ def get_issue_metadata(fuzz_target_path, extension): handle, delimiter='\n', strip=True, remove_empty=True) -def get_issue_labels(fuzz_target_path): +def get_issue_labels(fuzz_target_path: str) -> list[str]: """Return list of issue labels given a fuzz target path.""" return get_issue_metadata(fuzz_target_path, LABELS_FILE_EXTENSION) -def get_issue_components(fuzz_target_path): +def get_issue_components(fuzz_target_path: str) -> list[str]: """Return list of issue components given a fuzz target path.""" return get_issue_metadata(fuzz_target_path, COMPONENTS_FILE_EXTENSION) -def get_additional_issue_metadata(fuzz_target_path): +def get_additional_issue_metadata(fuzz_target_path: str) -> dict[str, Any]: """Return the additional metadata fields given a fuzz target path. The data will be a JSON-formatted dictionary.""" metadata_file_path = fuzzer_utils.get_supporting_file( @@ -431,9 +455,9 @@ def get_additional_issue_metadata(fuzz_target_path): return {} -def get_all_issue_metadata(fuzz_target_path): +def get_all_issue_metadata(fuzz_target_path: str) -> dict[str, Any]: """Get issue related metadata for a target.""" - metadata = {} + metadata: dict[str, Any] = {} issue_labels = get_issue_labels(fuzz_target_path) if issue_labels: @@ -454,7 +478,8 @@ def get_all_issue_metadata(fuzz_target_path): return metadata -def get_fuzz_target_issue_metadata(fuzz_target): +def get_fuzz_target_issue_metadata( + fuzz_target: data_types.FuzzTarget | None) -> dict[str, Any] | None: """Get issue related metadata given a testcase.""" if fuzz_target is None: return None @@ -463,6 +488,7 @@ def get_fuzz_target_issue_metadata(fuzz_target): # Not applicable. return None + assert fuzz_target.binary is not None build_dir = environment.get_value('BUILD_DIR') target_path = find_fuzzer_path(build_dir, fuzz_target.binary) if not target_path: @@ -472,13 +498,14 @@ def get_fuzz_target_issue_metadata(fuzz_target): return get_all_issue_metadata(target_path) -def get_project_qualified_fuzzer_name(target_path): +def get_project_qualified_fuzzer_name(target_path: str) -> str: """Return project qualified fuzzer name for a given target path.""" return data_types.fuzz_target_project_qualified_name( utils.current_project(), os.path.basename(target_path)) -def format_fuzzing_strategies(fuzzing_strategies): +def format_fuzzing_strategies( + fuzzing_strategies: list[str] | dict[str, Any] | None) -> str: """Format the strategies used for logging purposes.""" if not fuzzing_strategies: return '' @@ -494,25 +521,25 @@ def format_fuzzing_strategies(fuzzing_strategies): return 'cf::fuzzing_strategies: ' + value -def random_choice(sequence): +def random_choice(sequence: Sequence[_T]) -> _T: """Return a random element from the non-empty sequence.""" return random.SystemRandom().choice(sequence) -def read_data_from_file(file_path): +def read_data_from_file(file_path: str) -> bytes: """Read data from file.""" with open(file_path, 'rb') as file_handle: return file_handle.read() -def recreate_directory(directory_path): +def recreate_directory(directory_path: str) -> None: """Delete directory if exists, create empty directory. Throw an exception if either fails.""" if not shell.remove_directory(directory_path, recreate=True): raise OSError('Failed to recreate directory: ' + directory_path) -def write_data_to_file(content, file_path): +def write_data_to_file(content: Any, file_path: str) -> None: """Writes data to file.""" with open(file_path, 'wb') as file_handle: file_handle.write(str(content).encode('utf-8')) @@ -522,7 +549,7 @@ class MinijailEngineFuzzerRunner(minijail.MinijailProcessRunner): """Minijail runner for engine fuzzers.""" @contextlib.contextmanager - def _chroot_testcase(self, testcase_path): + def _chroot_testcase(self, testcase_path: str) -> Iterator[str]: """Context manager for testcases. Args: testcase_path: Host path to testcase. @@ -543,7 +570,7 @@ def _chroot_testcase(self, testcase_path): os.remove(copied_testcase_path) -def signal_term_handler(sig, frame): # pylint: disable=unused-argument +def signal_term_handler(sig: int, frame: types.FrameType | None) -> NoReturn: # pylint: disable=unused-argument try: print('SIGTERMed') except OSError: # Pipe may already be closed and we may not be able to print. @@ -553,7 +580,7 @@ def signal_term_handler(sig, frame): # pylint: disable=unused-argument sys.exit(0) -def get_seed_corpus_path(fuzz_target_path): +def get_seed_corpus_path(fuzz_target_path: str) -> str | None: """Returns the path of the seed corpus if one exists. Otherwise returns None. Logs an error if multiple seed corpora exist for the same target.""" archive_path_without_extension = fuzzer_utils.get_supporting_file( @@ -577,7 +604,7 @@ def get_seed_corpus_path(fuzz_target_path): return archive_paths[0] -def process_sanitizer_options_overrides(fuzzer_path): +def process_sanitizer_options_overrides(fuzzer_path: str) -> None: """Applies sanitizer option overrides from .options file.""" fuzzer_options = options.get_fuzz_target_options(fuzzer_path) if not fuzzer_options: @@ -609,11 +636,12 @@ def process_sanitizer_options_overrides(fuzzer_path): environment.set_memory_tool_options('HWASAN_OPTIONS', hwasan_options) -def unpack_seed_corpus_if_needed(fuzz_target_path, - corpus_directory, - max_bytes=float('inf'), - force_unpack=False, - max_files_for_unpack=MAX_FILES_FOR_UNPACK): +def unpack_seed_corpus_if_needed( + fuzz_target_path: str, + corpus_directory: str, + max_bytes: float | int = float('inf'), + force_unpack: bool = False, + max_files_for_unpack: int = MAX_FILES_FOR_UNPACK) -> None: """If seed corpus available, unpack it into the corpus directory if needed, ie: if corpus exists and either |force_unpack| is True, or the number of files in corpus_directory is less than |max_files_for_unpack|. Uses @@ -649,7 +677,7 @@ def unpack_seed_corpus_if_needed(fuzz_target_path, output_filename = '%016d' % idx output_file_path = os.path.join(corpus_directory, output_filename) with open(output_file_path, 'wb') as file_handle: - with reader.open(file.name) as file: + with cast(IO[bytes], reader.open(file.name)) as file: shutil.copyfileobj(file, file_handle) idx += 1 @@ -658,13 +686,14 @@ def unpack_seed_corpus_if_needed(fuzz_target_path, (idx, seed_corpus_archive_path)) -def get_log_header(command, time_executed): +def get_log_header(command: Sequence[str], + time_executed: float | int | str) -> str: """Get the log header.""" quoted_command = get_command_quoted(command) return f'Command: {quoted_command}\nTime ran: {time_executed}\n' -def is_sha1_hash(possible_hash): +def is_sha1_hash(possible_hash: str) -> bool: """Returns True if |possible_hash| looks like a valid sha1 hash.""" if len(possible_hash) != 40: return False @@ -672,7 +701,7 @@ def is_sha1_hash(possible_hash): return all(char in HEXDIGITS_SET for char in possible_hash) -def move_mergeable_units(merge_directory, corpus_directory): +def move_mergeable_units(merge_directory: str, corpus_directory: str) -> None: """Move new units in |merge_directory| into |corpus_directory|.""" initial_units = { os.path.basename(filename) @@ -687,7 +716,7 @@ def move_mergeable_units(merge_directory, corpus_directory): shell.move(unit_path, dest_path) -def create_temp_fuzzing_dir(name): +def create_temp_fuzzing_dir(name: str) -> str: """Create a temporary directory for fuzzing.""" new_corpus_directory = os.path.join(fuzzer_utils.get_temp_dir(), name) recreate_directory(new_corpus_directory) diff --git a/src/clusterfuzz/_internal/bot/fuzzers/googlefuzztest/engine.py b/src/clusterfuzz/_internal/bot/fuzzers/googlefuzztest/engine.py index 7c3d033f22e..8eb448d1419 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/googlefuzztest/engine.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/googlefuzztest/engine.py @@ -22,7 +22,7 @@ _CRASH_REGEX = re.compile(r'.*Reproducer file written to:\s*(.*)$') -def _get_reproducer_path(line): +def _get_reproducer_path(line: str) -> str | None: """Get the reproducer path, if any.""" crash_match = _CRASH_REGEX.match(line) if not crash_match: @@ -35,10 +35,11 @@ class Engine(engine.Engine): """GFT engine implementation.""" @property - def name(self): + def name(self) -> str: return 'googlefuzztest' - def prepare(self, corpus_dir, target_path, build_dir): # pylint: disable=unused-argument + def prepare(self, corpus_dir: str, target_path: str, + build_dir: str) -> engine.FuzzOptions: # pylint: disable=unused-argument """Prepare for a fuzzing session, by generating options. Returns a FuzzOptions object. @@ -53,7 +54,8 @@ def prepare(self, corpus_dir, target_path, build_dir): # pylint: disable=unused os.chmod(target_path, 0o775) return engine.FuzzOptions(corpus_dir, [], {}) - def fuzz(self, target_path, options, reproducers_dir, max_time): + def fuzz(self, target_path: str, options: engine.FuzzOptions, + reproducers_dir: str, max_time: int) -> engine.FuzzResult: """Run a fuzz session. Args: @@ -78,10 +80,11 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): additional_args=['--logtostderr', '--minloglevel=3']) log_lines = fuzz_result.output.splitlines() - crashes = [] + crashes: list[engine.Crash] = [] for line in log_lines: reproducer_path = _get_reproducer_path(line) if reproducer_path: + assert fuzz_result.time_executed is not None crashes.append( engine.Crash( reproducer_path, @@ -91,11 +94,12 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): continue # TODO(ochang): Implement stats parsing. - stats = {} + stats: dict[str, int] = {} return engine.FuzzResult(fuzz_result.output, fuzz_result.command, crashes, stats, fuzz_result.time_executed) - def reproduce(self, target_path, input_path, arguments, max_time): # pylint: disable=unused-argument + def reproduce(self, target_path: str, input_path: str, arguments: list[str], + max_time: int) -> engine.ReproduceResult: # pylint: disable=unused-argument """Reproduce a crash given an input. Args: @@ -115,8 +119,10 @@ def reproduce(self, target_path, input_path, arguments, max_time): # pylint: di return engine.ReproduceResult(result.command, result.return_code, result.time_executed, result.output) - def minimize_corpus(self, target_path, arguments, input_dirs, output_dir, - reproducers_dir, max_time): + def minimize_corpus(self, target_path: str, arguments: list[str], + input_dirs: list[str], output_dir: str, + reproducers_dir: str | None, + max_time: int) -> engine.FuzzResult: """Optional (but recommended): run corpus minimization. Args: @@ -137,8 +143,9 @@ def minimize_corpus(self, target_path, arguments, input_dirs, output_dir, """ raise NotImplementedError - def minimize_testcase(self, target_path, arguments, input_path, output_path, - max_time): + def minimize_testcase(self, target_path: str, arguments: list[str], + input_path: str, output_path: str, + max_time: int) -> engine.ReproduceResult: """Optional (but recommended): Minimize a testcase. Args: @@ -156,7 +163,8 @@ def minimize_testcase(self, target_path, arguments, input_path, output_path, """ raise NotImplementedError - def cleanse(self, target_path, arguments, input_path, output_path, max_time): + def cleanse(self, target_path: str, arguments: list[str], input_path: str, + output_path: str, max_time: int) -> engine.ReproduceResult: """Optional (but recommended): Cleanse a testcase. Args: diff --git a/src/clusterfuzz/_internal/bot/fuzzers/honggfuzz/engine.py b/src/clusterfuzz/_internal/bot/fuzzers/honggfuzz/engine.py index 2f7700dfb0e..1dcdc694dbf 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/honggfuzz/engine.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/honggfuzz/engine.py @@ -55,7 +55,7 @@ class HonggfuzzError(Exception): """Base exception class.""" -def _get_runner(): +def _get_runner() -> new_process.UnicodeProcessRunner: """Get the honggfuzz runner.""" honggfuzz_path = os.path.join(environment.get_value('BUILD_DIR'), 'honggfuzz') if not os.path.exists(honggfuzz_path): @@ -68,7 +68,7 @@ def _get_runner(): return new_process.UnicodeProcessRunner(honggfuzz_path) -def _find_sanitizer_stacktrace(reproducers_dir): +def _find_sanitizer_stacktrace(reproducers_dir: str) -> str | None: """Find the sanitizer stacktrace from the reproducers dir.""" for stacktrace_path in glob.glob( os.path.join(reproducers_dir, _HF_SANITIZER_LOG_PREFIX + '*')): @@ -78,7 +78,7 @@ def _find_sanitizer_stacktrace(reproducers_dir): return None -def _get_reproducer_path(line): +def _get_reproducer_path(line: str) -> str | None: """Get the reproducer path, if any.""" crash_match = _CRASH_REGEX.match(line) if not crash_match: @@ -87,7 +87,7 @@ def _get_reproducer_path(line): return crash_match.group(1) -def _get_stats(line): +def _get_stats(line: str) -> dict[str, int] | None: """Get stats, if any.""" if not line.startswith(_STATS_PREFIX): return None @@ -108,7 +108,7 @@ def _get_stats(line): return stats -def _contains_netdriver(target_path): +def _contains_netdriver(target_path: str) -> bool: """Returns whether |target_path| contains netdriver string.""" with open(target_path, 'rb') as file_handle: data = file_handle.read() @@ -119,10 +119,11 @@ class Engine(engine.Engine): """honggfuzz engine implementation.""" @property - def name(self): + def name(self) -> str: return 'honggfuzz' - def prepare(self, corpus_dir, target_path, build_dir): # pylint: disable=unused-argument + def prepare(self, corpus_dir: str, target_path: str, + build_dir: str) -> engine.FuzzOptions: # pylint: disable=unused-argument """Prepare for a fuzzing session, by generating options. Returns a FuzzOptions object. @@ -135,14 +136,15 @@ def prepare(self, corpus_dir, target_path, build_dir): # pylint: disable=unused A FuzzOptions object. """ os.chmod(target_path, 0o775) - arguments = [] + arguments: list[str] = [] dict_path = dictionary_manager.get_default_dictionary_path(target_path) if os.path.exists(dict_path): arguments.extend(['--dict', dict_path]) return engine.FuzzOptions(corpus_dir, arguments, {}) - def fuzz(self, target_path, options, reproducers_dir, max_time): + def fuzz(self, target_path: str, options: engine.FuzzOptions, + reproducers_dir: str, max_time: int) -> engine.FuzzResult: """Run a fuzz session. Args: @@ -177,10 +179,13 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): additional_args=arguments, timeout=max_time + _CLEAN_EXIT_SECS, extra_env=honggfuzz_env) + assert fuzz_result.output is not None + assert fuzz_result.command is not None + assert fuzz_result.time_executed is not None log_lines = fuzz_result.output.splitlines() sanitizer_stacktrace = _find_sanitizer_stacktrace(reproducers_dir) - crashes = [] + crashes: list[engine.Crash] = [] stats = None for line in log_lines: reproducer_path = _get_reproducer_path(line) @@ -198,7 +203,8 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): return engine.FuzzResult(fuzz_result.output, fuzz_result.command, crashes, stats, fuzz_result.time_executed) - def reproduce(self, target_path, input_path, arguments, max_time): # pylint: disable=unused-argument + def reproduce(self, target_path: str, input_path: str, arguments: list[str], + max_time: int) -> engine.ReproduceResult: # pylint: disable=unused-argument """Reproduce a crash given an input. Args: @@ -215,11 +221,16 @@ def reproduce(self, target_path, input_path, arguments, max_time): # pylint: di with open(input_path, 'rb') as f: result = runner.run_and_wait(timeout=max_time, stdin=f) + assert result.command is not None + assert result.return_code is not None + assert result.time_executed is not None return engine.ReproduceResult(result.command, result.return_code, result.time_executed, result.output) - def minimize_corpus(self, target_path, arguments, input_dirs, output_dir, - reproducers_dir, max_time): + def minimize_corpus(self, target_path: str, arguments: list[str], + input_dirs: list[str], output_dir: str, + reproducers_dir: str | None, + max_time: int) -> engine.FuzzResult: """Optional (but recommended): run corpus minimization. Args: @@ -267,13 +278,16 @@ def minimize_corpus(self, target_path, arguments, input_dirs, output_dir, extra_env=honggfuzz_env) # TODO(DavidKorczynski): Assign merge_stats output appropriately. - merge_stats = {} + merge_stats: dict[str, int] = {} + assert minimise_result.command is not None + assert minimise_result.time_executed is not None return engine.FuzzResult(minimise_result.output, minimise_result.command, [], merge_stats, minimise_result.time_executed) - def minimize_testcase(self, target_path, arguments, input_path, output_path, - max_time): + def minimize_testcase(self, target_path: str, arguments: list[str], + input_path: str, output_path: str, + max_time: int) -> engine.ReproduceResult: """Optional (but recommended): Minimize a testcase. Args: target_path: Path to the target. @@ -288,7 +302,8 @@ def minimize_testcase(self, target_path, arguments, input_path, output_path, """ raise NotImplementedError - def cleanse(self, target_path, arguments, input_path, output_path, max_time): + def cleanse(self, target_path: str, arguments: list[str], input_path: str, + output_path: str, max_time: int) -> engine.ReproduceResult: """Optional (but recommended): Cleanse a testcase. Args: target_path: Path to the target. diff --git a/src/clusterfuzz/_internal/bot/fuzzers/init.py b/src/clusterfuzz/_internal/bot/fuzzers/init.py index 916aa18b315..0fcc7a34843 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/init.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/init.py @@ -19,7 +19,7 @@ from clusterfuzz.fuzz import engine -def run(include_lowercase=False): +def run(include_lowercase: bool = False) -> None: """Initialise builtin fuzzing engines.""" engines = fuzzing.ENGINES diff --git a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/constants.py b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/constants.py index 80ec3c6c9f4..c60da802774 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/constants.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/constants.py @@ -20,67 +20,67 @@ """ # libFuzzer flags. -ARTIFACT_PREFIX_FLAG = '-artifact_prefix=' -ARTIFACT_PREFIX_FLAGNAME = 'artifact_prefix' +ARTIFACT_PREFIX_FLAG: str = '-artifact_prefix=' +ARTIFACT_PREFIX_FLAGNAME: str = 'artifact_prefix' -DATA_FLOW_TRACE_FLAGNAME = 'data_flow_trace' +DATA_FLOW_TRACE_FLAGNAME: str = 'data_flow_trace' -DICT_FLAGNAME = 'dict' +DICT_FLAGNAME: str = 'dict' -FOCUS_FUNCTION_FLAGNAME = 'focus_function' +FOCUS_FUNCTION_FLAGNAME: str = 'focus_function' -FORK_FLAGNAME = 'fork' +FORK_FLAGNAME: str = 'fork' -MAX_LEN_FLAGNAME = 'max_len' +MAX_LEN_FLAGNAME: str = 'max_len' -MAX_TOTAL_TIME_FLAGNAME = 'max_total_time' +MAX_TOTAL_TIME_FLAGNAME: str = 'max_total_time' -RSS_LIMIT_FLAGNAME = 'rss_limit_mb' +RSS_LIMIT_FLAGNAME: str = 'rss_limit_mb' -RUNS_FLAGNAME = 'runs' +RUNS_FLAGNAME: str = 'runs' -TIMEOUT_FLAGNAME = 'timeout' +TIMEOUT_FLAGNAME: str = 'timeout' -EXACT_ARTIFACT_PATH_FLAGNAME = 'exact_artifact_path' +EXACT_ARTIFACT_PATH_FLAGNAME: str = 'exact_artifact_path' -CLEANSE_CRASH_FLAGNAME = 'cleanse_crash' +CLEANSE_CRASH_FLAGNAME: str = 'cleanse_crash' -MERGE_FLAGNAME = 'merge' +MERGE_FLAGNAME: str = 'merge' -MERGE_CONTROL_FILE_FLAGNAME = 'merge_control_file' +MERGE_CONTROL_FILE_FLAGNAME: str = 'merge_control_file' -MINIMIZE_CRASH_FLAGNAME = 'minimize_crash' +MINIMIZE_CRASH_FLAGNAME: str = 'minimize_crash' -PRINT_FINAL_STATS_FLAGNAME = 'print_final_stats' +PRINT_FINAL_STATS_FLAGNAME: str = 'print_final_stats' -DETECT_LEAKS_FLAGNAME = 'detect_leaks' +DETECT_LEAKS_FLAGNAME: str = 'detect_leaks' -TMP_ARTIFACT_PREFIX_ARGUMENT = '/tmp/' +TMP_ARTIFACT_PREFIX_ARGUMENT: str = '/tmp/' -VALUE_PROFILE_FLAGNAME = 'use_value_profile' +VALUE_PROFILE_FLAGNAME: str = 'use_value_profile' # Default value for rss_limit_mb flag to catch OOM.s -DEFAULT_RSS_LIMIT_MB = 2560 +DEFAULT_RSS_LIMIT_MB: int = 2560 # Memory overhead we want to keep to ensure we're not going OOM.s -MEMORY_OVERHEAD = 1024 # 1 GB +MEMORY_OVERHEAD: int = 1024 # 1 GB # Default value for timeout flag to catch timeouts. -DEFAULT_TIMEOUT_LIMIT = 25 +DEFAULT_TIMEOUT_LIMIT: int = 25 # Buffer for processing crash reports. -REPORT_PROCESSING_TIME = 5 +REPORT_PROCESSING_TIME: int = 5 # libFuzzer's exit code if a bug occurred in libFuzzer. -LIBFUZZER_ERROR_EXITCODE = 1 +LIBFUZZER_ERROR_EXITCODE: int = 1 # Defines value of runs argument when loading a testcase. -RUNS_TO_REPRODUCE = 100 +RUNS_TO_REPRODUCE: int = 100 # libFuzzer's exit code if a bug was found in the target code. -TARGET_ERROR_EXITCODE = 77 +TARGET_ERROR_EXITCODE: int = 77 -NONCRASH_RETURN_CODES = { +NONCRASH_RETURN_CODES: set[int] = { # Code when LibFuzzer exits due to SIGTERM cancellation (timeout exceeded). -15, 0, diff --git a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py index 3ceb78f3538..43e36af7d0f 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/engine.py @@ -13,9 +13,11 @@ # limitations under the License. """libFuzzer engine interface.""" +from collections.abc import Sequence import os import re import tempfile +from typing import Any from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot.fuzzers import dictionary_manager @@ -39,7 +41,7 @@ MULTISTEP_MERGE_SUPPORT_TOKEN = b'fuzz target overwrites its const input' -def _is_multistep_merge_supported(target_path): +def _is_multistep_merge_supported(target_path: str) -> bool: """Checks whether a particular binary support multistep merge.""" # TODO(Dor1s): implementation below a temporary workaround, do not tell any # body that we are doing this. The real solution would be to execute a @@ -63,8 +65,9 @@ class MergeError(engine.Error): class LibFuzzerOptions(engine.FuzzOptions): """LibFuzzer engine options.""" - def __init__(self, corpus_dir, arguments, strategies, fuzz_corpus_dirs, - extra_env, is_mutations_run): + def __init__(self, corpus_dir: str, arguments: list[str], + strategies: dict[str, Any], fuzz_corpus_dirs: list[str], + extra_env: dict[str, str] | None, is_mutations_run: bool): super().__init__(corpus_dir, arguments, strategies) self.fuzz_corpus_dirs = fuzz_corpus_dirs self.extra_env = extra_env @@ -75,15 +78,16 @@ def __init__(self, corpus_dir, arguments, strategies, fuzz_corpus_dirs, class Engine(engine.Engine): """LibFuzzer engine implementation.""" - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._merge_control_file = None + self._merge_control_file: str | None = None @property - def name(self): + def name(self) -> str: return 'libFuzzer' - def fuzz_additional_processing_timeout(self, options): + def fuzz_additional_processing_timeout(self, + options: LibFuzzerOptions) -> int: """Return the maximum additional timeout in seconds for additional operations in fuzz() (e.g. merging back new items). @@ -96,9 +100,10 @@ def fuzz_additional_processing_timeout(self, options): fuzz_timeout = libfuzzer.get_fuzz_timeout( options.is_mutations_run, total_timeout=0) # get_fuzz_timeout returns a negative value. - return -fuzz_timeout + return -int(fuzz_timeout) - def prepare(self, corpus_dir, target_path, build_dir): + def prepare(self, corpus_dir: str, target_path: str, + build_dir: str) -> LibFuzzerOptions: """Prepare for a fuzzing session, by generating options. Returns a FuzzOptions object. @@ -182,12 +187,12 @@ def prepare(self, corpus_dir, target_path, build_dir): strategy_info.extra_env, strategy_info.is_mutations_run) - def _create_empty_testcase_file(self, reproducers_dir): + def _create_empty_testcase_file(self, reproducers_dir: str) -> str: """Create an empty testcase file in temporary directory.""" _, path = tempfile.mkstemp(dir=reproducers_dir) return path - def _create_temp_dir(self, name): + def _create_temp_dir(self, name: str) -> str: """Create a temporary directory suitable for putting into the TMPDIR environment variable, which practically speaking sometimes needs to be shortish.""" @@ -196,12 +201,14 @@ def _create_temp_dir(self, name): engine_common.recreate_directory(new_temp_dir) return new_temp_dir - def _create_merge_corpus_dir(self): + def _create_merge_corpus_dir(self) -> str: """Create merge corpus directory.""" return engine_common.create_temp_fuzzing_dir('merge-corpus') - def _merge_new_units(self, target_path, corpus_dir, new_corpus_dir, - fuzz_corpus_dirs, arguments, stat_overrides): + def _merge_new_units(self, target_path: str, corpus_dir: str, + new_corpus_dir: str, fuzz_corpus_dirs: list[str], + arguments: list[str], + stat_overrides: dict[str, Any]) -> None: """Merge new units.""" # Make a decision on whether merge step is needed at all. If there are no # new units added by libFuzzer run, then no need to do merge at all. @@ -254,13 +261,15 @@ def _merge_new_units(self, target_path, corpus_dir, new_corpus_dir, else: logs.info('No new units found.') - def _fuzz_output_contains_trusty_kernel_panic(self, log_lines): + def _fuzz_output_contains_trusty_kernel_panic( + self, log_lines: Sequence[str]) -> bool: for line in log_lines: if 'panic notifier - trusty version' in line: return True return False - def fuzz(self, target_path, options, reproducers_dir, max_time): + def fuzz(self, target_path: str, options: LibFuzzerOptions, + reproducers_dir: str, max_time: int) -> engine.FuzzResult: """Run a fuzz session. Args: @@ -277,6 +286,7 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): runner = libfuzzer.get_runner(target_path) # Directory to place new units. + new_corpus_dir = None if options.merge_back_new_testcases: new_corpus_dir = engine_common.create_temp_fuzzing_dir('new') corpus_directories = [new_corpus_dir] + options.fuzz_corpus_dirs @@ -327,7 +337,7 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): reproducers_dir) # Parse stats information based on libFuzzer output. - parsed_stats = libfuzzer.parse_log_stats(log_lines) + parsed_stats: dict[str, Any] = libfuzzer.parse_log_stats(log_lines) # Extend parsed stats by additional performance features. parsed_stats.update( @@ -335,10 +345,12 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): options.arguments)) args = fuzzer_options.FuzzerArguments.from_list(options.arguments) + assert args is not None # Set some initial stat overrides. timeout_limit = args.get( constants.TIMEOUT_FLAGNAME, default=None, constructor=int) + assert fuzz_result.time_executed is not None actual_duration = int(fuzz_result.time_executed) fuzzing_time_percent = 100 * actual_duration / float(max_time) parsed_stats.update({ @@ -353,6 +365,7 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): args.list(), is_merge=True) if options.merge_back_new_testcases: + assert new_corpus_dir is not None self._merge_new_units(target_path, options.corpus_dir, new_corpus_dir, options.fuzz_corpus_dirs, non_fuzz_arguments, parsed_stats) @@ -372,11 +385,13 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): engine.Crash(crash_testcase_file_path, fuzz_logs, reproduce_arguments, actual_duration)) + assert fuzz_result.command is not None return engine.FuzzResult(fuzz_logs, fuzz_result.command, crashes, parsed_stats, fuzz_result.time_executed, fuzz_result.timed_out) - def reproduce(self, target_path, input_path, arguments, max_time): + def reproduce(self, target_path: str, input_path: str, arguments: list[str], + max_time: int) -> engine.ReproduceResult: """Reproduce a crash given an input. Args: @@ -397,23 +412,29 @@ def reproduce(self, target_path, input_path, arguments, max_time): # Remove fuzzing specific arguments. This is only really needed for legacy # testcases, and can be removed in the distant future. arguments = libfuzzer.strip_fuzzing_arguments(arguments) - arguments = fuzzer_options.FuzzerArguments.from_list(arguments) + args = fuzzer_options.FuzzerArguments.from_list(arguments) + assert args is not None - arguments[constants.RUNS_FLAGNAME] = int(constants.RUNS_TO_REPRODUCE) + args[constants.RUNS_FLAGNAME] = int(constants.RUNS_TO_REPRODUCE) result = runner.run_single_testcase( - input_path, timeout=max_time, additional_args=arguments.list()) + input_path, timeout=max_time, additional_args=args.list()) if result.timed_out: logs.warning('Reproducing timed out.', fuzzer_output=result.output) raise TimeoutError('Reproducing timed out.') + assert result.command is not None + assert result.return_code is not None + assert result.time_executed is not None return engine.ReproduceResult(result.command, result.return_code, result.time_executed, result.output) - def _minimize_corpus_two_step(self, target_path, arguments, - existing_corpus_dirs, new_corpus_dir, - output_corpus_dir, reproducers_dir, max_time): + def _minimize_corpus_two_step(self, target_path: str, arguments: list[str], + existing_corpus_dirs: list[str], + new_corpus_dir: str, output_corpus_dir: str, + reproducers_dir: str | None, + max_time: float | int) -> engine.FuzzResult: """Optional (but recommended): run corpus minimization. Args: @@ -446,7 +467,7 @@ def _minimize_corpus_two_step(self, target_path, arguments, # Two step merge process to obtain accurate stats for the new corpus units. # See https://reviews.llvm.org/D66107 for a more detailed description. - merge_stats = {} + merge_stats: dict[str, Any] = {} # Step 1. Use only existing corpus and collect "initial" stats. result_1 = self.minimize_corpus(target_path, arguments, @@ -492,8 +513,10 @@ def _minimize_corpus_two_step(self, target_path, arguments, return engine.FuzzResult(output, result_2.command, [], merge_stats, result_1.time_executed + result_2.time_executed) - def minimize_corpus(self, target_path, arguments, input_dirs, output_dir, - reproducers_dir, max_time): + def minimize_corpus(self, target_path: str, arguments: list[str], + input_dirs: list[str], output_dir: str, + reproducers_dir: str | None, + max_time: float | int) -> engine.FuzzResult: """Optional (but recommended): run corpus minimization. Args: @@ -544,11 +567,14 @@ def minimize_corpus(self, target_path, arguments, input_dirs, output_dir, merge_output = result.output merge_stats = stats.parse_stats_from_merge_log(merge_output.splitlines()) + assert result.command is not None + assert result.time_executed is not None return engine.FuzzResult(merge_output, result.command, [], merge_stats, result.time_executed) - def minimize_testcase(self, target_path, arguments, input_path, output_path, - max_time): + def minimize_testcase(self, target_path: str, arguments: list[str], + input_path: str, output_path: str, + max_time: int) -> engine.ReproduceResult: """Optional (but recommended): Minimize a testcase. Args: @@ -579,10 +605,14 @@ def minimize_testcase(self, target_path, arguments, input_path, output_path, logs.error('Minimization timed out.', fuzzer_output=result.output) raise TimeoutError('Minimization timed out.') + assert result.command is not None + assert result.return_code is not None + assert result.time_executed is not None return engine.ReproduceResult(result.command, result.return_code, result.time_executed, result.output) - def cleanse(self, target_path, arguments, input_path, output_path, max_time): + def cleanse(self, target_path: str, arguments: list[str], input_path: str, + output_path: str, max_time: int) -> engine.ReproduceResult: """Optional (but recommended): Cleanse a testcase. Args: @@ -613,5 +643,8 @@ def cleanse(self, target_path, arguments, input_path, output_path, max_time): logs.error('Cleanse timed out.', fuzzer_output=result.output) raise TimeoutError('Cleanse timed out.') + assert result.command is not None + assert result.return_code is not None + assert result.time_executed is not None return engine.ReproduceResult(result.command, result.return_code, result.time_executed, result.output) diff --git a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/fuzzer.py b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/fuzzer.py index 3ca11ca582b..9b4d948bb79 100755 --- a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/fuzzer.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/fuzzer.py @@ -20,7 +20,7 @@ from clusterfuzz._internal.bot.fuzzers.libFuzzer import constants -def get_extra_env(fuzzer_path): +def get_extra_env(fuzzer_path: str) -> dict[str, str] | None: """Get environment variables for a given fuzz target if any (or None).""" fuzzer_options = options.get_fuzz_target_options(fuzzer_path) if fuzzer_options: @@ -29,11 +29,11 @@ def get_extra_env(fuzzer_path): return None -def get_arguments(fuzzer_path) -> options.FuzzerArguments: +def get_arguments(fuzzer_path: str) -> options.FuzzerArguments: """Get arguments for a given fuzz target.""" arguments = options.FuzzerArguments() - rss_limit_mb = None - timeout = None + rss_limit_mb: int | None = None + timeout: int | None = None fuzzer_options = options.get_fuzz_target_options(fuzzer_path) @@ -68,6 +68,6 @@ def get_arguments(fuzzer_path) -> options.FuzzerArguments: class LibFuzzer(builtin.EngineFuzzer): """Builtin libFuzzer fuzzer.""" - def generate_arguments(self, fuzzer_path): + def generate_arguments(self, fuzzer_path: str) -> str: """Generate arguments for fuzzer using .options file or default values.""" return ' '.join(get_arguments(fuzzer_path).list()) diff --git a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/stats.py b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/stats.py index 61bbfb78791..b416cb38bee 100755 --- a/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/stats.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/libFuzzer/stats.py @@ -13,7 +13,11 @@ # limitations under the License. """Performance stats constants and helpers for libFuzzer.""" +from collections.abc import Callable +from collections.abc import Sequence import re +from typing import Any +from typing import cast from clusterfuzz._internal.bot.fuzzers import dictionary_manager from clusterfuzz._internal.bot.fuzzers import options as fuzzer_options @@ -65,7 +69,7 @@ r' than (\d+) bytes.*') -def calculate_log_lines(log_lines): +def calculate_log_lines(log_lines: Sequence[str]) -> tuple[int, int, int]: """Calculate number of logs lines of different kind in the given log.""" # Counters to be returned. libfuzzer_lines_count = 0 @@ -116,12 +120,15 @@ def calculate_log_lines(log_lines): return other_lines_count, libfuzzer_lines_count, ignored_lines_count -def strategy_column_name(strategy_name): +def strategy_column_name(strategy_name: str) -> str: """Convert the strategy name into stats column name.""" return 'strategy_%s' % strategy_name -def parse_fuzzing_strategies(log_lines, strategies): +def parse_fuzzing_strategies( + log_lines: Sequence[str], + strategies: Sequence[str] | None, +) -> dict[str, Any]: """Extract stats for fuzzing strategies used.""" if not strategies: # Extract strategies from the log. @@ -131,14 +138,17 @@ def parse_fuzzing_strategies(log_lines, strategies): strategies = match.group(1).split(',') break - return process_strategies(strategies) + return process_strategies(cast(Sequence[str], strategies)) -def process_strategies(strategies, name_modifier=strategy_column_name): +def process_strategies( + strategies: Sequence[str], + name_modifier: Callable[[str], str] = strategy_column_name, +) -> dict[str, Any]: """Process strategies, parsing any stored values.""" - stats = {} + stats: dict[str, Any] = {} - def parse_line_for_strategy_prefix(line, strategy_name): + def parse_line_for_strategy_prefix(line: str, strategy_name: str) -> None: """Parse log line to find the value of a strategy with a prefix.""" strategy_prefix = strategy_name + '_' if not line.startswith(strategy_prefix): @@ -165,11 +175,15 @@ def parse_line_for_strategy_prefix(line, strategy_name): return stats -def parse_performance_features(log_lines, strategies, arguments): +def parse_performance_features( + log_lines: Sequence[str], + strategies: Sequence[str] | None, + arguments: Any, +) -> dict[str, Any]: """Extract stats for performance analysis.""" # TODO(ochang): Remove include_strategies once refactor is complete. # Initialize stats with default values. - stats = { + stats: dict[str, Any] = { 'bad_instrumentation': 0, 'corpus_crash_count': 0, 'corpus_size': 0, @@ -196,7 +210,8 @@ def parse_performance_features(log_lines, strategies, arguments): 'timeout_count': 0, } - arguments = fuzzer_options.FuzzerArguments.from_list(arguments) + arguments = cast(fuzzer_options.FuzzerArguments, + fuzzer_options.FuzzerArguments.from_list(arguments)) # Extract strategy selection method. # TODO(ochang): Move to more general place? stats['strategy_selection_method'] = environment.get_value( @@ -288,7 +303,7 @@ def parse_performance_features(log_lines, strategies, arguments): return stats -def parse_stats_from_merge_log(log_lines): +def parse_stats_from_merge_log(log_lines: Sequence[str]) -> dict[str, int]: """Extract stats from a log produced by libFuzzer run with -merge=1.""" stats = { 'edge_coverage': 0, diff --git a/src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py b/src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py index 952b06cbe7d..f77d2b2da60 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/libfuzzer.py @@ -13,18 +13,22 @@ # limitations under the License. """libFuzzer runners.""" -import collections import contextlib import copy import os import random import re import shutil +from typing import Any +from typing import cast +from typing import Iterator +from typing import NamedTuple from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot import testcase_manager from clusterfuzz._internal.bot.fuzzers import engine_common from clusterfuzz._internal.bot.fuzzers import options as fuzzer_options +from clusterfuzz._internal.bot.fuzzers import strategy_selection from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils from clusterfuzz._internal.bot.fuzzers.libFuzzer import constants from clusterfuzz._internal.fuzzing import strategy @@ -37,21 +41,22 @@ from clusterfuzz._internal.system import shell # Maximum length of a random chosen length for `-max_len`. -MAX_VALUE_FOR_MAX_LENGTH = 10000 +MAX_VALUE_FOR_MAX_LENGTH: int = 10000 -StrategyInfo = collections.namedtuple('StrategiesInfo', [ - 'fuzzing_strategies', - 'arguments', - 'additional_corpus_dirs', - 'extra_env', - 'is_mutations_run', -]) -MAX_OUTPUT_LEN = 1 * 1024 * 1024 # 1 MB +class StrategyInfo(NamedTuple): + fuzzing_strategies: list[str] + arguments: fuzzer_options.FuzzerArguments + additional_corpus_dirs: list[str] + extra_env: dict[str, str] + is_mutations_run: bool + + +MAX_OUTPUT_LEN: int = 1 * 1024 * 1024 # 1 MB # Regex to find testcase path from a crash. -CRASH_TESTCASE_REGEX = (r'.*Test unit written to\s*' - r'(.*(crash|oom|timeout|leak)-.*)') +CRASH_TESTCASE_REGEX: str = (r'.*Test unit written to\s*' + r'(.*(crash|oom|timeout|leak)-.*)') # pylint: disable=no-member @@ -64,24 +69,25 @@ class LibFuzzerCommon: """Provides common libFuzzer functionality.""" # Window of time for libFuzzer to exit gracefully before we KILL it. - LIBFUZZER_CLEAN_EXIT_TIME = 10.0 + LIBFUZZER_CLEAN_EXIT_TIME: float = 10.0 # Additional window of time for libFuzzer fork mode to exit gracefully. - LIBFUZZER_FORK_MODE_CLEAN_EXIT_TIME = 100.0 + LIBFUZZER_FORK_MODE_CLEAN_EXIT_TIME: float = 100.0 # Time to wait for SIGTERM handler. - SIGTERM_WAIT_TIME = 10.0 + SIGTERM_WAIT_TIME: float = 10.0 - def __init__(self): + def __init__(self) -> None: pass - def _normalize_artifact_prefix(self, artifact_prefix, sep=os.sep): + def _normalize_artifact_prefix(self, artifact_prefix: str, + sep: str = os.sep) -> str: if artifact_prefix.endswith(sep): return artifact_prefix return artifact_prefix + sep - def get_testcase_path(self, log_lines): + def get_testcase_path(self, log_lines: list[str]) -> str | None: """Get testcase path from log lines.""" for line in log_lines: match = re.match(CRASH_TESTCASE_REGEX, line) @@ -90,7 +96,7 @@ def get_testcase_path(self, log_lines): return None - def get_total_timeout(self, timeout): + def get_total_timeout(self, timeout: float | int) -> int: """Calculate the total process timeout. Args: @@ -99,7 +105,7 @@ def get_total_timeout(self, timeout): timeout = timeout + self.LIBFUZZER_CLEAN_EXIT_TIME return int(timeout) - def get_minimize_total_time(self, timeout): + def get_minimize_total_time(self, timeout: float | int) -> float | int: # We do timeout / 2 here because libFuzzer uses max_total_time for # individual runs of the target and not for the entire minimization. # Internally, libFuzzer does 2 runs of the target every iteration. This is @@ -108,12 +114,26 @@ def get_minimize_total_time(self, timeout): assert max_total_time > 0 return max_total_time - def fuzz(self, - corpus_directories, - fuzz_timeout, - artifact_prefix=None, - additional_args=None, - extra_env=None): + def run_and_wait( + self, + additional_args: list[str] | None = None, + timeout: float | None = None, + terminate_before_kill: bool = False, + terminate_wait_time: float | None = None, + max_stdout_len: int | None = None, + extra_env: dict[str, str] | None = None, + **kwargs: Any, + ) -> new_process.ProcessResult: + """Runs the process and waits for it to finish.""" + raise NotImplementedError + + def fuzz( + self, + corpus_directories: list[str], + fuzz_timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None, + extra_env: dict[str, str] | None = None) -> new_process.ProcessResult: """Running fuzzing command. Args: @@ -131,7 +151,8 @@ def fuzz(self, Returns: A process.ProcessResult. """ - arguments = fuzzer_options.FuzzerArguments.from_list(additional_args) + arguments = fuzzer_options.FuzzerArguments.from_list( + cast(list[str], additional_args)) assert arguments is not None max_total_time = fuzz_timeout @@ -159,12 +180,12 @@ def fuzz(self, extra_env=extra_env) def merge(self, - corpus_directories, - merge_timeout, - artifact_prefix=None, - tmp_dir=None, - additional_args=None, - merge_control_file=None): + corpus_directories: list[str], + merge_timeout: float | int, + artifact_prefix: str | None = None, + tmp_dir: str | None = None, + additional_args: list[str] | None = None, + merge_control_file: str | None = None) -> new_process.ProcessResult: """Runs a corpus merge command. Args: @@ -182,7 +203,8 @@ def merge(self, Returns: A process.ProcessResult. """ - arguments = fuzzer_options.FuzzerArguments.from_list(additional_args) + arguments = fuzzer_options.FuzzerArguments.from_list( + cast(list[str], additional_args)) assert arguments is not None arguments[constants.MERGE_FLAGNAME] = 1 @@ -193,7 +215,7 @@ def merge(self, if merge_control_file: arguments[constants.MERGE_CONTROL_FILE_FLAGNAME] = merge_control_file - extra_env = {} + extra_env: dict[str, str] = {} if tmp_dir: extra_env['TMPDIR'] = tmp_dir @@ -203,10 +225,11 @@ def merge(self, max_stdout_len=MAX_OUTPUT_LEN, extra_env=extra_env) - def run_single_testcase(self, - testcase_path, - timeout=None, - additional_args=None): + def run_single_testcase( + self, + testcase_path: str, + timeout: float | int | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """Runs a single testcase. Args: @@ -229,12 +252,13 @@ def run_single_testcase(self, timeout=timeout, max_stdout_len=MAX_OUTPUT_LEN) - def minimize_crash(self, - testcase_path, - output_path, - timeout, - artifact_prefix=None, - additional_args=None): + def minimize_crash( + self, + testcase_path: str, + output_path: str, + timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """Minimize crasher with libFuzzer. Args: @@ -244,7 +268,8 @@ def minimize_crash(self, additional_args: A sequence of additional arguments to be passed to the executable. """ - arguments = fuzzer_options.FuzzerArguments.from_list(additional_args) + arguments = fuzzer_options.FuzzerArguments.from_list( + cast(list[str], additional_args)) assert arguments is not None max_total_time = self.get_minimize_total_time(timeout) @@ -261,12 +286,13 @@ def minimize_crash(self, timeout=timeout, max_stdout_len=MAX_OUTPUT_LEN) - def cleanse_crash(self, - testcase_path, - output_path, - timeout, - artifact_prefix=None, - additional_args=None): + def cleanse_crash( + self, + testcase_path: str, + output_path: str, + timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """Cleanse crasher with libFuzzer. This attempts to remove non-essential bits of the testcase by replacing them with garbage. @@ -277,7 +303,8 @@ def cleanse_crash(self, additional_args: A sequence of additional arguments to be passed to the executable. """ - arguments = fuzzer_options.FuzzerArguments.from_list(additional_args) + arguments = fuzzer_options.FuzzerArguments.from_list( + cast(list[str], additional_args)) assert arguments is not None arguments[constants.CLEANSE_CRASH_FLAGNAME] = 1 @@ -299,7 +326,10 @@ class LibFuzzerRunner(new_process.ModifierProcessRunnerMixin, new_process.UnicodeProcessRunner, LibFuzzerCommon): """libFuzzer runner (when minijail is not used).""" - def __init__(self, executable_path, default_args=None, cwd=None): + def __init__(self, + executable_path: str, + default_args: list[str] | None = None, + cwd: str | None = None) -> None: """Inits the LibFuzzerRunner. Args: @@ -310,12 +340,13 @@ def __init__(self, executable_path, default_args=None, cwd=None): super().__init__( executable_path=executable_path, default_args=default_args, cwd=cwd) - def fuzz(self, - corpus_directories, - fuzz_timeout, - artifact_prefix=None, - additional_args=None, - extra_env=None): + def fuzz( + self, + corpus_directories: list[str], + fuzz_timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None, + extra_env: dict[str, str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.fuzz override.""" additional_args = copy.copy(additional_args) if additional_args is None: @@ -337,36 +368,43 @@ class FuchsiaUndercoatLibFuzzerRunner(new_process.UnicodeProcessRunner, # go much higher than that to account for variability in the virtualized # environment, and because this is after all intended solely as a second line # of defense. - TIMEOUT_PADDING = 30 * 60 + TIMEOUT_PADDING: int = 30 * 60 + + handle: str - def __init__(self, executable_path, instance_handle, default_args=None): + def __init__(self, + executable_path: str, + instance_handle: str, + default_args: list[str] | None = None) -> None: # An instance_handle from undercoat is required, and should be set up by the # build_manager. # Note: In this case executable_path is simply `package/fuzzer` super().__init__(executable_path=executable_path, default_args=default_args) self.handle = instance_handle - def _corpus_directories_libfuzzer(self, corpus_directories): + def _corpus_directories_libfuzzer(self, + corpus_directories: list[str]) -> list[str]: """Returns the corpus directory paths expected by libfuzzer itself.""" return [ self._target_corpus_path(os.path.basename(corpus_dir)) for corpus_dir in corpus_directories ] - def _new_corpus_dir_host(self, corpus_directories): + def _new_corpus_dir_host(self, corpus_directories: list[str]) -> str: """Returns the path of the 'new' corpus directory on the host.""" return corpus_directories[0] - def _new_corpus_dir_target(self, corpus_directories): + def _new_corpus_dir_target(self, corpus_directories: list[str]) -> str: """Returns the path of the 'new' corpus directory on the target.""" return self._target_corpus_path( os.path.basename(self._new_corpus_dir_host(corpus_directories))) - def _target_corpus_path(self, corpus_name): + def _target_corpus_path(self, corpus_name: str) -> str: """Returns the path of a given corpus directory on the target.""" return 'data/corpus/' + corpus_name - def _push_corpora_from_host_to_target(self, corpus_directories): + def _push_corpora_from_host_to_target(self, + corpus_directories: list[str]) -> None: # Push corpus directories to the device. self._clear_all_target_corpora() logs.info('Push corpora from host to target.') @@ -374,7 +412,8 @@ def _push_corpora_from_host_to_target(self, corpus_directories): undercoat.put_data(self.handle, self.executable_path, corpus_dir, 'data/corpus') - def _pull_new_corpus_from_target_to_host(self, corpus_directories): + def _pull_new_corpus_from_target_to_host( + self, corpus_directories: list[str]) -> None: """Pull corpus directories from device to host.""" # Appending '/*' indicates we want all the *files* in the target's # directory, rather than the directory itself. @@ -385,13 +424,13 @@ def _pull_new_corpus_from_target_to_host(self, corpus_directories): new_corpus_files_target, self._new_corpus_dir_host(corpus_directories)) - def _clear_all_target_corpora(self): + def _clear_all_target_corpora(self) -> None: """Clears out all the corpora on the target.""" logs.info('Clearing corpora on target') # prepare_fuzzer resets the data/ directory undercoat.prepare_fuzzer(self.handle, self.executable_path) - def _ensure_target_exists(self): + def _ensure_target_exists(self) -> None: """Check that the target fuzzer exists, raising an error if it does not. We do this check by looking at the list of fuzzers, instead of relying on @@ -410,18 +449,20 @@ def _ensure_target_exists(self): raise testcase_manager.TargetNotFoundError( f'Failed to find target {self.executable_path}') - def get_total_timeout(self, timeout): + def get_total_timeout(self, timeout: float | int) -> int: """LibFuzzerCommon.fuzz override.""" return super().get_total_timeout(timeout) + self.TIMEOUT_PADDING - def fuzz(self, - corpus_directories, - fuzz_timeout, - artifact_prefix=None, - additional_args=None, - extra_env=None): + def fuzz( + self, + corpus_directories: list[str], + fuzz_timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None, + extra_env: dict[str, str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.fuzz override.""" - arguments = fuzzer_options.FuzzerArguments.from_list(additional_args) + arguments = fuzzer_options.FuzzerArguments.from_list( + cast(list[str], additional_args)) assert arguments is not None undercoat.prepare_fuzzer(self.handle, self.executable_path) @@ -451,17 +492,18 @@ def fuzz(self, return result def merge(self, - corpus_directories, - merge_timeout, - artifact_prefix=None, - tmp_dir=None, - additional_args=None, - merge_control_file=None): + corpus_directories: list[str], + merge_timeout: float | int, + artifact_prefix: str | None = None, + tmp_dir: str | None = None, + additional_args: list[str] | None = None, + merge_control_file: str | None = None) -> new_process.ProcessResult: undercoat.prepare_fuzzer(self.handle, self.executable_path) self._push_corpora_from_host_to_target(corpus_directories) - arguments = fuzzer_options.FuzzerArguments.from_list(additional_args) + arguments = fuzzer_options.FuzzerArguments.from_list( + cast(list[str], additional_args)) assert arguments is not None arguments[constants.MAX_TOTAL_TIME_FLAGNAME] = int(merge_timeout) @@ -492,10 +534,11 @@ def merge(self, self._clear_all_target_corpora() return result - def run_single_testcase(self, - testcase_path, - timeout=None, - additional_args=None): + def run_single_testcase( + self, + testcase_path: str, + timeout: float | int | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """Run a single testcase.""" # TODO(eep): Are all these copy.copy() calls still necessary? additional_args = copy.copy(additional_args) @@ -520,14 +563,16 @@ def run_single_testcase(self, timeout=timeout) return result - def minimize_crash(self, - testcase_path, - output_path, - timeout, - artifact_prefix=None, - additional_args=None): + def minimize_crash( + self, + testcase_path: str, + output_path: str, + timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: - arguments = fuzzer_options.FuzzerArguments.from_list(additional_args) + arguments = fuzzer_options.FuzzerArguments.from_list( + cast(list[str], additional_args)) assert arguments is not None arguments[constants.MINIMIZE_CRASH_FLAGNAME] = 1 @@ -566,7 +611,10 @@ class MinijailLibFuzzerRunner(new_process.UnicodeProcessRunnerMixin, LibFuzzerCommon): """Minijail libFuzzer runner.""" - def __init__(self, executable_path, chroot, default_args=None): + def __init__(self, + executable_path: str, + chroot: minijail.MinijailChroot, + default_args: list[str] | None = None) -> None: """Inits the LibFuzzerRunner. Args: @@ -579,7 +627,7 @@ def __init__(self, executable_path, chroot, default_args=None): chroot=chroot, default_args=default_args) - def get_testcase_path(self, log_lines): + def get_testcase_path(self, log_lines: list[str]) -> str | None: """Get testcase path from log lines.""" path = LibFuzzerCommon.get_testcase_path(self, log_lines) if not path: @@ -592,7 +640,8 @@ def get_testcase_path(self, log_lines): raise LibFuzzerError('Invalid testcase path ' + path) - def _get_chroot_corpus_paths(self, corpus_directories): + def _get_chroot_corpus_paths(self, + corpus_directories: list[str]) -> list[str]: """Return chroot relative paths for the given corpus directories. Args: @@ -603,7 +652,7 @@ def _get_chroot_corpus_paths(self, corpus_directories): """ return [self._get_chroot_directory(path) for path in corpus_directories] - def _get_chroot_directory(self, directory_path): + def _get_chroot_directory(self, directory_path: str) -> str: """Return chroot relative path for the given directory. Args: @@ -618,7 +667,7 @@ def _get_chroot_directory(self, directory_path): f'Failed to get chroot binding for "{directory_path}".') return binding.dest_path - def _bind_corpus_dirs(self, corpus_directories): + def _bind_corpus_dirs(self, corpus_directories: list[str]) -> None: """Bind corpus directories to the minijail chroot. Also makes sure that the directories are world writeable. @@ -631,18 +680,19 @@ def _bind_corpus_dirs(self, corpus_directories): self.chroot.add_binding( minijail.ChrootBinding(corpus_directory, target_dir, writeable=True)) - def fuzz(self, - corpus_directories, - fuzz_timeout, - artifact_prefix=None, - additional_args=None, - extra_env=None): + def fuzz( + self, + corpus_directories: list[str], + fuzz_timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None, + extra_env: dict[str, str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.fuzz override.""" bind_directories = copy.copy(corpus_directories) if artifact_prefix: bind_directories.append(artifact_prefix) - ld_preload = None + ld_preload: str | None = None if extra_env and 'LD_PRELOAD' in extra_env: ld_preload = extra_env['LD_PRELOAD'] bind_directories.append(os.path.dirname(ld_preload)) @@ -651,6 +701,7 @@ def fuzz(self, corpus_directories = self._get_chroot_corpus_paths(corpus_directories) if ld_preload: + assert extra_env is not None extra_env['LD_PRELOAD'] = os.path.join( self._get_chroot_directory(os.path.dirname(ld_preload)), os.path.basename(ld_preload)) @@ -667,12 +718,12 @@ def fuzz(self, extra_env=extra_env) def merge(self, - corpus_directories, - merge_timeout, - artifact_prefix=None, - tmp_dir=None, - additional_args=None, - merge_control_file=None): + corpus_directories: list[str], + merge_timeout: float | int, + artifact_prefix: str | None = None, + tmp_dir: str | None = None, + additional_args: list[str] | None = None, + merge_control_file: str | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.merge override.""" additional_args = copy.copy(additional_args) if additional_args is None: @@ -706,21 +757,23 @@ def merge(self, additional_args=additional_args, merge_control_file=chroot_merge_control_file) - def run_single_testcase(self, - testcase_path, - timeout=None, - additional_args=None): + def run_single_testcase( + self, + testcase_path: str, + timeout: float | int | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.run_single_testcase override.""" with self._chroot_testcase(testcase_path) as chroot_testcase_path: return LibFuzzerCommon.run_single_testcase(self, chroot_testcase_path, timeout, additional_args) - def minimize_crash(self, - testcase_path, - output_path, - timeout, - artifact_prefix=None, - additional_args=None): + def minimize_crash( + self, + testcase_path: str, + output_path: str, + timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.minimize_crash override.""" with self._chroot_testcase(testcase_path) as chroot_testcase_path: chroot_output_name = 'minimized_crash' @@ -739,12 +792,13 @@ def minimize_crash(self, return result - def cleanse_crash(self, - testcase_path, - output_path, - timeout, - artifact_prefix=None, - additional_args=None): + def cleanse_crash( + self, + testcase_path: str, + output_path: str, + timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.cleanse_crash override.""" with self._chroot_testcase(testcase_path) as chroot_testcase_path: chroot_output_name = 'cleanse_crash' @@ -767,9 +821,12 @@ def cleanse_crash(self, class AndroidLibFuzzerRunner(new_process.UnicodeProcessRunner, LibFuzzerCommon): """Android libFuzzer runner.""" # This temp directory is used by libFuzzer merge tool. DONT CHANGE. - LIBFUZZER_TEMP_DIR = '/data/local/tmp' + LIBFUZZER_TEMP_DIR: str = '/data/local/tmp' - def __init__(self, executable_path, build_directory, default_args=None): + def __init__(self, + executable_path: str, + build_directory: str, + default_args: list[str] | None = None) -> None: """Inits the AndroidLibFuzzerRunner. Args: @@ -784,7 +841,8 @@ def __init__(self, executable_path, build_directory, default_args=None): android.adb.create_directory_if_needed(self.LIBFUZZER_TEMP_DIR) self.copy_local_directory_to_device(build_directory) - def _get_default_args(self, executable_path, extra_args): + def _get_default_args(self, executable_path: str, + extra_args: list[str] | None) -> list[str]: """Return a set of default arguments to pass to adb binary.""" default_args = ['shell'] @@ -812,17 +870,18 @@ def _get_default_args(self, executable_path, extra_args): return default_args - def _get_device_corpus_paths(self, corpus_directories): + def _get_device_corpus_paths(self, + corpus_directories: list[str]) -> list[str]: """Return device paths for the given corpus directories.""" return [self._get_device_path(path) for path in corpus_directories] - def _get_device_path(self, local_path): + def _get_device_path(self, local_path: str) -> str: """Return device path for the given local path.""" root_directory = environment.get_root_directory() return os.path.join(android.constants.DEVICE_FUZZING_DIR, os.path.relpath(local_path, root_directory)) - def _get_local_path(self, device_path): + def _get_local_path(self, device_path: str) -> str | None: """Return local path for the given device path.""" if not device_path.startswith(android.constants.DEVICE_FUZZING_DIR + '/'): logs.error('Bad device path: ' + device_path) @@ -833,19 +892,21 @@ def _get_local_path(self, device_path): root_directory, os.path.relpath(device_path, android.constants.DEVICE_FUZZING_DIR)) - def _copy_local_directories_to_device(self, local_directories): + def _copy_local_directories_to_device(self, + local_directories: list[str]) -> None: """Copies local directories to device.""" for local_directory in sorted(set(local_directories)): self.copy_local_directory_to_device(local_directory) - def copy_local_directory_to_device(self, local_directory): + def copy_local_directory_to_device(self, local_directory: str) -> None: """Copy local directory to device.""" device_directory = self._get_device_path(local_directory) android.adb.remove_directory(device_directory, recreate=True) android.adb.copy_local_directory_to_remote(local_directory, device_directory) - def _copy_local_directories_from_device(self, local_directories): + def _copy_local_directories_from_device(self, + local_directories: list[str]) -> None: """Copies directories from device to local.""" for local_directory in sorted(set(local_directories)): device_directory = self._get_device_path(local_directory) @@ -853,9 +914,9 @@ def _copy_local_directories_from_device(self, local_directories): android.adb.copy_remote_directory_to_local(device_directory, local_directory) - def _extract_trusty_stacktrace_from_logcat(self, logcat): + def _extract_trusty_stacktrace_from_logcat(self, logcat: str) -> str: """Finds and returns a TA stacktrace from a logcat""" - begin = android.constants.TRUSTY_STACKTRACE_BEGIN + begin: Any = android.constants.TRUSTY_STACKTRACE_BEGIN end = android.constants.TRUSTY_STACKTRACE_END target_idx = logcat.rfind(begin) @@ -880,13 +941,13 @@ def _extract_trusty_stacktrace_from_logcat(self, logcat): if target_idx == -1: return 'No TA crash stacktrace found in logcat.\n' - begin_idx = logcat[:target_idx].rfind(begin) + begin = logcat[:target_idx].rfind(begin) end_idx = target_idx + logcat[target_idx:].find(end) end_idx += logcat[end_idx:].find('\n') - return logcat[begin_idx:end_idx] + return logcat[begin:end_idx] - def _add_trusty_stacktrace_if_needed(self, output): + def _add_trusty_stacktrace_if_needed(self, output: str) -> str: """Add trusty stacktrace to beginning of output if found in logcat.""" if android.adb.get_device_state() == 'is-ramdump-mode:yes': @@ -905,9 +966,9 @@ def _add_trusty_stacktrace_if_needed(self, output): return ('+-- Logcat excerpt: Trusted App crash stacktrace --+' f'\n{ta_stacktrace}\n\n{output}\n\nLogcat:\n{logcat}') - def _extract_mte_stacktrace_from_logcat(self, logcat): + def _extract_mte_stacktrace_from_logcat(self, logcat: str) -> str: """Finds and returns the MTE stacktrace from a logcat.""" - begin = android.constants.MTE_STACKTRACE_BEGIN + begin: Any = android.constants.MTE_STACKTRACE_BEGIN end = android.constants.MTE_STACKTRACE_END target_idx = logcat.rfind(begin) @@ -930,13 +991,13 @@ def _extract_mte_stacktrace_from_logcat(self, logcat): if target_idx == -1: return 'No MTE crash stacktrace found in logcat.\n' - begin_idx = logcat[:target_idx].rfind(begin) + begin = logcat[:target_idx].rfind(begin) end_idx = target_idx + logcat[target_idx:].find(end) end_idx += logcat[end_idx:].find('\n') - return logcat[begin_idx:end_idx] + return logcat[begin:end_idx] - def _add_mte_stacktrace_if_needed(self, output): + def _add_mte_stacktrace_if_needed(self, output: str) -> str: """Add mte stacktrace to beginning of output if found in logcat.""" logcat = android.logger.log_output() mte_stacktrace = self._extract_mte_stacktrace_from_logcat(logcat) @@ -944,7 +1005,7 @@ def _add_mte_stacktrace_if_needed(self, output): return ('+-- Logcat excerpt: MTE crash stacktrace --+' f'\n{mte_stacktrace}\n\n{output}\n\nLogcat:\n{logcat}') - def _add_logcat_output_if_needed(self, output): + def _add_logcat_output_if_needed(self, output: str) -> str: """Add logcat output to end of output to capture crashes from related processes if current output has no sanitizer crash.""" if 'Sanitizer: ' in output: @@ -959,7 +1020,7 @@ def _add_logcat_output_if_needed(self, output): return f'{output}\n\nLogcat:\n{android.logger.log_output()}' @contextlib.contextmanager - def _device_file(self, file_path): + def _device_file(self, file_path: str) -> Iterator[str]: """Context manager for device files. Args: file_path: Host path to file. @@ -972,7 +1033,7 @@ def _device_file(self, file_path): # Cleanup android.adb.remove_file(device_file_path) - def get_testcase_path(self, log_lines): + def get_testcase_path(self, log_lines: list[str]) -> str | None: """Get testcase path from log lines.""" path = LibFuzzerCommon.get_testcase_path(self, log_lines) if not path: @@ -980,12 +1041,13 @@ def get_testcase_path(self, log_lines): return self._get_local_path(path) - def fuzz(self, - corpus_directories, - fuzz_timeout, - artifact_prefix=None, - additional_args=None, - extra_env=None): + def fuzz( + self, + corpus_directories: list[str], + fuzz_timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None, + extra_env: dict[str, str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.fuzz override.""" android.logger.clear_log() @@ -1000,7 +1062,8 @@ def fuzz(self, artifact_prefix = self._get_device_path(artifact_prefix) # Extract local dict path from arguments list and subsitute with device one. - arguments = fuzzer_options.FuzzerArguments.from_list(additional_args) + arguments = fuzzer_options.FuzzerArguments.from_list( + cast(list[str], additional_args)) assert arguments is not None dict_path = arguments.get( @@ -1025,12 +1088,12 @@ def fuzz(self, return result def merge(self, - corpus_directories, - merge_timeout, - artifact_prefix=None, - tmp_dir=None, - additional_args=None, - merge_control_file=None): + corpus_directories: list[str], + merge_timeout: float | int, + artifact_prefix: str | None = None, + tmp_dir: str | None = None, + additional_args: list[str] | None = None, + merge_control_file: str | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.merge override.""" additional_args = copy.copy(additional_args) if additional_args is None: @@ -1064,10 +1127,11 @@ def merge(self, self._copy_local_directories_from_device(sync_directories) return result - def run_single_testcase(self, - testcase_path, - timeout=None, - additional_args=None): + def run_single_testcase( + self, + testcase_path: str, + timeout: float | int | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.run_single_testcase override.""" android.logger.clear_log() @@ -1077,12 +1141,13 @@ def run_single_testcase(self, result.output = self._add_logcat_output_if_needed(result.output) return result - def minimize_crash(self, - testcase_path, - output_path, - timeout, - artifact_prefix=None, - additional_args=None): + def minimize_crash( + self, + testcase_path: str, + output_path: str, + timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.minimize_crash override.""" with self._device_file(testcase_path) as device_testcase_path: device_output_path = self._get_device_path(output_path) @@ -1099,12 +1164,13 @@ def minimize_crash(self, return result - def cleanse_crash(self, - testcase_path, - output_path, - timeout, - artifact_prefix=None, - additional_args=None): + def cleanse_crash( + self, + testcase_path: str, + output_path: str, + timeout: float | int, + artifact_prefix: str | None = None, + additional_args: list[str] | None = None) -> new_process.ProcessResult: """LibFuzzerCommon.cleanse_crash override.""" with self._device_file(testcase_path) as device_testcase_path: device_output_path = self._get_device_path(output_path) @@ -1122,7 +1188,13 @@ def cleanse_crash(self, return result -def get_runner(fuzzer_path, temp_dir=None, use_minijail=None): +def get_runner( + fuzzer_path: str, + temp_dir: str | None = None, + use_minijail: bool | None = None) -> (MinijailLibFuzzerRunner + | FuchsiaUndercoatLibFuzzerRunner + | AndroidLibFuzzerRunner + | LibFuzzerRunner): """Get a libfuzzer runner.""" if use_minijail is None: use_minijail = environment.get_value('USE_MINIJAIL') @@ -1148,12 +1220,20 @@ def get_runner(fuzzer_path, temp_dir=None, use_minijail=None): is_chromeos_system_job = environment.is_chromeos_system_job() + minijail_chroot: minijail.MinijailChroot | minijail.ChromeOSChroot | None = ( + None) if is_chromeos_system_job: minijail_chroot = minijail.ChromeOSChroot(build_dir) elif use_minijail: minijail_chroot = minijail.MinijailChroot(base_dir=temp_dir) + runner: ( + MinijailLibFuzzerRunner + | FuchsiaUndercoatLibFuzzerRunner + | AndroidLibFuzzerRunner + | LibFuzzerRunner) if use_minijail or is_chromeos_system_job: + assert minijail_chroot is not None # While it's possible for dynamic binaries to run without this, they need # to be accessible for symbolization etc. For simplicity we bind BUILD_DIR # to the same location within the chroot, which leaks the directory @@ -1175,6 +1255,7 @@ def get_runner(fuzzer_path, temp_dir=None, use_minijail=None): llvm_symbolizer_destination_path = os.path.join(minijail_bin, 'llvm-symbolizer') if not os.path.exists(llvm_symbolizer_destination_path): + assert llvm_symbolizer_source_path is not None shutil.copy(llvm_symbolizer_source_path, llvm_symbolizer_destination_path) # copy /bin/sh, necessary for system(). @@ -1201,10 +1282,11 @@ def get_runner(fuzzer_path, temp_dir=None, use_minijail=None): return runner -def copy_from_corpus(dest_corpus_path, src_corpus_path, num_testcases): +def copy_from_corpus(dest_corpus_path: str, src_corpus_path: str, + num_testcases: int) -> None: """Choose |num_testcases| testcases from the src corpus directory (and its subdirectories) and copy it into the dest directory.""" - src_corpus_files = [] + src_corpus_files: list[str] = [] for root, _, files in shell.walk(src_corpus_path): for f in files: src_corpus_files.append(os.path.join(root, f)) @@ -1214,9 +1296,11 @@ def copy_from_corpus(dest_corpus_path, src_corpus_path, num_testcases): shutil.copy(os.path.join(to_copy), os.path.join(dest_corpus_path, str(i))) -def strip_fuzzing_arguments(arguments, is_merge=False): +def strip_fuzzing_arguments(arguments: list[str], + is_merge: bool = False) -> list[str]: """Remove arguments used during fuzzing.""" args = fuzzer_options.FuzzerArguments.from_list(arguments) + assert args is not None for argument in [ # Remove as it overrides `-merge` argument. constants.FORK_FLAGNAME, # It overrides `-merge` argument. @@ -1244,10 +1328,11 @@ def strip_fuzzing_arguments(arguments, is_merge=False): return args.list() -def fix_timeout_argument_for_reproduction(arguments): +def fix_timeout_argument_for_reproduction(arguments: list[str]) -> list[str]: """Changes timeout argument for reproduction. This is slightly less than the |TEST_TIMEOUT| value for the job.""" args = fuzzer_options.FuzzerArguments.from_list(arguments) + assert args is not None if constants.TIMEOUT_FLAGNAME in args: del args[constants.TIMEOUT_FLAGNAME] @@ -1260,9 +1345,9 @@ def fix_timeout_argument_for_reproduction(arguments): return args.list() -def parse_log_stats(log_lines): +def parse_log_stats(log_lines: list[str]) -> dict[str, int]: """Parse libFuzzer log output.""" - log_stats = {} + log_stats: dict[str, int] = {} # Parse libFuzzer generated stats (`-print_final_stats=1`). stats_regex = re.compile(r'stat::([A-Za-z_]+):\s*([^\s]+)') @@ -1271,7 +1356,7 @@ def parse_log_stats(log_lines): if not match: continue - value = match.group(2) + value: Any = match.group(2) if not value.isdigit(): # We do not expect any non-numeric stats from libFuzzer, skip those. logs.error('Corrupted stats reported by libFuzzer: "%s".' % line) @@ -1289,18 +1374,20 @@ def parse_log_stats(log_lines): return log_stats -def set_sanitizer_options(fuzzer_path): +def set_sanitizer_options(fuzzer_path: str) -> None: """Sets sanitizer options based on .options file overrides and what this script requires.""" engine_common.process_sanitizer_options_overrides(fuzzer_path) sanitizer_options_var = environment.get_current_memory_tool_var() + assert sanitizer_options_var is not None sanitizer_options = environment.get_memory_tool_options( sanitizer_options_var, {}) sanitizer_options['exitcode'] = constants.TARGET_ERROR_EXITCODE environment.set_memory_tool_options(sanitizer_options_var, sanitizer_options) -def get_fuzz_timeout(is_mutations_run, total_timeout=None): +def get_fuzz_timeout(is_mutations_run: bool, + total_timeout: float | int | None = None) -> float | int: """Get the fuzz timeout.""" fuzz_timeout = ( engine_common.get_hard_timeout(total_timeout=total_timeout) - @@ -1312,12 +1399,14 @@ def get_fuzz_timeout(is_mutations_run, total_timeout=None): return fuzz_timeout -def pick_strategies(strategy_pool, fuzzer_path, corpus_directory, - existing_arguments): +def pick_strategies( + strategy_pool: strategy_selection.StrategyPool, fuzzer_path: str, + corpus_directory: str, + existing_arguments: fuzzer_options.FuzzerArguments) -> StrategyInfo: """Pick strategies.""" - fuzzing_strategies = [] + fuzzing_strategies: list[str] = [] arguments = fuzzer_options.FuzzerArguments({}) - additional_corpus_dirs = [] + additional_corpus_dirs: list[str] = [] # Select a generator to attempt to use for existing testcase mutations. candidate_generator = engine_common.select_generator(strategy_pool, @@ -1356,7 +1445,7 @@ def pick_strategies(strategy_pool, fuzzer_path, corpus_directory, fuzzing_strategies.append( '%s_%d' % (strategy.FORK_STRATEGY.name, num_fuzz_processes)) - extra_env = {} + extra_env: dict[str, str] = {} if (environment.platform() == 'LINUX' and utils.is_oss_fuzz() and strategy_pool.do_strategy(strategy.USE_EXTRA_SANITIZERS_STRATEGY)): @@ -1366,7 +1455,9 @@ def pick_strategies(strategy_pool, fuzzer_path, corpus_directory, extra_env, is_mutations_run) -def should_set_fork_flag(existing_arguments, strategy_pool): +def should_set_fork_flag( + existing_arguments: fuzzer_options.FuzzerArguments, + strategy_pool: strategy_selection.StrategyPool) -> bool: """Returns True if fork flag should be set.""" # FIXME: Disable for now to avoid severe battery drainage. Stabilize and # re-enable with a lower process count. diff --git a/src/clusterfuzz/_internal/bot/fuzzers/options.py b/src/clusterfuzz/_internal/bot/fuzzers/options.py index 111f217845e..906ac88db61 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/options.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/options.py @@ -17,6 +17,11 @@ import os import random import re +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Optional from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils from clusterfuzz._internal.bot.fuzzers.afl import constants as afl_constants @@ -38,22 +43,25 @@ class FuzzerArguments: PARSING_REGEX = re.compile(r'^[-]{1,2}([a-zA-Z0-9_]+)(=|\ )(.*)$') - def __init__(self, flags=None): - self.flags = flags if flags is not None else {} + def __init__(self, flags: Optional[Dict[str, Any]] = None) -> None: + self.flags: Dict[str, Any] = flags if flags is not None else {} - def __contains__(self, key): + def __contains__(self, key: str) -> bool: return key in self.flags - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return self.flags[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: self.flags[key] = value - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self.flags[key] - def get(self, key, default=None, constructor=None): + def get(self, + key: str, + default: Any = None, + constructor: Optional[Callable[[Any], Any]] = None) -> Any: """Return value for |key|, calling the |constructor| on it, or |default| if the key does not exist, or if the constructor threw an exception.""" try: @@ -67,22 +75,22 @@ def get(self, key, default=None, constructor=None): return default - def dict(self): + def dict(self) -> Dict[str, Any]: """Return arguments as a dict.""" return self.flags - def list(self): + def list(self) -> List[str]: """Return arguments as a list.""" return [f'-{key}={value}' for key, value in self.flags.items()] - def extend(self, flags): + def extend(self, flags: 'FuzzerArguments') -> None: """Extends the existing flags with the provided ones. In case of both containing the same key, `flag[key]` is the value that will be used.""" for key, value in flags.flags.items(): self.flags[key] = value @staticmethod - def from_list(arguments): + def from_list(arguments: List[str]) -> Optional['FuzzerArguments']: res = FuzzerArguments() for arg in arguments: match = FuzzerArguments.PARSING_REGEX.match(arg) @@ -98,7 +106,7 @@ class FuzzerOptions: OPTIONS_RANDOM_REGEX = re.compile( r'^\s*random\(\s*(\d+)\s*,\s*(\d+)\s*\)\s*$') - def __init__(self, options_file_path, cwd=None): + def __init__(self, options_file_path: str, cwd: Optional[str] = None) -> None: if not os.path.exists(options_file_path): raise FuzzerOptionsError('fuzzer options file does not exist.') @@ -114,18 +122,18 @@ def __init__(self, options_file_path, cwd=None): except configparser.Error: raise FuzzerOptionsError('Failed to parse fuzzer options file.') - def _get_dict_path(self, relative_dict_path): + def _get_dict_path(self, relative_dict_path: str) -> str: """Return a full path to the dictionary.""" return os.path.join(self._cwd, relative_dict_path) - def _get_option_section(self, section): + def _get_option_section(self, section: str) -> dict[str, str]: """Get an option section.""" if not self._config.has_section(section): return {} return dict(self._config.items(section)) - def get_env(self): + def get_env(self) -> dict[str, str]: """Returns dict containing env variables and their values set by "env" section. Only includes env variables permitted by |ENV_VAR_WHITELIST|. Variables are assumed to contain no lower case letters. @@ -139,7 +147,7 @@ def get_env(self): return env - def get_engine_arguments(self, engine): + def get_engine_arguments(self, engine: str) -> FuzzerArguments: """Return a list of fuzzer options.""" arguments = {} for option_name, option_value in self._get_option_section(engine).items(): @@ -157,24 +165,24 @@ def get_engine_arguments(self, engine): return FuzzerArguments(arguments) - def get_asan_options(self): + def get_asan_options(self) -> dict[str, str]: """Return a list of ASAN_OPTIONS overrides.""" return self._get_option_section('asan') - def get_msan_options(self): + def get_msan_options(self) -> dict[str, str]: """Return a list of MSAN_OPTIONS overrides.""" return self._get_option_section('msan') - def get_ubsan_options(self): + def get_ubsan_options(self) -> dict[str, str]: """Return a list of UBSAN_OPTIONS overrides.""" return self._get_option_section('ubsan') - def get_hwasan_options(self): + def get_hwasan_options(self) -> dict[str, str]: """Return a list of HWSAN_OPTIONS overrides.""" return self._get_option_section('hwasan') -def get_fuzz_target_options(fuzz_target_path): +def get_fuzz_target_options(fuzz_target_path: str) -> Optional[FuzzerOptions]: """Return a FuzzerOptions for the given target, or None if it does not exist.""" options_file_path = fuzzer_utils.get_supporting_file(fuzz_target_path, diff --git a/src/clusterfuzz/_internal/bot/fuzzers/strategy_selection.py b/src/clusterfuzz/_internal/bot/fuzzers/strategy_selection.py index ac857fe9c14..922ab8493ea 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/strategy_selection.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/strategy_selection.py @@ -17,7 +17,8 @@ that because of compatibility issues, the exact set of strategies generated here may be modified in the launcher before being launched.""" -from collections import namedtuple +from collections.abc import Sequence +from typing import NamedTuple from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot.fuzzers import engine_common @@ -25,35 +26,37 @@ from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -GENERATORS = [ +GENERATORS: list[strategy.Strategy] = [ strategy.CORPUS_MUTATION_RADAMSA_STRATEGY, ] -StrategyCombination = namedtuple('StrategyCombination', - 'strategy_name probability') + +class StrategyCombination(NamedTuple): + strategy_name: str + probability: float class StrategyPool: """Object used to keep track of which strategies the launcher should attempt to enable.""" - def __init__(self): + def __init__(self) -> None: """Empty set representing empty strategy pool.""" - self.strategy_names = set() + self.strategy_names: set[str] = set() - def add_strategy(self, strategy_tuple): + def add_strategy(self, strategy_tuple: strategy.Strategy) -> None: """Add a strategy into our existing strategy pool unless it is disabled.""" if strategy_tuple.name not in environment.get_value('DISABLED_STRATEGIES', '').split(','): self.strategy_names.add(strategy_tuple.name) - def do_strategy(self, strategy_tuple): + def do_strategy(self, strategy_tuple: strategy.Strategy) -> bool: """Boolean value representing whether or not a strategy is in our strategy pool.""" return strategy_tuple.name in self.strategy_names -def choose_generator(strategy_pool): +def choose_generator(strategy_pool: StrategyPool) -> None: """Chooses whether to use radamsa or no generator and updates the strategy pool.""" @@ -65,14 +68,15 @@ def choose_generator(strategy_pool): strategy_pool.add_strategy(strategy.CORPUS_MUTATION_RADAMSA_STRATEGY) -def do_strategy(strategy_tuple): +def do_strategy(strategy_tuple: strategy.Strategy) -> bool: """Return whether or not to use a given strategy.""" return engine_common.decide_with_probability( engine_common.get_strategy_probability(strategy_tuple.name, strategy_tuple.probability)) -def generate_default_strategy_pool(strategy_list, use_generator): +def generate_default_strategy_pool(strategy_list: Sequence[strategy.Strategy], + use_generator: bool) -> StrategyPool: """Return a strategy pool representing a selection of strategies for launcher to consider. @@ -98,7 +102,9 @@ def generate_default_strategy_pool(strategy_list, use_generator): return pool -def generate_weighted_strategy_pool(strategy_list, use_generator, engine_name): +def generate_weighted_strategy_pool(strategy_list: Sequence[strategy.Strategy], + use_generator: bool, + engine_name: str) -> StrategyPool: """Generate a strategy pool based on probability distribution from multi armed bandit experimentation.""" diff --git a/src/clusterfuzz/_internal/bot/fuzzers/utils.py b/src/clusterfuzz/_internal/bot/fuzzers/utils.py index e21affc2918..613448f28fa 100644 --- a/src/clusterfuzz/_internal/bot/fuzzers/utils.py +++ b/src/clusterfuzz/_internal/bot/fuzzers/utils.py @@ -13,13 +13,15 @@ # limitations under the License. """Fuzzer utils.""" +from collections.abc import Callable +from collections.abc import Sequence import functools import json import os import re import stat import tempfile -from typing import Callable +from typing import Any from typing import Optional from clusterfuzz._internal.base import utils @@ -27,17 +29,21 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import shell -ALLOWED_FUZZ_TARGET_EXTENSIONS = ['', '.exe', '.par'] -FUZZ_TARGET_SEARCH_BYTES = [b'LLVMFuzzerTestOneInput', b'LLVMFuzzerRunDriver'] -VALID_TARGET_NAME_REGEX = re.compile(r'^[a-zA-Z0-9@_.-]+$') -BLOCKLISTED_TARGET_NAME_REGEX = re.compile(r'^(jazzer_driver.*|jazzerjs)$') -EXTRA_BUILD_DIR = '__extra_build' +ALLOWED_FUZZ_TARGET_EXTENSIONS: list[str] = ['', '.exe', '.par'] +FUZZ_TARGET_SEARCH_BYTES: list[bytes] = [ + b'LLVMFuzzerTestOneInput', b'LLVMFuzzerRunDriver' +] +VALID_TARGET_NAME_REGEX: re.Pattern[str] = re.compile(r'^[a-zA-Z0-9@_.-]+$') +BLOCKLISTED_TARGET_NAME_REGEX: re.Pattern[str] = re.compile( + r'^(jazzer_driver.*|jazzerjs)$') +EXTRA_BUILD_DIR: str = '__extra_build' # Manifest file present in Chrome build archives. Specifies the archive # schema version along with optional build metadata (e.g., fuzz targets). -CHROME_MANIFEST_FILENAME = 'clusterfuzz_manifest.json' +CHROME_MANIFEST_FILENAME: str = 'clusterfuzz_manifest.json' -def is_fuzz_target(file_path, file_opener: Optional[Callable] = None): +def is_fuzz_target(file_path: str, + file_opener: Optional[Callable[[str], Any]] = None) -> bool: """Returns whether |file_path| is a fuzz target binary (local path).""" if '@' in file_path: # GFT targets often have periods in the name that get misinterpreted as an @@ -198,7 +204,7 @@ def _get_manifest_fuzz_targets(build_dir: str) -> list[str] | None: return fuzz_target_paths -def get_fuzz_targets_local(path): +def get_fuzz_targets_local(path: str) -> list[str]: """Gets the list of local fuzz target paths. If clusterfuzz_manifest.json is present and defines `fuzz_targets`, those @@ -214,7 +220,6 @@ def get_fuzz_targets_local(path): manifest_fuzz_targets = _get_manifest_fuzz_targets(path) if manifest_fuzz_targets is not None: return manifest_fuzz_targets - fuzz_target_paths = [] for root, _, files in shell.walk(path): @@ -230,7 +235,7 @@ def get_fuzz_targets_local(path): return fuzz_target_paths -def get_fuzz_targets(path): +def get_fuzz_targets(path: str) -> Sequence[str]: """Get list of fuzz targets paths.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host @@ -238,7 +243,8 @@ def get_fuzz_targets(path): return get_fuzz_targets_local(path) -def extract_argument(arguments, prefix, remove=True): +def extract_argument(arguments: list[str], prefix: str, + remove: bool = True) -> Optional[str]: """Extract argument from arguments.""" for argument in arguments[:]: if argument.startswith(prefix): @@ -249,7 +255,7 @@ def extract_argument(arguments, prefix, remove=True): return None -def get_build_revision(): +def get_build_revision() -> int: """Get build revision.""" try: build_revision = int(environment.get_value('APP_REVISION')) @@ -259,7 +265,7 @@ def get_build_revision(): return build_revision -def get_supporting_file(fuzz_target_path, extension_or_suffix): +def get_supporting_file(fuzz_target_path: str, extension_or_suffix: str) -> str: """Get supporting file for a fuzz target with the provided extension.""" base_fuzz_target_path = fuzz_target_path @@ -275,7 +281,7 @@ def get_supporting_file(fuzz_target_path, extension_or_suffix): return base_fuzz_target_path + extension_or_suffix -def get_temp_dir(use_fuzz_inputs_disk=True): +def get_temp_dir(use_fuzz_inputs_disk: bool = True) -> str: """Return the temp dir.""" temp_dirname = 'temp-' + str(os.getpid()) if use_fuzz_inputs_disk: @@ -287,7 +293,7 @@ def get_temp_dir(use_fuzz_inputs_disk=True): return temp_directory -def get_file_from_untrusted_worker(worker_file_path): +def get_file_from_untrusted_worker(worker_file_path: str) -> str: """Gets file from an untrusted worker to local. Local file stays in the temp folder until the end of task or can be explicitly deleted by the caller.""" from clusterfuzz._internal.bot.untrusted_runner import file_host @@ -299,12 +305,12 @@ def get_file_from_untrusted_worker(worker_file_path): return local_file_path -def cleanup(): +def cleanup() -> None: """Clean up temporary metadata.""" shell.remove_directory(get_temp_dir()) -def normalize_target_name(target_path): +def normalize_target_name(target_path: str) -> str: """Normalize target path, removing file extensions.""" target_name = os.path.basename(target_path) if '@' in target_name: diff --git a/src/clusterfuzz/_internal/bot/init_scripts/android.py b/src/clusterfuzz/_internal/bot/init_scripts/android.py index 64293ee76a0..99881741936 100644 --- a/src/clusterfuzz/_internal/bot/init_scripts/android.py +++ b/src/clusterfuzz/_internal/bot/init_scripts/android.py @@ -18,10 +18,10 @@ from clusterfuzz._internal.platforms import android from clusterfuzz._internal.system import environment -TIME_SINCE_REBOOT_MIN_THRESHOLD = 60 * 60 * 24 * 1 # 1 day. +TIME_SINCE_REBOOT_MIN_THRESHOLD: int = 60 * 60 * 24 * 1 # 1 day. -def run(): +def run() -> None: """Run Android initialization.""" init_runner.run() diff --git a/src/clusterfuzz/_internal/bot/init_scripts/chromeos.py b/src/clusterfuzz/_internal/bot/init_scripts/chromeos.py index 373b1038b7d..44edbea9678 100644 --- a/src/clusterfuzz/_internal/bot/init_scripts/chromeos.py +++ b/src/clusterfuzz/_internal/bot/init_scripts/chromeos.py @@ -16,6 +16,6 @@ from clusterfuzz._internal.bot.init_scripts import init_runner -def run(): +def run() -> None: """Run Chrome OS initialization.""" init_runner.run() diff --git a/src/clusterfuzz/_internal/bot/init_scripts/fuchsia.py b/src/clusterfuzz/_internal/bot/init_scripts/fuchsia.py index 3e78716cbd2..47038428677 100644 --- a/src/clusterfuzz/_internal/bot/init_scripts/fuchsia.py +++ b/src/clusterfuzz/_internal/bot/init_scripts/fuchsia.py @@ -16,7 +16,7 @@ from clusterfuzz._internal.bot.init_scripts import init_runner -def run(): +def run() -> None: """Initialize a device before running a task.""" init_runner.run() # TODO(mbarbella): Implement this. diff --git a/src/clusterfuzz/_internal/bot/init_scripts/init_runner.py b/src/clusterfuzz/_internal/bot/init_scripts/init_runner.py index acc85d617a7..e512ee48591 100644 --- a/src/clusterfuzz/_internal/bot/init_scripts/init_runner.py +++ b/src/clusterfuzz/_internal/bot/init_scripts/init_runner.py @@ -19,10 +19,10 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import process_handler -SCRIPT_DIR = os.path.join('bot', 'init') +SCRIPT_DIR: str = os.path.join('bot', 'init') -def _extension(platform): +def _extension(platform: str) -> str: """Get the init extension for a platform.""" if platform == 'windows': return '.ps1' @@ -30,7 +30,7 @@ def _extension(platform): return '.bash' -def run(): +def run() -> None: """Run custom platform specific init scripts.""" platform = environment.platform().lower() platform = environment.base_platform(platform) diff --git a/src/clusterfuzz/_internal/bot/init_scripts/linux.py b/src/clusterfuzz/_internal/bot/init_scripts/linux.py index 969ae08a9d9..150c944050c 100644 --- a/src/clusterfuzz/_internal/bot/init_scripts/linux.py +++ b/src/clusterfuzz/_internal/bot/init_scripts/linux.py @@ -16,6 +16,6 @@ from clusterfuzz._internal.bot.init_scripts import init_runner -def run(): +def run() -> None: """Run Linux initialization.""" init_runner.run() diff --git a/src/clusterfuzz/_internal/bot/init_scripts/mac.py b/src/clusterfuzz/_internal/bot/init_scripts/mac.py index f8dd5d612e1..e9473dc185d 100644 --- a/src/clusterfuzz/_internal/bot/init_scripts/mac.py +++ b/src/clusterfuzz/_internal/bot/init_scripts/mac.py @@ -13,6 +13,7 @@ # limitations under the License. """The initialization script for Mac. It is run before running a task.""" +from collections.abc import Iterator import os import re import shutil @@ -28,10 +29,11 @@ '/lsregister -dump') -def _execute(cmd): +def _execute(cmd: str) -> Iterator[str]: """Execute command and return output as an iterator.""" proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + assert proc.stdout is not None try: for line in iter(proc.stdout.readline, b''): yield line.decode('utf-8') @@ -39,7 +41,7 @@ def _execute(cmd): proc.kill() -def get_launch_service_path(): +def get_launch_service_path() -> str | None: """Get launch service path from lsregister.""" for line in _execute(LSREGISTER_CMD): m = LAUNCH_SERVICE_PATH_REGEX.match(line) @@ -51,7 +53,7 @@ def get_launch_service_path(): return None -def clear_launch_service_data(): +def clear_launch_service_data() -> None: """See crbug.com/661221 for more info.""" path = get_launch_service_path() if not path or not os.path.exists(path): @@ -62,7 +64,7 @@ def clear_launch_service_data(): shutil.rmtree(os.path.join(path, 'T'), ignore_errors=True) -def run(): +def run() -> None: """Run the initialization for Mac.""" init_runner.run() clear_launch_service_data() diff --git a/src/clusterfuzz/_internal/bot/init_scripts/windows.py b/src/clusterfuzz/_internal/bot/init_scripts/windows.py index b3ee1dae14f..a56f511c837 100644 --- a/src/clusterfuzz/_internal/bot/init_scripts/windows.py +++ b/src/clusterfuzz/_internal/bot/init_scripts/windows.py @@ -29,7 +29,7 @@ ] -def clean_temp_directories(): +def clean_temp_directories() -> None: """Clean temporary directories.""" for temp_directory in TEMP_DIRECTORIES: temp_directory_full_path = os.path.abspath( @@ -38,7 +38,7 @@ def clean_temp_directories(): temp_directory_full_path, recreate=True, ignore_errors=True) -def run(): +def run() -> None: """Run the initialization for Windows.""" init_runner.run() clean_temp_directories() diff --git a/src/clusterfuzz/_internal/bot/minimizer/basic_minimizers.py b/src/clusterfuzz/_internal/bot/minimizer/basic_minimizers.py index f5221436ed0..9b055ac2fcf 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/basic_minimizers.py +++ b/src/clusterfuzz/_internal/bot/minimizer/basic_minimizers.py @@ -13,13 +13,17 @@ # limitations under the License. """Simple minimizers for common tasks.""" +from collections.abc import Callable +from typing import Any +from typing import cast + from . import minimizer class SinglePassMinimizer(minimizer.Minimizer): # pylint:disable=abstract-method """Do a single pass over the token list.""" - def _execute(self, data): + def _execute(self, data: Any) -> minimizer.Testcase: """Attempt to remove each token starting from the last one.""" testcase = minimizer.Testcase(data, self) for i in reversed(list(range(len(testcase.tokens)))): @@ -33,16 +37,18 @@ def _execute(self, data): class EmptyTokenRemover(minimizer.Minimizer): # pylint:disable=abstract-method """Attempt to remove empty tokens.""" - def __init__(self, *args, **kwargs): - self.is_empty = self._handle_constructor_argument( - 'is_empty', kwargs, default=lambda s: not s.strip()) + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.is_empty: Callable[[Any], bool] = cast( + Callable[[Any], bool], + self._handle_constructor_argument( + 'is_empty', kwargs, default=lambda s: not s.strip())) minimizer.Minimizer.__init__(self, *args, **kwargs) - def _execute(self, data): + def _execute(self, data: Any) -> minimizer.Testcase: """Try to remove all blank tokens, then individual ones.""" testcase = minimizer.Testcase(data, self) tokens = testcase.tokens - empty_tokens = [] + empty_tokens: list[int] = [] for i, token in enumerate(tokens): if self.is_empty(token): diff --git a/src/clusterfuzz/_internal/bot/minimizer/chunk_minimizer.py b/src/clusterfuzz/_internal/bot/minimizer/chunk_minimizer.py index 47970d3f1e6..65ff86fd90f 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/chunk_minimizer.py +++ b/src/clusterfuzz/_internal/bot/minimizer/chunk_minimizer.py @@ -13,7 +13,8 @@ # limitations under the License. """Fast minimizer that attempts to remove tokens grouped in chunks.""" -import functools +from typing import Any +from typing import cast from . import errors from . import minimizer @@ -23,12 +24,14 @@ class ChunkMinimizer(minimizer.Minimizer): """Minimizer to replicate the old ClusterFuzz minimization strategy.""" - def __init__(self, *args, **kwargs): - self.chunk_sizes = self._handle_constructor_argument( - 'chunk_sizes', kwargs, default=[10, 4, 1]) + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.chunk_sizes: list[int] = cast( + list[int], + self._handle_constructor_argument( + 'chunk_sizes', kwargs, default=[10, 4, 1])) minimizer.Minimizer.__init__(self, *args, **kwargs) - def _execute(self, data): + def _execute(self, data: Any) -> minimizer.Testcase: """Minimize |data| using the algorithm from CF (but backwards).""" testcase = minimizer.Testcase(data, self) if not self.validate_tokenizer(data, testcase): @@ -46,7 +49,11 @@ def _execute(self, data): return testcase @staticmethod - def run(data, thread_count=minimizer.DEFAULT_THREAD_COUNT, file_extension=''): + def run( + data: Any, + thread_count: int = minimizer.DEFAULT_THREAD_COUNT, + file_extension: str = '', + ) -> Any: """Minimize |data| using the old strategy from CF.""" minimizer_round_1 = ChunkMinimizer( utils.test, @@ -56,11 +63,10 @@ def run(data, thread_count=minimizer.DEFAULT_THREAD_COUNT, file_extension=''): chunk_sizes=[80, 40, 20], file_extension=file_extension) - full_tokenizer = functools.partial(utils.tokenize, level=1) minimizer_round_2 = ChunkMinimizer( utils.test, max_threads=thread_count, - tokenizer=full_tokenizer, + tokenizer=utils.tokenize, token_combiner=utils.token_combiner, chunk_sizes=[10, 4, 1], file_extension=file_extension) diff --git a/src/clusterfuzz/_internal/bot/minimizer/delta_minimizer.py b/src/clusterfuzz/_internal/bot/minimizer/delta_minimizer.py index f79505951bc..05ea618e5f8 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/delta_minimizer.py +++ b/src/clusterfuzz/_internal/bot/minimizer/delta_minimizer.py @@ -13,6 +13,8 @@ # limitations under the License. """Minimizer based on the delta debugging algorithm.""" +from typing import Any + from . import errors from . import minimizer from . import utils @@ -21,7 +23,8 @@ class DeltaTestcase(minimizer.Testcase): """Test case for the delta minimizer.""" - def _process_test_result(self, test_passed, hypothesis): + def _process_test_result(self, test_passed: bool, + hypothesis: list[int]) -> None: """Update state based on test_passed and hypothesis.""" # If we crashed or cannot split the test into smaller chunks, we're done. if not test_passed or len(hypothesis) <= 1: @@ -40,7 +43,7 @@ def _process_test_result(self, test_passed, hypothesis): class DeltaMinimizer(minimizer.Minimizer): """Minimizer based on the delta algorithm.""" - def _execute(self, data): + def _execute(self, data: Any) -> DeltaTestcase: """Prepare tests for delta minimization and process.""" testcase = DeltaTestcase(data, self) if not self.validate_tokenizer(data, testcase): @@ -58,7 +61,11 @@ def _execute(self, data): return testcase @staticmethod - def run(data, thread_count=minimizer.DEFAULT_THREAD_COUNT, file_extension=''): + def run( + data: Any, + thread_count: int = minimizer.DEFAULT_THREAD_COUNT, + file_extension: str = '', + ) -> Any: """Try to minimize |data| using a simple line tokenizer.""" delta_minimizer = DeltaMinimizer( utils.test, max_threads=thread_count, file_extension=file_extension) diff --git a/src/clusterfuzz/_internal/bot/minimizer/errors.py b/src/clusterfuzz/_internal/bot/minimizer/errors.py index b1ee6bd9a11..72d371ee458 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/errors.py +++ b/src/clusterfuzz/_internal/bot/minimizer/errors.py @@ -13,26 +13,31 @@ # limitations under the License. """Exception classes for minimizers.""" +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from clusterfuzz._internal.bot.minimizer import minimizer + class MinimizationDeadlineExceededError(Exception): """Exception thrown if the deadline for minimization has been exceeded.""" - def __init__(self, testcase): - Exception.__init__(self, 'Deadline exceeded.') + def __init__(self, testcase: 'minimizer.Testcase') -> None: + super().__init__('Deadline exceeded.') self.testcase = testcase class NoCommandError(Exception): """Exception thrown if no command is configured for test runs.""" - def __init__(self): - Exception.__init__(self, 'Attempting to run with no command configured.') + def __init__(self) -> None: + super().__init__('Attempting to run with no command configured.') class TokenizationFailureError(Exception): - def __init__(self, minimization_type): - Exception.__init__(self, 'Unable to perform ' + minimization_type + '.') + def __init__(self, minimization_type: str) -> None: + super().__init__('Unable to perform ' + minimization_type + '.') class AntlrDecodeError(Exception): diff --git a/src/clusterfuzz/_internal/bot/minimizer/html_minimizer.py b/src/clusterfuzz/_internal/bot/minimizer/html_minimizer.py index 4d7f71f6f7b..67870e0359b 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/html_minimizer.py +++ b/src/clusterfuzz/_internal/bot/minimizer/html_minimizer.py @@ -14,7 +14,12 @@ """Minimizer that attempts to use specialized minimizers on different parts of an HTML test case.""" +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Sequence import functools +from typing import Any +from typing import cast from clusterfuzz._internal.bot.tokenizer.antlr_tokenizer import AntlrTokenizer from clusterfuzz._internal.bot.tokenizer.grammars.HTMLLexer import HTMLLexer @@ -27,8 +32,8 @@ from . import minimizer from . import utils -SCRIPT_START_STRING = b'' +SCRIPT_START_STRING: bytes = b'' class HTMLMinimizer(minimizer.Minimizer): # pylint:disable=abstract-method @@ -40,34 +45,37 @@ class HTMLMinimizer(minimizer.Minimizer): # pylint:disable=abstract-method class Token: """Helper class to represent a single token.""" - TYPE_HTML = 0 - TYPE_SCRIPT = 1 + TYPE_HTML: int = 0 + TYPE_SCRIPT: int = 1 - def __init__(self, data, token_type): - self.data = data - self.token_type = token_type + def __init__(self, data: bytes, token_type: int) -> None: + self.data: bytes = data + self.token_type: int = token_type class TokenizerState: """Enum for tokenizer states.""" - SEARCHING_FOR_SCRIPT = 0 - SEARCHING_FOR_TAG_END = 1 - SEARCHING_FOR_CLOSE_SCRIPT = 2 + SEARCHING_FOR_SCRIPT: int = 0 + SEARCHING_FOR_TAG_END: int = 1 + SEARCHING_FOR_CLOSE_SCRIPT: int = 2 - HTMLTOKENIZER = AntlrTokenizer(HTMLLexer).tokenize - JSTOKENIZER = AntlrTokenizer(JavaScriptLexer).tokenize + HTMLTOKENIZER: Callable[[bytes], list[str]] = AntlrTokenizer( + HTMLLexer).tokenize + JSTOKENIZER: Callable[[bytes], list[str]] = AntlrTokenizer( + JavaScriptLexer).tokenize - TOKENIZER_MAP = { + TOKENIZER_MAP: dict[int, list[Callable[[bytes], list[str]]]] = { Token.TYPE_HTML: [HTMLTOKENIZER, HTMLTOKENIZER, HTMLTOKENIZER], Token.TYPE_SCRIPT: [JSTOKENIZER, JSTOKENIZER], } - CHUNK_SIZES = [ + CHUNK_SIZES: list[list[int]] = [ [400, 100, 20, 5], [400, 100, 20, 5, 2], [400, 100, 20, 5, 1], ] - def __init__(self, test_function, *args, **kwargs): + def __init__(self, test_function: Callable[..., Any], *args: Any, + **kwargs: Any) -> None: # The HTML minimizer will not be used directly. Instead, preserve its # arguments and pass them along when creating subminimizers. super().__init__(lambda: False) @@ -76,10 +84,10 @@ def __init__(self, test_function, *args, **kwargs): assert 'tokenizer' not in kwargs, 'Custom tokenizers not supported.' assert 'token_combiner' not in kwargs, 'Custom tokenizers not supported.' - self.test_function = test_function - self.kwargs = kwargs + self.test_function: Callable[..., Any] = test_function + self.kwargs: dict[str, Any] = kwargs - def minimize(self, data): + def minimize(self, data: bytes) -> bytes: """Wrapper to perform common tasks and call |_execute|.""" # Do an initial line-by-line minimization to filter out noise. line_minimizer = delta_minimizer.DeltaMinimizer(self.test_function, @@ -87,7 +95,7 @@ def minimize(self, data): # Do two line minimizations to make up for the fact that minimzations on # bots don't always minimize as much as they can. for _ in range(2): - data = line_minimizer.minimize(data) + data = cast(bytes, line_minimizer.minimize(data)) tokens = self.get_tokens_and_metadata(data) for index, token in enumerate(tokens): @@ -103,12 +111,13 @@ def minimize(self, data): # portions of the test to the combined tokens. if token.token_type == HTMLMinimizer.Token.TYPE_HTML: - current_minimizer = chunk_minimizer.ChunkMinimizer( - self.test_function, - chunk_sizes=HTMLMinimizer.CHUNK_SIZES[level], - token_combiner=token_combiner, - tokenizer=current_tokenizer, - **self.kwargs) + current_minimizer: minimizer.Minimizer = ( + chunk_minimizer.ChunkMinimizer( + self.test_function, + chunk_sizes=HTMLMinimizer.CHUNK_SIZES[level], + token_combiner=token_combiner, + tokenizer=current_tokenizer, + **self.kwargs)) else: current_minimizer = js_minimizer.JSMinimizer( self.test_function, @@ -116,7 +125,7 @@ def minimize(self, data): tokenizer=current_tokenizer, **self.kwargs) - result_data = current_minimizer.minimize(token.data) + result_data = cast(bytes, current_minimizer.minimize(token.data)) start = len(prefix) end = len(result_data) - len(suffix) token.data = result_data[start:end] @@ -124,12 +133,12 @@ def minimize(self, data): # TODO(mbarbella): Remove this once other minimizers are improved. # Do a final line-by-line minimization pass. data = self.combine_tokens(tokens) - return line_minimizer.minimize(data) + return cast(bytes, line_minimizer.minimize(data)) @staticmethod - def get_tokens_and_metadata(data): + def get_tokens_and_metadata(data: bytes) -> list['HTMLMinimizer.Token']: """Get the token list with associated metadata.""" - tokens = [] + tokens: list[HTMLMinimizer.Token] = [] state = HTMLMinimizer.TokenizerState.SEARCHING_FOR_SCRIPT current_token_start = 0 current_token_type = HTMLMinimizer.Token.TYPE_HTML @@ -182,7 +191,9 @@ def get_tokens_and_metadata(data): return tokens @staticmethod - def combine_worker_tokens(tokens, prefix=b'', suffix=b''): + def combine_worker_tokens(tokens: Iterable[str | bytes], + prefix: bytes = b'', + suffix: bytes = b'') -> bytes: """Combine tokens for a worker minimizer.""" # The Antlr tokenizer decodes the bytes objects we originally pass to it. encoded_tokens = [ @@ -191,14 +202,14 @@ def combine_worker_tokens(tokens, prefix=b'', suffix=b''): return prefix + b''.join(encoded_tokens) + suffix @staticmethod - def combine_tokens(tokens): + def combine_tokens(tokens: Sequence['HTMLMinimizer.Token']) -> bytes: """Combine tokens into a usable format, stripping metadata.""" return b''.join([t.data for t in tokens]) @staticmethod - def run(data, - thread_count=minimizer.DEFAULT_THREAD_COUNT, - file_extension='.html'): + def run(data: bytes, + thread_count: int = minimizer.DEFAULT_THREAD_COUNT, + file_extension: str = '.html') -> bytes: """Attempt to minimize an html test case.""" html_minimizer = HTMLMinimizer( utils.test, max_threads=thread_count, file_extension=file_extension) diff --git a/src/clusterfuzz/_internal/bot/minimizer/js_minimizer.py b/src/clusterfuzz/_internal/bot/minimizer/js_minimizer.py index 5c101776e6f..10096cc1db1 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/js_minimizer.py +++ b/src/clusterfuzz/_internal/bot/minimizer/js_minimizer.py @@ -13,6 +13,9 @@ # limitations under the License. """Minimizer used for additional reduction on javascript test cases.""" +from collections.abc import Callable +from typing import Any + from clusterfuzz._internal.bot.tokenizer.antlr_tokenizer import AntlrTokenizer from clusterfuzz._internal.bot.tokenizer.grammars.JavaScriptLexer import \ JavaScriptLexer @@ -23,7 +26,7 @@ from . import utils -def step_back_while(cur_index, condition): +def step_back_while(cur_index: int, condition: Callable[[int], bool]) -> int: """Helper function. Decreases index from cur while condition is satisfied.""" while cur_index >= 0 and condition(cur_index): cur_index -= 1 @@ -33,13 +36,13 @@ def step_back_while(cur_index, condition): class JSMinimizer(minimizer.Minimizer): """Intended as a second-pass minimizer to remove unneeded tokens from JS.""" - def _execute(self, data): + def _execute(self, data: Any) -> minimizer.Testcase: testcase = minimizer.Testcase(data, self) if not self.validate_tokenizer(data, testcase): raise errors.TokenizationFailureError('JS Minimizer') - brace_stack = [] - paren_stack = [] + brace_stack: list[int] = [] + paren_stack: list[int] = [] for index, token in enumerate(testcase.tokens): if token == '{': @@ -165,9 +168,11 @@ def _execute(self, data): return testcase @staticmethod - def run(data, - thread_count=minimizer.DEFAULT_THREAD_COUNT, - file_extension='.js'): + def run( + data: Any, + thread_count: int = minimizer.DEFAULT_THREAD_COUNT, + file_extension: str = '.js', + ) -> Any: """Attempt to minimize a javascript test case.""" line_minimizer = delta_minimizer.DeltaMinimizer( utils.test, max_threads=thread_count, file_extension=file_extension) diff --git a/src/clusterfuzz/_internal/bot/minimizer/minimizer.py b/src/clusterfuzz/_internal/bot/minimizer/minimizer.py index bb79c141bb8..1753ffdf29d 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/minimizer.py +++ b/src/clusterfuzz/_internal/bot/minimizer/minimizer.py @@ -13,69 +13,87 @@ # limitations under the License. """Base classes for other minimizers.""" +from collections.abc import Callable +from collections.abc import Sequence import copy import functools import os import tempfile import threading import time +import types +from typing import Any from clusterfuzz._internal.metrics import logs from . import errors -DEFAULT_CLEANUP_INTERVAL = 20 -DEFAULT_THREAD_COUNT = 8 -DEFAULT_TESTS_PER_THREAD = 4 +DEFAULT_CLEANUP_INTERVAL: int = 20 +DEFAULT_THREAD_COUNT: int = 8 +DEFAULT_TESTS_PER_THREAD: int = 4 -MAX_MERGE_BATCH_SIZE = 32 -PROGRESS_REPORT_INTERVAL = 300 +MAX_MERGE_BATCH_SIZE: int = 32 +PROGRESS_REPORT_INTERVAL: int = 300 + +TestQueueEntry = tuple[Any, Callable[..., bool], Callable[[bool], None], + Callable[[], bool]] class DummyLock: """Dummy to replace threading.Lock for single-threaded tests.""" - def __enter__(self): + def __enter__(self) -> None: pass - def __exit__(self, exec_type, value, traceback): + def __exit__( + self, + exec_type: type[BaseException] | None, + value: BaseException | None, + traceback: types.TracebackType | None, + ) -> None: pass - def __bool__(self): + def __bool__(self) -> bool: return False class TestQueue: """Queue to store commands that should be executed to test hypotheses.""" - def __init__(self, - thread_count, - deadline_check=None, - progress_report_function=None, - per_thread_cleanup_function=None): - self.thread_count = thread_count - self.deadline_check = deadline_check - self.progress_report_function = progress_report_function - self.per_thread_cleanup_function = per_thread_cleanup_function - - self.lock = threading.Lock() - self.queue = [] - - def _pop(self): + def __init__( + self, + thread_count: int, + deadline_check: Callable[..., bool] | None = None, + progress_report_function: Callable[[], None] | None = None, + per_thread_cleanup_function: Callable[[], None] | None = None, + ) -> None: + self.thread_count: int = thread_count + self.deadline_check: Callable[..., bool] | None = deadline_check + self.progress_report_function: Callable[[], None] | None = ( + progress_report_function) + self.per_thread_cleanup_function: Callable[[], None] | None = ( + per_thread_cleanup_function) + + self.lock: threading.Lock = threading.Lock() + self.queue: list[TestQueueEntry] = [] + + def _pop(self) -> TestQueueEntry | None: """Pull a single hypothesis to process from the queue.""" with self.lock: if not self.queue: return None return self.queue.pop(0) - def _work(self): + def _work(self) -> None: """Process items from the queue until it is empty.""" while not self.deadline_check or not self.deadline_check(soft_check=True): current_item = self._pop() if not current_item: break - test, test_function, completion_callback, should_run = current_item # pylint: disable=unpacking-non-sequence + test, test_function, completion_callback, should_run = ( + current_item # pylint: disable=unpacking-non-sequence + ) if not should_run(): continue @@ -89,35 +107,39 @@ def _work(self): if self.deadline_check and self.deadline_check(soft_check=True): break - def _cleanup(self): + def _cleanup(self) -> None: """Clean up the queue to be sure that no more tasks will be executed.""" with self.lock: self.queue = [] - def push(self, - test, - test_function, - completion_callback, - should_run=lambda: True): + def push( + self, + test: Any, + test_function: Callable[..., bool], + completion_callback: Callable[[bool], None], + should_run: Callable[[], bool] = lambda: True, + ) -> None: """Add a test to the queue and a callback to run on completion.""" with self.lock: self.queue.append((test, test_function, completion_callback, should_run)) - def force(self, - test, - test_function, - completion_callback, - should_run=lambda: True): + def force( + self, + test: Any, + test_function: Callable[..., bool], + completion_callback: Callable[[bool], None], + should_run: Callable[[], bool] = lambda: True, + ) -> None: """Force a test to the front of the queue.""" entry = (test, test_function, completion_callback, should_run) with self.lock: self.queue.insert(0, entry) - def size(self): + def size(self) -> int: """Return the number of unprocessed tasks in the queue.""" return len(self.queue) - def process(self): + def process(self) -> None: """Process all tests in the queue and block until completion.""" while self.queue: threads = [ @@ -139,34 +161,34 @@ def process(self): class Testcase: """Single test case to be minimized.""" - def __init__(self, data, minimizer): - self.minimizer = minimizer + def __init__(self, data: Any, minimizer: 'Minimizer') -> None: + self.minimizer: Minimizer = minimizer if minimizer.tokenize: try: - self.tokens = minimizer.tokenizer(data) + self.tokens: list[Any] = minimizer.tokenizer(data) except UnicodeDecodeError: raise errors.AntlrDecodeError else: - self.tokens = data + self.tokens: list[Any] = data - self.required_tokens = [True] * len(self.tokens) - self.tested_hypotheses = set() - self.unmerged_failing_hypotheses = [] - self.tests_to_queue = [] - self.currently_processing = False - self.last_progress_report_time = 0 - self.runs_since_last_cleanup = 0 - self.runs_executed = 0 + self.required_tokens: list[bool] = [True] * len(self.tokens) + self.tested_hypotheses: set[tuple[int, ...]] = set() + self.unmerged_failing_hypotheses: list[list[int]] = [] + self.tests_to_queue: list[list[int]] = [] + self.currently_processing: bool = False + self.last_progress_report_time: float = 0 + self.runs_since_last_cleanup: int = 0 + self.runs_executed: int = 0 if minimizer.max_threads > 1: - self.test_queue = TestQueue( + self.test_queue: TestQueue | None = TestQueue( minimizer.max_threads, deadline_check=self._deadline_exceeded, progress_report_function=self._report_progress) - self.merge_preparation_lock = threading.Lock() - self.merge_lock = threading.Lock() - self.cache_lock = threading.Lock() - self.tests_to_queue_lock = threading.Lock() + self.merge_preparation_lock: threading.Lock | DummyLock = threading.Lock() + self.merge_lock: threading.Lock | DummyLock = threading.Lock() + self.cache_lock: threading.Lock | DummyLock = threading.Lock() + self.tests_to_queue_lock: threading.Lock | DummyLock = threading.Lock() else: self.test_queue = None self.merge_preparation_lock = DummyLock() @@ -174,12 +196,16 @@ def __init__(self, data, minimizer): self.cache_lock = DummyLock() self.tests_to_queue_lock = DummyLock() - def get_current_testcase_data(self): + def get_current_testcase_data(self) -> Any: """Return the current test case data.""" return self.minimizer.token_combiner(self.get_required_tokens()) # Helper functions based on minimizer configuration. - def _deadline_exceeded(self, cleanup_function=None, soft_check=False): + def _deadline_exceeded( + self, + cleanup_function: Callable[[], None] | None = None, + soft_check: bool = False, + ) -> bool: """Check to see if we have exceeded the deadline for execution.""" if self.minimizer.deadline and time.time() > self.minimizer.deadline: if soft_check: @@ -199,7 +225,7 @@ def _deadline_exceeded(self, cleanup_function=None, soft_check=False): return False - def _delete_file_if_needed(self, input_file): + def _delete_file_if_needed(self, input_file: Any) -> None: """Deletes a temporary file if necessary.""" # If we are not running in a mode where we need to delete files, do nothing. if not self.minimizer.tokenize or not self.minimizer.delete_temp_files: @@ -210,7 +236,7 @@ def _delete_file_if_needed(self, input_file): except OSError: pass - def _report_progress(self, is_final_progress_report=False): + def _report_progress(self, is_final_progress_report: bool = False) -> None: """Call a function to report progress if the minimizer uses one.""" if not self.minimizer.progress_report_function: return @@ -228,13 +254,14 @@ def _report_progress(self, is_final_progress_report=False): self.minimizer.progress_report_function(message) # Functions used when preparing tests. - def _range_complement(self, current_range): + def _range_complement(self, current_range: Sequence[int]) -> list[int]: """Return required tokens in the complement of the specified range.""" result = list(range(len(self.tokens))) to_remove = set(current_range) return [i for i in result if i not in to_remove and self.required_tokens[i]] - def _prepare_test_input(self, tokens, tested_tokens): + def _prepare_test_input(self, tokens: list[Any], + tested_tokens: Sequence[int] | set[int]) -> Any: """Write the tokens currently being tested to a temporary file.""" tested_tokens = set(tested_tokens) current_tokens = [t for i, t in enumerate(tokens) if i in tested_tokens] @@ -255,13 +282,14 @@ def _prepare_test_input(self, tokens, tested_tokens): handle.close() return destination - def _get_test_file(self, hypothesis): + def _get_test_file(self, hypothesis: Sequence[int]) -> Any: """Return a test file for a hypothesis.""" complement = self._range_complement(hypothesis) return self._prepare_test_input(self.tokens, complement) - def _push_test_to_queue(self, hypothesis): + def _push_test_to_queue(self, hypothesis: list[int]) -> None: """Add a test for a hypothesis to a queue for processing.""" + assert self.test_queue is not None test_file = self._get_test_file(hypothesis) callback = functools.partial( self._handle_completed_test, @@ -280,7 +308,7 @@ def _push_test_to_queue(self, hypothesis): if self.test_queue.size() >= self.minimizer.batch_size: self._do_single_pass_process() - def prepare_test(self, hypothesis): + def prepare_test(self, hypothesis: list[int]) -> None: """Prepare the test based on the mode we are running in.""" # Check the cache to make sure we have not tested this before. if self._has_tested(hypothesis): @@ -315,7 +343,8 @@ def prepare_test(self, hypothesis): self._push_test_to_queue(hypothesis) # Functions used when processing test results. - def _handle_completed_test(self, test_passed, hypothesis, input_file): + def _handle_completed_test(self, test_passed: bool, hypothesis: list[int], + input_file: Any) -> None: """Update state based on the test result and hypothesis.""" # If the test failed, handle the result. if not test_passed: @@ -327,10 +356,11 @@ def _handle_completed_test(self, test_passed, hypothesis, input_file): # Minimizers may need to do something with the test result. self._process_test_result(test_passed, hypothesis) - def _process_test_result(self, test_passed, hypothesis): + def _process_test_result(self, test_passed: bool, + hypothesis: list[int]) -> None: """Additional processing of the result. Minimizers may override this.""" - def _handle_failing_hypothesis(self, hypothesis): + def _handle_failing_hypothesis(self, hypothesis: list[int]) -> None: """Update the token list for a failing hypothesis.""" if not self.test_queue: # We aren't multithreaded, so just update the list directly. @@ -351,13 +381,13 @@ def _handle_failing_hypothesis(self, hypothesis): with self.merge_lock: self._attempt_merge(hypotheses_to_merge) - def _attempt_merge(self, hypotheses): + def _attempt_merge(self, hypotheses: list[list[int]]) -> None: """Update the required token list if the queued changes don't conflict.""" # If there's nothing to merge, we're done. if not hypotheses: return - aggregate_tokens = set() + aggregate_tokens: set[int] = set() for hypothesis in hypotheses: for token in hypothesis: aggregate_tokens.add(token) @@ -389,8 +419,9 @@ def _attempt_merge(self, hypotheses): self._attempt_merge(front) self._attempt_merge(back) - def _do_single_pass_process(self): + def _do_single_pass_process(self) -> None: """Process through a single pass of our test queue.""" + assert self.test_queue is not None self.currently_processing = True self.test_queue.process() @@ -408,7 +439,7 @@ def _do_single_pass_process(self): # This may trigger another round of processing, so don't hold the lock. self._push_test_to_queue(hypothesis) - def process(self): + def process(self) -> None: """Start a test.""" if not self.test_queue: return @@ -424,7 +455,8 @@ def process(self): self._attempt_merge(hypotheses_to_merge) # Cache functions. - def _contains_required_tokens(self, hypothesis, test_file): + def _contains_required_tokens(self, hypothesis: Sequence[int], + test_file: Any) -> bool: """Check to see if this hypothesis contains untested tokens.""" # It is possible that we could copy this while it is being updated. We do # not block in this case because the worst case scenario is that we run an @@ -450,7 +482,7 @@ def _contains_required_tokens(self, hypothesis, test_file): self._delete_file_if_needed(test_file) return False - def _has_tested(self, hypothesis): + def _has_tested(self, hypothesis: Sequence[int]) -> bool: """Check to see if this hypothesis has been tested before.""" hypothesis_tuple = tuple(hypothesis) with self.cache_lock: @@ -462,7 +494,7 @@ def _has_tested(self, hypothesis): return False # Result checking functions. - def get_result(self): + def get_result(self) -> Any: """Get the result of minimization.""" # Done with minimization, output log one more time self._report_progress(is_final_progress_report=True) @@ -471,21 +503,21 @@ def get_result(self): return self.get_required_tokens() return self.get_current_testcase_data() - def get_required_tokens(self): + def get_required_tokens(self) -> list[Any]: """Return all required tokens for this test case.""" return [t for i, t in enumerate(self.tokens) if self.required_tokens[i]] - def get_required_token_indices(self): + def get_required_token_indices(self) -> list[int]: """Get the indices of all remaining required tokens.""" return [i for i, v in enumerate(self.required_tokens) if v] -def _default_tokenizer(s): +def _default_tokenizer(s: bytes) -> list[bytes]: """Default string tokenizer which splits on newlines.""" return s.split(b'\n') -def _default_combiner(tokens): +def _default_combiner(tokens: list[bytes]) -> bytes: """Default token combiner which assumes each token is a line.""" return b'\n'.join(tokens) @@ -493,40 +525,42 @@ def _default_combiner(tokens): class Minimizer: """Base class for minimizers.""" - def __init__(self, - test_function, - max_threads=1, - tokenizer=_default_tokenizer, - token_combiner=_default_combiner, - tokenize=True, - cleanup_function=None, - single_thread_cleanup_interval=DEFAULT_CLEANUP_INTERVAL, - deadline=None, - get_temp_file=None, - delete_temp_files=True, - batch_size=None, - progress_report_function=None, - file_extension=''): + def __init__( + self, + test_function: Callable[..., bool], + max_threads: int = 1, + tokenizer: Callable[[Any], list[Any]] = _default_tokenizer, + token_combiner: Callable[[list[Any]], Any] = _default_combiner, + tokenize: bool = True, + cleanup_function: Callable[[], None] | None = None, + single_thread_cleanup_interval: int = DEFAULT_CLEANUP_INTERVAL, + deadline: float | None = None, + get_temp_file: Callable[[], Any] | None = None, + delete_temp_files: bool = True, + batch_size: int | None = None, + progress_report_function: Callable[[str], None] | None = None, + file_extension: str = '', + ) -> None: """Initialize a minimizer. A minimizer object can be used multiple times.""" - self.test_function = test_function - self.max_threads = max_threads - self.tokenizer = tokenizer - self.token_combiner = token_combiner - self.tokenize = tokenize - self.cleanup_function = cleanup_function - self.single_thread_cleanup_interval = single_thread_cleanup_interval - self.deadline = deadline - self.get_temp_file = get_temp_file - self.delete_temp_files = delete_temp_files - self.progress_report_function = progress_report_function + self.test_function: Callable[..., bool] = test_function + self.max_threads: int = max_threads + self.tokenizer: Callable[[Any], list[Any]] = tokenizer + self.token_combiner: Callable[[list[Any]], Any] = token_combiner + self.tokenize: bool = tokenize + self.cleanup_function: Callable[[], None] | None = cleanup_function + self.single_thread_cleanup_interval: int = single_thread_cleanup_interval + self.deadline: float | None = deadline + self.delete_temp_files: bool = delete_temp_files + self.progress_report_function: Callable[[str], None] | None = ( + progress_report_function) if batch_size: - self.batch_size = batch_size + self.batch_size: int = batch_size else: self.batch_size = DEFAULT_TESTS_PER_THREAD * max_threads if not get_temp_file: - self.get_temp_file = functools.partial( + self.get_temp_file: Callable[[], Any] = functools.partial( tempfile.NamedTemporaryFile, mode='wb', delete=False, @@ -536,7 +570,9 @@ def __init__(self, self.get_temp_file = get_temp_file @staticmethod - def _handle_constructor_argument(key, kwargs, default=None): + def _handle_constructor_argument(key: str, + kwargs: dict[str, Any], + default: Any = None) -> Any: """Cleanup a keyword argument specific to a subclass and get the value.""" result = default try: @@ -547,11 +583,11 @@ def _handle_constructor_argument(key, kwargs, default=None): return result - def _execute(self, data): + def _execute(self, data: Any) -> Testcase: """Perform minimization on a test case.""" raise NotImplementedError - def minimize(self, data): + def minimize(self, data: Any) -> Any: """Wrapper to perform common tasks and call |_execute|.""" try: testcase = self._execute(data) @@ -573,7 +609,7 @@ def minimize(self, data): return testcase.get_result() - def validate_tokenizer(self, data, testcase): + def validate_tokenizer(self, data: Any, testcase: Testcase) -> bool: """Validate that the tokenizer correctly tokenized the data. This is necessary because if the tokenizer does not recognize a character, it will skip it.""" @@ -591,6 +627,10 @@ def validate_tokenizer(self, data, testcase): return testcase.get_current_testcase_data() == data @staticmethod - def run(data, thread_count=DEFAULT_THREAD_COUNT, file_extension=''): + def run( + data: Any, + thread_count: int = DEFAULT_THREAD_COUNT, + file_extension: str = '', + ) -> Any: """Minimize |data| using this minimizer's default configuration.""" raise NotImplementedError diff --git a/src/clusterfuzz/_internal/bot/minimizer/run.py b/src/clusterfuzz/_internal/bot/minimizer/run.py index 1cf5a964ffe..e97293ab8d6 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/run.py +++ b/src/clusterfuzz/_internal/bot/minimizer/run.py @@ -25,7 +25,7 @@ from . import utils -def main(): +def main() -> None: """Minimize a file.""" minimizers = { 'chunk': chunk_minimizer.ChunkMinimizer, diff --git a/src/clusterfuzz/_internal/bot/minimizer/unicode_minimizer.py b/src/clusterfuzz/_internal/bot/minimizer/unicode_minimizer.py index e1b3afdefb2..c5afc3cbd35 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/unicode_minimizer.py +++ b/src/clusterfuzz/_internal/bot/minimizer/unicode_minimizer.py @@ -15,6 +15,7 @@ unicode escapes with their corresponding UTF-8 encoded characters.""" import re +from typing import Any from . import errors from . import minimizer @@ -30,19 +31,19 @@ rb')') -def is_unicode_escape(s): +def is_unicode_escape(s: bytes) -> re.Match[bytes] | None: """Returns true if string matches \\uXXXX, \\u{...}, or \\xXX pattern.""" return re.fullmatch(UNICODE_TOKEN_PATTERN, s) -def split_and_decode_unicode_literal(s): +def split_and_decode_unicode_literal(s: bytes) -> list[bytes]: """Splits the input file by unicode escapes. Then for each of unicode escapes creates two tokens: (1) unicode escape itself (2) utf-8 encoding of unicode escape.""" intermediate_tokens = re.split(UNICODE_TOKEN_PATTERN, s) - tokens = [] + tokens: list[bytes] = [] for token in intermediate_tokens: tokens.append(token) if re.fullmatch(UNICODE_TOKEN_PATTERN, token): @@ -65,10 +66,10 @@ def split_and_decode_unicode_literal(s): return tokens -def combine_tokens(tokens): +def combine_tokens(tokens: list[bytes]) -> bytes: """If unicode token is still present, remove the token that was supposed to replace it.""" - final_tokens = [] + final_tokens: list[bytes] = [] i = 0 while i < len(tokens): final_tokens.append(tokens[i]) @@ -89,18 +90,18 @@ class UnicodeMinimizer(minimizer.Minimizer): We create a single hypothesis with all unicode escapes. """ - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: kwargs['tokenizer'] = split_and_decode_unicode_literal kwargs['token_combiner'] = combine_tokens minimizer.Minimizer.__init__(self, *args, **kwargs) - def _execute(self, data): + def _execute(self, data: Any) -> minimizer.Testcase: testcase = minimizer.Testcase(data, self) if not self.validate_tokenizer(data, testcase): raise errors.TokenizationFailureError('Unicode minimizer') tokens = testcase.tokens - unicode_hypothesis = [] + unicode_hypothesis: list[int] = [] for i, token in enumerate(tokens): if is_unicode_escape(token): unicode_hypothesis.append(i) @@ -111,7 +112,11 @@ def _execute(self, data): return testcase @staticmethod - def run(data, thread_count=minimizer.DEFAULT_THREAD_COUNT, file_extension=''): + def run( + data: Any, + thread_count: int = minimizer.DEFAULT_THREAD_COUNT, + file_extension: str = '', + ) -> Any: """Try to minimize |data| using a simple line tokenizer.""" unicode_minimizer = UnicodeMinimizer( utils.test, max_threads=thread_count, file_extension=file_extension) diff --git a/src/clusterfuzz/_internal/bot/minimizer/utils.py b/src/clusterfuzz/_internal/bot/minimizer/utils.py index 77463ee647b..45c881d136a 100644 --- a/src/clusterfuzz/_internal/bot/minimizer/utils.py +++ b/src/clusterfuzz/_internal/bot/minimizer/utils.py @@ -17,18 +17,19 @@ import shlex import subprocess +from clusterfuzz._internal.base import utils as base_utils from clusterfuzz._internal.bot.tokenizer.antlr_tokenizer import AntlrTokenizer from clusterfuzz._internal.bot.tokenizer.grammars.HTMLLexer import HTMLLexer from . import errors # TODO(mbarbella): Improve configuration of the test function. -attempts = 1 -test_command = None +attempts: int = 1 +test_command: list[str] | None = None # This list of markers is a copy of code in ClusterFuzz. # This is kept here to keep the code standalone. -STACKTRACE_TOOL_MARKERS = [ +STACKTRACE_TOOL_MARKERS: list[str] = [ ' runtime error: ', 'AddressSanitizer', 'ASAN:', @@ -41,7 +42,7 @@ 'UndefinedBehaviorSanitizer', 'UndefinedSanitizer', ] -STACKTRACE_END_MARKERS = [ +STACKTRACE_END_MARKERS: list[str] = [ 'ABORTING', 'END MEMORY TOOL REPORT', 'End of process memory map.', @@ -52,7 +53,7 @@ '\nExiting', 'minidump has been written', ] -CHECK_FAILURE_MARKERS = [ +CHECK_FAILURE_MARKERS: list[str] = [ 'Check failed:', 'Device rebooted', 'Fatal error in', @@ -62,7 +63,7 @@ ] -def get_size_string(size): +def get_size_string(size: int) -> str: """Return string representation for size.""" if size < 1 << 10: return '%d B' % size @@ -74,7 +75,7 @@ def get_size_string(size): return '%d GB' % (size >> 30) -def has_marker(stacktrace, marker_list): +def has_marker(stacktrace: str, marker_list: list[str]) -> bool: """Return true if the stacktrace has atleast one marker in the marker list.""" for marker in marker_list: @@ -84,19 +85,19 @@ def has_marker(stacktrace, marker_list): return False -def set_test_command(new_test_command): +def set_test_command(new_test_command: str) -> None: """Set the command used for testing.""" global test_command test_command = shlex.split(new_test_command) -def set_test_attempts(new_attempts): +def set_test_attempts(new_attempts: int) -> None: """Set the number of times to attempt the test.""" global attempts attempts = new_attempts -def test(test_path): +def test(test_path: str) -> bool: """Wrapper function to verify that a test does not fail for multiple runs.""" for _ in range(attempts): if not single_test_run(test_path): @@ -104,7 +105,7 @@ def test(test_path): return True -def single_test_run(test_path): +def single_test_run(test_path: str) -> bool: """Hacky test function that checks for certain common errors.""" if not test_command: raise errors.NoCommandError @@ -116,9 +117,11 @@ def single_test_run(test_path): console_output = error.output # If we meet one of these conditions, assume we crashed. - if ((has_marker(console_output, STACKTRACE_TOOL_MARKERS) and - has_marker(console_output, STACKTRACE_END_MARKERS)) or - has_marker(console_output, CHECK_FAILURE_MARKERS)): + stacktrace = base_utils.decode_to_unicode( + console_output) if console_output else '' + if ((has_marker(stacktrace, STACKTRACE_TOOL_MARKERS) and + has_marker(stacktrace, STACKTRACE_END_MARKERS)) or + has_marker(stacktrace, CHECK_FAILURE_MARKERS)): print('Crashed, current test size %s.' % (get_size_string( os.path.getsize(test_path)))) return False @@ -129,11 +132,11 @@ def single_test_run(test_path): return True -def tokenize(data): +def tokenize(data: bytes) -> list[str]: """HTML tokenizer.""" return AntlrTokenizer(HTMLLexer).tokenize(data) -def token_combiner(tokens): +def token_combiner(tokens: list[str]) -> str: """Dummy token combiner.""" return ''.join(tokens) diff --git a/src/clusterfuzz/_internal/bot/tasks/blame_task.py b/src/clusterfuzz/_internal/bot/tasks/blame_task.py index 690e0b079b0..0475cbe7b27 100644 --- a/src/clusterfuzz/_internal/bot/tasks/blame_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/blame_task.py @@ -15,6 +15,7 @@ import json import re +from typing import Any from clusterfuzz._internal.build_management import build_manager from clusterfuzz._internal.build_management import revisions @@ -32,7 +33,10 @@ UNINTERESTING_LINES_REGEX = re.compile(r'.*:(VERBOSE\d|INFO|WARNING):') -def _compute_rolls(start_revisions_dict, end_revisions_dict): +def _compute_rolls( + start_revisions_dict: dict[str, Any], + end_revisions_dict: dict[str, Any], +) -> list[dict[str, Any]]: """Compute rolls between the start and end revision.""" result = [] for path, entry in end_revisions_dict.items(): @@ -61,7 +65,7 @@ def _compute_rolls(start_revisions_dict, end_revisions_dict): return result -def _extract_url_and_sha_from_deps_entry(entry): +def _extract_url_and_sha_from_deps_entry(entry: Any) -> tuple[str, str]: """Split a DEPS file entry into a URL and git sha.""" assert 'url' in entry and 'rev' in entry, 'Unexpected format: %s' % entry url = entry['url'] @@ -74,7 +78,8 @@ def _extract_url_and_sha_from_deps_entry(entry): return url, sha -def _format_component_revisions_for_predator(component_revisions): +def _format_component_revisions_for_predator( + component_revisions: dict[str, Any]) -> list[dict[str, Any]]: """Convert a dict of dependency rolls to the format Predator expects.""" result = [] for path, entry in component_revisions.items(): @@ -88,7 +93,8 @@ def _format_component_revisions_for_predator(component_revisions): return result -def _is_predator_testcase(testcase): +def _is_predator_testcase(testcase: data_types.Testcase, + ) -> tuple[bool, str | None]: """Return bool and error message for whether this testcase is applicable to predator or not.""" if build_manager.is_custom_binary(): @@ -104,7 +110,7 @@ def _is_predator_testcase(testcase): return True, None -def _filter_stacktrace(stacktrace): +def _filter_stacktrace(stacktrace: str) -> str: """Reduces noise from stacktrace and limit its size to avoid pubsub request limit of one megabyte.""" filtered_stacktrace_size = 0 @@ -125,7 +131,10 @@ def _filter_stacktrace(stacktrace): return '\n'.join(reversed(filtered_stacktrace_lines)) -def _prepare_component_revisions_dict(revision, job_type): +def _prepare_component_revisions_dict( + revision: int | str | None, + job_type: str | None, +) -> tuple[dict[str, Any] | None, str | None]: """Get a component revisions dict and git sha for a revision and job type. Revision is expected to be a commit position.""" @@ -140,7 +149,8 @@ def _prepare_component_revisions_dict(revision, job_type): return revisions_dict, revisions_dict['src']['rev'] -def _set_predator_result_with_error(testcase, error_message): +def _set_predator_result_with_error(testcase: data_types.Testcase, + error_message: str | None) -> None: """Sets predator result with error.""" predator_result = { 'result': { @@ -153,14 +163,17 @@ def _set_predator_result_with_error(testcase, error_message): } } - testcase = data_handler.get_testcase_by_id(testcase.key.id()) + testcase_id = testcase.key.id() + assert testcase_id is not None + testcase = data_handler.get_testcase_by_id(testcase_id) testcase.set_metadata( 'predator_result', predator_result, update_testcase=False) testcase.delete_metadata('blame_pending', update_testcase=False) testcase.put() -def _prepare_predator_message(testcase): +def _prepare_predator_message(testcase: data_types.Testcase, + ) -> pubsub.Message | None: """Prepare the json sent to the Predator service for the given test case.""" result, error_message = _is_predator_testcase(testcase) if not result: @@ -172,13 +185,15 @@ def _prepare_predator_message(testcase): # Do a None check since we can return {} for revision = 0. if crash_revisions_dict is None: _set_predator_result_with_error( - testcase, 'Failed to fetch component revisions for revision %s.' % - testcase.crash_revision) + testcase, + 'Failed to fetch component revisions for revision %s.' % + testcase.crash_revision, + ) return None dependency_rolls = [] start_revision_hash = end_revision_hash = None - if ':' in testcase.regression: + if testcase.regression and ':' in testcase.regression: regression_parts = testcase.regression.split(':', 1) start_revision = int(regression_parts[0]) end_revision = int(regression_parts[1]) @@ -188,20 +203,25 @@ def _prepare_predator_message(testcase): # Do a None check since we can return {} for revision = 0. if start_revisions_dict is None: _set_predator_result_with_error( - testcase, 'Failed to fetch component revisions for revision %s.' % - start_revision) + testcase, + 'Failed to fetch component revisions for revision %s.' % + start_revision, + ) return None - end_revisions_dict, end_revision_hash = ( - _prepare_component_revisions_dict(end_revision, testcase.job_type)) + end_revisions_dict, end_revision_hash = _prepare_component_revisions_dict( + end_revision, testcase.job_type) # Do a None check since we can return {} for revision = 0. if end_revisions_dict is None: _set_predator_result_with_error( testcase, - 'Failed to fetch component revisions for revision %s.' % end_revision) + 'Failed to fetch component revisions for revision %s.' % end_revision, + ) return None if start_revision != 0: + assert start_revisions_dict is not None + assert end_revisions_dict is not None dependency_rolls = _compute_rolls(start_revisions_dict, end_revisions_dict) @@ -225,7 +245,8 @@ def _prepare_predator_message(testcase): real_dep_path = SRC_COMPONENT_OVERRIDES.get(repo_url, 'src') if real_dep_path != 'src': for dependency_list in [ - dependency_rolls, crash_revision_component_revisions_list + dependency_rolls, + crash_revision_component_revisions_list, ]: for entry in dependency_list: if entry['dep_path'] == 'src': @@ -254,7 +275,7 @@ def _prepare_predator_message(testcase): 'sanitizer': environment.get_memory_tool_name(testcase.job_type), 'security_flag': testcase.security_flag, 'job_type': testcase.job_type, - 'testcase_id': testcase.key.id() + 'testcase_id': testcase.key.id(), }, 'platform': testcase.platform, 'client_id': 'clusterfuzz', @@ -262,14 +283,15 @@ def _prepare_predator_message(testcase): }).encode('utf-8')) -def _clear_blame_result_and_set_pending_flag(testcase): +def _clear_blame_result_and_set_pending_flag(testcase: data_types.Testcase, + ) -> None: """Clear blame result and set pending bit.""" testcase.set_metadata('blame_pending', True, update_testcase=False) testcase.set_metadata('predator_result', None, update_testcase=False) testcase.put() -def _execute_task(testcase_id): +def _execute_task(testcase_id: int | str) -> None: """Attempt to find the CL introducing the bug associated with testcase_id.""" # Locate the testcase associated with the id. testcase = data_handler.get_testcase_by_id(testcase_id) @@ -287,8 +309,10 @@ def _execute_task(testcase_id): if not message: testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment( - testcase, data_types.TaskState.ERROR, - 'Failed to generate request for Predator') + testcase, + data_types.TaskState.ERROR, + 'Failed to generate request for Predator', + ) return # Clear existing results and mark blame result as pending. @@ -303,7 +327,7 @@ def _execute_task(testcase_id): data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED) -def execute_task(testcase_id, _): +def execute_task(testcase_id: int | str, _: str = '') -> None: """Set logs context and execute blame task.""" testcase = data_handler.get_testcase_by_id(testcase_id) with logs.testcase_log_context(testcase, testcase.get_fuzz_target()): diff --git a/src/clusterfuzz/_internal/bot/tasks/commands.py b/src/clusterfuzz/_internal/bot/tasks/commands.py index 5e9468cd0dc..9b869552138 100644 --- a/src/clusterfuzz/_internal/bot/tasks/commands.py +++ b/src/clusterfuzz/_internal/bot/tasks/commands.py @@ -13,10 +13,13 @@ # limitations under the License. """Run command based on the current task.""" +from collections.abc import Callable import functools import os import sys import time +import types +from typing import Any import uuid from clusterfuzz._internal.base import errors @@ -45,9 +48,9 @@ from clusterfuzz._internal.system import process_handler from clusterfuzz._internal.system import shell -TASK_RETRY_WAIT_LIMIT = 5 * 60 # 5 minutes. +TASK_RETRY_WAIT_LIMIT: int = 5 * 60 # 5 minutes. -_COMMAND_MODULE_MAP = { +_COMMAND_MODULE_MAP: dict[str, types.ModuleType | None] = { 'analyze': analyze_task, 'blame': blame_task, 'corpus_pruning': corpus_pruning_task, @@ -64,7 +67,7 @@ } assert set(_COMMAND_MODULE_MAP.keys()) == set(task_types.COMMAND_TYPES.keys()) -COMMAND_MAP = { +COMMAND_MAP: dict[str, task_types.BaseTask] = { command: task_cls(_COMMAND_MODULE_MAP[command]) for command, task_cls in task_types.COMMAND_TYPES.items() } @@ -78,7 +81,7 @@ class AlreadyRunningError(Error): """Exception raised for a task that is already running on another bot.""" -def cleanup_task_state(): +def cleanup_task_state() -> None: """Cleans state before and after a task is executed.""" # Cleanup stale processes. if environment.is_tworker(): @@ -100,7 +103,7 @@ def cleanup_task_state(): utils.python_gc() -def is_supported_cpu_arch_for_job(): +def is_supported_cpu_arch_for_job() -> bool: """Return true if the current cpu architecture can run this job.""" cpu_arch = environment.get_cpu_arch() if not cpu_arch: @@ -118,7 +121,7 @@ def is_supported_cpu_arch_for_job(): return cpu_arch in supported_cpu_arch_list -def update_environment_for_job(environment_string): +def update_environment_for_job(environment_string: str) -> dict[str, Any]: """Process the environment variable string included with a job.""" # Now parse the job's environment definition. env = environment.parse_environment_definition(environment_string) @@ -148,11 +151,12 @@ def update_environment_for_job(environment_string): return uworker_env -def set_task_payload(func): +def set_task_payload(func: Callable[..., Any]) -> Callable[..., Any]: """Set TASK_PAYLOAD and unset TASK_PAYLOAD.""" @functools.wraps(func) - def wrapper(task_name, task_argument, job_name, *args, **kwargs): + def wrapper(task_name: str, task_argument: str, job_name: str, *args: Any, + **kwargs: Any) -> Any: """Wrapper.""" payload = tasks.construct_payload(task_name, task_argument, job_name) environment.set_value('TASK_PAYLOAD', payload) @@ -160,7 +164,8 @@ def wrapper(task_name, task_argument, job_name, *args, **kwargs): return func(task_name, task_argument, job_name, *args, **kwargs) except: # Truly catch *all* exceptions. e = sys.exc_info()[1] - e.extras = {'task_payload': environment.get_value('TASK_PAYLOAD')} + setattr(e, 'extras', + {'task_payload': environment.get_value('TASK_PAYLOAD')}) raise finally: environment.remove_key('TASK_PAYLOAD') @@ -168,7 +173,7 @@ def wrapper(task_name, task_argument, job_name, *args, **kwargs): return wrapper -def should_update_task_status(task_name): +def should_update_task_status(task_name: str) -> bool: """Whether the task status should be automatically handled.""" return task_name not in [ # Multiple fuzz tasks are expected to run in parallel. @@ -182,7 +187,7 @@ def should_update_task_status(task_name): ] -def start_web_server_if_needed(): +def start_web_server_if_needed() -> None: """Start web server for blackbox fuzzer jobs (non-engine fuzzer jobs).""" if environment.is_engine_fuzzer_job(): return @@ -193,7 +198,7 @@ def start_web_server_if_needed(): logs.error('Failed to start web server, skipping.') -def get_command_object(task_name): +def get_command_object(task_name: str) -> task_types.BaseTask | None: """Returns the command object that execute can be called on.""" task = COMMAND_MAP.get(task_name) if not environment.is_tworker(): @@ -210,7 +215,8 @@ def get_command_object(task_name): return task_types.UTask(_COMMAND_MODULE_MAP[task_name]) -def run_command(task_name, task_argument, job_name, uworker_env): +def run_command(task_name: str, task_argument: str, job_name: str, + uworker_env: dict[str, Any] | None) -> Any: """Runs the command.""" task = get_command_object(task_name) if not task: @@ -267,7 +273,7 @@ def run_command(task_name, task_argument, job_name, uworker_env): return result -def process_command(task): +def process_command(task: tasks.Task) -> Any: """Figures out what to do with the given task and executes the command.""" logs.info(f'Executing command "{task.payload()}"') if not task.payload().strip(): @@ -282,21 +288,21 @@ def process_command(task): # pylint: disable=too-many-nested-blocks # TODO(mbarbella): Rewrite this function to avoid nesting issues. @set_task_payload -def process_command_impl(task_name, - task_argument, - job_name, - high_end, - is_command_override, - queue=None): +def process_command_impl(task_name: str, + task_argument: str, + job_name: str, + high_end: bool, + is_command_override: bool, + queue: str | None = None) -> Any: """Implementation of process_command.""" - uworker_env = {} + uworker_env: dict[str, Any] = {} environment.set_value('TASK_NAME', task_name) environment.set_value('TASK_ARGUMENT', task_argument) environment.set_value('JOB_NAME', job_name) if task_name in {'uworker_main', 'postprocess'}: # We want the id of the task we are processing, not "uworker_main", or # "postprocess". - task_id = None + task_id: uuid.UUID | None = None else: task_id = uuid.uuid4() environment.set_value('CF_TASK_ID', task_id) @@ -358,7 +364,7 @@ def process_command_impl(task_name, time.sleep(environment.get_value('FAIL_WAIT')) return None - testcase = None + testcase: data_types.Testcase | None = None if task_utils.is_testcase_based_task(task_name): testcase = data_handler.get_entity_by_type_and_id(data_types.Testcase, task_argument) diff --git a/src/clusterfuzz/_internal/bot/tasks/impact_task.py b/src/clusterfuzz/_internal/bot/tasks/impact_task.py index b1f86cb4844..f4713e4f016 100644 --- a/src/clusterfuzz/_internal/bot/tasks/impact_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/impact_task.py @@ -14,6 +14,10 @@ """Impact task. Determine whether or not a test case affects production branches.""" +from collections.abc import Mapping +from typing import Any +from typing import cast + from clusterfuzz._internal.base import utils from clusterfuzz._internal.build_management import build_manager from clusterfuzz._internal.build_management import revisions @@ -28,20 +32,23 @@ class Impact: """Represents impact on a build type.""" - def __init__(self, version='', likely=False, extra_trace=''): + def __init__(self, + version: str | int = '', + likely: bool = False, + extra_trace: str = '') -> None: self.version = str(version) self.likely = likely self.extra_trace = extra_trace - def __str__(self): + def __str__(self) -> str: return (f'version: {self.version}, likely: {self.likely}, ' f'extra_trace: {self.extra_trace}') - def is_empty(self): + def is_empty(self) -> bool: """Return True if empty.""" return not self.version - def __eq__(self, other): + def __eq__(self, other: 'Impact') -> bool: # type: ignore return (self.version == other.version and self.likely == other.likely and self.extra_trace == other.extra_trace) @@ -49,29 +56,34 @@ def __eq__(self, other): class Impacts: """Represents impacts on different release channels.""" - def __init__(self, stable=None, beta=None, extended_stable=None, head=None): + def __init__(self, + stable: Impact | None = None, + beta: Impact | None = None, + extended_stable: Impact | None = None, + head: Impact | None = None) -> None: self.stable = stable or Impact() self.beta = beta or Impact() self.extended_stable = extended_stable or Impact() self.head = head or Impact() - def is_empty(self): + def is_empty(self) -> bool: return (self.extended_stable.is_empty() and self.stable.is_empty() and self.beta.is_empty() and self.head.is_empty()) - def get_extra_trace(self): + def get_extra_trace(self) -> str: return ( self.extended_stable.extra_trace + '\n' + self.stable.extra_trace + '\n' + self.beta.extra_trace + '\n' + self.head.extra_trace).strip() - def __eq__(self, other): + def __eq__(self, other: 'Impacts') -> bool: # type: ignore return (self.extended_stable == other.extended_stable and self.stable == other.stable and self.beta == other.beta and self.head == other.head) -def get_chromium_component_start_and_end_revision(start_revision, end_revision, - job_type): +def get_chromium_component_start_and_end_revision( + start_revision: int, end_revision: int, + job_type: str | None) -> tuple[int, int]: """Get revisions from chromium component.""" component_rev_list = revisions.get_component_range_list( start_revision, end_revision, job_type) @@ -84,7 +96,8 @@ def get_chromium_component_start_and_end_revision(start_revision, end_revision, return start_revision, end_revision -def get_start_and_end_revision(regression_range, job_type): +def get_start_and_end_revision(regression_range: str | None, + job_type: str | None) -> tuple[int, int]: """Get start and end revision.""" start_revision, end_revision = revisions.get_start_and_end_revision( regression_range) @@ -97,14 +110,16 @@ def get_start_and_end_revision(regression_range, job_type): return start_revision, end_revision -def is_valid_regression_range(regression_range, job_type): +def is_valid_regression_range(regression_range: str | None, + job_type: str | None) -> bool: """Return whether we have a valid regression range.""" start, end = get_start_and_end_revision(regression_range, job_type) return start != 0 or end != 0 -def get_component_information_by_name(chromium_revision, - component_display_name): +def get_component_information_by_name( + chromium_revision: str | int, + component_display_name: str) -> dict[str, Any] | None: """Returns a dictionary with information about a component at a revision.""" lower_name = component_display_name.lower() component_revisions = revisions.get_component_revisions_dict( @@ -122,10 +137,10 @@ def get_component_information_by_name(chromium_revision, return None -def get_component_impacts_from_url(component_name, - regression_range, - job_type, - platform=None): +def get_component_impacts_from_url(component_name: str, + regression_range: str | None, + job_type: str | None, + platform: str | None = None) -> Impacts: """Gets component impact string using the build information url.""" logs.info('Getting component impacts from URL. Component name %s, ' 'regression range %s, job type %s, platform %s.' % @@ -140,7 +155,7 @@ def get_component_impacts_from_url(component_name, if not build_revision_mappings: return Impacts() - found_impacts = {} + found_impacts: dict[str, Impact] = {} for build in ['extended_stable', 'stable', 'beta', 'canary']: mapping = build_revision_mappings.get(build) logs.info('Considering impacts for %s.' % (build)) @@ -177,11 +192,13 @@ def get_component_impacts_from_url(component_name, found_impacts['extended_stable'], found_impacts['canary']) -def get_impacts_from_url(regression_range, job_type, platform=None): +def get_impacts_from_url(regression_range: str | None, + job_type: str | None, + platform: str | None = None) -> Impacts: """Gets impact string using the build information url.""" logs.info('Get component impacts from URL: range %s, ' 'job type %s.' % (regression_range, str(job_type))) - component_name = data_handler.get_component_name(job_type) + component_name = data_handler.get_component_name(cast(str, job_type)) if component_name: return get_component_impacts_from_url(component_name, regression_range, job_type, platform) @@ -211,10 +228,10 @@ def get_impacts_from_url(regression_range, job_type, platform=None): return Impacts(stable, beta, extended_stable, head) -def get_impact(build_revision, - start_revision, - end_revision, - is_last_possible_build=False): +def get_impact(build_revision: Mapping[str, Any] | None, + start_revision: int, + end_revision: int, + is_last_possible_build: bool = False) -> Impact: """Return a Impact object represents the impact on a given build_type. Or return None.""" if not build_revision: @@ -245,7 +262,8 @@ def get_impact(build_revision, return Impact(version, likely=True) -def get_head_impact(build_revision_mappings, start_revision, end_revision): +def get_head_impact(build_revision_mappings: Mapping[str, Any], + start_revision: int, end_revision: int) -> Impact: """Return the impact on 'head', i.e. the latest build we can find.""" latest_build = build_revision_mappings.get('canary') if latest_build is None: @@ -254,7 +272,8 @@ def get_head_impact(build_revision_mappings, start_revision, end_revision): latest_build, start_revision, end_revision, is_last_possible_build=True) -def set_testcase_with_impacts(testcase, impacts): +def set_testcase_with_impacts(testcase: data_types.Testcase, + impacts: Impacts) -> None: """Set testcase's impact-related fields given impacts.""" testcase.impact_extended_stable_version = impacts.extended_stable.version testcase.impact_extended_stable_version_likely = \ @@ -268,7 +287,7 @@ def set_testcase_with_impacts(testcase, impacts): testcase.is_impact_set_flag = True -def _execute_task(testcase_id, job_type): +def _execute_task(testcase_id: int | str, job_type: str) -> None: """Attempt to find if the testcase affects release branches on Chromium.""" # We don't need job_type but it's supplied to all tasks. del job_type @@ -328,12 +347,12 @@ def _execute_task(testcase_id, job_type): testcase = data_handler.get_testcase_by_id(testcase_id) set_testcase_with_impacts(testcase, impacts) testcase_utils.emit_testcase_triage_duration_metric( - testcase_id, + cast(int, testcase_id), testcase_utils.TESTCASE_TRIAGE_DURATION_IMPACT_COMPLETED_STEP) data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED) -def execute_task(testcase_id, job_type): +def execute_task(testcase_id: int | str, job_type: str) -> None: """Set logs context and execute Chromium impact task.""" testcase = data_handler.get_testcase_by_id(testcase_id) with logs.testcase_log_context(testcase, testcase.get_fuzz_target()): diff --git a/src/clusterfuzz/_internal/bot/tasks/setup.py b/src/clusterfuzz/_internal/bot/tasks/setup.py index 9e8aab54316..1bedaef6788 100644 --- a/src/clusterfuzz/_internal/bot/tasks/setup.py +++ b/src/clusterfuzz/_internal/bot/tasks/setup.py @@ -17,7 +17,8 @@ import os import shlex import time -from typing import Optional +from typing import Any +from typing import cast import zipfile from clusterfuzz._internal.base import dates @@ -44,15 +45,16 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import shell -_BOT_DIR = 'bot' -_DATA_BUNDLE_CACHE_COUNT = 10 -_DATA_BUNDLE_SYNC_INTERVAL_IN_SECONDS = 6 * 60 * 60 -_SYNC_FILENAME = '.sync' -_TESTCASE_ARCHIVE_EXTENSION = '.zip' -_EXECUTABLE_PERMISSIONS = 0o750 +_BOT_DIR: str = 'bot' +_DATA_BUNDLE_CACHE_COUNT: int = 10 +_DATA_BUNDLE_SYNC_INTERVAL_IN_SECONDS: int = 6 * 60 * 60 +_SYNC_FILENAME: str = '.sync' +_TESTCASE_ARCHIVE_EXTENSION: str = '.zip' +_EXECUTABLE_PERMISSIONS: int = 0o750 -def _set_timeout_value_from_user_upload(testcase_id, uworker_env): +def _set_timeout_value_from_user_upload(testcase_id: int | str, + uworker_env: dict[str, Any]) -> None: """Get the timeout associated with this testcase.""" metadata = data_types.TestcaseUploadMetadata.query( data_types.TestcaseUploadMetadata.testcase_id == int(testcase_id)).get() @@ -60,15 +62,15 @@ def _set_timeout_value_from_user_upload(testcase_id, uworker_env): uworker_env['TEST_TIMEOUT'] = str(metadata.timeout) -def set_default_app_args(job_type, uworker_env): +def set_default_app_args(job_type: str, uworker_env: dict[str, Any]) -> None: """Sets default app args for the job in the uworker environment.""" job_app_args = data_handler.get_value_from_job_definition( job_type, 'APP_ARGS', default='') uworker_env['JOB_DEFAULT_APP_ARGS'] = job_app_args -def _copy_testcase_to_device_and_setup_environment(testcase, - testcase_file_path): +def _copy_testcase_to_device_and_setup_environment( + testcase: data_types.Testcase, testcase_file_path: str) -> None: """Android specific setup steps for testcase.""" # Copy test(s) to device. android.device.push_testcases_to_device() @@ -105,7 +107,8 @@ def _copy_testcase_to_device_and_setup_environment(testcase, android.adb.run_shell_command(['chmod', '0755', device_testcase_file_path]) -def _get_application_arguments(testcase, job_type, task_name): +def _get_application_arguments(testcase: data_types.Testcase, job_type: str, + task_name: str) -> str | None: """Get application arguments to use for setting up |testcase|. Use minimized arguments if available. For variant task, where we run a testcase against another job type, use both minimized arguments and application arguments @@ -140,12 +143,13 @@ def _get_application_arguments(testcase, job_type, task_name): return app_args -def _setup_memory_tools_environment(testcase): +def _setup_memory_tools_environment(testcase: data_types.Testcase) -> None: """Set up environment for various memory tools used.""" env = testcase.get_metadata('env') if not env: environment.reset_current_memory_tool_options( - redzone_size=testcase.redzone, disable_ubsan=testcase.disable_ubsan) + redzone_size=testcase.redzone or 0, + disable_ubsan=bool(testcase.disable_ubsan)) return for options_name, options_value in env.items(): @@ -155,7 +159,7 @@ def _setup_memory_tools_environment(testcase): environment.set_memory_tool_options(options_name, options_value) -def prepare_environment_for_testcase(testcase): +def prepare_environment_for_testcase(testcase: data_types.Testcase) -> None: """Set various environment variables based on the test case.""" _setup_memory_tools_environment(testcase) @@ -176,7 +180,7 @@ def prepare_environment_for_testcase(testcase): environment.set_value('FUZZ_TARGET', fuzz_target) -def handle_setup_testcase_error(uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_setup_testcase_error(uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Error handler for setup_testcase that is called by uworker_postprocess.""" # Get the testcase again because it is too hard to set the testcase for # partially migrated tasks. @@ -199,18 +203,21 @@ def handle_setup_testcase_error(uworker_output: uworker_msg_pb2.Output): # pyli wait_time=testcase_fail_wait) -ERROR_HANDLER = uworker_handle_errors.CompositeErrorHandler({ - uworker_msg_pb2.ErrorType.TESTCASE_SETUP: handle_setup_testcase_error, # pylint: disable=no-member -}) +ERROR_HANDLER: uworker_handle_errors.CompositeErrorHandler = ( + uworker_handle_errors.CompositeErrorHandler({ + uworker_msg_pb2.ErrorType.TESTCASE_SETUP: handle_setup_testcase_error, # pylint: disable=no-member + })) -def preprocess_setup_testcase(testcase, - uworker_env, - fuzzer_override=None, - with_deps=True): +def preprocess_setup_testcase( + testcase: data_types.Testcase, + uworker_env: dict[str, Any], + fuzzer_override: str | None = None, + with_deps: bool = True) -> uworker_msg_pb2.SetupInput: # pylint: disable=no-member """Preprocessing for setup_testcase function.""" fuzzer_name = fuzzer_override or testcase.fuzzer_name testcase_id = testcase.key.id() + assert testcase_id is not None if fuzzer_name and not with_deps: logs.info(f'Skipping fuzzer preprocess: {fuzzer_name}.') if fuzzer_name and with_deps: @@ -254,11 +261,15 @@ def preprocess_setup_testcase(testcase, return setup_input -def setup_testcase(testcase: data_types.Testcase, job_type: str, - setup_input: uworker_msg_pb2.SetupInput): # pylint: disable=no-member +def setup_testcase( + testcase: data_types.Testcase, + job_type: str, + setup_input: uworker_msg_pb2.SetupInput # pylint: disable=no-member +) -> tuple[list[str] | None, str | None, uworker_msg_pb2.Output | None]: # pylint: disable=no-member """Sets up the testcase and needed dependencies like fuzzer, data bundle, etc.""" testcase_id = testcase.key.id() + assert testcase_id is not None # Prepare an error result to return in case of error. # Only include uworker_input for callers that aren't deserializing the output # and thus, uworker_io is not adding the input to. @@ -312,9 +323,11 @@ def setup_testcase(testcase: data_types.Testcase, job_type: str, return file_list, testcase_file_path, None -def _get_testcase_file_and_path(testcase): +def _get_testcase_file_and_path( + testcase: data_types.Testcase) -> tuple[str, str]: """Figure out the relative path and input directory for this testcase.""" testcase_absolute_path = testcase.absolute_path + assert testcase_absolute_path is not None # This hack is needed so that we can run a testcase generated on windows, on # linux. os.path.isabs return false on paths like c:\a\b\c. @@ -329,12 +342,14 @@ def _get_testcase_file_and_path(testcase): # Default directory for testcases. input_directory = environment.get_value('FUZZ_INPUTS') + assert input_directory is not None if not testcase_path_is_absolute: testcase_path = os.path.join(input_directory, testcase_absolute_path) return input_directory, testcase_path # Root directory can be different on bots. Fix the path to account for this. root_directory = environment.get_value('ROOT_DIR') + assert root_directory is not None search_string = '%s%s%s' % (os.sep, _BOT_DIR, os.sep) search_index = testcase_absolute_path.find(search_string) relative_path = testcase_absolute_path[search_index + len(search_string):] @@ -343,35 +358,41 @@ def _get_testcase_file_and_path(testcase): return input_directory, testcase_path -def get_signed_testcase_download_url(testcase): +def get_signed_testcase_download_url(testcase: data_types.Testcase) -> str: """Returns a signed download URL for the testcase.""" key, _ = _get_testcase_key_and_archive_status(testcase) return blobs.get_signed_download_url(key) -def _get_testcase_key_and_archive_status(testcase): +def _get_testcase_key_and_archive_status( + testcase: data_types.Testcase) -> tuple[str, bool]: """Returns the testcase's key and whether or not it is archived.""" if _is_testcase_minimized(testcase): key = testcase.minimized_keys - archived = bool(testcase.archive_state & data_types.ArchiveStatus.MINIMIZED) + assert key is not None + archived = bool( + (testcase.archive_state or 0) & data_types.ArchiveStatus.MINIMIZED) return key, archived key = testcase.fuzzed_keys - archived = bool(testcase.archive_state & data_types.ArchiveStatus.FUZZED) + assert key is not None + archived = bool( + (testcase.archive_state or 0) & data_types.ArchiveStatus.FUZZED) return key, archived -def _is_testcase_minimized(testcase): +def _is_testcase_minimized(testcase: data_types.Testcase) -> bool: """Returns True if the testcase is minimized.""" - return testcase.minimized_keys and testcase.minimized_keys != 'NA' + return bool(testcase.minimized_keys and testcase.minimized_keys != 'NA') -def download_testcase(testcase_download_url, dst): +def download_testcase(testcase_download_url: str, dst: str) -> str: """Downloads a testcase from a signed url""" return storage.download_signed_url_to_filepath(testcase_download_url, dst) -def unpack_testcase(testcase, testcase_download_url): +def unpack_testcase(testcase: data_types.Testcase, + testcase_download_url: str) -> tuple[list[str] | None, str]: """Unpacks a testcase and returns all files it is composed of.""" # Figure out where the testcase file should be stored. input_directory, testcase_file_path = _get_testcase_file_and_path(testcase) @@ -382,6 +403,7 @@ def unpack_testcase(testcase, testcase_download_url): os.path.join(input_directory, str(testcase.key.id()) + _TESTCASE_ARCHIVE_EXTENSION)) elif archived: + assert testcase.archive_filename is not None temp_filename = os.path.join(input_directory, testcase.archive_filename) else: temp_filename = testcase_file_path @@ -416,21 +438,22 @@ def unpack_testcase(testcase, testcase_download_url): return file_list, testcase_file_path -def _get_data_bundle_update_lock_name(data_bundle_name): +def _get_data_bundle_update_lock_name(data_bundle_name: str) -> str: """Return the lock key name for the given data bundle.""" return f'update:data_bundle:{data_bundle_name}' -def _get_data_bundle_sync_file_path(data_bundle_directory): +def _get_data_bundle_sync_file_path(data_bundle_directory: str) -> str: """Return path to data bundle sync file.""" return os.path.join(data_bundle_directory, _SYNC_FILENAME) -def _clear_old_data_bundles_if_needed(): +def _clear_old_data_bundles_if_needed() -> None: """Clear old data bundles so as to keep the disk cache restricted to |_DATA_BUNDLE_CACHE_COUNT| data bundles and prevent potential out-of-disk spaces.""" data_bundles_directory = environment.get_value('DATA_BUNDLES_DIR') + assert data_bundles_directory is not None dirs = [] for filename in os.listdir(data_bundles_directory): @@ -447,7 +470,8 @@ def _clear_old_data_bundles_if_needed(): shell.remove_directory(dir_to_remove) -def _should_update_data_bundle(data_bundle, data_bundle_directory): +def _should_update_data_bundle(data_bundle: data_types.DataBundle, + data_bundle_directory: str) -> bool: """Returns True if the data bundle should be updated because it is out of date.""" # Check if data bundle is up to date. If yes, skip the update. @@ -464,7 +488,9 @@ def _should_update_data_bundle(data_bundle, data_bundle_directory): return True -def _prepare_update_data_bundle(fuzzer, data_bundle): +def _prepare_update_data_bundle( + fuzzer: data_types.Fuzzer, + data_bundle: data_types.DataBundle) -> str | None: """Create necessary directories to download the data bundle.""" data_bundle_directory = _get_data_bundle_directory(fuzzer, data_bundle) if not data_bundle_directory: @@ -479,7 +505,7 @@ def _prepare_update_data_bundle(fuzzer, data_bundle): return data_bundle_directory -def take_trusted_host_path(): +def take_trusted_host_path() -> bool: if environment.is_uworker(): return False return environment.is_trusted_host() @@ -493,7 +519,7 @@ def update_data_bundle( data_types.DataBundle) logs.info('Setting up data bundle %s.' % data_bundle) data_bundle_directory = _prepare_update_data_bundle(fuzzer, data_bundle) - + assert data_bundle_directory is not None if not _should_update_data_bundle(data_bundle, data_bundle_directory): return True @@ -501,6 +527,7 @@ def update_data_bundle( # No need to sync anything if this is a search index data bundle. In that # case, the fuzzer will generate testcases from a gcs bucket periodically. + assert data_bundle.name is not None if not _is_search_index_data_bundle(data_bundle.name): if not (take_trusted_host_path() and data_bundle.sync_to_worker): @@ -542,7 +569,7 @@ def update_data_bundle( return True -def _set_fuzzer_env_vars(fuzzer): +def _set_fuzzer_env_vars(fuzzer: data_types.Fuzzer) -> None: """Sets fuzzer env vars for fuzzer set up.""" environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content) # Adjust the test timeout, if user has provided one. @@ -567,7 +594,9 @@ def _set_fuzzer_env_vars(fuzzer): environment.set_value('FUZZ_INPUTS', testcase_disk_directory) -def preprocess_get_data_bundles(data_bundle_name, setup_input): +def preprocess_get_data_bundles( + data_bundle_name: str | None, + setup_input: uworker_msg_pb2.SetupInput) -> None: # pylint: disable=no-member """Gets the data bundels corresponding to data_bundle_name (if any) and adds them to setup_input.""" if not data_bundle_name: @@ -596,7 +625,7 @@ def preprocess_update_fuzzer_and_data_bundles( update_input = uworker_msg_pb2.SetupInput( # pylint: disable=no-member fuzzer_name=fuzzer_name, - fuzzer=uworker_io.entity_to_protobuf(fuzzer)) + fuzzer=cast(Any, uworker_io.entity_to_protobuf(fuzzer))) preprocess_get_data_bundles(fuzzer.data_bundle_name, update_input) update_input.fuzzer_log_upload_url = storage.get_signed_upload_url( fuzzer_logs.get_logs_gcs_path(fuzzer_name=fuzzer_name)) @@ -630,6 +659,7 @@ def _update_fuzzer( if fuzzer.builtin: return True + assert fuzzer.revision is not None if not revisions.needs_update(version_file, fuzzer.revision): return True @@ -641,6 +671,7 @@ def _update_fuzzer( return False # Copy the archive to local disk and unpack it. + assert fuzzer.filename is not None archive_path = os.path.join(fuzzer_directory, fuzzer.filename) if not storage.download_signed_url_to_filepath( update_input.fuzzer_download_url, archive_path): @@ -660,6 +691,7 @@ def _update_fuzzer( return False + assert fuzzer.executable_path is not None fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path) if not os.path.exists(fuzzer_path): error_message = ('Fuzzer executable %s not found. ' @@ -683,7 +715,7 @@ def _update_fuzzer( return True -def _set_up_data_bundles(update_input: uworker_msg_pb2.SetupInput): # pylint: disable=no-member +def _set_up_data_bundles(update_input: uworker_msg_pb2.SetupInput) -> bool: # pylint: disable=no-member """Sets up data bundles. Helper for update_fuzzer_and_data_bundles.""" # Setup data bundles associated with this fuzzer. logs.info('Setting up data bundles.') @@ -696,8 +728,8 @@ def _set_up_data_bundles(update_input: uworker_msg_pb2.SetupInput): # pylint: d return True -def update_fuzzer_and_data_bundles( - update_input: uworker_msg_pb2.SetupInput) -> Optional[data_types.Fuzzer]: # pylint: disable=no-member +def update_fuzzer_and_data_bundles(update_input: uworker_msg_pb2.SetupInput # pylint: disable=no-member + ) -> data_types.Fuzzer | None: """Updates the fuzzer specified by |update_input| and its data bundles.""" fuzzer = uworker_io.entity_from_protobuf(update_input.fuzzer, data_types.Fuzzer) @@ -736,13 +768,14 @@ def update_fuzzer_and_data_bundles( return fuzzer -def _is_search_index_data_bundle(data_bundle_name): +def _is_search_index_data_bundle(data_bundle_name: str) -> bool: """Return true on if this is a search index data bundle, false otherwise.""" return data_bundle_name.startswith( testcase_manager.SEARCH_INDEX_BUNDLE_PREFIX) -def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): +def _is_data_bundle_up_to_date(data_bundle: data_types.DataBundle, + data_bundle_directory: str) -> bool: """Return true if the data bundle is up to date, false otherwise.""" sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) @@ -765,6 +798,7 @@ def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): # For search index data bundle, we don't sync them from bucket. Instead, we # rely on the fuzzer to generate testcases periodically. + assert data_bundle.name is not None if _is_search_index_data_bundle(data_bundle.name): return False @@ -779,7 +813,9 @@ def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): return False -def get_data_bundle_directory(fuzzer, setup_input): +def get_data_bundle_directory( + fuzzer: data_types.Fuzzer, + setup_input: uworker_msg_pb2.SetupInput) -> str | None: # pylint: disable=no-member """Public interface for _get_data_bundle_directory.""" if not setup_input.data_bundle_corpuses: data_bundle = None @@ -793,7 +829,9 @@ def get_data_bundle_directory(fuzzer, setup_input): return _get_data_bundle_directory(fuzzer, data_bundle) -def _get_data_bundle_directory(fuzzer, data_bundle): +def _get_data_bundle_directory( + fuzzer: data_types.Fuzzer, + data_bundle: data_types.DataBundle | None) -> str | None: """Return data bundle data directory.""" # Store corpora for built-in fuzzers like libFuzzer in the same directory # as other local data bundles. This makes it easy to clear them when we run @@ -807,21 +845,25 @@ def _get_data_bundle_directory(fuzzer, data_bundle): # have their own data bundle. return environment.get_value('FUZZ_DATA') + assert local_data_bundles_directory is not None + assert data_bundle.name is not None local_data_bundle_directory = os.path.join(local_data_bundles_directory, data_bundle.name) return local_data_bundle_directory -def get_fuzzer_directory(fuzzer_name): +def get_fuzzer_directory(fuzzer_name: str) -> str: """Return directory used by a fuzzer.""" fuzzer_directory = environment.get_value('FUZZERS_DIR') + assert fuzzer_directory is not None fuzzer_directory = os.path.join(fuzzer_directory, fuzzer_name) return fuzzer_directory -def archive_testcase_and_dependencies_in_gcs(resource_list, testcase_path: str, - upload_url: str): +def archive_testcase_and_dependencies_in_gcs( + resource_list: list[str] | None, testcase_path: str, + upload_url: str) -> tuple[bool | None, str | None, str | None]: """Archive testcase and its dependencies, and store in blobstore. Returns whether it is archived, the absolute_filename, and the zip_filename.""" if not os.path.exists(testcase_path): @@ -940,6 +982,7 @@ def setup_local_testcase(testcase: data_types.Testcase) -> str | None: blob_key = testcase.minimized_keys else: blob_key = testcase.fuzzed_keys + assert blob_key is not None if not blobs.read_blob_to_disk(blob_key, testcase_file_path): logs.error(f'Failed to download testcase from blobstore: {blob_key}') # Returning None for path when download fails @@ -984,6 +1027,7 @@ def setup_local_fuzzer(fuzzer_name: str) -> bool: logs.info(f'Fuzzer {fuzzer_name} is builtin, no setup required.') return True + assert fuzzer.name is not None fuzzer_directory: str = get_fuzzer_directory(fuzzer.name) try: @@ -994,7 +1038,9 @@ def setup_local_fuzzer(fuzzer_name: str) -> bool: logs.error(f'Error clearing fuzzer directory {fuzzer_directory}: {e}') return False + assert fuzzer.filename is not None archive_path = os.path.join(fuzzer_directory, fuzzer.filename) + assert fuzzer.blobstore_key is not None if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path): logs.error('Failed to download fuzzer archive from blobstore: ' f'{fuzzer.blobstore_key}') @@ -1018,6 +1064,7 @@ def setup_local_fuzzer(fuzzer_name: str) -> bool: logs.warning( f'Failed to remove temporary archive file {archive_path}: {e}') + assert fuzzer.executable_path is not None fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path) if not os.path.exists(fuzzer_path): logs.error( diff --git a/src/clusterfuzz/_internal/bot/tasks/task_creation.py b/src/clusterfuzz/_internal/bot/tasks/task_creation.py index 0f1bdf26b09..f414e4cc8d6 100644 --- a/src/clusterfuzz/_internal/bot/tasks/task_creation.py +++ b/src/clusterfuzz/_internal/bot/tasks/task_creation.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Common functions for task creation for test cases.""" + from clusterfuzz._internal.base import bisection from clusterfuzz._internal.base import tasks from clusterfuzz._internal.base import utils @@ -24,8 +25,8 @@ from clusterfuzz._internal.system import environment -def mark_unreproducible_if_flaky(testcase, task_name, - potentially_flaky) -> None: +def mark_unreproducible_if_flaky(testcase: data_types.Testcase, task_name: str, + potentially_flaky: bool) -> None: """Check to see if a test case appears to be flaky.""" # If this run does not suggest that we are flaky, clear the flag and assume @@ -34,7 +35,7 @@ def mark_unreproducible_if_flaky(testcase, task_name, testcase.set_metadata('potentially_flaky', False) return - # If we have not been marked as potentially flaky in the past, don't mark + # If this is the first time that we are seeing potential flakiness, do not # mark the test case as unreproducible yet. It is now potentially flaky. if not testcase.get_metadata('potentially_flaky'): testcase.set_metadata('potentially_flaky', True) @@ -71,7 +72,7 @@ def mark_unreproducible_if_flaky(testcase, task_name, bisection.request_bisection(testcase) -def create_blame_task_if_needed(testcase): +def create_blame_task_if_needed(testcase: data_types.Testcase) -> None: """Creates a blame task if needed.""" # Blame doesn't work for non-chromium projects. if not utils.is_chromium(): @@ -113,7 +114,7 @@ def create_blame_task_if_needed(testcase): tasks.add_task('blame', testcase.key.id(), testcase.job_type) -def create_impact_task_if_needed(testcase): +def create_impact_task_if_needed(testcase: data_types.Testcase) -> None: """Creates an impact task if needed.""" # Impact doesn't make sense for non-chromium projects. if not utils.is_chromium(): @@ -131,12 +132,12 @@ def create_impact_task_if_needed(testcase): tasks.add_task('impact', testcase.key.id(), testcase.job_type) -def create_minimize_task_if_needed(testcase): +def create_minimize_task_if_needed(testcase: data_types.Testcase) -> None: """Creates a minimize task if needed.""" tasks.add_task('minimize', testcase.key.id(), testcase.job_type) -def create_regression_task_if_needed(testcase): +def create_regression_task_if_needed(testcase: data_types.Testcase) -> None: """Creates a regression task if needed.""" # We cannot run regression job for custom binaries since we don't have any # archived builds for previous revisions. We only track the last uploaded @@ -147,7 +148,7 @@ def create_regression_task_if_needed(testcase): tasks.add_task('regression', testcase.key.id(), testcase.job_type) -def create_variant_tasks_if_needed(testcase): +def create_variant_tasks_if_needed(testcase: data_types.Testcase) -> None: """Creates a variant task if needed.""" # TODO(https://b.corp.google.com/issues/328691756): Allow untrusted # testcases to only run untrusted variants. @@ -164,13 +165,14 @@ def create_variant_tasks_if_needed(testcase): if not testcase_job_is_engine: testcase_job = ( data_types.Job.query(data_types.Job.name == testcase.job_type).get()) + assert testcase_job is not None testcase_job_environment = testcase_job.get_environment() testcase_job_app_name = testcase_job_environment.get('APP_NAME') num_variant_tasks = 0 for job in jobs: # The variant needs to be tested in a different job type than us. job_type = job.name - if testcase.job_type == job_type: + if not job_type or testcase.job_type == job_type: continue # Don't try to reproduce engine fuzzer testcase with blackbox fuzzer @@ -205,7 +207,7 @@ def create_variant_tasks_if_needed(testcase): logs.info(f'Number of variant tasks: {num_variant_tasks}.') -def create_symbolize_task_if_needed(testcase): +def create_symbolize_task_if_needed(testcase: data_types.Testcase) -> None: """Creates a symbolize task if needed.""" # We cannot run symbolize job for custom binaries since we don't have any # archived symbolized builds. @@ -219,7 +221,7 @@ def create_symbolize_task_if_needed(testcase): tasks.add_task('symbolize', testcase.key.id(), testcase.job_type) -def create_tasks(testcase): +def create_tasks(testcase: data_types.Testcase) -> None: """Create tasks like minimization, regression, impact, progression, stack stack for a newly generated testcase.""" # No need to create progression task. It is automatically created by the cron @@ -264,7 +266,7 @@ def create_tasks(testcase): create_postminimize_tasks(testcase) -def create_postminimize_tasks(testcase): +def create_postminimize_tasks(testcase: data_types.Testcase) -> None: """Create assorted tasks needed after minimize task completes.""" if testcase.status == 'Duplicate' or testcase.duplicate_of: return diff --git a/src/clusterfuzz/_internal/bot/tasks/task_types.py b/src/clusterfuzz/_internal/bot/tasks/task_types.py index 9152ec98880..aac6d40b8c4 100644 --- a/src/clusterfuzz/_internal/bot/tasks/task_types.py +++ b/src/clusterfuzz/_internal/bot/tasks/task_types.py @@ -15,6 +15,8 @@ base/tasks.py depends on this module and many things commands.py imports depend on base/tasks.py (i.e. avoiding circular imports).""" +from typing import Any + from clusterfuzz._internal import swarming from clusterfuzz._internal.base import errors from clusterfuzz._internal.base import tasks @@ -31,14 +33,15 @@ class BaseTask: """Base module for tasks.""" @staticmethod - def is_execution_remote(command=None): + def is_execution_remote(command: str | None = None) -> bool: del command return False - def __init__(self, module): + def __init__(self, module: Any) -> None: self.module = module - def execute(self, task_argument, job_type, uworker_env): + def execute(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> None: """Executes a task.""" raise NotImplementedError('Child class must implement.') @@ -48,7 +51,8 @@ class TrustedTask(BaseTask): the original ones in ClusterFuzz.""" @logs.task_stage_context(logs.Stage.NA) - def execute(self, task_argument, job_type, uworker_env): + def execute(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> None: # Simple tasks can just use the environment they don't need the uworker env. del uworker_env assert not environment.is_tworker() @@ -76,11 +80,13 @@ class BaseUTask(BaseTask): """Base class representing an untrusted task. Children must decide to execute locally or remotely.""" - def execute(self, task_argument, job_type, uworker_env): + def execute(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> None: """Executes a task.""" raise NotImplementedError('Child class must implement.') - def execute_locally(self, task_argument, job_type, uworker_env): + def execute_locally(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> None: """Executes the utask locally (on this machine, not on batch).""" assert not environment.is_tworker() uworker_input = utasks.tworker_preprocess_no_io(self.module, task_argument, @@ -93,18 +99,19 @@ def execute_locally(self, task_argument, job_type, uworker_env): utasks.tworker_postprocess_no_io(self.module, uworker_output, uworker_input) logs.info('Utask local: done.') - def preprocess(self, task_argument, job_type, uworker_env): + def preprocess(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> str | None: """Executes preprocessing.""" raise NotImplementedError('Child class must implement.') -def is_no_privilege_workload(command, job): +def is_no_privilege_workload(command: str, job: str) -> bool: if not COMMAND_TYPES[command].is_execution_remote(command): return False return batch_service.is_remote_task(command, job) -def is_remote_utask(command, job): +def is_remote_utask(command: str, job: str) -> bool: if not COMMAND_TYPES[command].is_execution_remote(command): return False @@ -116,7 +123,7 @@ def is_remote_utask(command, job): job) or swarming.is_swarming_task(job) -def task_main_runs_on_uworker(): +def task_main_runs_on_uworker() -> bool: """This returns True if the uworker_main portion of this task is unprivileged.""" command = environment.get_value('TASK_NAME') @@ -128,11 +135,13 @@ class UTaskLocalExecutor(BaseUTask): """Represents an untrusted task. Executes it entirely locally and in memory.""" - def execute(self, task_argument, job_type, uworker_env): + def execute(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> None: """Executes a utask locally in-memory.""" self.execute_locally(task_argument, job_type, uworker_env) - def preprocess(self, task_argument, job_type, uworker_env): + def preprocess(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> str | None: """Executes preprocessing.""" raise NotImplementedError('Only needed for utasks.') @@ -143,10 +152,12 @@ class UTask(BaseUTask): opted-in. Otherwise executes locally.""" @staticmethod - def is_execution_remote(command=None): + def is_execution_remote(command: str | None = None) -> bool: + del command return task_utils.is_remotely_executing_utasks() - def execute(self, task_argument, job_type, uworker_env): + def execute(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> None: """Executes a utask.""" logs.info('Executing utask.') command = task_utils.get_command_from_module(self.module.__name__) @@ -187,7 +198,8 @@ def execute(self, task_argument, job_type, uworker_env): tasks.add_utask_main(command, download_url, job_type, None, queue.name) @logs.task_stage_context(logs.Stage.PREPROCESS) - def preprocess(self, task_argument, job_type, uworker_env): + def preprocess(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> str | None: result = utasks.tworker_preprocess(self.module, task_argument, job_type, uworker_env) if not result: @@ -204,14 +216,15 @@ def preprocess(self, task_argument, job_type, uworker_env): class PostprocessTask(BaseTask): """Represents postprocessing of an untrusted task.""" - def __init__(self, module): + def __init__(self, module: Any) -> None: del module # We don't need a module, postprocess isn't a real task, it's one part of # many different tasks. super().__init__('none') @logs.task_stage_context(logs.Stage.POSTPROCESS) - def execute(self, task_argument, job_type, uworker_env): + def execute(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> None: """Executes postprocessing of a utask.""" # These values are None for now. del job_type @@ -225,14 +238,15 @@ class UworkerMainTask(BaseTask): tasks that cannot use Google Cloud batch (e.g. Mac).""" # TODO(metzman): Merge with PostprocessTask. - def __init__(self, module): + def __init__(self, module: Any) -> None: # We don't need a module, uworker_main isn't a real task, it's one part of # many different tasks. del module super().__init__('none') @logs.task_stage_context(logs.Stage.MAIN) - def execute(self, task_argument, job_type, uworker_env): + def execute(self, task_argument: str, job_type: str, + uworker_env: dict[str, Any] | None) -> None: """Executes uworker_main of a utask.""" # These values are None for now. del job_type @@ -241,7 +255,7 @@ def execute(self, task_argument, job_type, uworker_env): utasks.uworker_main(input_path) -COMMAND_TYPES = { +COMMAND_TYPES: dict[str, type[BaseTask]] = { 'analyze': UTask, 'blame': TrustedTask, 'corpus_pruning': UTask, diff --git a/src/clusterfuzz/_internal/bot/tasks/trials.py b/src/clusterfuzz/_internal/bot/tasks/trials.py index 8e1b4d55029..d18464509e5 100644 --- a/src/clusterfuzz/_internal/bot/tasks/trials.py +++ b/src/clusterfuzz/_internal/bot/tasks/trials.py @@ -16,6 +16,13 @@ import json import os import random +from typing import Any +from typing import cast +from typing import Dict +from typing import List +from typing import Optional +from typing import Sequence +from typing import Set from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot.tasks.utasks import uworker_io @@ -28,12 +35,14 @@ class AppArgs: - def __init__(self, probability, contradicts=None): + def __init__(self, + probability: float, + contradicts: Optional[List[str]] = None) -> None: self.probability = probability self.contradicts = contradicts or [] -def preprocess_get_db_trials(): +def preprocess_get_db_trials() -> List[Any]: app_name = get_app_name() if not app_name: return [] @@ -41,7 +50,7 @@ def preprocess_get_db_trials(): return [uworker_io.entity_to_protobuf(trial) for trial in trial_query] -def get_app_name(): +def get_app_name() -> Optional[str]: """Gets a normalized APP_NAME""" app_name = environment.get_value('APP_NAME') if not app_name: @@ -59,8 +68,8 @@ def get_app_name(): class Trials: """Helper class for selecting app-specific extra flags.""" - def __init__(self, db_trials): - self.trials = {} + def __init__(self, db_trials: Sequence[Any]) -> None: + self.trials: Dict[str, AppArgs] = {} self._db_trials = [ uworker_io.entity_from_protobuf(trial, data_types.Trial) for trial in db_trials @@ -71,8 +80,9 @@ def __init__(self, db_trials): return for trial in self._db_trials: - self.trials[trial.app_args] = AppArgs(trial.probability, - trial.contradicts) + self.trials[cast(str, trial.app_args)] = AppArgs( + cast(float, trial.probability), + cast(Optional[List[str]], trial.contradicts)) app_dir = environment.get_value('APP_DIR') if not app_dir: @@ -94,10 +104,10 @@ def __init__(self, db_trials): logs.warning('Unable to parse config file: %s' % str(e)) return - def setup_additional_args_for_app(self, shuffle=True): + def setup_additional_args_for_app(self, shuffle: bool = True) -> None: """Select additional args for the specified app at random.""" - trial_args = [] - contradicts = set() + trial_args: List[str] = [] + contradicts: Set[str] = set() trial_keys = list(self.trials.keys()) diff --git a/src/clusterfuzz/_internal/bot/tasks/unpack_task.py b/src/clusterfuzz/_internal/bot/tasks/unpack_task.py index bd24f27efbc..34899ec494b 100644 --- a/src/clusterfuzz/_internal/bot/tasks/unpack_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/unpack_task.py @@ -16,6 +16,7 @@ import json import os +from typing import cast from google.cloud import ndb @@ -29,9 +30,11 @@ from clusterfuzz._internal.system import shell -def execute_task(metadata_id, job_type): +def execute_task(metadata_id: str | int, job_type: str) -> None: """Unpack a bundled testcase archive and create analyze jobs for each item.""" - metadata = ndb.Key(data_types.BundledArchiveMetadata, int(metadata_id)).get() + metadata = cast( + data_types.BundledArchiveMetadata | None, + ndb.Key(data_types.BundledArchiveMetadata, int(metadata_id)).get()) if not metadata: logs.error('Invalid bundle metadata id %s.' % metadata_id) return @@ -58,7 +61,8 @@ def execute_task(metadata_id, job_type): testcases_directory = environment.get_value('FUZZ_INPUTS_DISK') # Retrieve multi-testcase archive. - archive_path = os.path.join(testcases_directory, metadata.archive_filename) + archive_path = os.path.join(testcases_directory, + cast(str, metadata.archive_filename)) if not blobs.read_blob_to_disk(metadata.blobstore_key, archive_path): logs.error('Could not retrieve archive for bundle %d.' % metadata_id) tasks.add_task('unpack', metadata_id, job_type) @@ -101,14 +105,16 @@ def execute_task(metadata_id, job_type): data_handler.create_user_uploaded_testcase( blob_key, metadata.blobstore_key, archive_state, - metadata.archive_filename, filename, metadata.timeout, job, - metadata.job_queue, metadata.http_flag, metadata.gestures, + cast(str, metadata.archive_filename), filename, metadata.timeout, job, + metadata.job_queue, cast(bool, metadata.http_flag), + cast(list[str] | None, metadata.gestures), metadata.additional_arguments, metadata.bug_information, - metadata.crash_revision, metadata.uploader_email, metadata.platform_id, - metadata.app_launch_command, metadata.fuzzer_name, - metadata.overridden_fuzzer_name, metadata.fuzzer_binary_name, bundled, - upload_metadata.retries, upload_metadata.bug_summary_update_flag, - upload_metadata.quiet_flag, additional_metadata) + cast(int, metadata.crash_revision), cast(str, metadata.uploader_email), + metadata.platform_id, metadata.app_launch_command, + cast(str, metadata.fuzzer_name), metadata.overridden_fuzzer_name, + metadata.fuzzer_binary_name, bundled, upload_metadata.retries, + cast(bool, upload_metadata.bug_summary_update_flag), + cast(bool, upload_metadata.quiet_flag), additional_metadata) # The upload metadata for the archive is not needed anymore since we created # one for each testcase. diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index 115ef3ea5c6..6839075a3a6 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -41,17 +41,17 @@ from clusterfuzz._internal.system import process_handler from clusterfuzz._internal.system import shell -TESTS_LAST_UPDATE_KEY = 'tests_last_update' -TESTS_UPDATE_INTERVAL_DAYS = 1 +TESTS_LAST_UPDATE_KEY: str = 'tests_last_update' +TESTS_UPDATE_INTERVAL_DAYS: int = 1 -def _rename_dll_for_update(absolute_filepath): +def _rename_dll_for_update(absolute_filepath: str) -> None: """Rename a DLL to allow for updates.""" backup_filepath = absolute_filepath + '.bak.' + str(int(time.time())) os.rename(absolute_filepath, backup_filepath) -def _deployment_file_url(filename): +def _deployment_file_url(filename: str) -> str | None: """Helper to return deployment file url.""" deployment_bucket = local_config.ProjectConfig().get('deployment.bucket') if not deployment_bucket: @@ -62,7 +62,7 @@ def _deployment_file_url(filename): return f'gs://{deployment_bucket}/{filename}' -def get_source_url(): +def get_source_url() -> str | None: """Return the source URL.""" release = utils.get_clusterfuzz_release() platform_name = platform.system() @@ -76,13 +76,13 @@ def get_source_url(): utils.get_platform_deployment_filename(platform_name, release)) -def get_source_manifest_url(): +def get_source_manifest_url() -> str | None: """Return the source manifest URL.""" release = utils.get_clusterfuzz_release() return _deployment_file_url(utils.get_remote_manifest_filename(release)) -def clear_old_files(directory, extracted_file_set): +def clear_old_files(directory: str, extracted_file_set: set[str]) -> None: """Remove files from the directory that isn't in the given file list.""" for root_directory, _, filenames in shell.walk(directory): for filename in filenames: @@ -95,7 +95,7 @@ def clear_old_files(directory, extracted_file_set): shell.remove_empty_directories(directory) -def clear_pyc_files(directory): +def clear_pyc_files(directory: str) -> None: """Recursively remove all .pyc files from the given directory""" for root_directory, _, filenames in shell.walk(directory): for filename in filenames: @@ -106,7 +106,7 @@ def clear_pyc_files(directory): shell.remove_file(file_path) -def track_revision(): +def track_revision() -> None: """Get the local revision and report as a metric.""" source_code_revision = get_local_source_revision() or '' os_type = environment.platform() @@ -119,18 +119,20 @@ def track_revision(): monitoring_metrics.BOT_COUNT.set(1, labels) -def get_local_source_revision(): +def get_local_source_revision() -> str | None: """Return the local source revision.""" return utils.current_source_version() -def get_remote_source_revision(source_manifest_url): +def get_remote_source_revision(source_manifest_url: str) -> str: """Get remote revision. We refactor this method out, so that we can mock it.""" - return storage.read_data(source_manifest_url).decode('utf-8').strip() + data = storage.read_data(source_manifest_url) + assert data is not None + return data.decode('utf-8').strip() -def get_newer_source_revision(): +def get_newer_source_revision() -> str | None: """Returns the latest source revision if there is an update, or None if the current source is up to date.""" @@ -178,7 +180,7 @@ def get_newer_source_revision(): return source_version -def run_platform_init_scripts(): +def run_platform_init_scripts() -> None: """Run platform specific initialization scripts.""" logs.info('Running platform initialization scripts.') @@ -201,7 +203,7 @@ def run_platform_init_scripts(): logs.info('Completed running platform initialization scripts.') -def update_source_code(): +def update_source_code() -> None: """Updates source code files with latest version from appengine.""" process_handler.cleanup_stale_processes() shell.clear_temp_directory() @@ -212,7 +214,9 @@ def update_source_code(): temp_archive = os.path.join(cf_source_root_parent_dir, 'clusterfuzz-source.zip') try: - storage.copy_file_from(get_source_url(), temp_archive) + source_url = get_source_url() + assert source_url is not None + storage.copy_file_from(source_url, temp_archive) except Exception: logs.error('Could not retrieve source code archive from url.') return @@ -225,7 +229,7 @@ def update_source_code(): src_directory = os.path.join(root_directory, 'src') error_occurred = False - normalized_file_set = set() + normalized_file_set: set[str] = set() for file in reader.list_members(): filename = os.path.basename(file.name) @@ -268,6 +272,7 @@ def update_source_code(): pass extracted_path = reader.extract( file.name, cf_source_root_parent_dir, trusted=True) + assert extracted_path is not None os.chmod(extracted_path, 0o755) except Exception: error_occurred = True @@ -283,14 +288,16 @@ def update_source_code(): local_manifest_path = os.path.join(root_directory, utils.LOCAL_SOURCE_MANIFEST) - source_version = utils.read_data_from_file( - local_manifest_path, eval_data=False).decode('utf-8').strip() + manifest_data = utils.read_data_from_file( + local_manifest_path, eval_data=False) + assert manifest_data is not None + source_version = manifest_data.decode('utf-8').strip() os.remove(temp_archive) logs.info(f'Source code updated to {source_version} ' + f'(release = {utils.get_clusterfuzz_release()}).') -def update_tests_if_needed(tests_url=None): +def update_tests_if_needed(tests_url: str | None = None) -> None: """Updates layout tests every day.""" data_directory = environment.get_value('FUZZ_DATA') error_occured = False @@ -354,7 +361,7 @@ def update_tests_if_needed(tests_url=None): tasks.track_task_end() -def prepare_environment_for_new_task(): +def prepare_environment_for_new_task() -> None: """ It performs all requirements to cleanup and prepare the environment for receiving a new task. @@ -374,7 +381,7 @@ def prepare_environment_for_new_task(): logs.error('Error occurred while cleaning the environment before the task') -def run(): +def run() -> None: """Run update task.""" # Since this code is particularly sensitive for bot stability, continue # execution but store the exception if anything goes wrong during one of these diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py index fbf141a0a4f..9745e02fee9 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py @@ -13,13 +13,17 @@ # limitations under the License. """Module for executing the different parts of a utask.""" +from collections.abc import Mapping import contextlib import enum import importlib import time +import types +from typing import Any from typing import Optional -from google.protobuf import timestamp_pb2 +from google.protobuf.timestamp_pb2 import \ + Timestamp # pylint: disable=no-name-in-module # pyright: ignore[reportAttributeAccessIssue] from clusterfuzz._internal import swarming from clusterfuzz._internal.base import errors @@ -34,9 +38,6 @@ from clusterfuzz._internal.protos import uworker_msg_pb2 from clusterfuzz._internal.system import environment -# Define an alias to appease pylint. -Timestamp = timestamp_pb2.Timestamp # pylint: disable=no-member - class Mode(enum.Enum): """The execution mode of `uworker_main` tasks in a bot process.""" @@ -65,7 +66,7 @@ def _timestamp_now() -> Timestamp: return ts -def _get_execution_mode(job_type): +def _get_execution_mode(job_type: str) -> Mode: """Determines whether this task in executed on swarming or batch.""" if swarming.is_swarming_task(job_type): return Mode.SWARMING @@ -87,43 +88,48 @@ class _MetricRecorder(contextlib.AbstractContextManager): preprocess_returned: Indicates if preprocess returned the uworker_input. """ - def __init__(self, subtask: _Subtask): - self.start_time_ns = time.time_ns() - self._subtask = subtask - self._labels = None - self.utask_main_failure = None - self._utask_success_conditions = [ - None, # This can be a successful return value in, ie, fuzz task - uworker_msg_pb2.ErrorType.NO_ERROR, # pylint: disable=no-member - uworker_msg_pb2.ErrorType.ANALYZE_NO_CRASH, # pylint: disable=no-member - uworker_msg_pb2.ErrorType.PROGRESSION_BAD_STATE_MIN_MAX, # pylint: disable=no-member - uworker_msg_pb2.ErrorType.REGRESSION_NO_CRASH, # pylint: disable=no-member - uworker_msg_pb2.ErrorType.REGRESSION_LOW_CONFIDENCE_IN_REGRESSION_RANGE, # pylint: disable=no-member - uworker_msg_pb2.ErrorType.MINIMIZE_CRASH_TOO_FLAKY, # pylint: disable=no-member - uworker_msg_pb2.ErrorType.LIBFUZZER_MINIMIZATION_UNREPRODUCIBLE, # pylint: disable=no-member - uworker_msg_pb2.ErrorType.ANALYZE_CLOSE_INVALID_UPLOADED, # pylint: disable=no-member - ] + def __init__(self, subtask: _Subtask) -> None: + # pylint: disable=no-member + self.start_time_ns: int = time.time_ns() + self._subtask: _Subtask = subtask + self._labels: dict[str, str] | None = None + self.utask_main_failure: (uworker_msg_pb2.ErrorType.ValueType | None) = None + self._utask_success_conditions: list[ + uworker_msg_pb2.ErrorType.ValueType | None] = [ + None, # This can be a successful return value in, ie, fuzz task + uworker_msg_pb2.ErrorType.NO_ERROR, + uworker_msg_pb2.ErrorType.ANALYZE_NO_CRASH, + uworker_msg_pb2.ErrorType.PROGRESSION_BAD_STATE_MIN_MAX, + uworker_msg_pb2.ErrorType.REGRESSION_NO_CRASH, + uworker_msg_pb2.ErrorType. + REGRESSION_LOW_CONFIDENCE_IN_REGRESSION_RANGE, + uworker_msg_pb2.ErrorType.MINIMIZE_CRASH_TOO_FLAKY, + uworker_msg_pb2.ErrorType.LIBFUZZER_MINIMIZATION_UNREPRODUCIBLE, + uworker_msg_pb2.ErrorType.ANALYZE_CLOSE_INVALID_UPLOADED, + ] if subtask == _Subtask.PREPROCESS: - self._preprocess_start_time_ns = self.start_time_ns + self._preprocess_start_time_ns: int | None = self.start_time_ns else: self._preprocess_start_time_ns = None # Events-related metadata. - self._event_data = None + self._event_data: dict[str, Any] | None = None # Error type from utask main handled in postprocess. - self.post_utask_main_failure = None + self.post_utask_main_failure: (uworker_msg_pb2.ErrorType.ValueType + | None) = None # Whether preprocess returned the uworker input. self.preprocess_returned: bool | None = None def set_task_details( self, - utask_module, + utask_module: Any, job_type: str, execution_mode: Mode, platform: str, preprocess_start_time: Optional[Timestamp] = None, - task_argument: Optional[str | uworker_msg_pb2.Input] = None): # pylint: disable=no-member + task_argument: Optional[str | uworker_msg_pb2.Input] = None # pylint: disable=no-member + ) -> None: """Sets task details that might not be known at instantation time. Must be called once for metrics to be recorded when exiting the context. @@ -183,13 +189,18 @@ def emit_utask_events(self, task_status: str, task_stage=self._subtask.value, task_outcome=task_outcome) - def _infer_uworker_main_outcome(self, exc_type, uworker_error) -> bool: + def _infer_uworker_main_outcome( + self, + exc_type: type[BaseException] | None, + uworker_error: ( + uworker_msg_pb2.ErrorType.ValueType # pylint: disable=no-member + | None)) -> bool: """Returns True if task succeeded, False otherwise.""" if exc_type or uworker_error not in self._utask_success_conditions: return False return True - def _emit_event_on_exit(self, exc_type): + def _emit_event_on_exit(self, exc_type: type[BaseException] | None) -> None: """Resolves sending task execution events after a utask stage.""" if self._subtask == _Subtask.UWORKER_MAIN: # TODO(vtcosta): Currently, events can't be sent from uworkers main due @@ -220,7 +231,9 @@ def _emit_event_on_exit(self, exc_type): self.emit_utask_events(events.TaskStatus.POST_COMPLETED, task_outcome) return - def __exit__(self, _exc_type, _exc_value, _traceback): + def __exit__(self, _exc_type: type[BaseException] | None, + _exc_value: BaseException | None, + _traceback: types.TracebackType | None) -> None: # Ignore exception details, let Python continue unwinding the stack. if self._labels is None: @@ -233,6 +246,7 @@ def __exit__(self, _exc_type, _exc_value, _traceback): monitoring_metrics.UTASK_SUBTASK_DURATION_SECS.add(duration_secs, self._labels) + assert self._preprocess_start_time_ns is not None e2e_duration_secs = (now - self._preprocess_start_time_ns) / 10**9 monitoring_metrics.UTASK_SUBTASK_E2E_DURATION_SECS.add( e2e_duration_secs, self._labels) @@ -254,6 +268,7 @@ def __exit__(self, _exc_type, _exc_value, _traceback): else: error_condition = 'UNHANDLED_EXCEPTION' else: + assert self.utask_main_failure is not None error_condition = uworker_msg_pb2.ErrorType.Name( # pylint: disable=no-member self.utask_main_failure) self._emit_event_on_exit(_exc_type) @@ -269,7 +284,7 @@ def __exit__(self, _exc_type, _exc_value, _traceback): trimmed_labels) -def ensure_uworker_env_type_safety(uworker_env): +def ensure_uworker_env_type_safety(uworker_env: dict[str, Any]) -> None: """Converts all values in |uworker_env| to str types. ClusterFuzz parses env var values so that the type implied by the value (which in every OS I've seen is a string), is the Python type of the value. @@ -282,12 +297,14 @@ def ensure_uworker_env_type_safety(uworker_env): uworker_env[k] = str(uworker_env[k]) -def _preprocess(utask_module, task_argument, job_type, uworker_env, - recorder: _MetricRecorder, execution_mode: Mode): +def _preprocess(utask_module: Any, task_argument: str | None, job_type: str, + uworker_env: dict[str, Any] | None, recorder: _MetricRecorder, + execution_mode: Mode) -> Optional[uworker_msg_pb2.Input]: # pylint: disable=no-member """Shared logic for preprocessing between preprocess_no_io and the I/O tworker_preprocess.""" - ensure_uworker_env_type_safety(uworker_env) - set_uworker_env(uworker_env) + if uworker_env is not None: + ensure_uworker_env_type_safety(uworker_env) + set_uworker_env(uworker_env) task_utils.reset_task_stage_env() recorder.set_task_details( @@ -318,7 +335,7 @@ def _preprocess(utask_module, task_argument, job_type, uworker_env, return uworker_input -def _start_web_server_if_needed(job_type): +def _start_web_server_if_needed(job_type: str) -> None: """Start web server for blackbox fuzzer jobs (non-engine fuzzer jobs).""" if environment.is_engine_fuzzer_job(job_type): return @@ -330,8 +347,9 @@ def _start_web_server_if_needed(job_type): @logs.task_stage_context(logs.Stage.PREPROCESS) -def tworker_preprocess_no_io(utask_module, task_argument, job_type, - uworker_env): +def tworker_preprocess_no_io( + utask_module: Any, task_argument: str | None, job_type: str, + uworker_env: dict[str, Any] | None) -> bytes | None: """Executes the preprocessing step of the utask |utask_module| and returns the serialized output.""" with _MetricRecorder(_Subtask.PREPROCESS) as recorder: @@ -344,7 +362,8 @@ def tworker_preprocess_no_io(utask_module, task_argument, job_type, @logs.task_stage_context(logs.Stage.MAIN) -def uworker_main_no_io(utask_module, serialized_uworker_input): +def uworker_main_no_io(utask_module: Any, + serialized_uworker_input: bytes) -> bytes | None: """Executes the main part of a utask on the uworker (locally if not using remote executor).""" with _MetricRecorder(_Subtask.UWORKER_MAIN) as recorder: @@ -376,41 +395,47 @@ def uworker_main_no_io(utask_module, serialized_uworker_input): # TODO(metzman): Stop passing module to this function and `uworker_main_no_io`. # Make them consistent with the I/O versions. @logs.task_stage_context(logs.Stage.POSTPROCESS) -def tworker_postprocess_no_io(utask_module, uworker_output, uworker_input): +def tworker_postprocess_no_io(utask_module: Any, uworker_output: bytes, + uworker_input: bytes) -> None: """Executes the postprocess step on the trusted (t)worker (in this case it is the same bot as the uworker).""" logs.info('Starting postprocess on trusted worker.') with _MetricRecorder(_Subtask.POSTPROCESS) as recorder: - uworker_output = uworker_io.deserialize_uworker_output(uworker_output) + deserialized_uworker_output = uworker_io.deserialize_uworker_output( + uworker_output) # Do this to simulate out-of-band tamper-proof storage of the input. - uworker_input = uworker_io.deserialize_uworker_input(uworker_input) - uworker_output.uworker_input.CopyFrom(uworker_input) + deserialized_uworker_input = uworker_io.deserialize_uworker_input( + uworker_input) + deserialized_uworker_output.uworker_input.CopyFrom( + deserialized_uworker_input) - set_uworker_env(uworker_output.uworker_input.uworker_env) + set_uworker_env(deserialized_uworker_output.uworker_input.uworker_env) task_utils.reset_task_stage_env() recorder.set_task_details( utask_module, - uworker_input.job_type, + deserialized_uworker_input.job_type, Mode.QUEUE, environment.platform(), - uworker_input.preprocess_start_time, - task_argument=uworker_input) - recorder.post_utask_main_failure = uworker_output.error_type + deserialized_uworker_input.preprocess_start_time, + task_argument=deserialized_uworker_input) + recorder.post_utask_main_failure = deserialized_uworker_output.error_type # This emit is needed because we are not sending events from utask main. # Thus, this confirms that main finished and postprocess will start. recorder.emit_utask_events( events.TaskStatus.POST_STARTED, uworker_msg_pb2.ErrorType.Name( # pylint: disable=no-member - uworker_output.error_type)) + deserialized_uworker_output.error_type)) - utask_module.utask_postprocess(uworker_output) + utask_module.utask_postprocess(deserialized_uworker_output) @logs.task_stage_context(logs.Stage.PREPROCESS) -def tworker_preprocess(utask_module, task_argument, job_type, uworker_env): +def tworker_preprocess( + utask_module: Any, task_argument: str | None, job_type: str, + uworker_env: dict[str, Any] | None) -> tuple[str, str] | None: """Executes the preprocessing step of the utask |utask_module| and returns the signed download URL for the uworker's input and the (unsigned) download URL for its output.""" @@ -431,14 +456,14 @@ def tworker_preprocess(utask_module, task_argument, job_type, uworker_env): return uworker_io.serialize_and_upload_uworker_input(uworker_input) -def set_uworker_env(uworker_env: dict) -> None: +def set_uworker_env(uworker_env: Mapping[str, Any]) -> None: """Sets all env vars in |uworker_env| in the actual environment.""" for key, value in uworker_env.items(): environment.set_value(key, value) @logs.task_stage_context(logs.Stage.MAIN) -def uworker_main(input_download_url) -> None: +def uworker_main(input_download_url: str) -> bool: """Executes the main part of a utask on the uworker (locally if not using remote executor).""" with _MetricRecorder(_Subtask.UWORKER_MAIN) as recorder: @@ -482,20 +507,21 @@ def uworker_main(input_download_url) -> None: return True -def get_utask_module(module_name): +def get_utask_module(module_name: str) -> types.ModuleType: return importlib.import_module(module_name) -def uworker_bot_main(): +def uworker_bot_main() -> int: """The entrypoint for a uworker.""" logs.info('Starting utask_main on untrusted worker.') input_download_url = environment.get_value('UWORKER_INPUT_DOWNLOAD_URL') + assert input_download_url is not None uworker_main(input_download_url) return 0 @logs.task_stage_context(logs.Stage.POSTPROCESS) -def tworker_postprocess(output_download_url) -> None: +def tworker_postprocess(output_download_url: str) -> None: """Executes the postprocess step on the trusted (t)worker.""" logs.info('Starting postprocess untrusted worker.') with _MetricRecorder(_Subtask.POSTPROCESS) as recorder: diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/analyze_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/analyze_task.py index 9ed661a24ce..d0e893160c1 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/analyze_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/analyze_task.py @@ -14,8 +14,11 @@ """Analyze task for handling user uploads.""" import json +from typing import Any from typing import Dict from typing import Optional +from typing import Sequence +from typing import Tuple from clusterfuzz._internal.base import tasks from clusterfuzz._internal.base import utils @@ -30,6 +33,7 @@ from clusterfuzz._internal.common import testcase_utils from clusterfuzz._internal.crash_analysis import crash_analyzer from clusterfuzz._internal.crash_analysis import severity_analyzer +from clusterfuzz._internal.crash_analysis.crash_result import CrashResult from clusterfuzz._internal.datastore import data_handler from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.fuzzing import leak_blacklist @@ -38,10 +42,12 @@ from clusterfuzz._internal.metrics import monitoring_metrics from clusterfuzz._internal.protos import uworker_msg_pb2 from clusterfuzz._internal.system import environment +from clusterfuzz.stacktraces import CrashInfo -def _add_default_issue_metadata(testcase: data_types.Testcase, - fuzz_target_metadata: Dict): +def _add_default_issue_metadata( + testcase: data_types.Testcase, + fuzz_target_metadata: Optional[Dict[str, Any]]) -> None: """Adds the default issue metadata (e.g. components, labels) to testcase.""" if fuzz_target_metadata is None: return @@ -74,15 +80,19 @@ def _add_default_issue_metadata(testcase: data_types.Testcase, testcase.set_metadata(key, new_value) -def handle_analyze_no_revisions_list_error(output): +def handle_analyze_no_revisions_list_error( + output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Failed to fetch revision list') handle_build_setup_error(output) -def setup_build(testcase: data_types.Testcase, bad_revisions, - fuzz_target) -> Optional[uworker_msg_pb2.Output]: # pylint: disable=no-member +def setup_build( + testcase: data_types.Testcase, + bad_revisions: Optional[Sequence[int]], + fuzz_target: Optional[data_types.FuzzTarget], +) -> Optional[uworker_msg_pb2.Output]: # pylint: disable=no-member """Set up a custom or regular build based on revision. For regular builds, if a provided revision is not found, set up a build with the closest revision <= provided revision.""" @@ -91,7 +101,9 @@ def setup_build(testcase: data_types.Testcase, bad_revisions, if revision and not build_manager.is_custom_binary(): build_bucket_path = build_manager.get_primary_bucket_path() revision_list = build_manager.get_revisions_list( - build_bucket_path, bad_revisions, testcase=testcase) + build_bucket_path, + bad_revisions, # type: ignore + testcase=testcase) if not revision_list: return uworker_msg_pb2.Output( # pylint: disable=no-member error_type=uworker_msg_pb2.ErrorType.ANALYZE_NO_REVISIONS_LIST) # pylint: disable=no-member @@ -102,14 +114,14 @@ def setup_build(testcase: data_types.Testcase, bad_revisions, error_type=uworker_msg_pb2.ErrorType.ANALYZE_NO_REVISION_INDEX) # pylint: disable=no-member revision = revision_list[revision_index] - fuzz_target = fuzz_target.binary if fuzz_target else None - if not build_manager.setup_build(revision, fuzz_target): + fuzz_target_binary = fuzz_target.binary if fuzz_target else None + if not build_manager.setup_build(revision, fuzz_target_binary): return uworker_msg_pb2.Output( # pylint: disable=no-member error_type=uworker_msg_pb2.ErrorType.ANALYZE_BUILD_SETUP) # pylint: disable=no-member return None -def handle_analyze_no_revision_index(output): +def handle_analyze_no_revision_index(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) data_handler.update_testcase_comment( testcase, data_types.TaskState.ERROR, @@ -118,15 +130,18 @@ def handle_analyze_no_revision_index(output): handle_build_setup_error(output) -def handle_analyze_close_invalid_uploaded(output): +def handle_analyze_close_invalid_uploaded( + output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) testcase_upload_metadata = testcase_utils.get_testcase_upload_metadata( output.uworker_input.testcase_id) + assert testcase_upload_metadata is not None data_handler.close_invalid_uploaded_testcase( testcase, testcase_upload_metadata, 'Irrelevant') -def prepare_env_for_main(testcase_upload_metadata): +def prepare_env_for_main( + testcase_upload_metadata: data_types.TestcaseUploadMetadata) -> None: """Prepares the environment for execute_task.""" # Reset redzones. environment.reset_current_memory_tool_options(redzone_size=128) @@ -144,8 +159,12 @@ def prepare_env_for_main(testcase_upload_metadata): def setup_testcase_and_build( - testcase, job_type, setup_input, bad_revisions, - fuzz_target) -> (Optional[str], Optional[uworker_msg_pb2.Output]): # pylint: disable=no-member + testcase: data_types.Testcase, + job_type: str, + setup_input: uworker_msg_pb2.SetupInput, # pylint: disable=no-member + bad_revisions: Optional[Sequence[int]], + fuzz_target: Optional[data_types.FuzzTarget], +) -> Tuple[Optional[str], Optional[uworker_msg_pb2.Output]]: # pylint: disable=no-member """Sets up the |testcase| and builds. Returns the path to the testcase on success, None on error.""" # Set up testcase and get absolute testcase path. @@ -171,7 +190,7 @@ def setup_testcase_and_build( return testcase_file_path, None -def update_testcase_after_build_setup(testcase): +def update_testcase_after_build_setup(testcase: data_types.Testcase) -> None: """Updates the testcase entity with values from global state that was set during build setup.""" # NOTE: This must be done after setting up the build, which also sets @@ -191,7 +210,8 @@ def update_testcase_after_build_setup(testcase): testcase.minimized_arguments = minimized_arguments -def initialize_testcase_for_main(testcase, job_type): +def initialize_testcase_for_main(testcase: data_types.Testcase, + job_type: str) -> None: """Initializes a testcase for the crash testing phase.""" # Update initial testcase information. testcase.job_type = job_type @@ -200,13 +220,17 @@ def initialize_testcase_for_main(testcase, job_type): testcase.put() -def test_for_crash_with_retries(fuzz_target, testcase, testcase_file_path, - test_timeout): +def test_for_crash_with_retries( + fuzz_target: Optional[data_types.FuzzTarget], + testcase: data_types.Testcase, + testcase_file_path: str, + test_timeout: int | float, +) -> Tuple[CrashResult, bool]: """Tests for a crash with retries. Tries with HTTP (with retries) if initial attempts fail. Returns the most recent crash result and the possibly updated HTTP flag.""" # Get the crash output. - http_flag = testcase.http_flag + http_flag = bool(testcase.http_flag) result = testcase_manager.test_for_crash_with_retries( fuzz_target, testcase, @@ -236,11 +260,11 @@ def test_for_crash_with_retries(fuzz_target, testcase, testcase_file_path, return result, http_flag -def is_first_analyze_attempt(testcase): +def is_first_analyze_attempt(testcase: data_types.Testcase) -> bool: return data_handler.is_first_attempt_for_task('analyze', testcase) -def handle_noncrash(output): +def handle_noncrash(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles a non-crashing testcase. Either deletes the testcase or schedules another, final analysis.""" # Could not reproduce the crash. @@ -266,6 +290,7 @@ def handle_noncrash(output): return testcase_upload_metadata = testcase_utils.get_testcase_upload_metadata( output.uworker_input.testcase_id) + assert testcase_upload_metadata is not None rejection_event = events.TestcaseRejectionEvent( testcase=testcase, rejection_reason=events.RejectionReason.ANALYZE_NO_REPRO) @@ -274,8 +299,10 @@ def handle_noncrash(output): events.emit(rejection_event) -def update_testcase_after_crash(testcase, state, job_type, http_flag, - analyze_task_output): +def update_testcase_after_crash( + testcase: data_types.Testcase, state: CrashInfo, job_type: str, + http_flag: bool, + analyze_task_output: uworker_msg_pb2.AnalyzeTaskOutput) -> None: # pylint: disable=no-member """Updates |testcase| based on |state|.""" testcase.crash_type = state.crash_type testcase.crash_address = state.crash_address @@ -302,7 +329,9 @@ def update_testcase_after_crash(testcase, state, job_type, http_flag, analyze_task_output.security_severity = testcase.security_severity -def _utask_preprocess(testcase_id, job_type, uworker_env): +def _utask_preprocess( + testcase_id: str, job_type: str, + uworker_env: Dict[str, Any]) -> Optional[uworker_msg_pb2.Input]: # pylint: disable=no-member """Runs preprocessing for analyze task.""" # Get the testcase from the database and mark it as started. testcase = data_handler.get_testcase_by_id(testcase_id) @@ -344,7 +373,9 @@ def _utask_preprocess(testcase_id, job_type, uworker_env): return uworker_input -def utask_preprocess(testcase_id, job_type, uworker_env): +def utask_preprocess( + testcase_id: str, job_type: str, + uworker_env: Dict[str, Any]) -> Optional[uworker_msg_pb2.Input]: # pylint: disable=no-member """Sets logs context and runs preprocessing for analyze task.""" # Get the testcase from the database. testcase = data_handler.get_testcase_by_id(testcase_id) @@ -352,7 +383,7 @@ def utask_preprocess(testcase_id, job_type, uworker_env): return _utask_preprocess(testcase_id, job_type, uworker_env) -def get_analyze_task_input(): +def get_analyze_task_input() -> uworker_msg_pb2.AnalyzeTaskInput: # pylint: disable=no-member return uworker_msg_pb2.AnalyzeTaskInput( # pylint: disable=no-member bad_revisions=build_manager.get_job_bad_revisions()) @@ -362,9 +393,9 @@ def _build_task_output( """Copies the testcase updated fields to analyze_task_output to be updated in postprocess.""" analyze_task_output = uworker_msg_pb2.AnalyzeTaskOutput() # pylint: disable=no-member - analyze_task_output.crash_revision = int(testcase.crash_revision) - analyze_task_output.absolute_path = testcase.absolute_path - analyze_task_output.minimized_arguments = testcase.minimized_arguments + analyze_task_output.crash_revision = int(testcase.crash_revision or 0) + analyze_task_output.absolute_path = testcase.absolute_path or '' + analyze_task_output.minimized_arguments = (testcase.minimized_arguments or '') if testcase.get_metadata('build_key'): analyze_task_output.build_key = testcase.get_metadata('build_key') if testcase.get_metadata('build_url'): @@ -378,7 +409,8 @@ def _build_task_output( return analyze_task_output -def _utask_main(uworker_input): +def _utask_main( + uworker_input: uworker_msg_pb2.Input) -> Optional[uworker_msg_pb2.Output]: # pylint: disable=no-member """Executes the untrusted part of analyze_task.""" testcase_upload_metadata = uworker_io.entity_from_protobuf( uworker_input.testcase_upload_metadata, data_types.TestcaseUploadMetadata) @@ -459,7 +491,8 @@ def _utask_main(uworker_input): test_for_reproducibility(fuzz_target, testcase, testcase_file_path, state, test_timeout) - analyze_task_output.one_time_crasher_flag = testcase.one_time_crasher_flag + analyze_task_output.one_time_crasher_flag = bool( + testcase.one_time_crasher_flag) monitoring_metrics.ANALYZE_TASK_REPRODUCIBILITY.increment( labels={ @@ -480,7 +513,8 @@ def _utask_main(uworker_input): issue_metadata=json.dumps(fuzz_target_metadata)) -def utask_main(uworker_input): +def utask_main( + uworker_input: uworker_msg_pb2.Input) -> Optional[uworker_msg_pb2.Output]: # pylint: disable=no-member """Sets logs context and runs the untrusted part of analyze_task.""" testcase = uworker_io.entity_from_protobuf(uworker_input.testcase, data_types.Testcase) @@ -489,16 +523,18 @@ def utask_main(uworker_input): return _utask_main(uworker_input) -def test_for_reproducibility(fuzz_target, testcase, testcase_file_path, state, - test_timeout): +def test_for_reproducibility(fuzz_target: Optional[data_types.FuzzTarget], + testcase: data_types.Testcase, + testcase_file_path: str, state: CrashInfo, + test_timeout: int | float) -> None: one_time_crasher_flag = not testcase_manager.test_for_reproducibility( - fuzz_target, testcase_file_path, state.crash_type, state.crash_state, - testcase.security_flag, test_timeout, testcase.http_flag, - testcase.gestures) + fuzz_target, testcase_file_path, state.crash_type, + state.crash_state, testcase.security_flag, test_timeout, + bool(testcase.http_flag), testcase.gestures) testcase.one_time_crasher_flag = one_time_crasher_flag -def handle_build_setup_error(output): +def handle_build_setup_error(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles errors for scenarios where build setup fails.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, @@ -515,28 +551,30 @@ def handle_build_setup_error(output): return testcase_upload_metadata = testcase_utils.get_testcase_upload_metadata( output.uworker_input.testcase_id) + assert testcase_upload_metadata is not None data_handler.mark_invalid_uploaded_testcase( testcase, testcase_upload_metadata, 'Build setup failed') -_ERROR_HANDLER = uworker_handle_errors.CompositeErrorHandler({ - uworker_msg_pb2.ErrorType.ANALYZE_BUILD_SETUP: # pylint: disable=no-member - handle_build_setup_error, - uworker_msg_pb2.ErrorType.ANALYZE_NO_CRASH: # pylint: disable=no-member - handle_noncrash, - uworker_msg_pb2.ErrorType.ANALYZE_NO_REVISION_INDEX: # pylint: disable=no-member - handle_analyze_no_revision_index, - uworker_msg_pb2.ErrorType.ANALYZE_NO_REVISIONS_LIST: # pylint: disable=no-member - handle_analyze_no_revisions_list_error, - uworker_msg_pb2.ErrorType.ANALYZE_CLOSE_INVALID_UPLOADED: # pylint: disable=no-member - handle_analyze_close_invalid_uploaded, -}).compose_with( - setup.ERROR_HANDLER, - uworker_handle_errors.UNHANDLED_ERROR_HANDLER, -) - - -def _update_testcase(output): +_ERROR_HANDLER: uworker_handle_errors.CompositeErrorHandler = ( + uworker_handle_errors.CompositeErrorHandler({ + uworker_msg_pb2.ErrorType.ANALYZE_BUILD_SETUP: # pylint: disable=no-member + handle_build_setup_error, + uworker_msg_pb2.ErrorType.ANALYZE_NO_CRASH: # pylint: disable=no-member + handle_noncrash, + uworker_msg_pb2.ErrorType.ANALYZE_NO_REVISION_INDEX: # pylint: disable=no-member + handle_analyze_no_revision_index, + uworker_msg_pb2.ErrorType.ANALYZE_NO_REVISIONS_LIST: # pylint: disable=no-member + handle_analyze_no_revisions_list_error, + uworker_msg_pb2.ErrorType.ANALYZE_CLOSE_INVALID_UPLOADED: # pylint: disable=no-member + handle_analyze_close_invalid_uploaded, + }).compose_with( + setup.ERROR_HANDLER, + uworker_handle_errors.UNHANDLED_ERROR_HANDLER, + )) + + +def _update_testcase(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Updates the testcase using the info passed from utask_main.""" if not output.HasField('analyze_task_output'): return @@ -583,7 +621,7 @@ def _update_testcase(output): testcase.put() -def _utask_postprocess(output): +def _utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Trusted: Cleans up after a uworker execute_task, writing anything needed to the db.""" testcase_utils.emit_testcase_triage_duration_metric( @@ -597,6 +635,7 @@ def _utask_postprocess(output): testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) testcase_upload_metadata = testcase_utils.get_testcase_upload_metadata( output.uworker_input.testcase_id) + assert testcase_upload_metadata is not None if output.analyze_task_output.one_time_crasher_flag: one_time_crasher_message = ', but is flaky' @@ -657,7 +696,7 @@ def _utask_postprocess(output): task_creation.create_tasks(testcase) -def utask_postprocess(output): +def utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Sets logs context and runs postprocess of the analyze_task.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) with logs.testcase_log_context(testcase, testcase.get_fuzz_target()): diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py index 50d872162bb..4471f85fbbf 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/corpus_pruning_task.py @@ -13,20 +13,24 @@ # limitations under the License. """Corpus pruning task.""" import asyncio -import collections +from collections.abc import Iterable +from collections.abc import Sequence import datetime import json import os import random import shutil import time -from typing import List +from typing import Any +from typing import cast +from typing import NamedTuple import zipfile import aiohttp from google.auth.transport import requests as google_auth_requests from google.cloud import ndb -from google.protobuf import timestamp_pb2 +from google.protobuf.timestamp_pb2 import \ + Timestamp # pylint: disable=no-name-in-module # pyright: ignore[reportAttributeAccessIssue] from clusterfuzz._internal.base import feature_flags from clusterfuzz._internal.base import utils @@ -62,65 +66,73 @@ # TODO(ochang): Move common libFuzzer code from fuzzer into CF. # Redzone size for running testcase. -DEFAULT_REDZONE = 32 +DEFAULT_REDZONE: int = 32 # Minimum redzone size for use during merging. -MIN_REDZONE = 16 +MIN_REDZONE: int = 16 # Timeout for corpus pruning. Note that our priority is to make sure fuzzer's # own corpus is minimized. If time remains, we spend time on using units from # shared corpus. This combined timeout should be lower than task lease timeout. -CORPUS_PRUNING_TIMEOUT = 22 * 60 * 60 +CORPUS_PRUNING_TIMEOUT: int = 22 * 60 * 60 # Time to allow libFuzzer to timeout on its own. -SINGLE_UNIT_TIMEOUT = 5 -TIMEOUT_FLAG = f'-timeout={SINGLE_UNIT_TIMEOUT}' +SINGLE_UNIT_TIMEOUT: int = 5 +TIMEOUT_FLAG: str = f'-timeout={SINGLE_UNIT_TIMEOUT}' # Corpus files limit for cases when corpus pruning task failed in the last # execution. -CORPUS_FILES_LIMIT_FOR_FAILURES = 50_000 +CORPUS_FILES_LIMIT_FOR_FAILURES: int = 50_000 # Corpus total size limit for cases when corpus pruning task failed in the last # execution. -CORPUS_SIZE_LIMIT_FOR_FAILURES = 10 * 1024 * 1024 * 1024 # 10 GB. +CORPUS_SIZE_LIMIT_FOR_FAILURES: int = 10 * 1024 * 1024 * 1024 # 10 GB. # Maximum number of units to restore from quarantine in one run. -MAX_QUARANTINE_UNITS_TO_RESTORE = 128 +MAX_QUARANTINE_UNITS_TO_RESTORE: int = 128 # Memory limits for testcase. -RSS_LIMIT = 2560 -RSS_LIMIT_MB_FLAG = '-rss_limit_mb=%d' +RSS_LIMIT: int = 2560 +RSS_LIMIT_MB_FLAG: str = '-rss_limit_mb=%d' # Flag to enforce length limit for a single corpus element. -MAX_LEN_FLAG = '-max_len=%d' +MAX_LEN_FLAG: str = '-max_len=%d' # Flag to control memory leaks detection. -DETECT_LEAKS_FLAG = '-detect_leaks=%d' +DETECT_LEAKS_FLAG: str = '-detect_leaks=%d' # Longer than default sync timeout to fix broken (overly large) corpora without # losing coverage. -SYNC_TIMEOUT = 2 * 60 * 60 +SYNC_TIMEOUT: int = 2 * 60 * 60 # Number of fuzz targets whose backup corpus is used to cross pollinate with our # current fuzz target corpus. -CROSS_POLLINATE_FUZZER_COUNT = 3 +CROSS_POLLINATE_FUZZER_COUNT: int = 3 # From https://cloud.google.com/storage/docs/batch -GOOGLE_CLOUD_MAX_BATCH_SIZE = 100 +GOOGLE_CLOUD_MAX_BATCH_SIZE: int = 100 -CorpusPruningResult = collections.namedtuple('CorpusPruningResult', [ - 'coverage_info', 'crashes', 'fuzzer_binary_name', 'revision', - 'cross_pollination_stats' -]) -CrossPollinationStats = collections.namedtuple('CrossPollinationStats', [ - 'project_qualified_name', 'sources', 'initial_corpus_size', 'corpus_size', - 'initial_edge_coverage', 'edge_coverage', 'initial_feature_coverage', - 'feature_coverage' -]) +class CrossPollinationStats(NamedTuple): + project_qualified_name: str + sources: str + initial_corpus_size: int + corpus_size: int + initial_edge_coverage: int + edge_coverage: int + initial_feature_coverage: int + feature_coverage: int -def _get_corpus_file_paths(corpus_path): +class CorpusPruningResult(NamedTuple): + coverage_info: data_types.CoverageInformation | None + crashes: list[uworker_msg_pb2.CrashInfo] # pylint: disable=no-member + fuzzer_binary_name: str + revision: int + cross_pollination_stats: CrossPollinationStats | None + + +def _get_corpus_file_paths(corpus_path: str) -> list[str]: """Return full paths to corpus files in |corpus_path|.""" return [ os.path.join(corpus_path, filename) @@ -128,7 +140,7 @@ def _get_corpus_file_paths(corpus_path): ] -async def _limit_corpus_sizes(corpus_urls): +async def _limit_corpus_sizes(corpus_urls: Sequence[str]) -> None: try: await asyncio.gather( *[_limit_corpus_size_async(corpus_url) for corpus_url in corpus_urls]) @@ -136,21 +148,25 @@ async def _limit_corpus_sizes(corpus_urls): logs.error(f'Error in _limit_corpus_sizes: {e}') -async def _limit_corpus_size_async(corpus_url): +async def _limit_corpus_size_async(corpus_url: str) -> None: """Limits corpus size using async listing and deleting.""" logs.info(f'Limiting corpus size {corpus_url} asynchronously.') creds, _ = credentials.get_default() + assert creds is not None creds.refresh(google_auth_requests.Request()) bucket, path = storage.get_bucket_name_and_path(corpus_url) - async def _delete_corpus_element(name, session): + async def _delete_corpus_element(name: str, + session: aiohttp.ClientSession) -> None: + assert creds is not None and creds.token is not None await fast_http.delete_blob_async(bucket, name, session, creds.token) deleting = False corpus_size = 0 num_deleted = 0 - delete_tasks = [] + delete_tasks: list[asyncio.Task[None]] = [] + assert creds.token is not None async with aiohttp.ClientSession() as session: start_time = time.time() idx = 0 @@ -192,7 +208,7 @@ async def _delete_corpus_element(name, session): f'Took {time.time() - start_time} seconds.') -def _get_time_remaining(start_time): +def _get_time_remaining(start_time: datetime.datetime) -> int: """Return time remaining.""" time_used = int((datetime.datetime.utcnow() - start_time).total_seconds()) return CORPUS_PRUNING_TIMEOUT - time_used @@ -205,7 +221,8 @@ class CorpusPruningError(Exception): class CrossPollinateFuzzer: """Cross Pollinate Fuzzer.""" - def __init__(self, fuzz_target, backup_bucket_name, corpus_engine_name): + def __init__(self, fuzz_target: data_types.FuzzTarget, + backup_bucket_name: str, corpus_engine_name: str) -> None: self.fuzz_target = fuzz_target self.backup_bucket_name = backup_bucket_name self.corpus_engine_name = corpus_engine_name @@ -214,18 +231,24 @@ def __init__(self, fuzz_target, backup_bucket_name, corpus_engine_name): class Context: """Pruning state.""" - def __init__(self, uworker_input, fuzz_target, cross_pollinate_fuzzers): + def __init__( + self, + uworker_input: uworker_msg_pb2.Input, # pylint: disable=no-member + fuzz_target: data_types.FuzzTarget, + cross_pollinate_fuzzers: list[CrossPollinateFuzzer]) -> None: self.fuzz_target = fuzz_target self.cross_pollinate_fuzzers = cross_pollinate_fuzzers - self.merge_tmp_dir = None - self.engine = engine.get(self.fuzz_target.engine) - if not self.engine: + self.merge_tmp_dir: str | None = None + assert self.fuzz_target.engine is not None + engine_impl = engine.get(self.fuzz_target.engine) + if not engine_impl: raise CorpusPruningError( f'Engine not found for fuzz_target: {fuzz_target}') + self.engine: engine.Engine = engine_impl - self._created_directories = [] + self._created_directories: list[str] = [] # Set up temporary directories where corpora will be synced to. # Initial synced corpus. @@ -256,7 +279,7 @@ def __init__(self, uworker_input, fuzz_target, cross_pollinate_fuzzers): self.dated_backup_signed_url = ( uworker_input.corpus_pruning_task_input.dated_backup_signed_url) - def restore_quarantined_units(self): + def restore_quarantined_units(self) -> None: """Restore units from the quarantine.""" logs.info('Restoring units from quarantine.') # Limit the number of quarantine units to restore, in case there are a lot. @@ -271,17 +294,18 @@ def restore_quarantined_units(self): shutil.move(unit_path, os.path.join(self.initial_corpus_path, unit_filename)) - def _create_temp_corpus_directory(self, name): + def _create_temp_corpus_directory(self, name: str) -> str: """Create temporary corpus directory. Returns path to the created directory.""" testcases_directory = environment.get_value('FUZZ_INPUTS_DISK') + assert testcases_directory is not None directory_path = os.path.join(testcases_directory, name) shell.create_directory(directory_path) self._created_directories.append(directory_path) return directory_path - def sync_to_disk(self): + def sync_to_disk(self) -> None: """Sync required corpora to disk.""" if not self.corpus.rsync_to_disk(self.initial_corpus_path): raise CorpusPruningError('Failed to sync corpus to disk.') @@ -293,17 +317,17 @@ def sync_to_disk(self): self._cross_pollinate_other_fuzzer_corpuses() - def sync_to_gcs(self): + def sync_to_gcs(self) -> None: """Sync corpora to GCS post merge.""" if not self.corpus.rsync_from_disk(self.minimized_corpus_path): raise CorpusPruningError('Failed to sync minimized corpus to gcs.') - def cleanup(self): + def cleanup(self) -> None: """Cleanup state.""" for path in self._created_directories: shell.remove_directory(path) - def _cross_pollinate_other_fuzzer_corpuses(self): + def _cross_pollinate_other_fuzzer_corpuses(self) -> None: """Add other fuzzer corpuses to shared corpus path for cross-pollination.""" corpus_backup_date = utils.utcnow().date() - datetime.timedelta( days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS) @@ -317,6 +341,7 @@ def _cross_pollinate_other_fuzzer_corpuses(self): corpus_backup_url = corpus_manager.gcs_url_for_backup_file( backup_bucket_name, corpus_engine_name, project_qualified_name, corpus_backup_date) + assert corpus_backup_url is not None corpus_backup_local_filename = '%s-%s' % ( project_qualified_name, os.path.basename(corpus_backup_url)) corpus_backup_local_path = os.path.join(self.shared_corpus_path, @@ -354,21 +379,23 @@ def _cross_pollinate_other_fuzzer_corpuses(self): class BaseRunner: """Base Runner""" - def __init__(self, build_directory, context): + def __init__(self, build_directory: str, context: Context) -> None: self.build_directory = build_directory self.context = context - self.target_path = engine_common.find_fuzzer_path( + assert self.context.fuzz_target.binary is not None + target_path = engine_common.find_fuzzer_path( self.build_directory, self.context.fuzz_target.binary) - if not self.target_path: + if not target_path: raise CorpusPruningError( f'Failed to get fuzzer path for {self.context.fuzz_target.binary}') + self.target_path: str = target_path self.fuzzer_options = options.get_fuzz_target_options(self.target_path) - def get_fuzzer_flags(self): + def get_fuzzer_flags(self) -> list[str]: return [] - def process_sanitizer_options(self): + def process_sanitizer_options(self) -> None: """Process sanitizer options overrides.""" if not self.fuzzer_options: return @@ -384,12 +411,14 @@ def process_sanitizer_options(self): asan_options.update(overrides) environment.set_memory_tool_options('ASAN_OPTIONS', asan_options) - def reproduce(self, input_path, arguments, max_time): + def reproduce(self, input_path: str, arguments: list[str], + max_time: int) -> engine.ReproduceResult: return self.context.engine.reproduce(self.target_path, input_path, arguments, max_time) - def minimize_corpus(self, arguments, input_dirs, output_dir, reproducers_dir, - max_time): + def minimize_corpus(self, arguments: list[str], input_dirs: list[str], + output_dir: str, reproducers_dir: str, + max_time: int) -> engine.FuzzResult: return self.context.engine.minimize_corpus(self.target_path, arguments, input_dirs, output_dir, reproducers_dir, max_time) @@ -398,7 +427,7 @@ def minimize_corpus(self, arguments, input_dirs, output_dir, reproducers_dir, class LibFuzzerRunner(BaseRunner): """Runner for libFuzzer.""" - def get_fuzzer_flags(self): + def get_fuzzer_flags(self) -> list[str]: """Get default libFuzzer options for pruning.""" rss_limit = RSS_LIMIT max_len = engine_common.CORPUS_INPUT_SIZE_LIMIT @@ -434,12 +463,14 @@ def get_fuzzer_flags(self): return arguments.list() - def reproduce(self, input_path, arguments, max_time): + def reproduce(self, input_path: str, arguments: list[str], + max_time: int) -> engine.ReproduceResult: return self.context.engine.reproduce(self.target_path, input_path, arguments, max_time) - def minimize_corpus(self, arguments, input_dirs, output_dir, reproducers_dir, - max_time): + def minimize_corpus(self, arguments: list[str], input_dirs: list[str], + output_dir: str, reproducers_dir: str, + max_time: int) -> engine.FuzzResult: return self.context.engine.minimize_corpus(self.target_path, arguments, input_dirs, output_dir, reproducers_dir, max_time) @@ -452,11 +483,12 @@ class CentipedeRunner(BaseRunner): class CorpusPrunerBase: """Base class for corpus pruning that is engine‐agnostic.""" - def __init__(self, runner): + def __init__(self, runner: BaseRunner) -> None: self.runner = runner self.context = runner.context - def run(self, initial_corpus_path, minimized_corpus_path, bad_units_path): + def run(self, initial_corpus_path: str, minimized_corpus_path: str, + bad_units_path: str) -> dict[str, Any] | None: """Running generic corpus prunning""" if not shell.get_directory_file_count(initial_corpus_path): # Empty corpus, nothing to do. @@ -497,7 +529,8 @@ def run(self, initial_corpus_path, minimized_corpus_path, bad_units_path): logs.info('Corpus merge finished successfully.', output=symbolized_output) return result.stats - def process_bad_units(self, bad_units_path, quarantine_corpus_path): + def process_bad_units(self, bad_units_path: str, quarantine_corpus_path: str + ) -> dict[str, uworker_msg_pb2.CrashInfo]: # pylint: disable=no-member del bad_units_path del quarantine_corpus_path return {} @@ -509,23 +542,25 @@ class LibFuzzerPruner(CorpusPrunerBase): quarantining of problematic units and related special cases. """ - def _run_single_unit(self, unit_path): + def _run_single_unit(self, unit_path: str) -> engine.ReproduceResult: arguments = self.runner.get_fuzzer_flags() # Expect libFuzzer flags. return self.runner.reproduce(unit_path, arguments, SINGLE_UNIT_TIMEOUT) - def _quarantine_unit(self, unit_path, quarantine_corpus_path): + def _quarantine_unit(self, unit_path: str, + quarantine_corpus_path: str) -> str: quarantined_unit_path = os.path.join(quarantine_corpus_path, os.path.basename(unit_path)) shutil.move(unit_path, quarantined_unit_path) return quarantined_unit_path - def process_bad_units(self, bad_units_path, quarantine_corpus_path): + def process_bad_units(self, bad_units_path: str, quarantine_corpus_path: str + ) -> dict[str, uworker_msg_pb2.CrashInfo]: # pylint: disable=no-member """ Process bad units by running each test case individually, quarantining those that timeout, OOM, or crash due to memory sanitizer errors. """ - crashes = {} + crashes: dict[str, uworker_msg_pb2.CrashInfo] = {} # pylint: disable=no-member environment.reset_current_memory_tool_options(redzone_size=DEFAULT_REDZONE) self.runner.process_sanitizer_options() @@ -586,11 +621,11 @@ class CentipedePruner(CorpusPrunerBase): class CrossPollinator: """Cross pollination.""" - def __init__(self, runner): + def __init__(self, runner: BaseRunner) -> None: self.runner = runner self.context = self.runner.context - def run(self, timeout): + def run(self, timeout: int) -> dict[str, Any] | None: """Merge testcases from corpus from other fuzz targets.""" if not shell.get_directory_file_count(self.context.shared_corpus_path): logs.info('No files found in shared corpus, skip merge.') @@ -627,7 +662,8 @@ def run(self, timeout): return result.stats -def _fill_cross_pollination_stats(stats, output): +def _fill_cross_pollination_stats(stats: CrossPollinationStats | None, + output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Fills the cross pollination statistics in the corpus pruning output.""" if not stats: return @@ -645,7 +681,7 @@ def _fill_cross_pollination_stats(stats, output): output.corpus_pruning_task_output.cross_pollination_stats.CopyFrom(statistics) -def _record_cross_pollination_stats(output): +def _record_cross_pollination_stats(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Log stats about cross pollination in BigQuery.""" # If no stats were gathered due to a timeout or lack of corpus, return. @@ -675,13 +711,16 @@ def _record_cross_pollination_stats(output): client.insert([big_query.Insert(row=bigquery_row, insert_id=None)]) -def _get_pruner_and_runner(context): +def _get_pruner_and_runner( + context: Context) -> tuple[CorpusPrunerBase, BaseRunner]: """Get pruner and runner object acording with the FuzzTarget into the context """ build_directory = environment.get_value('BUILD_DIR') + assert build_directory is not None + assert context.fuzz_target.engine is not None if context.fuzz_target.engine.lower() == 'libfuzzer': - runner = LibFuzzerRunner(build_directory, context) - pruner = LibFuzzerPruner(runner) + runner: BaseRunner = LibFuzzerRunner(build_directory, context) + pruner: CorpusPrunerBase = LibFuzzerPruner(runner) elif context.fuzz_target.engine.lower() == 'centipede': runner = CentipedeRunner(build_directory, context) pruner = CentipedePruner(runner) @@ -691,7 +730,8 @@ def _get_pruner_and_runner(context): return pruner, runner -def do_corpus_pruning(context, revision) -> CorpusPruningResult: +def do_corpus_pruning(context: Context, + revision: int) -> CorpusPruningResult | None: """Run corpus pruning.""" # Set |FUZZ_TARGET| environment variable to help with unarchiving only fuzz # target and its related files. @@ -815,16 +855,17 @@ def do_corpus_pruning(context, revision) -> CorpusPruningResult: coverage_info=coverage_info, crashes=list(crashes.values()), fuzzer_binary_name=fuzzer_binary_name, - revision=environment.get_value('APP_REVISION'), + revision=int(environment.get_value('APP_REVISION', 0)), cross_pollination_stats=cross_pollination_stats) def _upload_corpus_crashes_zip(result: CorpusPruningResult, - corpus_crashes_blob_name, - corpus_crashes_upload_url): + corpus_crashes_blob_name: str, + corpus_crashes_upload_url: str) -> None: """Packs the corpus crashes in a zip file. The file is then uploaded using the signed upload url from the input.""" temp_dir = environment.get_value('BOT_TMPDIR') + assert temp_dir is not None zip_filename = os.path.join(temp_dir, corpus_crashes_blob_name) with zipfile.ZipFile(zip_filename, 'w') as zip_file: for crash in result.crashes: @@ -837,7 +878,7 @@ def _upload_corpus_crashes_zip(result: CorpusPruningResult, os.remove(zip_filename) -def _process_corpus_crashes(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def _process_corpus_crashes(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Process crashes found in the corpus.""" # TODO(metzman): Fix this function after the holiday break. # if not output.corpus_pruning_task_output.crashes: @@ -846,7 +887,9 @@ def _process_corpus_crashes(output: uworker_msg_pb2.Output): # pylint: disable= corpus_pruning_output = output.corpus_pruning_task_output crash_revision = corpus_pruning_output.crash_revision fuzz_target = data_handler.get_fuzz_target(output.uworker_input.fuzzer_name) + assert fuzz_target is not None job_type = environment.get_value('JOB_NAME') + assert job_type is not None minimized_arguments = f'%TESTCASE% {fuzz_target.binary}' project_name = data_handler.get_project_name(job_type) @@ -856,6 +899,7 @@ def _process_corpus_crashes(output: uworker_msg_pb2.Output): # pylint: disable= # Copy the crashes zip file from cloud storage into a temporary directory. temp_dir = environment.get_value('BOT_TMPDIR') + assert temp_dir is not None corpus_crashes_blob_name = ( output.uworker_input.corpus_pruning_task_input.corpus_crashes_blob_name) corpus_crashes_zip_local_path = os.path.join( @@ -884,8 +928,9 @@ def _process_corpus_crashes(output: uworker_msg_pb2.Output): # pylint: disable= # Set the absolute_path property of the Testcase to a file in FUZZ_INPUTS # instead of the local quarantine directory. - absolute_testcase_path = os.path.join( - environment.get_value('FUZZ_INPUTS'), 'testcase') + fuzz_inputs = environment.get_value('FUZZ_INPUTS') + assert fuzz_inputs is not None + absolute_testcase_path = os.path.join(fuzz_inputs, 'testcase') # TODO(https://b.corp.google.com/issues/328691756): Set trusted based on # the job when we start doing untrusted fuzzing. @@ -915,6 +960,7 @@ def _process_corpus_crashes(output: uworker_msg_pb2.Output): # pylint: disable= # Set fuzzer_binary_name in testcase metadata. testcase = data_handler.get_testcase_by_id(testcase_id) + assert testcase is not None testcase.set_metadata('fuzzer_binary_name', corpus_pruning_output.fuzzer_binary_name) events.emit( @@ -930,6 +976,7 @@ def _process_corpus_crashes(output: uworker_msg_pb2.Output): # pylint: disable= # Create additional tasks for testcase (starting with minimization). testcase = data_handler.get_testcase_by_id(testcase_id) + assert testcase is not None task_creation.create_tasks(testcase) os.remove(corpus_crashes_zip_local_path) @@ -937,12 +984,14 @@ def _process_corpus_crashes(output: uworker_msg_pb2.Output): # pylint: disable= blobs.delete_blob(corpus_crashes_blob_name) -def _select_targets_and_jobs_for_pollination(engine_name, current_fuzzer_name): +def _select_targets_and_jobs_for_pollination( + engine_name: str, current_fuzzer_name: str +) -> list[tuple[data_types.FuzzTarget, data_types.FuzzTargetJob]]: """Select jobs to use for cross pollination.""" target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(engine=engine_name)) targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs) - targets_and_jobs = [(target, target_job) + targets_and_jobs = [(cast(data_types.FuzzTarget, target), target_job) for target, target_job in zip(targets, target_jobs) if target_job.fuzz_target_name != current_fuzzer_name] selected_targets_and_jobs = random.SystemRandom().sample( @@ -954,9 +1003,9 @@ def _select_targets_and_jobs_for_pollination(engine_name, current_fuzzer_name): def _get_cross_pollinate_fuzzers( engine_name: str, current_fuzzer_name: str -) -> List[uworker_msg_pb2.CrossPollinateFuzzerProto]: # pylint: disable=no-member +) -> list[uworker_msg_pb2.CrossPollinateFuzzerProto]: # pylint: disable=no-member """Return a list of fuzzer objects to use for cross pollination.""" - cross_pollinate_fuzzers = [] + cross_pollinate_fuzzers: list[uworker_msg_pb2.CrossPollinateFuzzerProto] = [] # pylint: disable=no-member selected_targets_and_jobs = _select_targets_and_jobs_for_pollination( engine_name, current_fuzzer_name) @@ -985,7 +1034,10 @@ def _get_cross_pollinate_fuzzers( return cross_pollinate_fuzzers -def _get_cross_pollinate_fuzzers_from_protos(cross_pollinate_fuzzers_protos): +def _get_cross_pollinate_fuzzers_from_protos( + cross_pollinate_fuzzers_protos: Iterable[ + uworker_msg_pb2.CrossPollinateFuzzerProto], # pylint: disable=no-member +) -> list[CrossPollinateFuzzer]: return [ CrossPollinateFuzzer( uworker_io.entity_from_protobuf(proto.fuzz_target, @@ -996,7 +1048,7 @@ def _get_cross_pollinate_fuzzers_from_protos(cross_pollinate_fuzzers_protos): ] -def _save_coverage_information(output): +def _save_coverage_information(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Saves coverage information in datastore using an atomic transaction.""" if not output.corpus_pruning_task_output.HasField('coverage_info'): return @@ -1004,12 +1056,13 @@ def _save_coverage_information(output): cov_info = output.corpus_pruning_task_output.coverage_info # Use ndb.transaction with retries below to mitigate risk of a race condition. - def _try_save_coverage_information(): + def _try_save_coverage_information() -> None: """Implements save_coverage_information function.""" coverage_info = data_handler.get_coverage_information( cov_info.project_name, cov_info.timestamp.ToDatetime().date(), create_if_needed=True) + assert coverage_info is not None # Intentionally skip edge and function coverage values as those would come # from fuzzer coverage cron task (see src/go/server/cron/coverage.go). @@ -1024,7 +1077,7 @@ def _try_save_coverage_information(): coverage_info.put() try: - ndb.transaction( + ndb.transaction( # pyright: ignore[reportAttributeAccessIssue] _try_save_coverage_information, retries=data_handler.DEFAULT_FAIL_RETRIES) except Exception as e: @@ -1034,35 +1087,45 @@ def _try_save_coverage_information(): 'Failed to save corpus pruning result: %s.' % repr(e)) -def _get_proto_timestamp(initial_timestamp): - timestamp = timestamp_pb2.Timestamp() # pylint: disable=no-member - timestamp.FromDatetime(initial_timestamp) +def _get_proto_timestamp(initial_timestamp: datetime.datetime | datetime.date, + ) -> Timestamp: + timestamp = Timestamp() + timestamp.FromDatetime(cast(datetime.datetime, initial_timestamp)) return timestamp -def _extract_coverage_information(context, result): +def _extract_coverage_information( + context: Context, + result: CorpusPruningResult, +) -> uworker_msg_pb2.CoverageInformation: # pylint: disable=no-member """Extracts and stores the coverage information in a proto.""" coverage_info = uworker_msg_pb2.CoverageInformation() # pylint: disable=no-member coverage_info.project_name = context.fuzz_target.project_qualified_name() + assert result.coverage_info is not None + assert result.coverage_info.date is not None proto_timestamp = _get_proto_timestamp(result.coverage_info.date) coverage_info.timestamp.CopyFrom(proto_timestamp) # Intentionally skip edge and function coverage values as those would come # from fuzzer coverage cron task. - coverage_info.corpus_size_units = result.coverage_info.corpus_size_units - coverage_info.corpus_size_bytes = result.coverage_info.corpus_size_bytes - coverage_info.corpus_location = result.coverage_info.corpus_location + coverage_info.corpus_size_units = cast(int, + result.coverage_info.corpus_size_units) + coverage_info.corpus_size_bytes = cast(int, + result.coverage_info.corpus_size_bytes) + coverage_info.corpus_location = cast(str, + result.coverage_info.corpus_location) if result.coverage_info.corpus_backup_location: coverage_info.corpus_backup_location = ( result.coverage_info.corpus_backup_location) - coverage_info.quarantine_size_units = ( - result.coverage_info.quarantine_size_units) - coverage_info.quarantine_size_bytes = ( - result.coverage_info.quarantine_size_bytes) - coverage_info.quarantine_location = result.coverage_info.quarantine_location + coverage_info.quarantine_size_units = cast( + int, result.coverage_info.quarantine_size_units) + coverage_info.quarantine_size_bytes = cast( + int, result.coverage_info.quarantine_size_bytes) + coverage_info.quarantine_location = cast( + str, result.coverage_info.quarantine_location) return coverage_info -def _utask_main(uworker_input): +def _utask_main(uworker_input: uworker_msg_pb2.Input) -> uworker_msg_pb2.Output: # pylint: disable=no-member """Execute corpus pruning task.""" fuzz_target = uworker_io.entity_from_protobuf( uworker_input.corpus_pruning_task_input.fuzz_target, @@ -1078,13 +1141,14 @@ def _utask_main(uworker_input): uworker_input.corpus_pruning_task_input.cross_pollinate_fuzzers) context = Context(uworker_input, fuzz_target, cross_pollinate_fuzzers) - if uworker_input.global_blacklisted_functions: + if uworker_input.setup_input.global_blacklisted_functions: leak_blacklist.copy_global_to_local_blacklist( - uworker_input.corpus_task_input.global_blacklisted_functions) + uworker_input.setup_input.global_blacklisted_functions) uworker_output = None try: result = do_corpus_pruning(context, revision) + assert result is not None issue_metadata = engine_common.get_fuzz_target_issue_metadata(fuzz_target) issue_metadata = issue_metadata or {} # TODO(metzman): Fix this issue. @@ -1092,6 +1156,7 @@ def _utask_main(uworker_input): # result, # uworker_input.corpus_pruning_task_input.corpus_crashes_blob_name, # uworker_input.corpus_pruning_task_input.corpus_crashes_upload_url) + assert result.coverage_info is not None uworker_output = uworker_msg_pb2.Output( # pylint: disable=no-member corpus_pruning_task_output=uworker_msg_pb2.CorpusPruningTaskOutput( # pylint: disable=no-member coverage_info=_extract_coverage_information(context, result), @@ -1114,7 +1179,7 @@ def _utask_main(uworker_input): return uworker_output -def utask_main(uworker_input): +def utask_main(uworker_input: uworker_msg_pb2.Input) -> uworker_msg_pb2.Output: # pylint: disable=no-member """Sets logs context and executes corpus pruning task.""" fuzz_target = uworker_io.entity_from_protobuf( uworker_input.corpus_pruning_task_input.fuzz_target, @@ -1124,14 +1189,16 @@ def utask_main(uworker_input): return _utask_main(uworker_input) -def handle_corpus_pruning_failures(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_corpus_pruning_failures(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member task_name = (f'corpus_pruning_{output.uworker_input.fuzzer_name}_' f'{output.uworker_input.job_type}') data_handler.update_task_status(task_name, data_types.TaskState.ERROR) -def _create_backup_urls(fuzz_target: data_types.FuzzTarget, - corpus_pruning_task_input): +def _create_backup_urls( + fuzz_target: data_types.FuzzTarget, + corpus_pruning_task_input: uworker_msg_pb2.CorpusPruningTaskInput # pylint: disable=no-member +) -> None: """Creates the backup urls if a backup bucket is provided.""" backup_bucket_name = environment.get_value('BACKUP_BUCKET') if not backup_bucket_name: @@ -1147,8 +1214,11 @@ def _create_backup_urls(fuzz_target: data_types.FuzzTarget, latest_backup_gcs_url = corpus_manager.gcs_url_for_backup_file( backup_bucket_name, engine_name, fuzz_target.project_qualified_name(), corpus_manager.LATEST_BACKUP_TIMESTAMP) + assert dated_backup_gcs_url is not None + assert latest_backup_gcs_url is not None dated_backup_signed_url = storage.get_signed_upload_url(dated_backup_gcs_url) + assert dated_backup_signed_url is not None corpus_pruning_task_input.dated_backup_gcs_url = dated_backup_gcs_url corpus_pruning_task_input.latest_backup_gcs_url = latest_backup_gcs_url @@ -1184,9 +1254,15 @@ def _should_use_threaded_ops(fuzzer_name: str) -> bool: return False -def _utask_preprocess(fuzzer_name, job_type, uworker_env): +def _utask_preprocess( + fuzzer_name: str, + job_type: str, + uworker_env: dict[str, str] | None, +) -> uworker_msg_pb2.Input | None: # pylint: disable=no-member """Runs preprocessing for corpus pruning task.""" fuzz_target = data_handler.get_fuzz_target(fuzzer_name) + assert fuzz_target is not None + assert fuzz_target.engine is not None task_name = f'corpus_pruning_{fuzzer_name}_{job_type}' @@ -1258,22 +1334,27 @@ def _utask_preprocess(fuzzer_name, job_type, uworker_env): corpus_pruning_task_input=corpus_pruning_task_input) -def utask_preprocess(fuzzer_name, job_type, uworker_env): +def utask_preprocess( + fuzzer_name: str, + job_type: str, + uworker_env: dict[str, str] | None, +) -> uworker_msg_pb2.Input | None: # pylint: disable=no-member """Sets logs context and runs preprocessing for corpus pruning task.""" fuzz_target = data_handler.get_fuzz_target(fuzzer_name) with logs.fuzzer_log_context(fuzzer_name, job_type, fuzz_target): return _utask_preprocess(fuzzer_name, job_type, uworker_env) -_ERROR_HANDLER = uworker_handle_errors.CompositeErrorHandler({ - uworker_msg_pb2.ErrorType.CORPUS_PRUNING_FUZZER_SETUP_FAILED: # pylint: disable=no-member - uworker_handle_errors.noop_handler, - uworker_msg_pb2.ErrorType.CORPUS_PRUNING_ERROR: # pylint: disable=no-member - handle_corpus_pruning_failures, -}) +_ERROR_HANDLER: uworker_handle_errors.CompositeErrorHandler = ( + uworker_handle_errors.CompositeErrorHandler({ + uworker_msg_pb2.ErrorType.CORPUS_PRUNING_FUZZER_SETUP_FAILED: # pylint: disable=no-member + uworker_handle_errors.noop_handler, + uworker_msg_pb2.ErrorType.CORPUS_PRUNING_ERROR: # pylint: disable=no-member + handle_corpus_pruning_failures, + })) -def _update_latest_backup(output): +def _update_latest_backup(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Updates the latest_backup with the dated_backup uploaded in utask_main if any.""" if not output.corpus_pruning_task_output.corpus_backup_uploaded: @@ -1293,7 +1374,7 @@ def _update_latest_backup(output): f'{latest_backup_gcs_url}.') -def _utask_postprocess(output): +def _utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Trusted: Handles errors and writes anything needed to the db.""" if output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR: # pylint: disable=no-member _ERROR_HANDLER.handle(output) @@ -1308,7 +1389,7 @@ def _utask_postprocess(output): data_handler.update_task_status(task_name, data_types.TaskState.FINISHED) -def utask_postprocess(output): +def utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Sets logs context and runs postprocess for corpus pruning task.""" fuzzer_name = output.uworker_input.fuzzer_name job_type = output.uworker_input.job_type diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py index 419590bc597..4eafcd2b143 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py @@ -13,7 +13,9 @@ # limitations under the License. """Fuzz task for handling fuzzing.""" -import collections +from collections.abc import Generator +from collections.abc import MutableMapping +from collections.abc import Sequence import datetime import itertools import json @@ -24,9 +26,13 @@ import threading import time from typing import Any +from typing import cast from typing import Dict from typing import List +from typing import NamedTuple from typing import Optional +from typing import Set +from typing import Tuple from google.cloud import ndb @@ -76,17 +82,17 @@ # 10 MB limit for proto testcase payload field (see FuzzerRunOutput testcase in # src/clusterfuzz/_internal/protos/uworker_msg.proto:L248). -MAX_TESTCASE_PAYLOAD_SIZE = 10 * 1024 * 1024 +MAX_TESTCASE_PAYLOAD_SIZE: int = 10 * 1024 * 1024 # PROTO msg limit is 2 GB, reserving 200 MB for safety # https://protobuf.dev/programming-guides/proto-limits/ -PROTOBUF_MSG_LIMIT = 1.8 * 1024 * 1024 * 1024 -FUZZER_METADATA_REGEX = re.compile(r'metadata::(\w+):\s*(.*)') -FUZZER_FAILURE_THRESHOLD = 0.33 -MAX_NEW_CORPUS_FILES = 500 -THREAD_WAIT_TIMEOUT = 1 -MAX_CRASHES_UPLOADED = 64 +PROTOBUF_MSG_LIMIT: float = 1.8 * 1024 * 1024 * 1024 +FUZZER_METADATA_REGEX: re.Pattern = re.compile(r'metadata::(\w+):\s*(.*)') +FUZZER_FAILURE_THRESHOLD: float = 0.33 +MAX_NEW_CORPUS_FILES: int = 500 +THREAD_WAIT_TIMEOUT: int = 1 +MAX_CRASHES_UPLOADED: int = 64 -ENGINE_OUTPUT_LIMIT = 10 * 2**20 +ENGINE_OUTPUT_LIMIT: int = 10 * 2**20 class FuzzTaskError(Exception): @@ -98,26 +104,42 @@ class FuzzTargetNotFoundError(Exception): class FuzzErrorCode: - FUZZER_TIMEOUT = -1 - FUZZER_SETUP_FAILED = -2 - FUZZER_EXECUTION_FAILED = -3 - DATA_BUNDLE_SETUP_FAILED = -4 - BUILD_SETUP_FAILED = -5 - - -Context = collections.namedtuple('Context', [ - 'project_name', 'bot_name', 'job_type', 'fuzz_target', 'redzone', - 'disable_ubsan', 'platform_id', 'crash_revision', 'fuzzer_name', - 'window_argument', 'fuzzer_metadata', 'testcases_metadata', - 'timeout_multiplier', 'test_timeout', 'data_directory' -]) - -GenerateBlackboxTestcasesResult = collections.namedtuple( - 'GenerateBlackboxTestcasesResult', - ['success', 'testcase_file_paths', 'fuzzer_metadata']) - - -def get_unsymbolized_crash_stacktrace(stack_file_path): + FUZZER_TIMEOUT: int = -1 + FUZZER_SETUP_FAILED: int = -2 + FUZZER_EXECUTION_FAILED: int = -3 + DATA_BUNDLE_SETUP_FAILED: int = -4 + BUILD_SETUP_FAILED: int = -5 + + +class Context(NamedTuple): + """Fuzz task context.""" + + project_name: Optional[str] + bot_name: Optional[str] + job_type: str + fuzz_target: Optional[data_types.FuzzTarget] + redzone: int + disable_ubsan: bool + platform_id: Optional[str] + crash_revision: Any + fuzzer_name: str + window_argument: Optional[str] + fuzzer_metadata: Dict[str, Any] + testcases_metadata: Dict[str, Any] + timeout_multiplier: float + test_timeout: int + data_directory: Optional[str] + + +class GenerateBlackboxTestcasesResult(NamedTuple): + """Result from generating blackbox testcases.""" + + success: bool + testcase_file_paths: Optional[List[str]] + fuzzer_metadata: Optional[Dict[str, Any]] + + +def get_unsymbolized_crash_stacktrace(stack_file_path: str) -> str: """Read unsymbolized crash stacktrace.""" with open(stack_file_path, 'rb') as f: return utils.decode_to_unicode(f.read()) @@ -131,7 +153,7 @@ class UploadUrlCollection: """Upload URLs collection.""" def __init__(self, upload_urls: List[uworker_msg_pb2.BlobUploadUrl]): - self.upload_urls = upload_urls + self.upload_urls: List[uworker_msg_pb2.BlobUploadUrl] = upload_urls def get(self) -> uworker_msg_pb2.BlobUploadUrl: if not self.upload_urls: @@ -146,7 +168,7 @@ def _get_max_testcases() -> int: return environment.get_value('MAX_TESTCASES', 1) -def _get_max_corpus_uploads_per_task(): +def _get_max_corpus_uploads_per_task() -> int: number_of_fuzzer_runs = _get_max_testcases() return MAX_NEW_CORPUS_FILES * number_of_fuzzer_runs @@ -155,7 +177,8 @@ class Crash: """Represents a crash (before creating a testcase).""" @classmethod - def from_testcase_manager_crash(cls, crash): + def from_testcase_manager_crash( + cls, crash: testcase_manager.Crash) -> Optional['Crash']: """Create a Crash from a testcase_manager.Crash.""" try: orig_unsymbolized_crash_stacktrace = ( @@ -174,11 +197,13 @@ def from_testcase_manager_crash(cls, crash): crash.file_path, needs_http=needs_http)) # TODO(ochang): Remove once all engines are migrated to new pipeline. - fuzzing_strategies = libfuzzer_stats.LIBFUZZER_FUZZING_STRATEGIES.search( - orig_unsymbolized_crash_stacktrace) - if fuzzing_strategies: - assert len(fuzzing_strategies.groups()) == 1 - fuzzing_strategies_string = fuzzing_strategies.groups()[0] + fuzzing_strategies_match = ( + libfuzzer_stats.LIBFUZZER_FUZZING_STRATEGIES.search( + orig_unsymbolized_crash_stacktrace)) + fuzzing_strategies = None + if fuzzing_strategies_match: + assert len(fuzzing_strategies_match.groups()) == 1 + fuzzing_strategies_string = fuzzing_strategies_match.groups()[0] fuzzing_strategies = [ strategy.strip() for strategy in fuzzing_strategies_string.split(',') ] @@ -196,7 +221,8 @@ def from_testcase_manager_crash(cls, crash): fuzzing_strategies=fuzzing_strategies) @classmethod - def from_engine_crash(cls, crash, fuzzing_strategies): + def from_engine_crash(cls, crash: engine.Crash, + fuzzing_strategies: Optional[List[str]]) -> 'Crash': """Create a Crash from a engine.Crash.""" return Crash( file_path=crash.input_path, @@ -211,62 +237,63 @@ def from_engine_crash(cls, crash, fuzzing_strategies): fuzzing_strategies=fuzzing_strategies) def __init__(self, - file_path, - crash_time, - return_code, - resource_list, - gestures, - unsymbolized_crash_stacktrace, - arguments, - application_command_line, - http_flag=False, - fuzzing_strategies=None): - self.file_path = file_path - self.crash_time = crash_time - self.return_code = return_code - self.resource_list = resource_list - self.gestures = gestures - self.arguments = arguments - self.fuzzing_strategies = fuzzing_strategies - - self.security_flag = False - self.should_be_ignored = False - - self.http_flag = http_flag - self.application_command_line = application_command_line - self.unsymbolized_crash_stacktrace = unsymbolized_crash_stacktrace + file_path: str, + crash_time: Any, + return_code: Any, + resource_list: Any, + gestures: Any, + unsymbolized_crash_stacktrace: str, + arguments: str, + application_command_line: str, + http_flag: bool = False, + fuzzing_strategies: Optional[Sequence[str]] = None): + self.file_path: str = file_path + self.crash_time: Any = crash_time + self.return_code: Any = return_code + self.resource_list: Any = resource_list + self.gestures: Any = gestures + self.arguments: str = arguments + self.fuzzing_strategies: Optional[List[str]] = ( + list(fuzzing_strategies) if fuzzing_strategies else None) + + self.security_flag: bool = False + self.should_be_ignored: bool = False + + self.http_flag: bool = http_flag + self.application_command_line: str = application_command_line + self.unsymbolized_crash_stacktrace: str = unsymbolized_crash_stacktrace state = stack_analyzer.get_crash_data(self.unsymbolized_crash_stacktrace) - self.crash_type = state.crash_type - self.crash_address = state.crash_address - self.crash_state = state.crash_state - self.crash_stacktrace = utils.get_crash_stacktrace_output( + self.crash_type: str = state.crash_type + self.crash_address: str = state.crash_address + self.crash_state: str = state.crash_state + self.crash_stacktrace: str = utils.get_crash_stacktrace_output( self.application_command_line, state.crash_stacktrace, self.unsymbolized_crash_stacktrace) - self.crash_categories = state.crash_categories + self.crash_categories: Set[str] = state.crash_categories self.security_flag = crash_analyzer.is_security_issue( self.unsymbolized_crash_stacktrace, self.crash_type, self.crash_address) - self.key = f'{self.crash_type},{self.crash_state},{self.security_flag}' + self.key: str = f'{self.crash_type},{self.crash_state},{self.security_flag}' self.should_be_ignored = crash_analyzer.ignore_stacktrace( state.crash_stacktrace) # self.crash_info gets populated in create_testcase; save what we need. - self.crash_frames = state.frames - self.crash_info = None - self.fuzzed_key = None - self.absolute_path = None - self.archive_filename = None - self.archived = False + self.crash_frames: Optional[List[Any]] = state.frames + self.crash_info: Optional[Any] = None + self.fuzzed_key: Optional[str] = None + self.absolute_path: Optional[str] = None + self.archive_filename: Optional[str] = None + self.archived: Optional[bool] = False @property - def filename(self): + def filename(self) -> str: return os.path.basename(self.file_path) - def is_uploaded(self): + def is_uploaded(self) -> bool: """Return true if archive_testcase_in_blobstore(..) was performed.""" return self.fuzzed_key is not None - def archive_testcase_in_blobstore(self, - upload_url: uworker_msg_pb2.BlobUploadUrl): + def archive_testcase_in_blobstore( + self, upload_url: uworker_msg_pb2.BlobUploadUrl) -> None: """Calling setup.archive_testcase_and_dependencies_in_gcs(..) and hydrate certain attributes. We single out this method because it's expensive and we want to do it at the very last minute.""" @@ -281,11 +308,11 @@ def archive_testcase_in_blobstore(self, setup.archive_testcase_and_dependencies_in_gcs( self.resource_list, self.file_path, upload_url.url)) - def is_valid(self): + def is_valid(self) -> bool: """Return true if the crash is valid for processing.""" return self.get_error() is None - def get_error(self): + def get_error(self) -> Optional[str]: """Return the reason why the crash is invalid.""" filter_functional_bugs = environment.get_value('FILTER_FUNCTIONAL_BUGS') if filter_functional_bugs and not self.security_flag: @@ -304,16 +331,23 @@ def get_error(self): return None - def to_proto(self): + def to_proto(self) -> uworker_msg_pb2.FuzzTaskCrash: """Converts this object to a proto.""" is_valid = self.is_valid() crash = uworker_msg_pb2.FuzzTaskCrash() crash.is_valid = is_valid crash.file_path = self.file_path - crash.crash_time = self.crash_time - crash.return_code = self.return_code - crash.resource_list.extend(self.resource_list) - crash.gestures.extend(self.gestures) + if self.crash_time is not None: + crash.crash_time = self.crash_time + if self.return_code is not None: + crash.return_code = self.return_code + if self.resource_list: + crash.resource_list.extend(self.resource_list) + if self.gestures: + if isinstance(self.gestures, (list, tuple)): + crash.gestures.extend(self.gestures) + else: + crash.gestures.append(self.gestures) crash.arguments = self.arguments if self.fuzzing_strategies: crash.fuzzing_strategies.extend(self.fuzzing_strategies) @@ -328,18 +362,20 @@ def to_proto(self): crash.security_flag = self.security_flag crash.key = self.key crash.should_be_ignored = self.should_be_ignored - crash.archived = self.archived + crash.archived = bool(self.archived) if self.fuzzed_key: crash.fuzzed_key = self.fuzzed_key + assert self.absolute_path is not None crash.absolute_path = self.absolute_path if self.archive_filename: crash.archive_filename = self.archive_filename return crash -def find_main_crash(crashes: List[Crash], - fuzz_target: Optional[data_types.FuzzTarget], - test_timeout: int, upload_urls: UploadUrlCollection): +def find_main_crash( + crashes: List[Crash], fuzz_target: Optional[data_types.FuzzTarget], + test_timeout: int, + upload_urls: UploadUrlCollection) -> Tuple[Optional[Crash], Optional[bool]]: """Find the first reproducible crash or the first valid crash. And return the crash and the one_time_crasher_flag.""" for crash in crashes: @@ -380,22 +416,25 @@ class CrashGroup: """Represent a group of identical crashes. The key is (crash_type, crash_state, security_flag).""" - def __init__(self, crashes: List[Crash], context, + def __init__(self, crashes: List[Crash], context: Context, upload_urls: UploadUrlCollection): for c in crashes: assert crashes[0].crash_type == c.crash_type assert crashes[0].crash_state == c.crash_state assert crashes[0].security_flag == c.security_flag - self.crashes = crashes + self.crashes: List[Crash] = crashes + self.main_crash: Optional[Crash] + self.one_time_crasher_flag: Optional[bool] self.main_crash, self.one_time_crasher_flag = find_main_crash( crashes, context.fuzz_target, context.test_timeout, upload_urls) - self.newly_created_testcase = None + self.newly_created_testcase: Optional[data_types.Testcase] = None -def _should_create_testcase(group: uworker_msg_pb2.FuzzTaskCrashGroup, - existing_testcase): +def _should_create_testcase( + group: uworker_msg_pb2.FuzzTaskCrashGroup, + existing_testcase: Optional[data_types.Testcase]) -> bool: """Returns True if this crash should create a testcase.""" if not existing_testcase: return True @@ -415,15 +454,15 @@ def _should_create_testcase(group: uworker_msg_pb2.FuzzTaskCrashGroup, class _TrackFuzzTime: """Track the actual fuzzing time (e.g. excluding preparing binary).""" - _active_trackers = set() - _lock = threading.Lock() - _callback_registered = False - _accumulated_wasted_time = 0 - _session_fuzzer = None - _session_job = None + _active_trackers: Set['_TrackFuzzTime'] = set() + _lock: threading.Lock = threading.Lock() + _callback_registered: bool = False + _accumulated_wasted_time: float = 0 + _session_fuzzer: Optional[str] = None + _session_job: Optional[str] = None @classmethod - def reset_session(cls, fuzzer_name, job_type): + def reset_session(cls, fuzzer_name: str, job_type: str) -> None: """Resets the accumulated time at the start of a session.""" with cls._lock: cls._accumulated_wasted_time = 0 @@ -431,23 +470,23 @@ def reset_session(cls, fuzzer_name, job_type): cls._session_job = job_type @classmethod - def add_wasted_time(cls, duration): + def add_wasted_time(cls, duration: float) -> None: """Accumulates time from finished rounds.""" with cls._lock: cls._accumulated_wasted_time += duration @classmethod - def get_active_trackers(cls): + def get_active_trackers(cls) -> List['_TrackFuzzTime']: with cls._lock: return list(cls._active_trackers) @classmethod def record_metrics(cls, - duration, - fuzzer_name, - job_type, - timeout=None, - is_preempted=False): + duration: float, + fuzzer_name: str, + job_type: str, + timeout: Optional[bool] = None, + is_preempted: bool = False) -> None: """Helper to record metrics consistently.""" common_labels = { 'platform': environment.platform(), @@ -473,7 +512,7 @@ def record_metrics(cls, job_metric.increment_by(int(duration), job_labels) @classmethod - def _preemption_callback(cls): + def _preemption_callback(cls) -> None: """Callback to record wasted fuzzing time on preemption. We track `_accumulated_wasted_time` because if a task is preempted at any @@ -489,6 +528,7 @@ def _preemption_callback(cls): for tracker in active_trackers: tracker.preempted = True # Mark to avoid double counting in __exit__ + assert tracker.start_time is not None current_duration = tracker.time.time() - tracker.start_time total_wasted = cls._accumulated_wasted_time + current_duration @@ -521,15 +561,15 @@ def _preemption_callback(cls): cls._session_job, is_preempted=True) - def __init__(self, fuzzer_name, job_type, time_module=time): - self.fuzzer_name = fuzzer_name - self.job_type = job_type - self.time = time_module - self.start_time = None - self.timeout = None - self.preempted = False + def __init__(self, fuzzer_name: str, job_type: str, time_module: Any = time): + self.fuzzer_name: str = fuzzer_name + self.job_type: str = job_type + self.time: Any = time_module + self.start_time: Optional[float] = None + self.timeout: Optional[bool] = None + self.preempted: bool = False - def __enter__(self): + def __enter__(self) -> '_TrackFuzzTime': if not _TrackFuzzTime._callback_registered: monitor.register_on_sigterm(_TrackFuzzTime._preemption_callback) _TrackFuzzTime._callback_registered = True @@ -540,10 +580,11 @@ def __enter__(self): self._active_trackers.add(self) return self - def __exit__(self, exc_type, value, traceback): + def __exit__(self, exc_type: Any, value: Any, traceback: Any) -> None: with self._lock: self._active_trackers.discard(self) + assert self.start_time is not None duration = self.time.time() - self.start_time if not self.preempted: _TrackFuzzTime.record_metrics( @@ -558,15 +599,16 @@ def __exit__(self, exc_type, value, traceback): 'as it was already handled by preemption callback.') -def _track_fuzzer_run_result(fuzzer_name, job_type, generated_testcase_count, - expected_testcase_count, return_code): +def _track_fuzzer_run_result( + fuzzer_name: str, job_type: str, generated_testcase_count: int, + expected_testcase_count: int | float, return_code: int) -> None: """Track fuzzer run result""" if expected_testcase_count > 0: ratio = float(generated_testcase_count) / expected_testcase_count monitoring_metrics.FUZZER_TESTCASE_COUNT_RATIO.add(ratio, {'fuzzer': fuzzer_name}) - def clamp(val, minimum, maximum): + def clamp(val: int, minimum: int, maximum: int) -> int: return max(minimum, min(maximum, val)) # Clamp return code to max, min int 32-bit, otherwise it can get detected as @@ -585,7 +627,7 @@ def clamp(val, minimum, maximum): }) -def _track_build_run_result(job_type, _, is_bad_build): +def _track_build_run_result(job_type: str, _: Any, is_bad_build: bool) -> None: """Track build run result.""" # FIXME: Add support for |crash_revision| as part of state. monitoring_metrics.JOB_BAD_BUILD_COUNT.increment({ @@ -594,8 +636,8 @@ def _track_build_run_result(job_type, _, is_bad_build): }) -def _track_testcase_run_result(fuzzer, job_type, new_crash_count, - known_crash_count): +def _track_testcase_run_result(fuzzer: str, job_type: str, new_crash_count: int, + known_crash_count: int) -> None: """Track testcase run result.""" monitoring_metrics.FUZZER_KNOWN_CRASH_COUNT.increment_by( known_crash_count, { @@ -623,7 +665,7 @@ def _track_testcase_run_result(fuzzer, job_type, new_crash_count, }) -def _last_sync_time(sync_file_path): +def _last_sync_time(sync_file_path: str) -> Optional[datetime.datetime]: """Read and parse the last sync file for the GCS corpus.""" if not os.path.exists(sync_file_path): return None @@ -648,23 +690,24 @@ def _last_sync_time(sync_file_path): class GcsCorpus: """Sync state for a corpus.""" - def __init__(self, engine_name, project_qualified_target_name, - corpus_directory, data_directory, proto_corpus): + def __init__(self, engine_name: str, project_qualified_target_name: str, + corpus_directory: str, data_directory: str, + proto_corpus: uworker_msg_pb2.FuzzTargetCorpus): if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import \ corpus_manager as remote_corpus_manager - self.gcs_corpus = remote_corpus_manager.RemoteFuzzTargetCorpus( + self.gcs_corpus: Any = remote_corpus_manager.RemoteFuzzTargetCorpus( engine_name, project_qualified_target_name) else: self.gcs_corpus = corpus_manager.ProtoFuzzTargetCorpus.deserialize( proto_corpus) - self._corpus_directory = corpus_directory - self._data_directory = data_directory - self._project_qualified_target_name = project_qualified_target_name - self._synced_files = set() + self._corpus_directory: str = corpus_directory + self._data_directory: str = data_directory + self._project_qualified_target_name: str = project_qualified_target_name + self._synced_files: Set[str] = set() - def _walk(self): + def _walk(self) -> Generator[str, None, None]: if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host yield from file_host.list_files(self._corpus_directory, recursive=True) @@ -673,16 +716,16 @@ def _walk(self): for filename in files: yield os.path.join(root, filename) - def _get_gcs_url(self): + def _get_gcs_url(self) -> str: # TODO(https://github.com/google/clusterfuzz/issues/3726): Get rid of this # wrapper when untrusted runner is removed. return self.gcs_corpus.get_gcs_url() - def _sync_to_disk(self, corpus_directory): + def _sync_to_disk(self, corpus_directory: str) -> bool: self.gcs_corpus.rsync_to_disk(corpus_directory) return True - def sync_from_gcs(self): + def sync_from_gcs(self) -> bool: """Update sync state after a sync from GCS.""" already_synced = False sync_file_path = os.path.join( @@ -727,13 +770,13 @@ def sync_from_gcs(self): return result - def upload_files(self, new_files): + def upload_files(self, new_files: Sequence[str]) -> bool: """Update state after files are uploaded.""" result = self.gcs_corpus.upload_files(new_files) self._synced_files.update(new_files) return result - def get_new_files(self): + def get_new_files(self) -> List[str]: """Return list of new files in the directory that were generated by the fuzzer.""" new_files = [] @@ -744,20 +787,21 @@ def get_new_files(self): return new_files -def upload_testcase_run_stats(testcase_run): +def upload_testcase_run_stats(testcase_run: fuzzer_stats.BaseRun) -> None: """Upload TestcaseRun stats.""" fuzzer_stats.upload_stats([testcase_run]) -def add_additional_testcase_run_data(testcase_run, fully_qualified_fuzzer_name, - job_type, revision): +def add_additional_testcase_run_data(testcase_run: fuzzer_stats.TestcaseRun, + fully_qualified_fuzzer_name: str, + job_type: str, revision: Any) -> None: """Add additional testcase run data.""" testcase_run['fuzzer'] = fully_qualified_fuzzer_name testcase_run['job'] = job_type testcase_run['build_revision'] = revision -def get_fuzzer_metadata_from_output(fuzzer_output): +def get_fuzzer_metadata_from_output(fuzzer_output: str) -> Dict[str, str]: """Extract metadata from fuzzer output.""" metadata = {} for line in fuzzer_output.splitlines(): @@ -768,7 +812,8 @@ def get_fuzzer_metadata_from_output(fuzzer_output): return metadata -def get_testcases(testcase_count, testcase_directory, data_directory): +def get_testcases(testcase_count: int | float, testcase_directory: str, + data_directory: str) -> Tuple[List[str], int, str]: """Return fuzzed testcases from the data directories.""" logs.info('Locating generated test cases.') @@ -811,14 +856,14 @@ def get_testcases(testcase_count, testcase_directory, data_directory): generated_testcase_string) -def set_test_timeout(timeout, multipler): +def set_test_timeout(timeout: float, multipler: float) -> int: """Set the test timeout based on a timeout value and multiplier.""" test_timeout = int(timeout * multipler) environment.set_value('TEST_TIMEOUT', test_timeout) return test_timeout -def truncate_fuzzer_output(output, limit): +def truncate_fuzzer_output(output: str, limit: int) -> str: """Truncate output in the middle according to limit.""" if len(output) < limit: return output @@ -836,16 +881,16 @@ def truncate_fuzzer_output(output, limit): def upload_job_run_stats( fuzzer_name: str, job_type: str, - revision: int, + revision: Any, timestamp: float, new_crash_count: int, known_crash_count: int, testcases_executed: int, - groups: List[Dict[str, Any]], - testcases_generated: int = None, - testcase_generation_duration: datetime.timedelta = None, - testcase_execution_duration: datetime.timedelta = None, - fuzzing_duration: datetime.timedelta = None): + groups: Any, + testcases_generated: Optional[int] = None, + testcase_generation_duration: Optional[datetime.timedelta] = None, + testcase_execution_duration: Optional[datetime.timedelta] = None, + fuzzing_duration: Optional[datetime.timedelta] = None) -> None: """Upload job run stats.""" # New format. job_run = fuzzer_stats.JobRun(fuzzer_name, job_type, revision, timestamp, @@ -859,10 +904,13 @@ def upload_job_run_stats( known_crash_count) -def store_fuzzer_run_results(testcase_file_paths, fuzzer, fuzzer_command, - fuzzer_output, fuzzer_return_code, - generated_testcase_count, expected_testcase_count, - generated_testcase_string, fuzz_task_input): +def store_fuzzer_run_results( + testcase_file_paths: Sequence[str], fuzzer: data_types.Fuzzer, + fuzzer_command: str, fuzzer_output: str, fuzzer_return_code: Optional[int], + generated_testcase_count: int, expected_testcase_count: int | float, + generated_testcase_string: str, + fuzz_task_input: uworker_msg_pb2.FuzzTaskInput +) -> Optional[uworker_msg_pb2.StoreFuzzerRunResultsOutput]: """Store fuzzer run results in database.""" # Upload fuzzer script output to bucket. if environment.is_engine_fuzzer_job(): @@ -910,11 +958,13 @@ def store_fuzzer_run_results(testcase_file_paths, fuzzer, fuzzer_command, fuzzer_run_results_output.console_output = console_output fuzzer_run_results_output.generated_testcase_string = ( generated_testcase_string) - fuzzer_run_results_output.fuzzer_return_code = fuzzer_return_code + if fuzzer_return_code is not None: + fuzzer_run_results_output.fuzzer_return_code = fuzzer_return_code return fuzzer_run_results_output -def preprocess_store_fuzzer_run_results(fuzz_task_input): +def preprocess_store_fuzzer_run_results( + fuzz_task_input: uworker_msg_pb2.FuzzTaskInput) -> None: """Does preprocessing for store_fuzzer_run_results. More specifically, gets URLs to upload a sample testcase and the logs.""" if environment.is_engine_fuzzer_job(): @@ -927,7 +977,8 @@ def preprocess_store_fuzzer_run_results(fuzz_task_input): fuzz_task_input.script_log_upload_key) -def postprocess_store_fuzzer_run_results(output): +def postprocess_store_fuzzer_run_results( + output: uworker_msg_pb2.Output) -> None: """Postprocess store_fuzzer_run_results.""" if environment.is_engine_fuzzer_job(output.uworker_input.job_type): return @@ -953,6 +1004,7 @@ def postprocess_store_fuzzer_run_results(output): data_types.Fuzzer.name == output.uworker_input.fuzzer_name).get() if not fuzzer: logs.log_fatal_and_exit('Fuzzer does not exist, exiting.') + assert fuzzer is not None fuzzer_run_results = output.fuzz_task_output.fuzzer_run_results if fuzzer.revision != output.fuzz_task_output.fuzzer_revision: @@ -969,7 +1021,9 @@ def postprocess_store_fuzzer_run_results(output): logs.info('Finished storing results from fuzzer run.') -def _infer_fully_qualified_fuzzer_name(uworker_input: uworker_msg_pb2.Input): +def _infer_fully_qualified_fuzzer_name( + uworker_input: uworker_msg_pb2.Input) -> str: + """Infers the fully qualified fuzzer name from uworker input.""" fuzz_target = None if uworker_input.fuzz_task_input.HasField('fuzz_target'): fuzz_target = uworker_io.entity_from_protobuf( @@ -981,17 +1035,18 @@ def _infer_fully_qualified_fuzzer_name(uworker_input: uworker_msg_pb2.Input): return fully_qualified_fuzzer_name -def _get_sample_rate(): +def _get_sample_rate() -> int: project_config = local_config.ProjectConfig() return int(project_config.get('fuzz_task_sampling.sampling_rate', 0)) -def _get_replication_topic(): +def _get_replication_topic() -> Optional[str]: project_config = local_config.ProjectConfig() return project_config.get('fuzz_task_sampling.sampling_topic', None) -def _publish_to_pubsub(messages, topic_name): +def _publish_to_pubsub(messages: Sequence[Dict[str, Any]], + topic_name: str) -> None: pubsub_client = pubsub.PubSubClient() topic_name = pubsub.topic_name(utils.get_application_id(), topic_name) pubsub_messages = [ @@ -1000,8 +1055,9 @@ def _publish_to_pubsub(messages, topic_name): pubsub_client.publish(topic_name, pubsub_messages) -def postprocess_sample_testcases(uworker_input: uworker_msg_pb2.Input, - uworker_output: uworker_msg_pb2.Output): +def postprocess_sample_testcases( + uworker_input: uworker_msg_pb2.Input, + uworker_output: uworker_msg_pb2.Output) -> None: """Samples fuzz task testcases to reupload through the Upload Testcase endpoint in AppEngine. Meant to enable analyze task coverage in testing environments.""" @@ -1054,11 +1110,12 @@ def postprocess_sample_testcases(uworker_input: uworker_msg_pb2.Input, _publish_to_pubsub(messages, sampling_topic) -def postprocess_process_crashes(uworker_input: uworker_msg_pb2.Input, - uworker_output: uworker_msg_pb2.Output): +def postprocess_process_crashes( + uworker_input: uworker_msg_pb2.Input, uworker_output: uworker_msg_pb2.Output +) -> Tuple[int, int, List[uworker_msg_pb2.FuzzTaskCrashGroup]]: """Postprocess process_crashes""" processed_groups = [] - crash_groups_for_stats = [] + crash_groups_for_stats: Any = [] new_crash_count = 0 known_crash_count = 0 @@ -1075,7 +1132,7 @@ def postprocess_process_crashes(uworker_input: uworker_msg_pb2.Input, # condition among different machines. One machine might finish first and # prevent other machines from creating identical testcases. existing_testcase = data_handler.find_testcase( - uworker_input.uworker_env.get('PROJECT_NAME'), + uworker_input.uworker_env.get('PROJECT_NAME', ''), group.crashes[0].crash_type, group.crashes[0].crash_state, group.crashes[0].security_flag, @@ -1088,6 +1145,7 @@ def postprocess_process_crashes(uworker_input: uworker_msg_pb2.Input, uworker_output=uworker_output, fully_qualified_fuzzer_name=fully_qualified_fuzzer_name) else: + assert existing_testcase is not None _update_testcase_variant_if_needed(group, existing_testcase, fuzz_task_output.crash_revision, uworker_input.job_type) @@ -1137,19 +1195,20 @@ def postprocess_process_crashes(uworker_input: uworker_msg_pb2.Input, return new_crash_count, known_crash_count, processed_groups -def get_regression(one_time_crasher_flag): +def get_regression(one_time_crasher_flag: Optional[bool]) -> str: """Get the right regression value.""" if one_time_crasher_flag or build_manager.is_custom_binary(): return 'NA' return '' -def get_fixed_or_minimized_key(one_time_crasher_flag): +def get_fixed_or_minimized_key(one_time_crasher_flag: Optional[bool]) -> str: """Get the right fixed value.""" return 'NA' if one_time_crasher_flag else '' -def get_testcase_timeout_multiplier(timeout_multiplier, crash, test_timeout): +def get_testcase_timeout_multiplier(timeout_multiplier: float, crash: Any, + test_timeout: float) -> float: """Get testcase timeout multiplier.""" testcase_timeout_multiplier = timeout_multiplier if timeout_multiplier > 1 and (crash.crash_time + THREAD_WAIT_TIMEOUT) < ( @@ -1162,7 +1221,7 @@ def get_testcase_timeout_multiplier(timeout_multiplier, crash, test_timeout): def create_testcase(group: uworker_msg_pb2.FuzzTaskCrashGroup, uworker_input: uworker_msg_pb2.Input, uworker_output: uworker_msg_pb2.Output, - fully_qualified_fuzzer_name: str): + fully_qualified_fuzzer_name: str) -> data_types.Testcase: """Create a testcase based on crash.""" crash = group.main_crash fuzz_task_output = uworker_output.fuzz_task_output @@ -1257,18 +1316,20 @@ def filter_crashes(crashes: List[Crash]) -> List[Crash]: return filtered -def get_engine(context): +def get_engine(context: Context) -> str: """Get the fuzzing engine.""" - if context.fuzz_target: + if context.fuzz_target and context.fuzz_target.engine: return context.fuzz_target.engine return '' -def write_crashes_to_big_query(group, newly_created_testcase, existing_testcase, - uworker_input: uworker_msg_pb2.Input, - output: uworker_msg_pb2.Output, - fully_qualified_fuzzer_name): +def write_crashes_to_big_query( + group: uworker_msg_pb2.FuzzTaskCrashGroup, + newly_created_testcase: Optional[data_types.Testcase], + existing_testcase: Optional[data_types.Testcase], + uworker_input: uworker_msg_pb2.Input, output: uworker_msg_pb2.Output, + fully_qualified_fuzzer_name: str) -> None: """Write a group of crashes to BigQuery.""" # Many of ChromeOS fuzz targets run on Linux bots, so we incorrectly set the @@ -1358,13 +1419,16 @@ def write_crashes_to_big_query(group, newly_created_testcase, existing_testcase, row_count, {'success': False}) -def _update_testcase_variant_if_needed(group, existing_testcase, crash_revision, - job_type): +def _update_testcase_variant_if_needed( + group: uworker_msg_pb2.FuzzTaskCrashGroup, + existing_testcase: data_types.Testcase, crash_revision: Any, + job_type: str) -> None: """Update testcase variant if this is not already covered by existing testcase variant on this job.""" - variant = data_handler.get_or_create_testcase_variant( - existing_testcase.key.id(), job_type) + testcase_id = existing_testcase.key.id() + assert testcase_id is not None + variant = data_handler.get_or_create_testcase_variant(testcase_id, job_type) if not variant or variant.status == data_types.TestcaseVariantStatus.PENDING: # Either no variant created yet since minimization hasn't finished OR # variant analysis is not yet finished. Wait in both cases, since we @@ -1390,10 +1454,11 @@ def _update_testcase_variant_if_needed(group, existing_testcase, crash_revision, def process_crashes(crashes: List[Crash], context: Context, - upload_urls) -> List[uworker_msg_pb2.FuzzTaskCrashGroup]: + upload_urls: Sequence[uworker_msg_pb2.BlobUploadUrl] + ) -> List[uworker_msg_pb2.FuzzTaskCrashGroup]: """Process a list of crashes.""" - def key_fn(crash): + def key_fn(crash: Crash) -> str: return crash.key crash_groups = [] @@ -1402,10 +1467,10 @@ def key_fn(crash): crashes = filter_crashes(crashes) group_of_crashes = itertools.groupby(sorted(crashes, key=key_fn), key_fn) - upload_urls = UploadUrlCollection(upload_urls) + upload_url_collection = UploadUrlCollection(list(upload_urls)) for _, grouped_crashes in group_of_crashes: try: - group = CrashGroup(list(grouped_crashes), context, upload_urls) + group = CrashGroup(list(grouped_crashes), context, upload_url_collection) except NoMoreUploadUrlsError: # Ignore the remaining crashes. logs.error('Ran out of crash upload URLs.') @@ -1422,6 +1487,7 @@ def key_fn(crash): context.fuzzer_metadata['issue_metadata'] = json.dumps( context.fuzzer_metadata['issue_metadata']) + assert group.one_time_crasher_flag is not None group_proto = uworker_msg_pb2.FuzzTaskCrashGroup( context=uworker_msg_pb2.FuzzContext( redzone=context.redzone, @@ -1448,7 +1514,7 @@ def key_fn(crash): return crash_groups -def _get_issue_metadata_from_environment(variable_name): +def _get_issue_metadata_from_environment(variable_name: str) -> List[str]: """Get issue metadata from environment.""" values = str(environment.get_value_string(variable_name, '')).split(',') # Allow a variation with a '_1' to specified. This is needed in cases where @@ -1458,14 +1524,14 @@ def _get_issue_metadata_from_environment(variable_name): return [value.strip() for value in values if value.strip()] -def _add_issue_metadata_from_environment(metadata): +def _add_issue_metadata_from_environment(metadata: Dict[str, Any]) -> None: """Add issue metadata from environment.""" - def _append(old, new_values): + def _append(old: Optional[str], new_values: Sequence[str]) -> str: if not old: return ','.join(new_values) - return ','.join(old.split(',') + new_values) + return ','.join(old.split(',') + list(new_values)) components = _get_issue_metadata_from_environment('AUTOMATIC_COMPONENTS') if components: @@ -1477,8 +1543,9 @@ def _append(old, new_values): metadata['issue_labels'] = _append(metadata.get('issue_labels'), labels) -def run_engine_fuzzer(engine_impl, target_name, sync_corpus_directory, - testcase_directory): +def run_engine_fuzzer(engine_impl: Any, target_name: str, + sync_corpus_directory: str, + testcase_directory: str) -> Any: """Run engine for fuzzing.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import tasks_host @@ -1535,46 +1602,52 @@ def run_engine_fuzzer(engine_impl, target_name, sync_corpus_directory, class FuzzingSession: """Class for orchestrating fuzzing sessions.""" - def __init__(self, uworker_input, test_timeout): - self.fuzzer_name = uworker_input.fuzzer_name - self.job_type = uworker_input.job_type - self.uworker_input = uworker_input + def __init__(self, uworker_input: uworker_msg_pb2.Input, + test_timeout: Optional[float]): + self.fuzzer_name: str = uworker_input.fuzzer_name + self.job_type: str = uworker_input.job_type + self.uworker_input: uworker_msg_pb2.Input = uworker_input # Set up randomly selected fuzzing parameters. - self.redzone = fuzz_task_knobs.pick_redzone() - self.disable_ubsan = fuzz_task_knobs.pick_ubsan_disabled(self.job_type) - self.timeout_multiplier = fuzz_task_knobs.pick_timeout_multiplier() - self.window_argument = fuzz_task_knobs.pick_window_argument() - self.test_timeout = set_test_timeout(test_timeout, self.timeout_multiplier) + self.redzone: int = fuzz_task_knobs.pick_redzone() + self.disable_ubsan: bool = fuzz_task_knobs.pick_ubsan_disabled( + self.job_type) + self.timeout_multiplier: float = fuzz_task_knobs.pick_timeout_multiplier() + self.window_argument: Optional[str] = fuzz_task_knobs.pick_window_argument() + self.test_timeout: int = set_test_timeout(test_timeout or 0, + self.timeout_multiplier) # Set up during run(). - self.fuzzer = None - self.testcase_directory = None - self.data_directory = None + self.fuzzer: Optional[data_types.Fuzzer] = None + self.testcase_directory: Optional[str] = None + self.data_directory: Optional[str] = None # Fuzzing engine specific state. if uworker_input.fuzz_task_input.HasField('fuzz_target'): - self.fuzz_target = uworker_io.entity_from_protobuf( - uworker_input.fuzz_task_input.fuzz_target, data_types.FuzzTarget) + self.fuzz_target: Optional[ + data_types.FuzzTarget] = uworker_io.entity_from_protobuf( + uworker_input.fuzz_task_input.fuzz_target, data_types.FuzzTarget) else: # We take this branch when no fuzz target is picked. Such as on a new # build. self.fuzz_target = None - self.gcs_corpus = None + self.gcs_corpus: Optional[GcsCorpus] = None self.fuzz_task_output = uworker_msg_pb2.FuzzTaskOutput() # pylint: disable=no-member @property - def fully_qualified_fuzzer_name(self): + def fully_qualified_fuzzer_name(self) -> str: """Get the fully qualified fuzzer name.""" if self.fuzz_target: return self.fuzz_target.fully_qualified_name() return self.fuzzer_name - def sync_corpus(self, sync_corpus_directory): + def sync_corpus(self, sync_corpus_directory: str) -> None: """Sync corpus from GCS.""" # Corpus should always be set at this point. + assert self.fuzz_target is not None + assert self.data_directory is not None self.gcs_corpus = GcsCorpus(self.fuzzer_name, self.fuzz_target.project_qualified_name(), sync_corpus_directory, self.data_directory, @@ -1584,7 +1657,7 @@ def sync_corpus(self, sync_corpus_directory): 'Failed to sync corpus for fuzzer %s (job %s).' % (self.fuzz_target.project_qualified_name(), self.job_type)) - def _file_size(self, file_path): + def _file_size(self, file_path: str) -> Optional[int]: """Return file size depending on whether file is local or remote (untrusted worker).""" if environment.is_trusted_host(): @@ -1594,8 +1667,10 @@ def _file_size(self, file_path): return os.path.getsize(file_path) - def sync_new_corpus_files(self): + def sync_new_corpus_files(self) -> None: """Sync new files from corpus to GCS.""" + assert self.gcs_corpus is not None + assert self.fuzz_target is not None new_files = self.gcs_corpus.get_new_files() new_files_count = len(new_files) logs.info('%d new corpus files generated by fuzzer %s (job %s).' % @@ -1607,7 +1682,9 @@ def sync_new_corpus_files(self): for new_file in new_files: if filtered_new_files_count >= MAX_NEW_CORPUS_FILES: break - if self._file_size(new_file) > engine_common.CORPUS_INPUT_SIZE_LIMIT: + file_size = self._file_size(new_file) + if (file_size is not None and + file_size > engine_common.CORPUS_INPUT_SIZE_LIMIT): continue filtered_new_files.append(new_file) filtered_new_files_count += 1 @@ -1621,15 +1698,18 @@ def sync_new_corpus_files(self): self.gcs_corpus.upload_files(filtered_new_files) def generate_blackbox_testcases( - self, fuzzer, job_type, fuzzer_directory, - testcase_count) -> GenerateBlackboxTestcasesResult: + self, fuzzer: data_types.Fuzzer, job_type: str, fuzzer_directory: str, + testcase_count: int | float) -> GenerateBlackboxTestcasesResult: """Run the blackbox fuzzer and generate testcases.""" # Helper variables. + assert fuzzer.name is not None fuzzer_name = fuzzer.name error_return_value = GenerateBlackboxTestcasesResult(False, None, None) # Clear existing testcases (only if past task failed). + assert self.testcase_directory is not None + assert self.data_directory is not None testcase_directories = [self.testcase_directory] testcase_manager.remove_testcases_from_directories(testcase_directories) @@ -1738,6 +1818,7 @@ def generate_blackbox_testcases( output=fuzzer_output) # Store fuzzer run results. + assert fuzzer.name is not None fuzzer_run_results = store_fuzzer_run_results( testcase_file_paths, fuzzer, fuzzer_command, fuzzer_output, fuzzer_return_code, generated_testcase_count, testcase_count, @@ -1753,8 +1834,12 @@ def generate_blackbox_testcases( return GenerateBlackboxTestcasesResult(success, testcase_file_paths, fuzzer_metadata) - def do_engine_fuzzing(self, engine_impl): + def do_engine_fuzzing(self, + engine_impl: Any) -> Tuple[List[Crash], Dict[str, Any]]: """Run fuzzing engine.""" + assert self.fuzz_target is not None + assert self.data_directory is not None + assert self.testcase_directory is not None _TrackFuzzTime.reset_session(self.fully_qualified_fuzzer_name, self.job_type) @@ -1771,8 +1856,8 @@ def do_engine_fuzzing(self, engine_impl): redzone_size=self.redzone, disable_ubsan=self.disable_ubsan) revision = environment.get_value('APP_REVISION') - crashes = [] - fuzzer_metadata = {} + crashes: List[Crash] = [] + fuzzer_metadata: Dict[str, Any] = {} return_code = 1 # Vanilla return-code for engine crashes. self.fuzz_task_output.app_revision = environment.get_value('APP_REVISION') @@ -1782,6 +1867,7 @@ def do_engine_fuzzing(self, engine_impl): try: with _TrackFuzzTime(self.fully_qualified_fuzzer_name, self.job_type) as tracker: + assert self.fuzz_target.binary is not None result, cur_fuzzer_metadata, fuzzing_strategies = run_engine_fuzzer( engine_impl, self.fuzz_target.binary, sync_corpus_directory, self.testcase_directory) @@ -1843,8 +1929,9 @@ def do_engine_fuzzing(self, engine_impl): return crashes, fuzzer_metadata - def _emit_testcase_generation_time_metric(self, start_time, testcase_count, - fuzzer, job): + def _emit_testcase_generation_time_metric(self, start_time: float, + testcase_count: int | float, + fuzzer: str, job: str) -> None: """Emits the average testcase generation time metric to GCP.""" testcase_generation_finish = time.time() elapsed_testcase_generation_time = testcase_generation_finish @@ -1870,8 +1957,8 @@ def _is_fuzz_task_output_limit_exceeded(self) -> bool: return self.fuzz_task_output.ByteSize() >= PROTOBUF_MSG_LIMIT def _handle_unexecuted_testcases( - self, testcase_file_paths: list[str], testcases_metadata: dict[str, dict], - testcases_executed: int) -> tuple[list[str], dict[str, dict]]: + self, testcase_file_paths: List[str], testcases_metadata: Dict[str, Any], + testcases_executed: int) -> Tuple[List[str], Dict[str, Any]]: """Cleans up unexecuted testcase files and slices execution metadata. Args: @@ -1895,7 +1982,7 @@ def _handle_unexecuted_testcases( def _add_crashes_from_temp_queue( self, temp_queue: queue.Queue, - crashes: list[testcase_manager.Crash]) -> None: + crashes: List[testcase_manager.Crash]) -> None: """Pops items from temp_queue and processes crashes and run outputs. Args: @@ -1913,7 +2000,10 @@ def _add_crashes_from_temp_queue( if fuzzer_run_output and not self._is_fuzz_task_output_limit_exceeded(): self.fuzz_task_output.fuzzer_run_outputs.append(fuzzer_run_output) - def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type): + def do_blackbox_fuzzing( + self, fuzzer: data_types.Fuzzer, fuzzer_directory: str, + job_type: str) -> Tuple[Optional[Dict[str, Any]], Optional[List[str]], + Optional[Dict[str, Any]], Optional[List[Crash]]]: """Run blackbox fuzzing. Currently also used for engine fuzzing.""" _TrackFuzzTime.reset_session(self.fully_qualified_fuzzer_name, job_type) @@ -1947,17 +2037,19 @@ def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type): if not generate_result.success: return None, None, None, None + assert generate_result.testcase_file_paths is not None generation_time = datetime.timedelta(seconds=time.time() - testcase_generation_start) self.fuzz_task_output.testcase_generation_duration.FromTimedelta( generation_time) + assert fuzzer.name is not None self._emit_testcase_generation_time_metric( testcase_generation_start, testcase_count, fuzzer.name, job_type) environment.set_value('FUZZER_NAME', self.fully_qualified_fuzzer_name) # Initialize a list of crashes. - crashes = [] + crashes: List[testcase_manager.Crash] = [] # Helper variables. max_threads = utils.maximum_parallel_processes_allowed() @@ -1973,7 +2065,7 @@ def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type): redzone_size=self.redzone, disable_ubsan=self.disable_ubsan) # Create a dict to store metadata specific to each testcase. - testcases_metadata = {} + testcases_metadata: Dict[str, Any] = {} testcase_file_paths = generate_result.testcase_file_paths self.fuzz_task_output.testcases_generated = len(testcase_file_paths) @@ -1988,7 +2080,7 @@ def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type): db_trials = self.uworker_input.fuzz_task_input.trials # Setup trials from DB and build. # Prepare selecting trials in main loop below. - trial_selector = trials.Trials(db_trials) + trial_selector = trials.Trials(cast(List[str], db_trials)) # TODO(machenbach): Move this back to the main loop and make it test-case # specific in a way that get's persistet on crashes. @@ -2093,7 +2185,7 @@ def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type): if crashes: crashes = [ Crash.from_testcase_manager_crash(crash) for crash in crashes if crash - ] + ] # type: ignore executed_testcase_file_paths = testcase_file_paths executed_testcases_metadata = testcases_metadata if test_number < len(testcase_file_paths): @@ -2103,9 +2195,9 @@ def do_blackbox_fuzzing(self, fuzzer, fuzzer_directory, job_type): testcases_metadata=testcases_metadata, testcases_executed=test_number)) return (generate_result.fuzzer_metadata, executed_testcase_file_paths, - executed_testcases_metadata, crashes) + executed_testcases_metadata, cast(Optional[List[Crash]], crashes)) - def run(self): + def run(self) -> uworker_msg_pb2.Output: """Run the fuzzing session.""" start_time = time.time() # Update LSAN local blacklist with global blacklist. @@ -2145,6 +2237,7 @@ def run(self): environment.get_value('APP_REVISION'), fuzz_target=fuzz_target) _add_build_metadata_to_output(self.fuzz_task_output) + assert self.fuzzer.name is not None engine_impl = engine.get(self.fuzzer.name) if engine_impl and build_setup_result: # If we did not pick a fuzz target to fuzz with the engine, then return @@ -2188,6 +2281,9 @@ def run(self): return uworker_msg_pb2.Output( # pylint: disable=no-member error_type=uworker_msg_pb2.ErrorType.FUZZ_DATA_BUNDLE_SETUP_FAILURE) + testcase_file_paths: Optional[List[str]] + testcases_metadata: Optional[Dict[str, Any]] + crashes: Optional[List[Crash]] if engine_impl: crashes, fuzzer_metadata = self.do_engine_fuzzing(engine_impl) @@ -2223,6 +2319,8 @@ def run(self): # Process and save crashes to datastore. bot_name = environment.get_value('BOT_NAME') project_name = environment.get_value('PROJECT_NAME') + assert fuzzer_metadata is not None + assert testcases_metadata is not None crash_groups = process_crashes( crashes=crashes, context=Context( @@ -2246,6 +2344,7 @@ def run(self): # Delete the fuzzed testcases. This was once explicitly needed since some # testcases resided on NFS and would otherwise be left forever. Now it's # unclear if needed but it is kept because it is not harmful. + assert testcase_file_paths is not None for testcase_file_path in testcase_file_paths: shell.remove_file(testcase_file_path) @@ -2268,7 +2367,7 @@ def run(self): self.fuzz_task_output.crash_revision = str(crash_revision) self.fuzz_task_output.job_run_timestamp = time.time() self.fuzz_task_output.testcases_executed = testcases_executed - self.fuzz_task_output.fuzzer_revision = self.fuzzer.revision + self.fuzz_task_output.fuzzer_revision = cast(int, self.fuzzer.revision) self.fuzz_task_output.crash_groups.extend(crash_groups) fuzzing_session_duration_seconds = time.time() - start_time @@ -2285,7 +2384,7 @@ def run(self): return uworker_msg_pb2.Output(fuzz_task_output=self.fuzz_task_output) # pylint: disable=no-member - def postprocess(self, uworker_output): + def postprocess(self, uworker_output: uworker_msg_pb2.Output) -> None: """Handles postprocessing.""" # TODO(metzman): Finish this. fuzz_task_output = uworker_output.fuzz_task_output @@ -2316,7 +2415,7 @@ def postprocess(self, uworker_output): uworker_input.job_type, uworker_output.fuzz_task_output.build_data) -def _upload_testcase_run_jsons(testcase_run_jsons): +def _upload_testcase_run_jsons(testcase_run_jsons: Sequence[str]) -> None: for testcase_run in testcase_run_jsons: testcase_run = fuzzer_stats.BaseRun.from_json(testcase_run) if not testcase_run: @@ -2326,31 +2425,33 @@ def _upload_testcase_run_jsons(testcase_run_jsons): # TODO(metzman): Find out if this can be a single upload. -def handle_fuzz_build_setup_failure(output): +def handle_fuzz_build_setup_failure(output: uworker_msg_pb2.Output) -> None: _track_fuzzer_run_result(output.uworker_input.fuzzer_name, output.uworker_input.job_type, 0, 0, FuzzErrorCode.BUILD_SETUP_FAILED) -def handle_fuzz_data_bundle_setup_failure(output): +def handle_fuzz_data_bundle_setup_failure( + output: uworker_msg_pb2.Output) -> None: _track_fuzzer_run_result(output.uworker_input.fuzzer_name, output.uworker_input.job_type, 0, 0, FuzzErrorCode.DATA_BUNDLE_SETUP_FAILED) -def handle_fuzz_no_fuzzer(output): +def handle_fuzz_no_fuzzer(output: uworker_msg_pb2.Output) -> None: _track_fuzzer_run_result(output.uworker_input.fuzzer_name, output.uworker_input.job_type, 0, 0, FuzzErrorCode.FUZZER_SETUP_FAILED) -def handle_fuzz_bad_build(uworker_output): +def handle_fuzz_bad_build(uworker_output: uworker_msg_pb2.Output) -> None: testcase_manager.update_build_metadata( uworker_output.uworker_input.job_type, uworker_output.fuzz_task_output.build_data) -def _get_fuzz_target(uworker_input): +def _get_fuzz_target( + uworker_input: uworker_msg_pb2.Input) -> Optional[data_types.FuzzTarget]: if uworker_input.fuzz_task_input.HasField('fuzz_target'): fuzz_target = uworker_io.entity_from_protobuf( uworker_input.fuzz_task_input.fuzz_target, data_types.FuzzTarget) @@ -2359,7 +2460,8 @@ def _get_fuzz_target(uworker_input): return fuzz_target -def utask_main(uworker_input): +def utask_main( + uworker_input: uworker_msg_pb2.Input) -> Optional[uworker_msg_pb2.Output]: """Runs the given fuzzer for one round.""" if not engine.get(uworker_input.fuzzer_name): update_task.update_tests_if_needed( @@ -2373,7 +2475,7 @@ def utask_main(uworker_input): return session.run() -def handle_fuzz_no_fuzz_target_selected(output): +def handle_fuzz_no_fuzz_target_selected(output: uworker_msg_pb2.Output) -> None: save_fuzz_targets(output) # Try again now that there are some fuzz targets. utask_preprocess(output.uworker_input.fuzzer_name, @@ -2381,7 +2483,7 @@ def handle_fuzz_no_fuzz_target_selected(output): output.uworker_input.uworker_env) -def _make_session(uworker_input): +def _make_session(uworker_input: uworker_msg_pb2.Input) -> FuzzingSession: test_timeout = environment.get_value('TEST_TIMEOUT') return FuzzingSession( uworker_input, @@ -2403,7 +2505,7 @@ def _make_session(uworker_input): }).compose_with(uworker_handle_errors.UNHANDLED_ERROR_HANDLER) -def _pick_fuzz_target(): +def _pick_fuzz_target() -> Optional[str]: """Picks a random fuzz target from job_type for use in fuzzing.""" if not environment.is_engine_fuzzer_job(): logs.info('Not engine fuzzer. Not picking fuzz target.') @@ -2414,7 +2516,8 @@ def _pick_fuzz_target(): return build_manager.pick_random_fuzz_target(target_weights) -def _get_or_create_fuzz_target(engine_name, fuzz_target_binary, job_type): +def _get_or_create_fuzz_target(engine_name: str, fuzz_target_binary: str, + job_type: str) -> data_types.FuzzTarget: """Gets or creates a FuzzTarget db entity.""" project = data_handler.get_project_name(job_type) qualified_name = data_types.fuzz_target_fully_qualified_name( @@ -2429,14 +2532,17 @@ def _get_or_create_fuzz_target(engine_name, fuzz_target_binary, job_type): return fuzz_target -def _preprocess_get_fuzz_target(fuzzer_name, job_type): +def _preprocess_get_fuzz_target( + fuzzer_name: str, job_type: str) -> Optional[data_types.FuzzTarget]: fuzz_target_name = _pick_fuzz_target() if fuzz_target_name: return _get_or_create_fuzz_target(fuzzer_name, fuzz_target_name, job_type) return None -def _utask_preprocess(fuzzer_name, job_type, uworker_env): +def _utask_preprocess( + fuzzer_name: str, job_type: str, + uworker_env: MutableMapping[str, str]) -> uworker_msg_pb2.Input: """Preprocess untrusted task.""" setup_input = setup.preprocess_update_fuzzer_and_data_bundles(fuzzer_name) fuzz_task_knobs.do_multiarmed_bandit_strategy_selection(uworker_env) @@ -2485,7 +2591,9 @@ def _utask_preprocess(fuzzer_name, job_type, uworker_env): ) -def utask_preprocess(fuzzer_name, job_type, uworker_env): +def utask_preprocess( + fuzzer_name: str, job_type: str, + uworker_env: MutableMapping[str, str]) -> Optional[uworker_msg_pb2.Input]: """Set logs context and preprocess untrusted task.""" # Delay adding the fuzz target to logs context until it is chosen in # preprocess. @@ -2497,7 +2605,7 @@ def utask_preprocess(fuzzer_name, job_type, uworker_env): return None -def save_fuzz_targets(output): +def save_fuzz_targets(output: uworker_msg_pb2.Output) -> None: """Saves fuzz targets that were seen in the build to the database.""" if not output.fuzz_task_output.fuzz_targets: return @@ -2508,8 +2616,8 @@ def save_fuzz_targets(output): output.uworker_input.job_type) -def _to_fuzzer_run_output(data: testcase_manager.FuzzerRunOutputData - ) -> uworker_msg_pb2.FuzzerRunOutput | None: +def _to_fuzzer_run_output(data: Optional[testcase_manager.FuzzerRunOutputData] + ) -> Optional[uworker_msg_pb2.FuzzerRunOutput]: """Returns a FuzzerRunOutput proto from FuzzerRunOutputData.""" if not data: return None @@ -2522,6 +2630,7 @@ def _to_fuzzer_run_output(data: testcase_manager.FuzzerRunOutputData if len(output) != len(truncated_output): logs.warning('Fuzzer output truncated.') + assert data.log_time is not None proto_timestamp = uworker_io.timestamp_to_proto_timestamp(data.log_time) fuzzer_run_output = uworker_msg_pb2.FuzzerRunOutput( output=truncated_output.encode('utf-8'), @@ -2544,7 +2653,10 @@ def _add_build_metadata_to_output( fuzz_task_output.gn_args = data_handler.get_filtered_gn_args() or '' -def _upload_fuzzer_run_output(fuzzer_run_output, fuzzer_name): +def _upload_fuzzer_run_output( + fuzzer_run_output: uworker_msg_pb2.FuzzerRunOutput, + fuzzer_name: str) -> None: + """Uploads fuzzer run log and testcase.""" timestamp = uworker_io.proto_timestamp_to_timestamp( fuzzer_run_output.timestamp) testcase_manager.upload_log( @@ -2556,7 +2668,7 @@ def _upload_fuzzer_run_output(fuzzer_run_output, fuzzer_name): None, fuzzer_run_output.testcase, timestamp, fuzzer_name=fuzzer_name) -def _utask_postprocess(output): +def _utask_postprocess(output: uworker_msg_pb2.Output) -> None: """Postprocesses fuzz_task.""" if output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR: # pylint: disable=no-member _ERROR_HANDLER.handle(output) @@ -2575,7 +2687,7 @@ def _utask_postprocess(output): _upload_fuzzer_run_output(fuzzer_run_output, fuzzer_name) -def utask_postprocess(output): +def utask_postprocess(output: uworker_msg_pb2.Output) -> None: """Sets fuzzing logs context and postprocesses fuzz_task.""" fuzzer_name = output.uworker_input.fuzzer_name job_type = output.uworker_input.job_type diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task_knobs.py b/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task_knobs.py index 2f6e96a09e2..61f3a944b2e 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task_knobs.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task_knobs.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. """Handles knobs that can be randomly chosen for fuzz_task.""" -import collections +from collections.abc import MutableMapping +from typing import Any +from typing import NamedTuple from clusterfuzz._internal.base import utils from clusterfuzz._internal.datastore import data_types @@ -23,16 +25,25 @@ DEFAULT_CHOOSE_PROBABILITY = 9 # 10% MAX_GESTURES = 30 -Redzone = collections.namedtuple('Redzone', ['size', 'weight']) -SelectionMethod = collections.namedtuple('SelectionMethod', - 'method_name probability') -SELECTION_METHOD_DISTRIBUTION = [ + +class Redzone(NamedTuple): + size: int + weight: float + + +class SelectionMethod(NamedTuple): + method_name: str + probability: float + + +SELECTION_METHOD_DISTRIBUTION: list[SelectionMethod] = [ SelectionMethod('default', .7), SelectionMethod('multi_armed_bandit', .3) ] -def do_multiarmed_bandit_strategy_selection(uworker_env): +def do_multiarmed_bandit_strategy_selection( + uworker_env: MutableMapping[str, str] | None) -> None: """Set multi-armed bandit strategy selection during preprocessing. Set multi-armed bandit strategy selection distribution as an environment variable so we can access it in launcher.""" @@ -52,7 +63,7 @@ def do_multiarmed_bandit_strategy_selection(uworker_env): uworker_env) -def pick_gestures(test_timeout): +def pick_gestures(test_timeout: int) -> list[str]: """Return a list of random gestures.""" if not environment.get_value('ENABLE_GESTURES', True): # Gestures disabled. @@ -77,7 +88,7 @@ def pick_gestures(test_timeout): return gestures -def pick_redzone(): +def pick_redzone() -> int: """Return a random size for redzone.""" thread_multiplier = environment.get_value('THREAD_MULTIPLIER', 1) @@ -104,7 +115,7 @@ def pick_redzone(): return utils.random_weighted_choice(redzone_list).size -def pick_ubsan_disabled(job_type): +def pick_ubsan_disabled(job_type: str) -> bool: """Choose whether to disable UBSan in an ASan+UBSan build.""" # This is only applicable in an ASan build. memory_tool_name = environment.get_memory_tool_name(job_type) @@ -118,7 +129,7 @@ def pick_ubsan_disabled(job_type): return not utils.random_number(0, DEFAULT_CHOOSE_PROBABILITY) -def pick_timeout_multiplier(): +def pick_timeout_multiplier() -> float: """Return a random testcase timeout multiplier and adjust timeout.""" fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT') custom_timeout_multipliers = environment.get_value( @@ -139,7 +150,7 @@ def pick_timeout_multiplier(): return timeout_multiplier -def pick_window_argument(): +def pick_window_argument() -> str: """Return a window argument with random size and x,y position.""" default_window_argument = environment.get_value('WINDOW_ARG', '') window_argument_change_chance = not utils.random_number( @@ -174,7 +185,7 @@ def pick_window_argument(): return window_argument -def get_strategy_distribution_from_ndb(): +def get_strategy_distribution_from_ndb() -> list[dict[str, Any]]: """Queries and returns the distribution stored in the ndb table.""" query = data_types.FuzzStrategyProbability.query() distribution = [] diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/minimize_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/minimize_task.py index 67e7f04355f..3d1bff32ce6 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/minimize_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/minimize_task.py @@ -14,10 +14,16 @@ """Minimize task for handling testcase minimization.""" import binascii +from collections.abc import Callable +from collections.abc import Mapping +from collections.abc import Sequence import functools import os import threading import time +from typing import Any +from typing import BinaryIO +from typing import cast from typing import Dict from typing import List from typing import Optional @@ -61,14 +67,14 @@ from clusterfuzz._internal.system import shell from clusterfuzz.fuzz import engine -IPCDUMP_TIMEOUT = 60 -COMBINED_IPCDUMP_TIMEOUT = 60 * 3 -MAX_DEADLINE_EXCEEDED_ATTEMPTS = 3 -MAX_TEMPORARY_FILE_BASENAME_LENGTH = 32 -MINIMIZE_SANITIZER_OPTIONS_RETRIES = 3 -TOKENS_PER_IPCDUMP = 2000 +IPCDUMP_TIMEOUT: int = 60 +COMBINED_IPCDUMP_TIMEOUT: int = 60 * 3 +MAX_DEADLINE_EXCEEDED_ATTEMPTS: int = 3 +MAX_TEMPORARY_FILE_BASENAME_LENGTH: int = 32 +MINIMIZE_SANITIZER_OPTIONS_RETRIES: int = 3 +TOKENS_PER_IPCDUMP: int = 2000 -IPC_MESSAGE_UTIL_EXECUTABLE_FOR_PLATFORM = { +IPC_MESSAGE_UTIL_EXECUTABLE_FOR_PLATFORM: Dict[str, str] = { 'LINUX': 'ipc_message_util', 'WINDOWS': 'ipc_message_util.exe', } @@ -77,54 +83,55 @@ # unneeded when reproducing a given crash, but after the bug is fixed, the lack # of these options might prevent ClusterFuzz from verifying the fix and closing # the bug. See https://github.com/google/oss-fuzz/issues/3227 for example. -MANDATORY_OSS_FUZZ_OPTIONS = [ +MANDATORY_OSS_FUZZ_OPTIONS: List[str] = [ 'silence_unsigned_overflow', ] class MinimizationPhase: """Effectively an enum to represent the current phase of minimization.""" - GESTURES = 0 - MAIN_FILE = 1 - FILE_LIST = 2 - RESOURCES = 3 - ARGUMENTS = 4 + GESTURES: int = 0 + MAIN_FILE: int = 1 + FILE_LIST: int = 2 + RESOURCES: int = 3 + ARGUMENTS: int = 4 class TestRunner: """Helper class for running the same test multiple times.""" - def __init__(self, testcase, file_path, files, input_directory, arguments, - required_arguments, threads, deadline): - self.testcase = testcase - self.file_path = file_path - self.files = files - self.input_directory = input_directory - self.gestures = testcase.gestures - self.arguments = arguments - self.threads = threads - self.deadline = deadline - - self.cleanup_interval = environment.get_value( + def __init__(self, testcase: data_types.Testcase, file_path: str, + files: List[str], input_directory: str, arguments: str, + required_arguments: str, threads: int, deadline: float) -> None: + self.testcase: data_types.Testcase = testcase + self.file_path: str = file_path + self.files: List[str] = files + self.input_directory: str = input_directory + self.gestures: Optional[List[str]] = testcase.gestures + self.arguments: str = arguments + self.threads: int = threads + self.deadline: float = deadline + + self.cleanup_interval: int = environment.get_value( 'TESTCASES_BEFORE_STALE_PROCESS_CLEANUP', 1) - self.timeout = environment.get_value('TEST_TIMEOUT', 10) - self.full_timeout = self.timeout - self.last_failing_result = None - self.required_arguments = set(required_arguments.split()) + self.timeout: int | float = environment.get_value('TEST_TIMEOUT', 10) + self.full_timeout: int | float = self.timeout + self.last_failing_result: Optional[CrashResult] = None + self.required_arguments: set[str] = set(required_arguments.split()) - self.expected_security_flag = False - self.is_flaky = False - self.expected_state = None + self.expected_security_flag: bool = False + self.is_flaky: bool = False + self.expected_state: Optional[str] = None - self._profile_lock = threading.Lock() - self._available_profiles = [True] * threads + self._profile_lock: threading.Lock = threading.Lock() + self._available_profiles: List[bool] = [True] * threads - self._result_lock = threading.Lock() - self._results = [] + self._result_lock: threading.Lock = threading.Lock() + self._results: List[CrashResult] = [] - self._previous_arguments = None + self._previous_arguments: Optional[str] = None - def _get_profile_index(self): + def _get_profile_index(self) -> int: """Get the first available profile directory index.""" with self._profile_lock: for index, is_available in enumerate(self._available_profiles): @@ -135,12 +142,12 @@ def _get_profile_index(self): # Raise an exception rather than running in a bad state. raise errors.BadStateError('No profile directories available.') - def _release_profile(self, index): + def _release_profile(self, index: int) -> None: """Mark the specified profile as available.""" with self._profile_lock: self._available_profiles[index] = True - def _handle_test_result(self, result): + def _handle_test_result(self, result: CrashResult) -> bool: """Handle a test result, return True on pass (no crash), False on fail.""" if not result.is_crash(): return True @@ -162,7 +169,7 @@ def _handle_test_result(self, result): self.last_failing_result = result return False - def _repopulate_required_arguments(self, arguments): + def _repopulate_required_arguments(self, arguments: List[str]) -> List[str]: """Add required arguments back to the argument list.""" fixed_arguments = [] original_arguments = self.arguments.split() @@ -185,17 +192,17 @@ def _repopulate_required_arguments(self, arguments): return fixed_arguments - def get_argument_string(self, arguments): + def get_argument_string(self, arguments: List[str]) -> str: """Convert a list of argument tokens to a usable value.""" fixed_arguments = self._repopulate_required_arguments(arguments) return ' '.join(fixed_arguments) - def test_with_defaults(self, _): + def test_with_defaults(self, _: Any) -> bool: """Run a test with all default values.""" result = self.run() return self._handle_test_result(result) - def test_with_files(self, files): + def test_with_files(self, files: List[str]) -> bool: """Run the test with the specified file list.""" files_to_rename = list(set(self.files) - set(files)) files_to_skip = [] @@ -236,17 +243,17 @@ def test_with_files(self, files): return self._handle_test_result(result) - def test_with_file(self, file_path): + def test_with_file(self, file_path: str) -> bool: """Run the test with the specified contents for a particular file.""" result = self.run(file_path=file_path) return self._handle_test_result(result) - def test_with_gestures(self, gestures): + def test_with_gestures(self, gestures: List[str]) -> bool: """Run the test with the specified gesture list.""" result = self.run(gestures=gestures) return self._handle_test_result(result) - def test_with_command_line_arguments(self, arguments): + def test_with_command_line_arguments(self, arguments: List[str]) -> bool: """Run the test with the specified command line.""" fixed_arguments = self.get_argument_string(arguments) result = self.run( @@ -255,20 +262,20 @@ def test_with_command_line_arguments(self, arguments): use_fresh_profile=True) return self._handle_test_result(result) - def set_test_expectations(self, security_flag, is_flaky, - unsymbolized_crash_state): + def set_test_expectations(self, security_flag: bool, is_flaky: bool, + unsymbolized_crash_state: Optional[str]) -> None: """Set expectations when using this runner for tests.""" self.expected_security_flag = security_flag self.is_flaky = is_flaky self.expected_state = unsymbolized_crash_state def run(self, - file_path=None, - gestures=None, - arguments=None, - timeout=None, - log_command=False, - use_fresh_profile=False): + file_path: Optional[str] = None, + gestures: Optional[List[str]] = None, + arguments: Optional[str] = None, + timeout: Optional[int | float] = None, + log_command: bool = False, + use_fresh_profile: bool = False) -> CrashResult: """Run the test.""" if file_path is None: file_path = self.file_path @@ -283,7 +290,7 @@ def run(self, if timeout is None: timeout = self.timeout - needs_http = self.testcase.http_flag + needs_http = bool(self.testcase.http_flag) profile_index = self._get_profile_index() if use_fresh_profile and environment.get_value('USER_PROFILE_ARG'): @@ -314,7 +321,7 @@ def run(self, self._release_profile(profile_index) return CrashResult(return_code, crash_time, output) - def store_result_from_run(self, result): + def store_result_from_run(self, result: CrashResult) -> None: """Run and store the result for later processing.""" with self._result_lock: self._results.append(result) @@ -324,7 +331,8 @@ def store_result_from_run(self, result): if not self.last_failing_result and result.is_crash(): self.last_failing_result = result - def execute_parallel_runs(self, runs, instances=None): + def execute_parallel_runs( + self, runs: int, instances: Optional[int] = None) -> List[CrashResult]: """Run multiple instances of this test in parallel.""" if not instances: instances = self.threads @@ -340,7 +348,8 @@ def execute_parallel_runs(self, runs, instances=None): run_queue = minimizer.TestQueue( instances, per_thread_cleanup_function=cleanup_function) for _ in range(runs): - run_queue.push(self.file_path, self.run, self.store_result_from_run) + run_queue.push(self.file_path, cast(Callable[..., bool], self.run), + cast(Callable[[bool], None], self.store_result_from_run)) run_queue.process() @@ -355,7 +364,8 @@ def execute_parallel_runs(self, runs, instances=None): return results -def _get_minimize_task_input(testcase): +def _get_minimize_task_input( + testcase: data_types.Testcase) -> uworker_msg_pb2.MinimizeTaskInput: # pylint: disable=no-member testcase_blob_name, testcase_upload_url = blobs.get_blob_signed_upload_url() (stacktrace_blob_name, stacktrace_upload_url) = blobs.get_blob_signed_upload_url() @@ -369,7 +379,9 @@ def _get_minimize_task_input(testcase): arguments=arguments) -def utask_preprocess(testcase_id, job_type, uworker_env): +def utask_preprocess( + testcase_id: str, job_type: str, + uworker_env: Dict[str, Any]) -> Optional[uworker_msg_pb2.Input]: # pylint: disable=no-member """Preprocess in a trusted bot.""" # Locate the testcase associated with the id. try: @@ -403,7 +415,8 @@ def utask_preprocess(testcase_id, job_type, uworker_env): return uworker_input -def utask_main(uworker_input: uworker_msg_pb2.Input): # pylint: disable=no-member +def utask_main( + uworker_input: uworker_msg_pb2.Input) -> Optional[uworker_msg_pb2.Output]: # pylint: disable=no-member """Attempt to minimize a given testcase.""" testcase = uworker_io.entity_from_protobuf(uworker_input.testcase, data_types.Testcase) @@ -417,6 +430,9 @@ def utask_main(uworker_input: uworker_msg_pb2.Input): # pylint: disable=no-memb if uworker_error_output: return uworker_error_output + assert testcase_file_path is not None + assert file_list is not None + # Initialize variables. max_timeout = environment.get_value('TEST_TIMEOUT', 10) app_arguments = environment.get_value('APP_ARGS') @@ -425,10 +441,12 @@ def utask_main(uworker_input: uworker_msg_pb2.Input): # pylint: disable=no-memb last_tested_crash_revision = testcase.get_metadata( 'last_tested_crash_revision') - crash_revision = last_tested_crash_revision or testcase.crash_revision + crash_revision = int(last_tested_crash_revision or + testcase.crash_revision or 0) fuzz_target = testcase_manager.get_fuzz_target_from_input(uworker_input) - fuzz_target = fuzz_target.binary if fuzz_target else None - build_setup_result = build_manager.setup_build(crash_revision, fuzz_target) + fuzz_target_binary = fuzz_target.binary if fuzz_target else None + build_setup_result = build_manager.setup_build(crash_revision, + fuzz_target_binary) # Check if we have an application path. If not, our build failed # to setup correctly. @@ -526,8 +544,9 @@ def utask_main(uworker_input: uworker_msg_pb2.Input): # pylint: disable=no-memb minimize_task_output=minimize_task_output, error_type=uworker_msg_pb2.ErrorType.MINIMIZE_CRASH_TOO_FLAKY) # pylint: disable=no-member - test_runner.set_test_expectations(testcase.security_flag, flaky_stack, - saved_unsymbolized_crash_state) + test_runner.set_test_expectations( + bool(testcase.security_flag), flaky_stack, + saved_unsymbolized_crash_state) # Use the max crash time unless this would be greater than the max timeout. test_timeout = min(max(crash_times), max_timeout) + 1 @@ -543,7 +562,8 @@ def utask_main(uworker_input: uworker_msg_pb2.Input): # pylint: disable=no-memb # At this point, we do not have a test case to store, so we can't call # check_deadline_exceeded_and_store_partial_minimized_testcase. - if testcase.security_flag and len(testcase.gestures) != len(gestures): + if testcase.security_flag and len(testcase.gestures or + []) != len(gestures): # Re-run security severity analysis since gestures affect the severity. testcase.security_severity = severity_analyzer.get_security_severity( testcase.crash_type, data_handler.get_stacktrace(testcase), @@ -662,7 +682,7 @@ def utask_main(uworker_input: uworker_msg_pb2.Input): # pylint: disable=no-memb command = testcase_manager.get_command_line_for_application( testcase_file_path, app_args=app_arguments, - needs_http=testcase.http_flag) + needs_http=bool(testcase.http_flag)) last_crash_result = test_runner.last_failing_result store_minimized_testcase(testcase, input_directory, file_list, data, @@ -676,7 +696,7 @@ def utask_main(uworker_input: uworker_msg_pb2.Input): # pylint: disable=no-memb return uworker_msg_pb2.Output(minimize_task_output=minimize_task_output) # pylint: disable=no-member -def _cleanup_unused_blobs_from_storage(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def _cleanup_unused_blobs_from_storage(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Cleanup the blobs created in preprocess if they weren't used during utask_main.""" delete_testcase_blob = True @@ -702,7 +722,7 @@ def _cleanup_unused_blobs_from_storage(output: uworker_msg_pb2.Output): # pylin blobs.delete_blob(stacktrace_blob_name) -def update_testcase(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def update_testcase(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Updates the tescase using the values passed from utask_main. This is done at the beginning of utask_postprocess and before error handling is called.""" if not output.HasField('minimize_task_output'): @@ -746,7 +766,7 @@ def update_testcase(output: uworker_msg_pb2.Output): # pylint: disable=no-membe testcase.put() -def handle_minimize_setup_error(output): +def handle_minimize_setup_error(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles errors occuring during setup.""" build_fail_wait = environment.get_value('FAIL_WAIT') @@ -765,7 +785,8 @@ def handle_minimize_setup_error(output): wait_time=build_fail_wait) -def handle_minimize_unreproducible_crash(output): +def handle_minimize_unreproducible_crash( + output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles unreproducible crashes.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, @@ -773,7 +794,7 @@ def handle_minimize_unreproducible_crash(output): task_creation.mark_unreproducible_if_flaky(testcase, 'minimize', True) -def handle_minimize_crash_too_flaky(output): +def handle_minimize_crash_too_flaky(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Schedules postminimize tasks when the crash is too flaky.""" # We reproduced this crash at least once. It's too flaky to minimize, but # maybe we'll have more luck in the other jobs. @@ -785,7 +806,8 @@ def handle_minimize_crash_too_flaky(output): task_creation.create_postminimize_tasks(testcase) -def handle_minimize_deadline_exceeded_in_main_file_phase(output): +def handle_minimize_deadline_exceeded_in_main_file_phase( + output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Reschedules the minimize task when the deadline is exceeded just before starting the main file phase.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) @@ -796,7 +818,7 @@ def handle_minimize_deadline_exceeded_in_main_file_phase(output): output.uworker_input.job_type) -def handle_minimize_deadline_exceeded(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_minimize_deadline_exceeded(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Reschedules a minimize task when minimization deadline is exceeded or calls _skip_minimization when the number of reattempts is surpassed.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) @@ -816,7 +838,7 @@ def handle_minimize_deadline_exceeded(output: uworker_msg_pb2.Output): # pylint def handle_libfuzzer_minimization_unreproducible( - output: uworker_msg_pb2.Output): # pylint: disable=no-member + output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles libfuzzer minimization task's failure to reproduce the issue.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) # Be more lenient with marking testcases as unreproducible when this is a @@ -828,7 +850,8 @@ def handle_libfuzzer_minimization_unreproducible( task_creation.mark_unreproducible_if_flaky(testcase, 'minimize', True) -def handle_libfuzzer_minimization_failed(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_libfuzzer_minimization_failed( + output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles libfuzzer minimization task failure.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) _skip_minimization( @@ -858,7 +881,9 @@ def handle_libfuzzer_minimization_failed(output: uworker_msg_pb2.Output): # pyl ) -def finalize_testcase(testcase_id, last_crash_result_dict, flaky_stack=False): +def finalize_testcase(testcase_id: str, + last_crash_result_dict: Optional[Mapping[str, str]], + flaky_stack: bool = False) -> None: """Perform final updates on a test case and prepare it for other tasks.""" # Symbolize crash output if we have it. testcase = data_handler.get_testcase_by_id(testcase_id) @@ -880,7 +905,7 @@ def finalize_testcase(testcase_id, last_crash_result_dict, flaky_stack=False): task_creation.create_postminimize_tasks(testcase) -def utask_postprocess(output): +def utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Postprocess in a trusted bot.""" # Retrive the testcase early for logs context. try: @@ -905,7 +930,7 @@ def utask_postprocess(output): flaky_stack=output.minimize_task_output.flaky_stack) -def should_attempt_phase(testcase, phase): +def should_attempt_phase(testcase: data_types.Testcase, phase: int) -> bool: """Return true if we should we attempt a minimization phase.""" if (phase == MinimizationPhase.ARGUMENTS and environment.is_engine_fuzzer_job()): @@ -917,7 +942,8 @@ def should_attempt_phase(testcase, phase): return phase >= current_phase -def minimize_gestures(test_runner, testcase): +def minimize_gestures(test_runner: TestRunner, + testcase: data_types.Testcase) -> List[str]: """Minimize the gesture list for a test case.""" gestures = testcase.gestures if gestures: @@ -935,7 +961,8 @@ def minimize_gestures(test_runner, testcase): return gestures -def minimize_main_file(test_runner, testcase_file_path, data): +def minimize_main_file(test_runner: TestRunner, testcase_file_path: str, + data: bytes) -> bytes: """Minimize the main test case file.""" if not can_minimize_file(testcase_file_path): return data @@ -950,7 +977,8 @@ def minimize_main_file(test_runner, testcase_file_path, data): return data -def minimize_file_list(test_runner, file_list, input_directory, main_file): +def minimize_file_list(test_runner: TestRunner, file_list: List[str], + input_directory: str, main_file: str) -> List[str]: """Minimize the test case files.""" if len(file_list) <= 1: return file_list @@ -976,7 +1004,8 @@ def minimize_file_list(test_runner, file_list, input_directory, main_file): return file_list -def minimize_resource(test_runner, dependency, input_directory, main_file): +def minimize_resource(test_runner: TestRunner, dependency: str, + input_directory: str, main_file: str) -> None: """Minimize a resource for the test case.""" # TODO(mbarbella): Simplify this with refactoring of setup_testcase. offset = len(input_directory) + len(os.path.sep) @@ -1007,7 +1036,7 @@ def minimize_resource(test_runner, dependency, input_directory, main_file): logs.info(f'Minimized dependency file: {dependency}') -def minimize_arguments(test_runner, app_arguments): +def minimize_arguments(test_runner: TestRunner, app_arguments: str) -> str: """Minimize the argument list for a test case.""" argument_minimizer = delta_minimizer.DeltaMinimizer( test_runner.test_with_command_line_arguments, @@ -1027,10 +1056,10 @@ def store_minimized_testcase( testcase: data_types.Testcase, base_directory: str, file_list: List[str], - file_to_run_data: str, + file_to_run_data: bytes, file_to_run: str, minimize_task_input: uworker_msg_pb2.MinimizeTaskInput, # pylint: disable=no-member - minimize_task_output: uworker_msg_pb2.MinimizeTaskOutput): # pylint: disable=no-member + minimize_task_output: uworker_msg_pb2.MinimizeTaskOutput) -> None: # pylint: disable=no-member """Store all files that make up this testcase.""" # Write the main file data. utils.write_data_to_file(file_to_run_data, file_to_run) @@ -1067,7 +1096,8 @@ def store_minimized_testcase( if not filtered_file_list: # We minimized everything. The only thing needed to reproduce is the # interaction gesture. - with open(file_list[0], 'wb'): + testcase_filepath = os.path.join(base_directory, file_list[0]) + with open(testcase_filepath, 'wb'): # Open it to clear the file contents. pass else: @@ -1075,17 +1105,24 @@ def store_minimized_testcase( testcase.absolute_path = os.path.join( base_directory, os.path.basename(testcase_filepath)) minimize_task_output.absolute_path = testcase.absolute_path - testcase.archive_state &= ~data_types.ArchiveStatus.MINIMIZED - minimize_task_output.archive_state = testcase.archive_state + if testcase.archive_state: + testcase.archive_state = ( + int(testcase.archive_state) & + ~data_types.ArchiveStatus.MINIMIZED) + minimize_task_output.archive_state = testcase.archive_state else: absolute_filename = os.path.join(base_directory, file_list[0]) testcase_filepath = absolute_filename - testcase.archive_state &= ~data_types.ArchiveStatus.MINIMIZED - minimize_task_output.archive_state = testcase.archive_state + if testcase.archive_state: + testcase.archive_state = ( + int(testcase.archive_state) & ~data_types.ArchiveStatus.MINIMIZED) + minimize_task_output.archive_state = testcase.archive_state else: testcase_filepath = file_list[0] - testcase.archive_state &= ~data_types.ArchiveStatus.MINIMIZED - minimize_task_output.archive_state = testcase.archive_state + if testcase.archive_state: + testcase.archive_state = ( + int(testcase.archive_state) & ~data_types.ArchiveStatus.MINIMIZED) + minimize_task_output.archive_state = testcase.archive_state if not testcase_filepath: logs.error('Unable to find file to read') @@ -1109,11 +1146,11 @@ def store_minimized_testcase( def check_deadline_exceeded_and_store_partial_minimized_testcase( - deadline, + deadline: float, testcase: data_types.Testcase, input_directory: str, - file_list, - file_to_run_data, + file_list: List[str], + file_to_run_data: bytes, main_file_path: str, minimize_task_input: uworker_msg_pb2.MinimizeTaskInput, # pylint: disable=no-member minimize_task_output: uworker_msg_pb2.MinimizeTaskOutput) -> bool: # pylint: disable=no-member @@ -1125,9 +1162,11 @@ def check_deadline_exceeded_and_store_partial_minimized_testcase( return time.time() > deadline -def check_for_initial_crash(test_runner, crash_retries, testcase): +def check_for_initial_crash( + test_runner: TestRunner, crash_retries: int, + testcase: data_types.Testcase) -> tuple[Optional[str], bool, List[float]]: """Initial check to see how long it takes to reproduce a crash.""" - crash_times = [] + crash_times: List[float] = [] flaky_stack = False saved_crash_state = None saved_security_flag = None @@ -1152,7 +1191,8 @@ def check_for_initial_crash(test_runner, crash_retries, testcase): if security_flag != testcase.security_flag: continue - crash_times.append(result.crash_time) + if result.crash_time is not None: + crash_times.append(result.crash_time) if not saved_crash_state: saved_crash_state = crash_state @@ -1171,7 +1211,7 @@ def check_for_initial_crash(test_runner, crash_retries, testcase): return saved_unsymbolized_crash_state, flaky_stack, crash_times -def get_temporary_file_name(original_file): +def get_temporary_file_name(original_file: str) -> str: """Generate a temporary file name in the same directory as |original_file|.""" directory, basename = os.path.split(original_file) basename = basename[-MAX_TEMPORARY_FILE_BASENAME_LENGTH:] @@ -1182,7 +1222,8 @@ def get_temporary_file_name(original_file): return new_file_path -def get_temporary_file(original_file, no_modifications=False): +def get_temporary_file(original_file: str, + no_modifications: bool = False) -> BinaryIO: """Get a temporary file handle with a name based on an original file name.""" if no_modifications: handle = open(original_file, 'wb') @@ -1192,7 +1233,7 @@ def get_temporary_file(original_file, no_modifications=False): return handle -def get_ipc_message_util_executable(): +def get_ipc_message_util_executable() -> Optional[str]: """Return the ipc_message_util executable path for the current build.""" app_directory = environment.get_value('APP_DIR') platform = environment.platform() @@ -1206,7 +1247,8 @@ def get_ipc_message_util_executable(): return os.path.join(app_directory, executable) -def create_partial_ipc_dump(tokens, original_file_path): +def create_partial_ipc_dump(tokens: List[int], + original_file_path: str) -> Optional[str]: """Use the ipc_message_util utility to create a file for up to |TOKENS_PER_IPCDUMP| tokens.""" assert len(tokens) <= TOKENS_PER_IPCDUMP @@ -1215,6 +1257,7 @@ def create_partial_ipc_dump(tokens, original_file_path): temp_file_path = get_temporary_file_name(original_file_path) executable = get_ipc_message_util_executable() + assert executable is not None command_line = shell.get_command_line_from_argument_list( [executable, '--in=%s' % token_list, original_file_path, temp_file_path]) @@ -1228,10 +1271,12 @@ def create_partial_ipc_dump(tokens, original_file_path): return temp_file_path -def combine_ipc_dumps(ipcdumps, original_file_path): +def combine_ipc_dumps(ipcdumps: List[str], + original_file_path: str) -> Optional[str]: """Combines a list of ipcdump files into a single dump.""" input_file_string = ','.join(ipcdumps) executable = get_ipc_message_util_executable() + assert executable is not None output_file_path = get_temporary_file_name(original_file_path) command_line = shell.get_command_line_from_argument_list( [executable, input_file_string, output_file_path]) @@ -1248,7 +1293,7 @@ def combine_ipc_dumps(ipcdumps, original_file_path): return output_file_path -def supports_ipc_minimization(file_path): +def supports_ipc_minimization(file_path: str) -> bool: """Check to see if IPC minimization is supported for the current build.""" executable = get_ipc_message_util_executable() if not executable: @@ -1267,11 +1312,11 @@ def supports_ipc_minimization(file_path): if return_code or not output.startswith('0.'): return False - supports_ipc_minimization.is_supported = True + setattr(supports_ipc_minimization, 'is_supported', True) return True -def can_minimize_file(file_path): +def can_minimize_file(file_path: str) -> bool: """Check to see if we support minimization for this file.""" # If this is not a binary file, we should be able to minimize it in some way. if not utils.is_binary_file(file_path): @@ -1285,14 +1330,18 @@ def can_minimize_file(file_path): return False -def do_ipc_dump_minimization(test_function, get_temp_file, file_path, deadline, - threads, cleanup_interval, delete_temp_files): +def do_ipc_dump_minimization( + test_function: Callable[[str], bool], get_temp_file: Callable[[], BinaryIO], + file_path: str, deadline: float, threads: int, cleanup_interval: int, + delete_temp_files: bool) -> bytes: """IPC dump minimization strategy.""" - def tokenize(current_file_path): + def tokenize(current_file_path: str) -> List[int]: """Generate a token list for an IPC fuzzer test case.""" + executable = get_ipc_message_util_executable() + assert executable is not None command_line = shell.get_command_line_from_argument_list( - [get_ipc_message_util_executable(), '--dump', current_file_path]) + [executable, '--dump', current_file_path]) _, _, output = process_handler.run_process( command_line, testcase_run=False, timeout=IPCDUMP_TIMEOUT) output_lines = output.splitlines() @@ -1309,14 +1358,14 @@ def tokenize(current_file_path): return list(range(last_index + 1)) - def combine_tokens(tokens): + def combine_tokens(tokens: List[int]) -> bytes: """Use the ipc_message_util utility to create a file for these tokens.""" partial_ipcdumps = [] for start_index in range(0, len(tokens), TOKENS_PER_IPCDUMP): end_index = min(start_index + TOKENS_PER_IPCDUMP, len(tokens)) current_tokens = tokens[start_index:end_index] - partial_ipcdumps.append( - create_partial_ipc_dump(current_tokens, file_path)) + partial_ipcdump = create_partial_ipc_dump(current_tokens, file_path) + partial_ipcdumps.append(cast(str, partial_ipcdump)) combined_file_path = None if len(partial_ipcdumps) > 1: @@ -1350,8 +1399,10 @@ def combine_tokens(tokens): return current_minimizer.minimize(file_path) -def do_unicode_minimization(test_function, get_temp_file, data, deadline, - threads, cleanup_interval, delete_temp_files): +def do_unicode_minimization( + test_function: Callable[[bytes], bool], + get_temp_file: Callable[[], BinaryIO], data: bytes, deadline: float, + threads: int, cleanup_interval: int, delete_temp_files: bool) -> bytes: """Attempts unicode escapes minimization.""" try: @@ -1370,8 +1421,10 @@ def do_unicode_minimization(test_function, get_temp_file, data, deadline, return data -def do_js_minimization(test_function, get_temp_file, data, deadline, threads, - cleanup_interval, delete_temp_files): +def do_js_minimization(test_function: Callable[[bytes], bool], + get_temp_file: Callable[[], BinaryIO], data: bytes, + deadline: float, threads: int, cleanup_interval: int, + delete_temp_files: bool) -> bytes: """Javascript minimization strategy.""" # Start by using a generic line minimizer on the test. # Do two line minimizations to make up for the fact that minimzations on bots @@ -1412,10 +1465,10 @@ def do_js_minimization(test_function, get_temp_file, data, deadline, threads, threads, cleanup_interval, delete_temp_files) -def _run_libfuzzer_testcase(fuzz_target, +def _run_libfuzzer_testcase(fuzz_target: Optional[data_types.FuzzTarget], testcase: data_types.Testcase, testcase_file_path: str, - crash_retries: int = 1) -> CrashResult: + crash_retries: Optional[int] = 1) -> CrashResult: """Run libFuzzer testcase, and return the CrashResult.""" # Cleanup any existing application instances and temp directories. process_handler.cleanup_stale_processes() @@ -1432,9 +1485,12 @@ def _run_libfuzzer_testcase(fuzz_target, crash_retries=crash_retries) -def run_libfuzzer_engine(tool_name, target_name, arguments, testcase_path, - output_path, timeout): +def run_libfuzzer_engine(tool_name: str, target_name: Optional[str], + arguments: Sequence[str], testcase_path: str, + output_path: str, + timeout: int) -> engine.ReproduceResult: """Run the libFuzzer engine.""" + assert target_name is not None target_path = engine_common.find_fuzzer_path( environment.get_value('BUILD_DIR'), target_name) if not target_path: @@ -1469,9 +1525,10 @@ def _run_libfuzzer_tool( """ memory_tool_options_var = environment.get_current_memory_tool_var() + assert memory_tool_options_var is not None saved_memory_tool_options = environment.get_value(memory_tool_options_var) - def _set_dedup_flags(): + def _set_dedup_flags() -> None: """Allow libFuzzer to do its own crash comparison during minimization.""" memory_tool_options = environment.get_memory_tool_options( memory_tool_options_var, default_value={}) @@ -1482,7 +1539,7 @@ def _set_dedup_flags(): environment.set_memory_tool_options(memory_tool_options_var, memory_tool_options) - def _unset_dedup_flags(): + def _unset_dedup_flags() -> None: """Reset memory tool options.""" # This is needed so that when we re-run, we can symbolize ourselves # (ignoring inline frames). @@ -1536,7 +1593,11 @@ def _unset_dedup_flags(): return output_file_path, crash_result, minimized_keys -def _extract_crash_result(crash_result, command, minimize_task_input): +def _extract_crash_result( + crash_result: Optional[CrashResult], + command: str, + minimize_task_input: uworker_msg_pb2.MinimizeTaskInput # pylint: disable=no-member +) -> Dict[str, str]: """Extract necessary data from CrashResult.""" if not crash_result: raise errors.BadStateError( @@ -1560,7 +1621,8 @@ def _extract_crash_result(crash_result, command, minimize_task_input): } -def _update_crash_result(testcase, crash_result_dict): +def _update_crash_result(testcase: data_types.Testcase, + crash_result_dict: Mapping[str, str]) -> None: """Update testcase with crash result.""" testcase.crash_type = crash_result_dict['crash_type'] testcase.crash_address = crash_result_dict['crash_address'] @@ -1568,9 +1630,10 @@ def _update_crash_result(testcase, crash_result_dict): testcase.crash_stacktrace = crash_result_dict['crash_stacktrace'] -def _skip_minimization(testcase: data_types.Testcase, - message: str, - crash_result_dict: Dict[str, str] = None): +def _skip_minimization( + testcase: data_types.Testcase, + message: str, + crash_result_dict: Optional[Mapping[str, str]] = None) -> None: """Skip minimization for a testcase, only called during postrocess.""" testcase.minimized_keys = testcase.fuzzed_keys @@ -1582,8 +1645,9 @@ def _skip_minimization(testcase: data_types.Testcase, task_creation.create_postminimize_tasks(testcase) -def _update_testcase_memory_tool_options(testcase: data_types.Testcase, - memory_tool_options: Dict[str, str]): +def _update_testcase_memory_tool_options( + testcase: data_types.Testcase, + memory_tool_options: Mapping[str, str]) -> None: """Updates the testcase metadata env with the values set during utask_main.""" env = {} for key, value in memory_tool_options.items(): @@ -1642,6 +1706,7 @@ def do_libfuzzer_minimization( environment.set_memory_tool_options(options_env_var, minimized_options) reproduced = False + crash_result = None for _ in range(MINIMIZE_SANITIZER_OPTIONS_RETRIES): crash_result = _run_libfuzzer_testcase(fuzz_target, testcase, testcase_file_path) @@ -1658,12 +1723,16 @@ def do_libfuzzer_minimization( options_env_var=options_env_var, options_name=options_name)) else: minimized_options[options_name] = options_value + assert crash_result is not None + crash_type = crash_result.get_type() + crash_state = crash_result.get_state() + security_flag = crash_result.is_security_issue() logs.info( 'Skipped needed {options_env_var} option: {options_name}'.format( options_env_var=options_env_var, options_name=options_name), - crash_type=crash_result.get_type(), - crash_state=crash_result.get_state(), - security_flag=crash_result.is_security_issue()) + crash_type=crash_type, + crash_state=crash_state, + security_flag=security_flag) environment.set_memory_tool_options(options_env_var, minimized_options) env[options_env_var] = environment.get_memory_tool_options(options_env_var) @@ -1696,7 +1765,7 @@ def do_libfuzzer_minimization( if not last_crash_result: repro_command = testcase_manager.get_command_line_for_application( - file_to_run=testcase_file_path, needs_http=testcase.http_flag) + file_to_run=testcase_file_path, needs_http=bool(testcase.http_flag)) crash_result_dict = _extract_crash_result( initial_crash_result, repro_command, minimize_task_input) minimize_task_output = uworker_msg_pb2.MinimizeTaskOutput( # pylint: disable=no-member @@ -1721,7 +1790,7 @@ def do_libfuzzer_minimization( # Finalize the test case if we were able to reproduce it. repro_command = testcase_manager.get_command_line_for_application( - file_to_run=current_testcase_path, needs_http=testcase.http_flag) + file_to_run=current_testcase_path, needs_http=bool(testcase.http_flag)) last_crash_result_dict = _extract_crash_result( last_crash_result, repro_command, minimize_task_input) @@ -1737,9 +1806,13 @@ def do_libfuzzer_minimization( return uworker_msg_pb2.Output(minimize_task_output=minimize_task_output) # pylint: disable=no-member -def do_libfuzzer_cleanse(fuzz_target: Optional[data_types.FuzzTarget], testcase, - testcase_file_path, expected_crash_state, - minimize_task_input): +def do_libfuzzer_cleanse( + fuzz_target: Optional[data_types.FuzzTarget], + testcase: data_types.Testcase, + testcase_file_path: str, + expected_crash_state: str, + minimize_task_input: uworker_msg_pb2.MinimizeTaskInput # pylint: disable=no-member +) -> tuple[Optional[str], Optional[str]]: """Cleanse testcase using libFuzzer.""" timeout = environment.get_value('LIBFUZZER_CLEANSE_TIMEOUT', 180) output_file_path, _, minimized_keys = _run_libfuzzer_tool( @@ -1752,8 +1825,10 @@ def do_libfuzzer_cleanse(fuzz_target: Optional[data_types.FuzzTarget], testcase, return output_file_path, minimized_keys -def do_line_minimization(test_function, get_temp_file, data, deadline, threads, - cleanup_interval, delete_temp_files): +def do_line_minimization(test_function: Callable[[bytes], bool], + get_temp_file: Callable[[], BinaryIO], data: bytes, + deadline: float, threads: int, cleanup_interval: int, + delete_temp_files: bool) -> bytes: """Line-by-line minimization strategy.""" current_minimizer = delta_minimizer.DeltaMinimizer( test_function, @@ -1767,8 +1842,10 @@ def do_line_minimization(test_function, get_temp_file, data, deadline, threads, return current_minimizer.minimize(data) -def do_html_minimization(test_function, get_temp_file, data, deadline, threads, - cleanup_interval, delete_temp_files): +def do_html_minimization(test_function: Callable[[bytes], bool], + get_temp_file: Callable[[], BinaryIO], data: bytes, + deadline: float, threads: int, cleanup_interval: int, + delete_temp_files: bool) -> bytes: """HTML minimization strategy.""" current_minimizer = html_minimizer.HTMLMinimizer( test_function, @@ -1788,14 +1865,14 @@ def do_html_minimization(test_function, get_temp_file, data, deadline, threads, threads, cleanup_interval, delete_temp_files) -def minimize_file(file_path, - test_function, - get_temp_file, - data, - deadline, - threads, - cleanup_interval, - delete_temp_files=True): +def minimize_file(file_path: str, + test_function: Callable[[Any], bool], + get_temp_file: Callable[..., BinaryIO], + data: bytes, + deadline: float, + threads: int, + cleanup_interval: int, + delete_temp_files: bool = True) -> bytes: """Attempt to minimize a single file.""" # Specialized minimization strategy for IPC dumps. if file_path.endswith(testcase_manager.IPCDUMP_EXTENSION): diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py index 2c742d12416..98fd19e5b15 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py @@ -15,9 +15,10 @@ import json import time +from typing import Any from typing import Dict -from typing import List from typing import Optional +from typing import Sequence from clusterfuzz._internal.base import bisection from clusterfuzz._internal.base import errors @@ -31,6 +32,7 @@ from clusterfuzz._internal.bot.tasks.utasks import uworker_io from clusterfuzz._internal.build_management import build_manager from clusterfuzz._internal.build_management import revisions +from clusterfuzz._internal.crash_analysis.crash_result import CrashResult from clusterfuzz._internal.datastore import data_handler from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.fuzzing import corpus_manager @@ -44,7 +46,8 @@ def _maybe_clear_progression_last_min_max_metadata( - testcase: data_types.Testcase, uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member + testcase: data_types.Testcase, + uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Clears last_progression_min and last_progression_max when clear_min_max_metadata is set to True""" if not uworker_output.HasField('progression_task_output'): @@ -57,15 +60,18 @@ def _maybe_clear_progression_last_min_max_metadata( testcase.put() -def _update_build_metadata(job_type: str, - build_data_list: List[uworker_msg_pb2.BuildData]): # pylint: disable=no-member +def _update_build_metadata( + job_type: str, + build_data_list: Sequence[uworker_msg_pb2.BuildData]) -> None: # pylint: disable=no-member """A helper method to update the build metadata corresponding to a job_type.""" for build_data in build_data_list: testcase_manager.update_build_metadata(job_type, build_data) -def _save_current_fixed_range_indices(testcase, uworker_output): +def _save_current_fixed_range_indices( + testcase: data_types.Testcase, + uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Save current fixed range indices in case we die in middle of task.""" task_output = uworker_output.progression_task_output last_progression_min = None @@ -82,7 +88,7 @@ def _save_current_fixed_range_indices(testcase, uworker_output): 'last_progression_max', last_progression_max, update_testcase=False) -def handle_progression_timeout(uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_progression_timeout(uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Job has exceeded the deadline. Recreate the task to pick up where we left off.""" testcase_id = uworker_output.uworker_input.testcase_id @@ -94,7 +100,8 @@ def handle_progression_timeout(uworker_output: uworker_msg_pb2.Output): # pylin tasks.add_task('progression', testcase_id, job_type) -def handle_progression_build_not_found(uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_progression_build_not_found( + uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles an expected build that no longer exists, we can't continue. Also, clears progression_pending testcase metadata""" testcase_id = uworker_output.uworker_input.testcase_id @@ -111,7 +118,7 @@ def handle_progression_build_not_found(uworker_output: uworker_msg_pb2.Output): def handle_progression_revision_list_error( - uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member + uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles revision list errors, in which case the testcase is closed with error.""" testcase_id = uworker_output.uworker_input.testcase_id @@ -120,7 +127,8 @@ def handle_progression_revision_list_error( 'Failed to fetch revision list') -def _cleanup_stacktrace_blob_from_storage(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def _cleanup_stacktrace_blob_from_storage( + output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Cleanup the blob created in preprocess if it wasn't used to store the filterd stacktrace.""" if output.HasField('progression_task_output'): @@ -135,7 +143,7 @@ def _cleanup_stacktrace_blob_from_storage(output: uworker_msg_pb2.Output): # py blobs.delete_blob(blob_name) -def crash_on_latest(uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member +def crash_on_latest(uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles crash on latest revision, or custom binary crashes. Saves the crash info for non-custom binaries.""" testcase_id = uworker_output.uworker_input.testcase_id @@ -160,7 +168,7 @@ def crash_on_latest(uworker_output: uworker_msg_pb2.Output): # pylint: disable= def handle_progression_bad_state_min_max( - uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member + uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles when we end up in a state having min and max versions the same during a progression.""" testcase = data_handler.get_testcase_by_id( @@ -184,7 +192,7 @@ def handle_progression_bad_state_min_max( bisection.request_bisection(testcase) -def handle_progression_no_crash(uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_progression_no_crash(uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Expected crash version doesn't crash. Retries once to confirm the result otherwise marks unreproducible if the testcase is flaky.""" testcase_id = uworker_output.uworker_input.testcase_id @@ -211,7 +219,7 @@ def handle_progression_no_crash(uworker_output: uworker_msg_pb2.Output): # pyli def handle_progression_build_setup_error( - uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member + uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles errors for scenarios where build setup fails.""" # If we failed to setup a build, it is likely a bot error. We can retry # the task in this case. @@ -225,7 +233,8 @@ def handle_progression_build_setup_error( 'progression', testcase_id, job_type, wait_time=build_fail_wait) -def handle_progression_bad_build(uworker_output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_progression_bad_build( + uworker_output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handles unrecoverable bad build errors.""" # Though bad builds when narrowing the range are recoverable, certain builds # being marked as bad may be unrecoverable. Recoverable ones should not @@ -237,8 +246,9 @@ def handle_progression_bad_build(uworker_output: uworker_msg_pb2.Output): # pyl error_message) -def _write_to_bigquery(testcase, progression_range_start, - progression_range_end): +def _write_to_bigquery(testcase: data_types.Testcase, + progression_range_start: int, + progression_range_end: int) -> None: """Write the fixed range to BigQuery.""" big_query.write_range( table_id='fixeds', @@ -248,7 +258,7 @@ def _write_to_bigquery(testcase, progression_range_start, end=progression_range_end) -def _log_output(revision, crash_result): +def _log_output(revision: int, crash_result: CrashResult) -> None: """Log process output.""" logs.info( f'Testing {revision}', @@ -256,9 +266,9 @@ def _log_output(revision, crash_result): output=crash_result.get_stacktrace(symbolized=True)) -def _check_fixed_for_custom_binary(testcase: data_types.Testcase, - testcase_file_path: str, - uworker_input: uworker_msg_pb2.Input): # pylint: disable=no-member +def _check_fixed_for_custom_binary( + testcase: data_types.Testcase, testcase_file_path: str, + uworker_input: uworker_msg_pb2.Input) -> uworker_msg_pb2.Output: # pylint: disable=no-member """Simplified fixed check for test cases using custom binaries.""" build_setup_result = build_manager.setup_build() # 'APP_REVISION' is set during setup_build(). @@ -279,7 +289,7 @@ def _check_fixed_for_custom_binary(testcase: data_types.Testcase, testcase, testcase_file_path, test_timeout, - http_flag=testcase.http_flag) + http_flag=bool(testcase.http_flag)) _log_output(revision, result) # If this still crashes on the most recent build, it's not fixed. The task @@ -287,7 +297,9 @@ def _check_fixed_for_custom_binary(testcase: data_types.Testcase, if result.is_crash(): app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( - testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) + testcase_file_path, + app_path=app_path, + needs_http=bool(testcase.http_flag)) symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) stacktrace = utils.get_crash_stacktrace_output( @@ -296,12 +308,12 @@ def _check_fixed_for_custom_binary(testcase: data_types.Testcase, last_tested_crash_stacktrace = data_handler.filter_stacktrace( stacktrace, progression_task_input.blob_name, progression_task_input.stacktrace_upload_url) - progression_task_output = uworker_msg_pb2.ProgressionTaskOutput( + progression_task_output = uworker_msg_pb2.ProgressionTaskOutput( # pylint: disable=no-member crash_on_latest=True, crash_on_latest_message='Still crashes on latest custom build.', crash_revision=int(revision), last_tested_crash_stacktrace=last_tested_crash_stacktrace) - return uworker_msg_pb2.Output( + return uworker_msg_pb2.Output( # pylint: disable=no-member progression_task_output=progression_task_output) progression_task_output = uworker_msg_pb2.ProgressionTaskOutput( # pylint: disable=no-member @@ -309,7 +321,9 @@ def _check_fixed_for_custom_binary(testcase: data_types.Testcase, return uworker_msg_pb2.Output(progression_task_output=progression_task_output) # pylint: disable=no-member -def _update_issue_metadata(testcase: data_types.Testcase, metadata: Dict): +def _update_issue_metadata(testcase: data_types.Testcase, + metadata: Optional[Dict[str, Any]]) -> None: + """Updates issue metadata in testcase.""" if not metadata: return @@ -322,9 +336,13 @@ def _update_issue_metadata(testcase: data_types.Testcase, metadata: Dict): def _testcase_reproduces_in_revision( - testcase: data_types.Testcase, testcase_file_path: str, job_type: str, - revision: int, fuzz_target: Optional[data_types.FuzzTarget], - progression_task_output: uworker_msg_pb2.ProgressionTaskOutput): # pylint: disable=no-member + testcase: Optional[data_types.Testcase], + testcase_file_path: str, + job_type: str, + revision: int, + fuzz_target: Optional[data_types.FuzzTarget], + progression_task_output: uworker_msg_pb2.ProgressionTaskOutput, # pylint: disable=no-member +) -> tuple[Optional[CrashResult], Optional[uworker_msg_pb2.Output]]: # pylint: disable=no-member """Tests to see if a test case reproduces in the specified revision. Returns a tuple containing the (result, error) depending on whether there was an error.""" @@ -359,17 +377,19 @@ def _testcase_reproduces_in_revision( error_type=uworker_msg_pb2.ErrorType.PROGRESSION_BAD_BUILD) # pylint: disable=no-member test_timeout = environment.get_value('TEST_TIMEOUT', 10) + assert testcase is not None result = testcase_manager.test_for_crash_with_retries( fuzz_target, testcase, testcase_file_path, test_timeout, - http_flag=testcase.http_flag) + http_flag=bool(testcase.http_flag)) _log_output(revision, result) return result, None -def _save_fixed_range(testcase_id, min_revision, max_revision): +def _save_fixed_range(testcase_id: str, min_revision: int, + max_revision: int) -> None: """Update a test case and other metadata with a fixed range.""" testcase = data_handler.get_testcase_by_id(testcase_id) testcase.fixed = f'{min_revision}:{max_revision}' @@ -385,7 +405,7 @@ def _save_fixed_range(testcase_id, min_revision, max_revision): def _store_testcase_for_regression_testing( testcase: data_types.Testcase, testcase_file_path: str, - progression_task_input: uworker_msg_pb2.ProgressionTaskInput): # pylint: disable=no-member + progression_task_input: uworker_msg_pb2.ProgressionTaskInput) -> None: # pylint: disable=no-member """Stores reproduction testcase for future regression testing in corpus pruning task.""" if testcase.open: @@ -414,7 +434,7 @@ def _store_testcase_for_regression_testing( def _set_regression_testcase_upload_url( progression_input: uworker_msg_pb2.ProgressionTaskInput, # pylint: disable=no-member - testcase: data_types.Testcase): + testcase: data_types.Testcase) -> None: """Determines and sets the signed regression_testcase_url (if any) in the progression task input. Raises RuntimeError in case of UUID collision on the generated filename. @@ -434,18 +454,22 @@ def _set_regression_testcase_upload_url( logs.error( 'Not saving uploaded testcase to regression corpus (uploaded not set).') return + testcase_id = testcase.key.id() upload_metadata = data_types.TestcaseUploadMetadata.query( - data_types.TestcaseUploadMetadata.testcase_id == testcase.key.id()).get() + data_types.TestcaseUploadMetadata.testcase_id == testcase_id).get() if upload_metadata: logs.error('Not saving uploaded testcase to regression corpus ' '(uploaded and email not set).') return + assert fuzz_target.engine is not None progression_input.regression_testcase_url = ( corpus_manager.get_regressions_signed_upload_url( fuzz_target.engine, fuzz_target.project_qualified_name())) -def utask_preprocess(testcase_id, job_type, uworker_env): +def utask_preprocess( + testcase_id: str, job_type: str, + uworker_env: Dict[str, Any]) -> Optional[uworker_msg_pb2.Input]: # pylint: disable=no-member """Runs preprocessing for progression task.""" testcase = data_handler.get_testcase_by_id(testcase_id) with logs.testcase_log_context(testcase, testcase.get_fuzz_target()): @@ -483,7 +507,8 @@ def utask_preprocess(testcase_id, job_type, uworker_env): return uworker_input -def find_fixed_range(uworker_input): +def find_fixed_range( + uworker_input: uworker_msg_pb2.Input) -> uworker_msg_pb2.Output: # pylint: disable=no-member """Attempt to find the revision range where a testcase was fixed.""" deadline = tasks.get_task_completion_deadline() testcase = uworker_io.entity_from_protobuf(uworker_input.testcase, @@ -497,6 +522,8 @@ def find_fixed_range(uworker_input): if error: return error + assert testcase_file_path is not None + # Custom binaries are handled as special cases. if build_manager.is_custom_binary(): return _check_fixed_for_custom_binary(testcase, testcase_file_path, @@ -545,6 +572,8 @@ def find_fixed_range(uworker_input): logs.info( f'min_revision is {min_revision} and max_revision is {max_revision}.') + assert min_revision is not None + assert max_revision is not None min_index = revisions.find_min_revision_index(revision_list, min_revision) if min_index is None: error_message = f'Build {min_revision} no longer exists.' @@ -572,12 +601,15 @@ def find_fixed_range(uworker_input): if error is not None: return error + assert result is not None if result.is_crash(): logs.info(f'Found crash with same signature on latest' f' revision r{max_revision}.') app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( - testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) + testcase_file_path, + app_path=app_path, + needs_http=bool(testcase.http_flag)) symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) stacktrace = utils.get_crash_stacktrace_output( @@ -594,7 +626,7 @@ def find_fixed_range(uworker_input): progression_task_output.crash_revision = int(max_revision) progression_task_output.last_tested_crash_stacktrace = ( last_tested_crash_stacktrace) - return uworker_msg_pb2.Output( + return uworker_msg_pb2.Output( # pylint: disable=no-member progression_task_output=progression_task_output, issue_metadata=json.dumps(issue_metadata)) @@ -615,8 +647,8 @@ def find_fixed_range(uworker_input): error_message=error_message, error_type=uworker_msg_pb2.ErrorType.PROGRESSION_NO_CRASH) # pylint: disable=no-member - last_progression_min = None - last_progression_max = None + last_progression_min: Optional[int] = None + last_progression_max: Optional[int] = None # Start a binary search to find last non-crashing revision. At this point, we # know that we do crash in the min_revision, and do not crash in max_revision. while time.time() < deadline: @@ -665,11 +697,14 @@ def find_fixed_range(uworker_input): max_index -= 1 continue # Only bad build errors are recoverable. - progression_task_output.last_progression_min = int(last_progression_min) - progression_task_output.last_progression_max = int(last_progression_max) + if last_progression_min is not None: + progression_task_output.last_progression_min = int(last_progression_min) + if last_progression_max is not None: + progression_task_output.last_progression_max = int(last_progression_max) error.progression_task_output.CopyFrom(progression_task_output) return error + assert result is not None if result.is_crash(): min_index = middle_index else: @@ -693,7 +728,8 @@ def find_fixed_range(uworker_input): error_type=uworker_msg_pb2.ErrorType.PROGRESSION_TIMEOUT) # pylint: disable=no-member -def utask_main(uworker_input): +def utask_main( + uworker_input: uworker_msg_pb2.Input) -> Optional[uworker_msg_pb2.Output]: # pylint: disable=no-member """Executes the untrusted part of progression_task.""" testcase = uworker_io.entity_from_protobuf(uworker_input.testcase, data_types.Testcase) @@ -703,28 +739,29 @@ def utask_main(uworker_input): return find_fixed_range(uworker_input) -_ERROR_HANDLER = uworker_handle_errors.CompositeErrorHandler({ - uworker_msg_pb2.ErrorType.PROGRESSION_BAD_BUILD: # pylint: disable=no-member - handle_progression_bad_build, - uworker_msg_pb2.ErrorType.PROGRESSION_BAD_STATE_MIN_MAX: # pylint: disable=no-member - handle_progression_bad_state_min_max, - uworker_msg_pb2.ErrorType.PROGRESSION_BUILD_NOT_FOUND: # pylint: disable=no-member - handle_progression_build_not_found, - uworker_msg_pb2.ErrorType.PROGRESSION_BUILD_SETUP_ERROR: # pylint: disable=no-member - handle_progression_build_setup_error, - uworker_msg_pb2.ErrorType.PROGRESSION_NO_CRASH: # pylint: disable=no-member - handle_progression_no_crash, - uworker_msg_pb2.ErrorType.PROGRESSION_REVISION_LIST_ERROR: # pylint: disable=no-member - handle_progression_revision_list_error, - uworker_msg_pb2.ErrorType.PROGRESSION_TIMEOUT: # pylint: disable=no-member - handle_progression_timeout, -}).compose_with( - setup.ERROR_HANDLER, - uworker_handle_errors.UNHANDLED_ERROR_HANDLER, -) - - -def utask_postprocess(output: uworker_msg_pb2.Output): # pylint: disable=no-member +_ERROR_HANDLER: uworker_handle_errors.CompositeErrorHandler = ( + uworker_handle_errors.CompositeErrorHandler({ + uworker_msg_pb2.ErrorType.PROGRESSION_BAD_BUILD: # pylint: disable=no-member + handle_progression_bad_build, + uworker_msg_pb2.ErrorType.PROGRESSION_BAD_STATE_MIN_MAX: # pylint: disable=no-member + handle_progression_bad_state_min_max, + uworker_msg_pb2.ErrorType.PROGRESSION_BUILD_NOT_FOUND: # pylint: disable=no-member + handle_progression_build_not_found, + uworker_msg_pb2.ErrorType.PROGRESSION_BUILD_SETUP_ERROR: # pylint: disable=no-member + handle_progression_build_setup_error, + uworker_msg_pb2.ErrorType.PROGRESSION_NO_CRASH: # pylint: disable=no-member + handle_progression_no_crash, + uworker_msg_pb2.ErrorType.PROGRESSION_REVISION_LIST_ERROR: # pylint: disable=no-member + handle_progression_revision_list_error, + uworker_msg_pb2.ErrorType.PROGRESSION_TIMEOUT: # pylint: disable=no-member + handle_progression_timeout, + }).compose_with( + setup.ERROR_HANDLER, + uworker_handle_errors.UNHANDLED_ERROR_HANDLER, + )) + + +def utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Trusted: Cleans up after a uworker execute_task, writing anything needed to the db.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) @@ -762,6 +799,7 @@ def utask_postprocess(output: uworker_msg_pb2.Output): # pylint: disable=no-mem 'progression', testcase, reset_after_retry=True): tasks.add_task('progression', output.uworker_input.testcase_id, output.uworker_input.job_type) + assert task_output is not None data_handler.update_progression_completion_metadata( testcase, task_output.crash_revision) return @@ -769,6 +807,7 @@ def utask_postprocess(output: uworker_msg_pb2.Output): # pylint: disable=no-mem # The bug is fixed. testcase.fixed = 'Yes' testcase.open = False + assert task_output is not None data_handler.update_progression_completion_metadata( testcase, task_output.crash_revision, @@ -780,6 +819,7 @@ def utask_postprocess(output: uworker_msg_pb2.Output): # pylint: disable=no-mem return testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) + assert task_output is not None if task_output.HasField('min_revision'): _save_fixed_range(output.uworker_input.testcase_id, task_output.min_revision, task_output.max_revision) diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/regression_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/regression_task.py index 32c685254f3..0f3429199ce 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/regression_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/regression_task.py @@ -16,6 +16,7 @@ import random import time +from typing import Any from typing import Dict from typing import List from typing import Optional @@ -40,16 +41,18 @@ # Number of revisions before the maximum to test before doing a bisect. This # is also used as a cap for revisions to test near the minimum if the minimum # happens to be a bad build. -EXTREME_REVISIONS_TO_TEST = 3 +EXTREME_REVISIONS_TO_TEST: int = 3 # Number of earlier revisions to check when validating ranges. -REVISIONS_TO_TEST_FOR_VALIDATION = 2 +REVISIONS_TO_TEST_FOR_VALIDATION: int = 2 # Maximum revisions to look back when validating. -EARLIER_REVISIONS_TO_CONSIDER_FOR_VALIDATION = 10 +EARLIER_REVISIONS_TO_CONSIDER_FOR_VALIDATION: int = 10 -def write_to_big_query(testcase, regression_range_start, regression_range_end): +def write_to_big_query(testcase: data_types.Testcase, + regression_range_start: int, + regression_range_end: int) -> None: """Write the regression range to BigQuery.""" big_query.write_range( table_id='regressions', @@ -60,7 +63,8 @@ def write_to_big_query(testcase, regression_range_start, regression_range_end): def _save_current_regression_range_indices( - task_output: uworker_msg_pb2.RegressionTaskOutput, testcase_id: str): # pylint: disable=no-member + task_output: uworker_msg_pb2.RegressionTaskOutput, # pylint: disable=no-member + testcase_id: str) -> None: """Save current regression range indices in case we die in middle of task.""" last_regression_min = None if task_output.HasField('last_regression_min'): @@ -84,7 +88,7 @@ def _save_current_regression_range_indices( testcase.put() -def save_regression_range(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def save_regression_range(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Saves the regression range and creates blame and impact task if needed.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) regression_range_start = output.regression_task_output.regression_range_start @@ -114,7 +118,8 @@ def _testcase_reproduces_in_revision( fuzz_target: Optional[data_types.FuzzTarget], should_log: bool = True, min_revision: Optional[int] = None, - max_revision: Optional[int] = None): + max_revision: Optional[int] = None, +) -> tuple[Optional[bool], Optional[uworker_msg_pb2.Output]]: # pylint: disable=no-member """Test to see if a test case reproduces in the specified revision. Returns a tuple containing the (result, error) depending on whether there was an error.""" @@ -149,7 +154,7 @@ def _testcase_reproduces_in_revision( testcase, testcase_file_path, test_timeout, - http_flag=testcase.http_flag) + http_flag=bool(testcase.http_flag)) return result.is_crash(), None @@ -284,6 +289,7 @@ def find_min_revision( # Make sure we account for MIN_REVISION. first_revision = revisions.get_first_revision_in_list(revision_list) first_index = revisions.find_min_revision_index(revision_list, first_revision) + assert first_index is not None while True: # If we fall off the end of the revision list, try the earliest revision. @@ -585,8 +591,9 @@ def find_regression_range( error_message=error_message) -def utask_preprocess(testcase_id: str, job_type: str, - uworker_env: Dict) -> Optional[uworker_msg_pb2.Input]: # pylint: disable=no-member +def utask_preprocess( + testcase_id: str, job_type: str, + uworker_env: Dict[str, Any]) -> Optional[uworker_msg_pb2.Input]: # pylint: disable=no-member """Prepares inputs for `utask_main()` to run on an untrusted worker. Runs on a trusted worker. @@ -640,13 +647,13 @@ def utask_main( return find_regression_range(uworker_input) -def handle_revision_list_error(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_revision_list_error(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) data_handler.close_testcase_with_error(testcase, 'Failed to fetch revision list') -def handle_build_not_found_error(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_build_not_found_error(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member # If an expected build no longer exists, we can't continue. testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) testcase.regression = 'NA' @@ -654,7 +661,7 @@ def handle_build_not_found_error(output: uworker_msg_pb2.Output): # pylint: dis output.error_message) -def handle_regression_build_setup_error(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_regression_build_setup_error(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member # If we failed to setup a build, it is likely a bot error. We can retry # the task in this case. uworker_input = output.uworker_input @@ -672,7 +679,7 @@ def handle_regression_build_setup_error(output: uworker_msg_pb2.Output): # pyli # TODO(https://crbug.com/396344382): Wait for all uworkers to run code past # https://github.com/google/clusterfuzz/pull/3934 for a week, then delete this. # This error type is obsolete. -def handle_regression_bad_build_error(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_regression_bad_build_error(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member # Though bad builds when narrowing the range are recoverable, certain builds # being marked as bad may be unrecoverable. Recoverable ones should not # reach this point. @@ -683,7 +690,7 @@ def handle_regression_bad_build_error(output: uworker_msg_pb2.Output): # pylint error_message) -def handle_regression_no_crash(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_regression_no_crash(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, output.error_message) @@ -691,7 +698,7 @@ def handle_regression_no_crash(output: uworker_msg_pb2.Output): # pylint: disab task_creation.mark_unreproducible_if_flaky(testcase, 'regression', True) -def handle_regression_timeout(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_regression_timeout(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, output.error_message) @@ -699,29 +706,31 @@ def handle_regression_timeout(output: uworker_msg_pb2.Output): # pylint: disabl output.uworker_input.job_type) -def handle_low_confidence_in_regression_range(output: uworker_msg_pb2.Output): # pylint: disable=no-member +def handle_low_confidence_in_regression_range( + output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) testcase.regression = 'NA' data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, output.error_message) -_ERROR_HANDLER = uworker_handle_errors.CompositeErrorHandler({ - uworker_msg_pb2.ErrorType.REGRESSION_BAD_BUILD_ERROR: # pylint: disable=no-member - handle_regression_bad_build_error, - uworker_msg_pb2.ErrorType.REGRESSION_BUILD_NOT_FOUND: # pylint: disable=no-member - handle_build_not_found_error, - uworker_msg_pb2.ErrorType.REGRESSION_BUILD_SETUP_ERROR: # pylint: disable=no-member - handle_regression_build_setup_error, - uworker_msg_pb2.ErrorType.REGRESSION_LOW_CONFIDENCE_IN_REGRESSION_RANGE: # pylint: disable=no-member - handle_low_confidence_in_regression_range, - uworker_msg_pb2.ErrorType.REGRESSION_NO_CRASH: # pylint: disable=no-member - handle_regression_no_crash, - uworker_msg_pb2.ErrorType.REGRESSION_REVISION_LIST_ERROR: # pylint: disable=no-member - handle_revision_list_error, - uworker_msg_pb2.ErrorType.REGRESSION_TIMEOUT_ERROR: # pylint: disable=no-member - handle_regression_timeout, -}).compose_with(setup.ERROR_HANDLER) +_ERROR_HANDLER: uworker_handle_errors.CompositeErrorHandler = ( + uworker_handle_errors.CompositeErrorHandler({ + uworker_msg_pb2.ErrorType.REGRESSION_BAD_BUILD_ERROR: # pylint: disable=no-member + handle_regression_bad_build_error, + uworker_msg_pb2.ErrorType.REGRESSION_BUILD_NOT_FOUND: # pylint: disable=no-member + handle_build_not_found_error, + uworker_msg_pb2.ErrorType.REGRESSION_BUILD_SETUP_ERROR: # pylint: disable=no-member + handle_regression_build_setup_error, + uworker_msg_pb2.ErrorType.REGRESSION_LOW_CONFIDENCE_IN_REGRESSION_RANGE: # pylint: disable=no-member + handle_low_confidence_in_regression_range, + uworker_msg_pb2.ErrorType.REGRESSION_NO_CRASH: # pylint: disable=no-member + handle_regression_no_crash, + uworker_msg_pb2.ErrorType.REGRESSION_REVISION_LIST_ERROR: # pylint: disable=no-member + handle_revision_list_error, + uworker_msg_pb2.ErrorType.REGRESSION_TIMEOUT_ERROR: # pylint: disable=no-member + handle_regression_timeout, + }).compose_with(setup.ERROR_HANDLER)) def utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member @@ -758,7 +767,8 @@ def utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disabl def _update_build_metadata( - job_type: str, build_data_list: Sequence[uworker_msg_pb2.BuildData]): # pylint: disable=no-member + job_type: str, + build_data_list: Sequence[uworker_msg_pb2.BuildData]) -> None: # pylint: disable=no-member """A helper method to update the build metadata corresponding to a job_type.""" for build_data in build_data_list: diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/symbolize_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/symbolize_task.py index de49ba3c043..92251cae9b2 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/symbolize_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/symbolize_task.py @@ -15,6 +15,7 @@ Add stack traces from non-optimized release and debug builds.""" import os +from typing import cast from clusterfuzz._internal.base import tasks from clusterfuzz._internal.base import utils @@ -33,13 +34,13 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import process_handler -DEFAULT_REDZONE = 128 -MAX_REDZONE = 1024 -MIN_REDZONE = 16 -STACK_FRAME_COUNT = 128 +DEFAULT_REDZONE: int = 128 +MAX_REDZONE: int = 1024 +MIN_REDZONE: int = 16 +STACK_FRAME_COUNT: int = 128 -def handle_build_setup_error(output): +def handle_build_setup_error(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member testcase_id = output.uworker_input.testcase_id job_type = output.uworker_input.job_type testcase = data_handler.get_testcase_by_id(testcase_id) @@ -49,7 +50,11 @@ def handle_build_setup_error(output): tasks.add_task('symbolize', testcase_id, job_type, wait_time=build_fail_wait) -def _utask_preprocess(testcase_id, job_type, uworker_env): +def _utask_preprocess( + testcase_id: str, + job_type: str, + uworker_env: dict, +) -> uworker_msg_pb2.Input | None: # pylint: disable=no-member """Run preprocessing for symbolize task.""" # Locate the testcase associated with the id. testcase = data_handler.get_testcase_by_id(testcase_id) @@ -74,7 +79,11 @@ def _utask_preprocess(testcase_id, job_type, uworker_env): old_crash_stacktrace=old_crash_stacktrace)) -def utask_preprocess(testcase_id, job_type, uworker_env): +def utask_preprocess( + testcase_id: str, + job_type: str, + uworker_env: dict, +) -> uworker_msg_pb2.Input | None: # pylint: disable=no-member """Set logs context and run preprocessing for symbolize task.""" # Locate the testcase associated with the id. testcase = data_handler.get_testcase_by_id(testcase_id) @@ -82,7 +91,9 @@ def utask_preprocess(testcase_id, job_type, uworker_env): return _utask_preprocess(testcase_id, job_type, uworker_env) -def _utask_main(uworker_input): +def _utask_main( + uworker_input: uworker_msg_pb2.Input, # pylint: disable=no-member +) -> uworker_msg_pb2.Output | None: # pylint: disable=no-member """Execute the untrusted part of a symbolize command.""" job_type = uworker_input.job_type testcase = uworker_io.entity_from_protobuf(uworker_input.testcase, @@ -96,6 +107,8 @@ def _utask_main(uworker_input): if error: return error + assert testcase_file_path is not None + # Initialize variables. old_crash_stacktrace = ( uworker_input.symbolize_task_input.old_crash_stacktrace) @@ -114,9 +127,10 @@ def _utask_main(uworker_input): build_revision = testcase.crash_revision fuzz_target = testcase_manager.get_fuzz_target_from_input(uworker_input) - fuzz_target = fuzz_target.binary if fuzz_target else None + fuzz_target_binary = fuzz_target.binary if fuzz_target else None # Set up a custom or regular build based on revision. - build = build_manager.setup_build(build_revision, fuzz_target) + build = build_manager.setup_build( + cast(int, build_revision), fuzz_target_binary) # Get crash revision used in setting up build. crash_revision = environment.get_value('APP_REVISION') @@ -130,7 +144,7 @@ def _utask_main(uworker_input): # See if we can get better stacks with higher redzone sizes. # A UAF might actually turn out to be OOB read/write with a bigger redzone. if environment.tool_matches('ASAN', job_type) and testcase.security_flag: - redzone = MAX_REDZONE + redzone: int | float = MAX_REDZONE while redzone >= MIN_REDZONE: logs.info(f'Trying to reproduce crash with ASAN redzone size {redzone}.') @@ -139,10 +153,11 @@ def _utask_main(uworker_input): process_handler.terminate_stale_application_instances() command = testcase_manager.get_command_line_for_application( - testcase_file_path, needs_http=testcase.http_flag) + testcase_file_path, needs_http=bool(testcase.http_flag)) + gestures = testcase.gestures return_code, crash_time, output = ( process_handler.run_process( - command, timeout=warmup_timeout, gestures=testcase.gestures)) + command, timeout=warmup_timeout, gestures=gestures)) crash_result = CrashResult(return_code, crash_time, output) if crash_result.is_crash() and 'AddressSanitizer' in output: @@ -232,7 +247,9 @@ def _utask_main(uworker_input): return uworker_msg_pb2.Output(symbolize_task_output=symbolize_task_output) # pylint: disable=no-member -def utask_main(uworker_input): +def utask_main( + uworker_input: uworker_msg_pb2.Input, # pylint: disable=no-member +) -> uworker_msg_pb2.Output | None: # pylint: disable=no-member """Set logs context and run the untrusted part of a symbolize command.""" testcase = uworker_io.entity_from_protobuf(uworker_input.testcase, data_types.Testcase) @@ -241,8 +258,12 @@ def utask_main(uworker_input): return _utask_main(uworker_input) -def get_symbolized_stacktraces(testcase_file_path, testcase, - old_crash_stacktrace, expected_state): +def get_symbolized_stacktraces( + testcase_file_path: str, + testcase: data_types.Testcase, + old_crash_stacktrace: str, + expected_state: str | None, +) -> tuple[bool, str]: """Use the symbolized builds to generate an updated stacktrace.""" # Initialize variables. app_path = environment.get_value('APP_PATH') @@ -254,6 +275,8 @@ def get_symbolized_stacktraces(testcase_file_path, testcase, debug_build_stacktrace = '' release_build_stacktrace = old_crash_stacktrace + gestures = testcase.gestures + # Symbolize using the debug build first so that the debug build stacktrace # comes after the more important release build stacktrace. if app_path_debug: @@ -262,10 +285,10 @@ def get_symbolized_stacktraces(testcase_file_path, testcase, command = testcase_manager.get_command_line_for_application( testcase_file_path, app_path=app_path_debug, - needs_http=testcase.http_flag) + needs_http=bool(testcase.http_flag)) return_code, crash_time, output = ( process_handler.run_process( - command, timeout=long_test_timeout, gestures=testcase.gestures)) + command, timeout=long_test_timeout, gestures=gestures)) crash_result = CrashResult(return_code, crash_time, output) if crash_result.is_crash(): @@ -289,10 +312,12 @@ def get_symbolized_stacktraces(testcase_file_path, testcase, for _ in range(retry_limit): process_handler.terminate_stale_application_instances() command = testcase_manager.get_command_line_for_application( - testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) + testcase_file_path, + app_path=app_path, + needs_http=bool(testcase.http_flag)) return_code, crash_time, output = ( process_handler.run_process( - command, timeout=long_test_timeout, gestures=testcase.gestures)) + command, timeout=long_test_timeout, gestures=gestures)) crash_result = CrashResult(return_code, crash_time, output) if crash_result.is_crash(): @@ -336,7 +361,7 @@ def get_symbolized_stacktraces(testcase_file_path, testcase, ) -def _utask_postprocess(output): +def _utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handle the output from utask_main.""" if output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR: # pylint: disable=no-member _ERROR_HANDLER.handle(output) @@ -377,7 +402,7 @@ def _utask_postprocess(output): task_creation.create_blame_task_if_needed(testcase) -def utask_postprocess(output): +def utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Set logs context and handle the output from utask_main.""" # Retrieve the testcase associated with the id. testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/uworker_handle_errors.py b/src/clusterfuzz/_internal/bot/tasks/utasks/uworker_handle_errors.py index 73ddcd7f699..f367c549f82 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/uworker_handle_errors.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/uworker_handle_errors.py @@ -13,6 +13,7 @@ # limitations under the License. """Module for handling errors in utasks.""" +from typing import Any from typing import Callable from typing import Dict @@ -32,7 +33,7 @@ class CompositeErrorHandler: """A handler for several different types of uworker errors.""" - def __init__(self, handlers: HandlerDict): + def __init__(self, handlers: HandlerDict) -> None: """Initializes a handler that delegates to the values in `handlers`. For example: @@ -69,7 +70,7 @@ def is_handled(self, error_type: ErrorType.ValueType) -> bool: """Returns whether the given error type is handled by this instance.""" return error_type in self._handlers - def handle(self, output: Output): + def handle(self, output: Output) -> None: """Handles the given `output`, delegating to underlying handlers. Raises: @@ -82,7 +83,7 @@ def handle(self, output: Output): handler(output) -def noop_handler(*args, **kwargs): +def noop_handler(*args: Any, **kwargs: Any) -> None: del args del kwargs diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/uworker_io.py b/src/clusterfuzz/_internal/bot/tasks/utasks/uworker_io.py index 96de5a8d0f4..0124bbf22b4 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/uworker_io.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/uworker_io.py @@ -11,8 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# pyright: reportAttributeAccessIssue=none """Module for dealing with input and output (I/O) to a uworker.""" +import datetime from typing import Tuple from typing import Type from typing import TypeVar @@ -23,18 +25,17 @@ from google.cloud.datastore_v1.types import entity as entity_pb2 from google.cloud.ndb import model from google.protobuf import any_pb2 -from google.protobuf import timestamp_pb2 import google.protobuf.message +from google.protobuf.timestamp_pb2 import \ + Timestamp # pylint: disable=no-name-in-module from clusterfuzz._internal.base.tasks import task_utils +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.google_cloud_utils import storage from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.protos import uworker_msg_pb2 from clusterfuzz._internal.system import environment -# Define an alias to appease pylint. -Timestamp = timestamp_pb2.Timestamp # pylint: disable=no-member - def generate_new_input_file_name() -> str: """Generates a new input file name.""" @@ -45,13 +46,14 @@ def get_uworker_input_gcs_path() -> str: """Returns a GCS path for uworker I/O.""" # Inspired by blobs.write_blob. io_bucket = storage.uworker_input_bucket() + assert io_bucket is not None io_file_name = generate_new_input_file_name() if storage.get(storage.get_cloud_storage_file_path(io_bucket, io_file_name)): raise RuntimeError(f'UUID collision found: {io_file_name}.') return f'/{io_bucket}/{io_file_name}' -def get_uworker_output_urls(input_gcs_path: str) -> str: +def get_uworker_output_urls(input_gcs_path: str) -> Tuple[str, str]: """Returns a signed download URL for the uworker to upload the output and a GCS url for the tworker to download the output. Make sure we can infer the actual input since the output is not trusted.""" @@ -61,23 +63,29 @@ def get_uworker_output_urls(input_gcs_path: str) -> str: def uworker_input_path_to_output_path(input_gcs_path: str) -> str: - return input_gcs_path.replace(storage.uworker_input_bucket(), - storage.uworker_output_bucket()) + input_bucket = storage.uworker_input_bucket() + output_bucket = storage.uworker_output_bucket() + assert input_bucket is not None + assert output_bucket is not None + return input_gcs_path.replace(input_bucket, output_bucket) def uworker_output_path_to_input_path(output_gcs_path: str) -> str: - return output_gcs_path.replace(storage.uworker_output_bucket(), - storage.uworker_input_bucket()) + input_bucket = storage.uworker_input_bucket() + output_bucket = storage.uworker_output_bucket() + assert input_bucket is not None + assert output_bucket is not None + return output_gcs_path.replace(output_bucket, input_bucket) -def get_uworker_input_urls(): +def get_uworker_input_urls() -> Tuple[str, str]: """Returns a signed download URL for the uworker to download the input and a GCS url for the tworker to upload it (this happens first).""" gcs_path = get_uworker_input_gcs_path() return storage.get_signed_download_url(gcs_path), gcs_path -def upload_uworker_input(uworker_input: bytes, gcs_path: str): +def upload_uworker_input(uworker_input: bytes, gcs_path: str) -> None: """Uploads input for the untrusted portion of a task.""" storage.write_data(uworker_input, gcs_path) @@ -125,6 +133,7 @@ def download_and_deserialize_uworker_input( """Downloads and deserializes the input to the uworker from the signed download URL.""" data = storage.download_signed_url(uworker_input_download_url) + assert data is not None try: data = zlib.decompress(data) except zlib.error: @@ -154,7 +163,7 @@ def deserialize_uworker_output(serialized: bytes) -> uworker_msg_pb2.Output: # def serialize_and_upload_uworker_output( uworker_output: uworker_msg_pb2.Output, # pylint: disable=no-member - upload_url: str): + upload_url: str) -> None: """Serializes |uworker_output| and uploads it to |upload_url.""" serialized_uworker_output = zlib.compress(uworker_output.SerializeToString()) storage.upload_signed_url(serialized_uworker_output, upload_url) @@ -166,6 +175,7 @@ def download_input_based_on_output_url( location) downloads the input based on the output_url.""" input_url = uworker_output_path_to_input_path(output_url) data = storage.read_data(input_url) + assert data is not None try: serialized_uworker_input = zlib.decompress(data) except zlib.error: @@ -173,6 +183,7 @@ def download_input_based_on_output_url( serialized_uworker_input = data if serialized_uworker_input is None: logs.error(f'No corresponding input for output: {output_url}.') + assert serialized_uworker_input is not None return deserialize_uworker_input(serialized_uworker_input) @@ -180,12 +191,14 @@ def download_and_deserialize_uworker_output( output_url: str) -> uworker_msg_pb2.Output: # pylint: disable=no-member """Downloads and deserializes uworker output.""" data = storage.read_data(output_url) + assert data is not None try: serialized_uworker_output = zlib.decompress(data) except zlib.error: # For backwards compatability support uncompressed. serialized_uworker_output = data + assert serialized_uworker_output is not None uworker_output = deserialize_uworker_output(serialized_uworker_output) # Now download the input, which is stored securely so that the uworker cannot @@ -196,19 +209,19 @@ def download_and_deserialize_uworker_output( return uworker_output -def entity_to_protobuf(entity: ndb.Model) -> entity_pb2.Entity: +def entity_to_protobuf(entity: ndb.Model) -> any_pb2.Any: # pylint: disable=no-member """Helper function to convert entity to protobuf format.""" #_entity_to_protobuf returns google.cloud.datastore_v1.types.Entity ndb_proto = model._entity_to_protobuf(entity) # pylint: disable=protected-access return entity_to_any_message(ndb_proto) -def db_entity_to_entity_message(entity): +def db_entity_to_entity_message(entity: entity_pb2.Entity) -> None: any_entity_message = entity_to_any_message(entity) - entity = uworker_msg_pb2.Entity(any_wrapper=any_entity_message) # pylint: disable=no-member + entity = uworker_msg_pb2.Entity(any_wrapper=any_entity_message) # ty: ignore[unresolved-attribute] # pylint: disable=no-member,unused-variable -def entity_to_any_message(entity_proto): +def entity_to_any_message(entity_proto: entity_pb2.Entity) -> any_pb2.Any: # pylint: disable=no-member any_entity_message = any_pb2.Any() # pylint: disable=no-member any_entity_message.Pack(entity_proto._pb) # pylint: disable=protected-access return any_entity_message # pylint: disable=protected-access @@ -231,7 +244,7 @@ def entity_from_protobuf(entity_proto: any_pb2.Any, model_type: Type[T]) -> T: return entity -def check_handling_testcase_safe(testcase): +def check_handling_testcase_safe(testcase: data_types.Testcase) -> None: """Exits when the current task execution model is trusted but the testcase is untrusted. This will allow uploading testcases to trusted jobs (e.g. Mac) more safely.""" @@ -243,7 +256,7 @@ def check_handling_testcase_safe(testcase): logs.warning(f'Cannot handle {testcase.key.id()} in trusted task.') -def check_running_fuzzer_safe(fuzzer): +def check_running_fuzzer_safe(fuzzer: data_types.Fuzzer) -> bool: """Exits when the fuzzer is untrusted but the execution environment is trusted.""" if fuzzer.trusted: @@ -256,11 +269,12 @@ def check_running_fuzzer_safe(fuzzer): return False -def timestamp_to_proto_timestamp(pydt) -> Timestamp: +def timestamp_to_proto_timestamp(pydt: datetime.datetime) -> Timestamp: proto_timestamp = Timestamp() proto_timestamp.FromDatetime(pydt) return proto_timestamp -def proto_timestamp_to_timestamp(proto_timestamp: Timestamp): +def proto_timestamp_to_timestamp( + proto_timestamp: Timestamp) -> datetime.datetime: return proto_timestamp.ToDatetime() diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/variant_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/variant_task.py index 8a7219479b8..198a830973b 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/variant_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/variant_task.py @@ -13,6 +13,8 @@ # limitations under the License. """Variant task for analyzing testcase variants with a different job.""" +from typing import cast + from clusterfuzz._internal.base import errors from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot import testcase_manager @@ -28,7 +30,8 @@ from clusterfuzz._internal.system import environment -def _get_variant_testcase_for_job(testcase, job_type): +def _get_variant_testcase_for_job(testcase: data_types.Testcase, + job_type: str) -> data_types.Testcase: """Return a testcase entity for variant task use. This changes the fuzz target params for a particular fuzzing engine.""" if testcase.job_type == job_type: @@ -40,6 +43,7 @@ def _get_variant_testcase_for_job(testcase, job_type): return testcase engine_name = environment.get_engine_for_job(job_type) + assert engine_name is not None project = data_handler.get_project_name(job_type) binary_name = testcase.get_metadata('fuzzer_binary_name') fully_qualified_fuzzer_name = data_types.fuzz_target_fully_qualified_name( @@ -52,12 +56,17 @@ def _get_variant_testcase_for_job(testcase, job_type): variant_testcase.job_type = job_type # Remove put() method to avoid updates. DO NOT REMOVE THIS. - variant_testcase.put = lambda: None + variant_testcase.put = ( # pyright: ignore[reportAttributeAccessIssue] + lambda: None) return variant_testcase -def utask_preprocess(testcase_id, job_type, uworker_env): +def utask_preprocess( + testcase_id: str, + job_type: str, + uworker_env: dict, +) -> uworker_msg_pb2.Input | None: # pylint: disable=no-member """Run a test case with a different job type to see if they reproduce.""" try: testcase = data_handler.get_testcase_by_id(testcase_id) @@ -103,7 +112,9 @@ def utask_preprocess(testcase_id, job_type, uworker_env): return uworker_input -def utask_main(uworker_input): +def utask_main( + uworker_input: uworker_msg_pb2.Input, # pylint: disable=no-member +) -> uworker_msg_pb2.Output | None: # pylint: disable=no-member """The main part of the variant task. Downloads the testcase and build checks if the build can reproduce the error.""" testcase = uworker_io.entity_from_protobuf(uworker_input.testcase, @@ -114,7 +125,7 @@ def utask_main(uworker_input): # Remove put() method to avoid updates. DO NOT REMOVE THIS. # Repeat this because the in-memory executor may allow puts. # TODO(metzman): Remove this when we use batch. - testcase.put = lambda: None + testcase.put = lambda: None # pyright: ignore[reportAttributeAccessIssue] # Setup testcase and its dependencies. _, testcase_file_path, error = setup.setup_testcase( @@ -122,12 +133,15 @@ def utask_main(uworker_input): if error: return error + assert testcase_file_path is not None + # Set up a custom or regular build. We explicitly omit the crash revision # since we want to test against the latest build here. try: fuzz_target = testcase_manager.get_fuzz_target_from_input(uworker_input) - fuzz_target = fuzz_target.binary if fuzz_target else None - build_setup_result = build_manager.setup_build(fuzz_target=fuzz_target) + fuzz_target_binary = fuzz_target.binary if fuzz_target else None + build_setup_result = build_manager.setup_build( + fuzz_target=fuzz_target_binary) except errors.BuildNotFoundError: logs.warning('Matching build not found.') return uworker_msg_pb2.Output( # pylint: disable=no-member @@ -146,7 +160,9 @@ def utask_main(uworker_input): # Reproduce the crash. app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( - testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) + testcase_file_path, + app_path=app_path, + needs_http=cast(bool, testcase.http_flag)) test_timeout = environment.get_value('TEST_TIMEOUT', 10) revision = environment.get_value('APP_REVISION') fuzz_target = testcase_manager.get_fuzz_target_from_input(uworker_input) @@ -156,7 +172,7 @@ def utask_main(uworker_input): testcase, testcase_file_path, test_timeout, - http_flag=testcase.http_flag, + http_flag=cast(bool, testcase.http_flag), use_gestures=use_gestures, compare_crash=False) except testcase_manager.TargetNotFoundError: @@ -169,11 +185,12 @@ def utask_main(uworker_input): crash_type = result.get_type() security_flag = result.is_security_issue() - gestures = testcase.gestures if use_gestures else None + gestures = cast(list[str] | None, + testcase.gestures) if use_gestures else None fuzz_target = testcase_manager.get_fuzz_target_from_input(uworker_input) one_time_crasher_flag = not testcase_manager.test_for_reproducibility( fuzz_target, testcase_file_path, crash_type, crash_state, - security_flag, test_timeout, testcase.http_flag, gestures) + security_flag, test_timeout, cast(bool, testcase.http_flag), gestures) if one_time_crasher_flag: status = data_types.TestcaseVariantStatus.FLAKY else: @@ -213,7 +230,7 @@ def utask_main(uworker_input): crash_stacktrace_output=crash_stacktrace_output) -def handle_build_setup_error(output): +def handle_build_setup_error(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) data_handler.update_testcase_comment( testcase, data_types.TaskState.ERROR, @@ -228,7 +245,7 @@ def handle_build_setup_error(output): ) -def utask_postprocess(output): +def utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disable=no-member """Handle the output from utask_main.""" try: testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) @@ -243,7 +260,7 @@ def utask_postprocess(output): if environment.is_engine_fuzzer_job(output.uworker_input.job_type): # Remove put() method to avoid updates. DO NOT REMOVE THIS. - testcase.put = lambda: None + testcase.put = lambda: None # pyright: ignore[reportAttributeAccessIssue] if (output.uworker_input.variant_task_input.original_job_type == output.uworker_input.job_type): diff --git a/src/clusterfuzz/_internal/bot/testcase_manager.py b/src/clusterfuzz/_internal/bot/testcase_manager.py index d62a9345819..fda3ff5c12e 100644 --- a/src/clusterfuzz/_internal/bot/testcase_manager.py +++ b/src/clusterfuzz/_internal/bot/testcase_manager.py @@ -14,14 +14,18 @@ """Functions for testcase management.""" import base64 -import collections +from collections.abc import Sequence import dataclasses import datetime import os import queue import re +from typing import cast +from typing import NamedTuple import zlib +from google.protobuf import message + from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot.fuzzers import engine_common from clusterfuzz._internal.bot.tasks.utasks import uworker_io @@ -45,24 +49,24 @@ from clusterfuzz.stacktraces import CrashInfo # Testcase filename prefixes and suffixes. -CRASH_PREFIX = 'crash-' -FUZZ_PREFIX = 'fuzz-' -FLAGS_PREFIX = 'flags-' -HTTP_PREFIX = 'http-' -RESOURCES_PREFIX = 'resources-' +CRASH_PREFIX: str = 'crash-' +FUZZ_PREFIX: str = 'fuzz-' +FLAGS_PREFIX: str = 'flags-' +HTTP_PREFIX: str = 'http-' +RESOURCES_PREFIX: str = 'resources-' # TODO(mbarbella): Once all fuzzers are converted to "resources-", remove this. -DEPENDENCY_PREFIX = 'cfdependency-' -APPS_PREFIX = 'fuzz-apps-' -EXTENSIONS_PREFIX = 'fuzz-extension-' -COVERAGE_SUFFIX = '.cov' - -INFO_FILE_EXTENSION = '.info' -IPCDUMP_EXTENSION = '.ipcdump' -REPRODUCIBILITY_FACTOR = 0.5 -SEARCH_INDEX_TESTCASES_DIRNAME = 'common' -SEARCH_INDEX_BUNDLE_PREFIX = '__%s_' % SEARCH_INDEX_TESTCASES_DIRNAME -TESTCASE_LIST_FILENAME = 'files.info' +DEPENDENCY_PREFIX: str = 'cfdependency-' +APPS_PREFIX: str = 'fuzz-apps-' +EXTENSIONS_PREFIX: str = 'fuzz-extension-' +COVERAGE_SUFFIX: str = '.cov' + +INFO_FILE_EXTENSION: str = '.info' +IPCDUMP_EXTENSION: str = '.ipcdump' +REPRODUCIBILITY_FACTOR: float = 0.5 +SEARCH_INDEX_TESTCASES_DIRNAME: str = 'common' +SEARCH_INDEX_BUNDLE_PREFIX: str = '__%s_' % SEARCH_INDEX_TESTCASES_DIRNAME +TESTCASE_LIST_FILENAME: str = 'files.info' CHROME_URL_LOAD_REGEX = re.compile( r'.*(NetworkDelegate::NotifyBeforeURLRequest|FileURLLoader::Start)' @@ -71,7 +75,7 @@ HTTP_URL_REGEX = re.compile( r'.*(localhost|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[^/]*[/]([^"#?]+)') -BAD_STATE_HINTS = [ +BAD_STATE_HINTS: list[str] = [ # X server issues. 'cannot open display', 'Maximum number of clients reached', @@ -82,7 +86,7 @@ ] # Flaky crash types for which crash comparison does not make sense. -IGNORE_STATE_CRASH_TYPES = [ +IGNORE_STATE_CRASH_TYPES: list[str] = [ 'Stack-overflow', ] @@ -95,7 +99,7 @@ class TargetNotFoundError(TestcaseManagerError): """Error when a fuzz target is not found.""" -def set_extra_sanitizers(crash_type): +def set_extra_sanitizers(crash_type: str | None) -> None: """Set extra sanitizers based on crash_type.""" if crash_type in crash_analyzer.EXTRA_SANITIZERS_SECURITY: environment.set_value('USE_EXTRA_SANITIZERS', True) @@ -104,7 +108,7 @@ def set_extra_sanitizers(crash_type): environment.set_value('USE_EXTRA_SANITIZERS', False) -def create_testcase_list_file(output_directory): +def create_testcase_list_file(output_directory: str) -> None: """Create a testcase list file for tests in a directory.""" files_list = [] files_list_file_path = os.path.join(output_directory, TESTCASE_LIST_FILENAME) @@ -125,7 +129,7 @@ def create_testcase_list_file(output_directory): utils.write_data_to_file('\n'.join(sorted(files_list)), files_list_file_path) -def get_testcases_from_directories(directories): +def get_testcases_from_directories(directories: Sequence[str]) -> list[str]: """Returns all testcases from testcase directories.""" testcase_paths = [] max_testcases = environment.get_value('MAX_TESTCASES') @@ -158,7 +162,7 @@ def get_testcases_from_directories(directories): return testcase_paths -def is_testcase_resource(filename): +def is_testcase_resource(filename: str) -> bool: """Returns true if this is a testcase or its resource dependency.""" if filename.startswith(FUZZ_PREFIX): return True @@ -178,7 +182,7 @@ def is_testcase_resource(filename): return False -def remove_testcases_from_directories(directories): +def remove_testcases_from_directories(directories: Sequence[str]) -> None: """Removes all testcases and their dependencies from testcase directories.""" generators = [] for directory in directories: @@ -209,7 +213,7 @@ def remove_testcases_from_directories(directories): shell.remove_file(file_path) -def read_resource_list(resource_file_path): +def read_resource_list(resource_file_path: str) -> list[str]: """Generate a resource list.""" if not os.path.exists(resource_file_path): return [] @@ -228,7 +232,8 @@ def read_resource_list(resource_file_path): return resources -def get_resource_dependencies(testcase_absolute_path, test_prefix=FUZZ_PREFIX): +def get_resource_dependencies(testcase_absolute_path: str, + test_prefix: str = FUZZ_PREFIX) -> list[str]: """Returns the list of testcase resource dependencies.""" resources = [] if not os.path.exists(testcase_absolute_path): @@ -285,7 +290,7 @@ def get_resource_dependencies(testcase_absolute_path, test_prefix=FUZZ_PREFIX): return resources -def get_command_line_flags(testcase_path): +def get_command_line_flags(testcase_path: str) -> str: """Returns command line flags to use for a testcase.""" arguments = environment.get_value('APP_ARGS') additional_arguments = get_additional_command_line_flags(testcase_path) @@ -297,7 +302,7 @@ def get_command_line_flags(testcase_path): return arguments.strip() -def get_additional_command_line_flags(testcase_path): +def get_additional_command_line_flags(testcase_path: str) -> str: """Returns additional command line flags to use for a testcase.""" # Get the initial flags list from the environment value. additional_command_line_flags = ( @@ -319,7 +324,9 @@ def get_additional_command_line_flags(testcase_path): return additional_command_line_flags.strip() -def run_testcase(thread_index, file_path, gestures, env_copy): +def run_testcase(thread_index: int, file_path: str, gestures: list[str] | None, + env_copy: dict[str, str] | None + ) -> tuple[int | None, float | None, str | None]: """Run a single testcase and return crash results in the crash queue.""" try: # Update environment with environment copy from parent. @@ -349,13 +356,16 @@ def run_testcase(thread_index, file_path, gestures, env_copy): return None, None, None -class Crash( - collections.namedtuple( - 'Crash', 'file_path crash_time return_code resource_list gestures ' - 'stack_file_path')): +class Crash(NamedTuple): """Represents a crash in a queue. This class is transformed into fuzz_task.Crash. Therefore, please be careful when adding/removing fields.""" + file_path: str + crash_time: float | None + return_code: int | None + resource_list: list[str] + gestures: list[str] | None + stack_file_path: str @dataclasses.dataclass @@ -371,15 +381,16 @@ class FuzzerRunOutputData: _output: str | bytes | None = None _file_path: str | None = None crash_path: str | None = None - return_code: int = 0 + return_code: int | None = 0 log_time: datetime.datetime | None = None @classmethod - def from_file_path(cls, - file_path: str, - crash_path: str | None = None, - return_code: int = 0, - log_time: datetime.datetime | None = None): + def from_file_path( + cls, + file_path: str, + crash_path: str | None = None, + return_code: int | None = 0, + log_time: datetime.datetime | None = None) -> 'FuzzerRunOutputData': return cls( _file_path=file_path, crash_path=crash_path, @@ -387,11 +398,12 @@ def from_file_path(cls, log_time=log_time) @classmethod - def from_memory(cls, - output: str | bytes, - crash_path: str | None = None, - return_code: int = 0, - log_time: datetime.datetime | None = None): + def from_memory( + cls, + output: str | bytes, + crash_path: str | None = None, + return_code: int | None = 0, + log_time: datetime.datetime | None = None) -> 'FuzzerRunOutputData': return cls( _output=output, crash_path=crash_path, @@ -422,7 +434,7 @@ class TestcaseRunResult: fuzzer_run_output_data: FuzzerRunOutputData | None = None -def get_resource_paths(output): +def get_resource_paths(output: str) -> list[str]: """Read the urls from the output.""" resource_paths = set() for line in output.splitlines(): @@ -438,7 +450,7 @@ def get_resource_paths(output): return list(resource_paths) -def convert_dependency_url_to_local_path(url): +def convert_dependency_url_to_local_path(url: str) -> str | None: """Convert a dependency URL to a corresponding local path.""" # Bot-specific import. from clusterfuzz._internal.bot.webserver import http_server @@ -481,7 +493,7 @@ def convert_dependency_url_to_local_path(url): return local_path -def _get_testcase_time(testcase_path): +def _get_testcase_time(testcase_path: str) -> datetime.datetime: """Returns the timestamp of a testcase.""" stats = fuzzer_stats.TestcaseRun.read_from_disk(testcase_path) if stats: @@ -490,7 +502,10 @@ def _get_testcase_time(testcase_path): return datetime.datetime.utcnow() -def upload_testcase(testcase_path, testcase_data, log_time, fuzzer_name=None): +def upload_testcase(testcase_path: str | None, + testcase_data: str | bytes | None, + log_time: datetime.datetime, + fuzzer_name: str | None = None) -> None: """Uploads testcase so that a log file can be matched with it folder.""" fuzz_logs_bucket = environment.get_value('FUZZ_LOGS_BUCKET') if not fuzz_logs_bucket: @@ -514,7 +529,7 @@ def upload_testcase(testcase_path, testcase_data, log_time, fuzzer_name=None): file_extension='.testcase') -def _get_crash_output(output): +def _get_crash_output(output: str | None) -> str | None: """Returns crash part of the output, excluding unrelated content (e.g. output from corpus merge, etc).""" if output is None: @@ -530,7 +545,7 @@ def _get_crash_output(output): def run_testcase_and_return_result_in_queue( crash_queue: queue.Queue, thread_index: int, file_path: str, - gestures: list[str], env_copy: dict) -> None: + gestures: list[str] | None, env_copy: dict[str, str] | None) -> None: """Run a single testcase and return crash results in the crash queue.""" # Since this is running in its own process, initialize the log handler again. # This is needed for Windows where instances are not shared across child @@ -548,7 +563,7 @@ def run_testcase_and_return_result_in_queue( def _do_run_testcase_and_return_result_in_queue( crash_queue: queue.Queue, thread_index: int, file_path: str, - gestures: list[str], env_copy: dict) -> None: + gestures: list[str] | None, env_copy: dict[str, str] | None) -> None: """Run a single testcase and return crash results in the crash queue.""" try: # Run testcase and check whether a crash occurred or not. @@ -572,6 +587,7 @@ def _do_run_testcase_and_return_result_in_queue( if crash_result.is_crash(): # Initialize resource list with the testcase path. resource_list = [file_path] + assert crash_output is not None resource_list += get_resource_paths(crash_output) # Store the crash stack file in the crash stacktrace directory @@ -613,8 +629,9 @@ def _do_run_testcase_and_return_result_in_queue( exception=e) -def engine_reproduce(engine_impl: engine.Engine, target_name, testcase_path, - arguments, timeout) -> engine.ReproduceResult: +def engine_reproduce(engine_impl: engine.Engine, target_name: str, + testcase_path: str, arguments: Sequence[str], + timeout: int | float) -> engine.ReproduceResult: """Do engine reproduction.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import tasks_host @@ -640,19 +657,30 @@ def engine_reproduce(engine_impl: engine.Engine, target_name, testcase_path, class TestcaseRunner: """Testcase runner.""" + _testcase_path: str + _test_timeout: int | float + _gestures: list[str] | None + _needs_http: bool + _is_black_box: bool + _engine_impl: engine.Engine + _arguments: list[str] + _fuzz_target: data_types.FuzzTarget + _command: str + def __init__(self, - fuzz_target, - testcase_path, - test_timeout, - gestures, - needs_http=False, - arguments=None): + fuzz_target: data_types.FuzzTarget | None, + testcase_path: str, + test_timeout: int | float, + gestures: list[str] | None, + needs_http: bool = False, + arguments: str | None = None): self._testcase_path = testcase_path self._test_timeout = test_timeout self._gestures = gestures self._needs_http = needs_http if fuzz_target: + assert fuzz_target.engine is not None engine_impl = engine.get(fuzz_target.engine) else: engine_impl = None @@ -690,6 +718,7 @@ def run(self, round_number: int) -> CrashResult: current_working_directory=app_directory) else: try: + assert self._fuzz_target.binary is not None result = engine_reproduce(self._engine_impl, self._fuzz_target.binary, self._testcase_path, self._arguments, run_timeout) @@ -715,7 +744,7 @@ def run(self, round_number: int) -> CrashResult: return crash_result - def _pre_run_cleanup(self): + def _pre_run_cleanup(self) -> None: """Common cleanup before running a testcase.""" # Cleanup any existing application instances and user profile directories. # Cleaning up temp user profile directories. Should be done before calling @@ -741,7 +770,7 @@ def _get_crash_state(self, round_number: int, def reproduce_with_retries(self, retries: int, - expected_state: CrashInfo | None = None, + expected_state: str | None = None, expected_security_flag: bool | None = None, flaky_stacktrace: bool = False) -> CrashResult: """Try reproducing a crash with retries.""" @@ -782,14 +811,15 @@ def reproduce_with_retries(self, logs.info('Crash stacktrace does not match original stacktrace.') logs.info('Didn\'t crash at all.') + assert crash_result is not None return CrashResult( return_code=0, crash_time=0, output=crash_result.output, unexpected_crash=unexpected_crash) - def test_reproduce_reliability(self, retries, expected_state, - expected_security_flag) -> bool: + def test_reproduce_reliability(self, retries: int, expected_state: str | None, + expected_security_flag: bool | None) -> bool: """Test to see if a crash is fully reproducible or is a one-time crasher.""" logs.info("Beginning a reproducibility test.") self._pre_run_cleanup() @@ -835,14 +865,15 @@ def test_reproduce_reliability(self, retries, expected_state, return False -def test_for_crash_with_retries(fuzz_target, - testcase, - testcase_path, - test_timeout, - http_flag=False, - use_gestures=True, - compare_crash=True, - crash_retries=None) -> CrashResult: +def test_for_crash_with_retries( + fuzz_target: data_types.FuzzTarget | None, + testcase: data_types.Testcase, + testcase_path: str, + test_timeout: int | float, + http_flag: bool = False, + use_gestures: bool = True, + compare_crash: bool = True, + crash_retries: int | None = None) -> CrashResult: """Test for a crash and return crash parameters like crash type, crash state, crash stacktrace, etc.""" logs.info('Testing for crash.') @@ -854,6 +885,8 @@ def test_for_crash_with_retries(fuzz_target, if crash_retries is None: crash_retries = environment.get_value('CRASH_RETRIES') + assert crash_retries is not None + if compare_crash and testcase.crash_type not in IGNORE_STATE_CRASH_TYPES: expected_state = testcase.crash_state expected_security_flag = testcase.security_flag @@ -864,20 +897,22 @@ def test_for_crash_with_retries(fuzz_target, try: return runner.reproduce_with_retries(crash_retries, expected_state, expected_security_flag, - testcase.flaky_stack) + bool(testcase.flaky_stack)) except TargetNotFoundError: # If a target isn't found, treat it as not crashing. return CrashResult(return_code=0, crash_time=0, output='') -def get_fuzz_target_from_input(uworker_input): +def get_fuzz_target_from_input(uworker_input: uworker_msg_pb2.Input | None # pylint: disable=no-member + ) -> data_types.FuzzTarget | None: if uworker_input and uworker_input.HasField('fuzz_target'): return uworker_io.entity_from_protobuf(uworker_input.fuzz_target, data_types.FuzzTarget) return None -def preprocess_testcase_manager(testcase, uworker_input): +def preprocess_testcase_manager(testcase: data_types.Testcase, + uworker_input: uworker_msg_pb2.Input) -> None: # pylint: disable=no-member """Preprocess function for users of test_for_reproducibility.""" # TODO(metzman): Make this work for test_for_crash_with_retries. fuzz_target = testcase.get_fuzz_target() @@ -888,18 +923,19 @@ def preprocess_testcase_manager(testcase, uworker_input): if not fuzz_target: return - uworker_input.fuzz_target.CopyFrom(uworker_io.entity_to_protobuf(fuzz_target)) + uworker_input.fuzz_target.CopyFrom( # pylint: disable=no-member + cast(message.Message, uworker_io.entity_to_protobuf(fuzz_target))) -def test_for_reproducibility(fuzz_target, - testcase_path, - crash_type, - expected_state, - expected_security_flag, - test_timeout, - http_flag, - gestures, - arguments=None) -> bool: +def test_for_reproducibility(fuzz_target: data_types.FuzzTarget | None, + testcase_path: str, + crash_type: str | None, + expected_state: str | None, + expected_security_flag: bool | None, + test_timeout: int | float, + http_flag: bool, + gestures: list[str] | None, + arguments: str | None = None) -> bool: """Test to see if a crash is fully reproducible or is a one-time crasher.""" set_extra_sanitizers(crash_type) runner = TestcaseRunner( @@ -911,11 +947,13 @@ def test_for_reproducibility(fuzz_target, arguments=arguments) crash_retries = environment.get_value('CRASH_RETRIES') + assert crash_retries is not None return runner.test_reproduce_reliability(crash_retries, expected_state, expected_security_flag) -def _prepare_log_for_upload(symbolized_output, return_code, app_revision): +def _prepare_log_for_upload(symbolized_output: str, return_code: int | None, + app_revision: str | int | None) -> bytes: """Prepare log for upload.""" # Add revision information to the logs. app_revision = app_revision or environment.get_value('APP_REVISION') @@ -940,11 +978,11 @@ def _prepare_log_for_upload(symbolized_output, return_code, app_revision): return result.encode('utf-8') -def upload_log(symbolized_output, - return_code, - log_time, - app_revision=None, - fuzzer_name=None): +def upload_log(symbolized_output: str, + return_code: int | None, + log_time: datetime.datetime, + app_revision: str | int | None = None, + fuzzer_name: str | None = None) -> None: """Upload the output into corresponding GCS logs bucket.""" log = _prepare_log_for_upload(symbolized_output, return_code, app_revision) fuzz_logs_bucket = environment.get_value('FUZZ_LOGS_BUCKET') @@ -955,7 +993,7 @@ def upload_log(symbolized_output, fuzz_logs_bucket, log, time=log_time, fuzzer_name=fuzzer_name) -def get_user_profile_directory(user_profile_index): +def get_user_profile_directory(user_profile_index: int) -> str: """Returns a user profile directory from a directory index.""" temp_directory = environment.get_value('BOT_TMPDIR') user_profile_in_memory = environment.get_value('USER_PROFILE_IN_MEMORY') @@ -971,13 +1009,13 @@ def get_user_profile_directory(user_profile_index): return user_profile_directory -def get_command_line_for_application(file_to_run='', - user_profile_index=0, - app_path=None, - app_args=None, - needs_http=False, - write_command_line_file=False, - get_arguments_only=False): +def get_command_line_for_application(file_to_run: str = '', + user_profile_index: int = 0, + app_path: str | None = None, + app_args: str | None = None, + needs_http: bool = False, + write_command_line_file: bool = False, + get_arguments_only: bool = False) -> str: """Returns the complete command line required to execute application.""" if app_args is None: app_args = environment.get_value('APP_ARGS') @@ -1149,7 +1187,7 @@ def get_command_line_for_application(file_to_run='', return str(command) -def setup_user_profile_directory_if_needed(user_profile_directory): +def setup_user_profile_directory_if_needed(user_profile_directory: str) -> None: """Set user profile directory if it does not exist.""" if os.path.exists(user_profile_directory): # User profile directory already exists. Bail out. @@ -1327,7 +1365,8 @@ def check_for_bad_build(job_type: str, build_run_console_output=build_run_console_output) -def update_build_metadata(job_type: str, build_data: uworker_msg_pb2.BuildData): # pylint: disable=no-member +def update_build_metadata(job_type: str, + build_data: uworker_msg_pb2.BuildData) -> None: # pylint: disable=no-member """ Updates the corresponding build metadata. diff --git a/src/clusterfuzz/_internal/bot/tokenizer/antlr_tokenizer.py b/src/clusterfuzz/_internal/bot/tokenizer/antlr_tokenizer.py index d4c61b4df64..927003843e7 100644 --- a/src/clusterfuzz/_internal/bot/tokenizer/antlr_tokenizer.py +++ b/src/clusterfuzz/_internal/bot/tokenizer/antlr_tokenizer.py @@ -13,6 +13,10 @@ # limitations under the License. """Antlr Tokenizer""" +from collections.abc import Callable +from collections.abc import Iterable +import typing + import antlr4 from clusterfuzz._internal.base import utils @@ -24,10 +28,13 @@ class AntlrTokenizer: $ antlr4 -Dlanguage=Pythonn and allows user to tokenize files using that grammar.""" - def __init__(self, lexer): + def __init__( + self, + lexer: type[antlr4.Lexer] | Callable[[antlr4.InputStream], antlr4.Lexer] + ) -> None: self._lexer = lexer - def fill(self, stream): + def fill(self, stream: antlr4.CommonTokenStream) -> int: """Helper function. antlr4.CommonTokenStream.fill should work, but it does not fetch all of the tokens. This is a replacement that works.""" i = 0 @@ -35,7 +42,7 @@ def fill(self, stream): i += 1 return i - def tokenize(self, data): + def tokenize(self, data: bytes) -> list[str]: """Takes in a file and uses the antlr lexer to return a list of tokens""" # Antlr expects a string, but test cases are not necessarily valid utf-8. try: @@ -46,11 +53,13 @@ def tokenize(self, data): stream = antlr4.CommonTokenStream(self._lexer(lexer_input)) end = self.fill(stream) tokens = stream.getTokens(0, end) + assert tokens is not None return [token.text for token in tokens] - def combine(self, tokens): + def combine(self, tokens: Iterable[str | bytes]) -> bytes: """Token combiner passed to minimizer""" # This tokenizer must handle either bytes or str inputs. Antlr works with # strings, but the tokenizer validation step uses the original data, which # is always raw bytes. - return b''.join(utils.encode_as_unicode(t) for t in tokens) + return b''.join( + typing.cast(bytes, utils.encode_as_unicode(t)) for t in tokens) diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/build_setup.py b/src/clusterfuzz/_internal/bot/untrusted_runner/build_setup.py index 2e62f96117a..aa2c4e55d77 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/build_setup.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/build_setup.py @@ -20,7 +20,8 @@ # pylint: disable=no-member -def _build_response(build, result): +def _build_response(build: build_manager.Build, + result: bool) -> untrusted_runner_pb2.SetupBuildResponse: if not result: return untrusted_runner_pb2.SetupBuildResponse(result=False) @@ -36,7 +37,8 @@ def _build_response(build, result): fuzz_targets=build.fuzz_targets or []) -def setup_regular_build(request): +def setup_regular_build(request: untrusted_runner_pb2.SetupRegularBuildRequest + ) -> untrusted_runner_pb2.SetupBuildResponse: """Set up a regular build.""" build = build_manager.RegularBuild(request.base_build_dir, request.revision, request.build_url, request.build_prefix, diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/build_setup_host.py b/src/clusterfuzz/_internal/bot/untrusted_runner/build_setup_host.py index dec46869ca2..4478b191800 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/build_setup_host.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/build_setup_host.py @@ -13,14 +13,20 @@ # limitations under the License. """Build setup host (client).""" +from typing import Any +from typing import cast + from clusterfuzz._internal.build_management import build_manager from clusterfuzz._internal.protos import untrusted_runner_pb2 +from clusterfuzz._internal.protos import untrusted_runner_pb2_grpc from clusterfuzz._internal.system import environment from . import host +# pylint: disable=no-member + -def _clear_env(): +def _clear_env() -> None: """Clear build env vars.""" environment.remove_key('APP_PATH') environment.remove_key('APP_REVISION') @@ -31,7 +37,8 @@ def _clear_env(): environment.remove_key('FUZZ_TARGET') -def _update_env_from_response(response): +def _update_env_from_response( + response: untrusted_runner_pb2.SetupBuildResponse) -> None: """Update environment variables from response.""" environment.set_value('APP_PATH', response.app_path) environment.set_value('APP_PATH_DEBUG', response.app_path_debug) @@ -45,11 +52,12 @@ def _update_env_from_response(response): class RemoteRegularBuild(build_manager.RegularBuild): """Remote regular build.""" - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._fuzz_targets = [] + self._fuzz_targets: list[str] = [] - def _handle_response(self, response): + def _handle_response( + self, response: untrusted_runner_pb2.SetupBuildResponse) -> bool: """Handle build setup response.""" if not response.result: _clear_env() @@ -67,15 +75,16 @@ def _handle_response(self, response): return True - def setup(self): - request = untrusted_runner_pb2.SetupRegularBuildRequest( # pylint: disable=no-member + def setup(self) -> bool: + request = untrusted_runner_pb2.SetupRegularBuildRequest( base_build_dir=self.base_build_dir, revision=self.revision, build_url=self.build_url, fuzz_target=self.fuzz_target, build_prefix=self.build_prefix) - return self._handle_response(host.stub().SetupRegularBuild(request)) + stub = cast(untrusted_runner_pb2_grpc.UntrustedRunnerStub, host.stub()) + return self._handle_response(stub.SetupRegularBuild(request)) @property - def fuzz_targets(self): + def fuzz_targets(self) -> list[str]: return self._fuzz_targets diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/config.py b/src/clusterfuzz/_internal/bot/untrusted_runner/config.py index 383b1d2e48b..8026195b8d0 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/config.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/config.py @@ -13,25 +13,25 @@ # limitations under the License. """Trusted host/untrusted instance configuration.""" -PORT = 9001 +PORT: int = 9001 -INITIAL_CONNECT_TIMEOUT_SECONDS = 5 * 60 -RECONNECT_TIMEOUT_SECONDS = 30 +INITIAL_CONNECT_TIMEOUT_SECONDS: int = 5 * 60 +RECONNECT_TIMEOUT_SECONDS: int = 30 -FILE_TRANSFER_CHUNK_SIZE = 4096 +FILE_TRANSFER_CHUNK_SIZE: int = 4096 -HEARTBEAT_INTERVAL_SECONDS = 60 -HEARTBEAT_TIMEOUT_SECONDS = 15 +HEARTBEAT_INTERVAL_SECONDS: int = 60 +HEARTBEAT_TIMEOUT_SECONDS: int = 15 -GET_STATUS_TIMEOUT_SECONDS = 15 -UPDATE_SOURCE_TIMEOUT_SECONDS = 15 +GET_STATUS_TIMEOUT_SECONDS: int = 15 +UPDATE_SOURCE_TIMEOUT_SECONDS: int = 15 -RPC_RETRY_ATTEMPTS = 1 +RPC_RETRY_ATTEMPTS: int = 1 -GRPC_OPTIONS = ( +GRPC_OPTIONS: tuple[tuple[str, int], ...] = ( ('grpc.max_send_message_length', -1), ('grpc.max_receive_message_length', -1), ('grpc.max_metadata_size', 32 * 1024 * 1024), ) -NUM_WORKER_THREADS = 4 +NUM_WORKER_THREADS: int = 4 diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/corpus_manager.py b/src/clusterfuzz/_internal/bot/untrusted_runner/corpus_manager.py index e291972de50..c41dcf9e998 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/corpus_manager.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/corpus_manager.py @@ -22,18 +22,24 @@ class RemoteGSUtilRunner(gsutil.GSUtilRunner): """Remote GSUtil runner.""" - def __init__(self): + def __init__(self) -> None: super().__init__(process_runner=remote_process_host.RemoteProcessRunner) class RemoteFuzzTargetCorpus(corpus_manager.FuzzTargetCorpus): """libFuzzer corpus sync that runs on untrusted bot.""" - def __init__(self, fuzzer_name, fuzzer_executable_name, quarantine=False): + def __init__( + self, + fuzzer_name: str, + fuzzer_executable_name: str, + quarantine: bool = False, + ) -> None: super().__init__( fuzzer_name, fuzzer_executable_name, quarantine, # Never log results for remote corpora since the state is on the worker. log_results=False, - gsutil_runner_func=RemoteGSUtilRunner) + gsutil_runner_func=RemoteGSUtilRunner, + ) diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/environment.py b/src/clusterfuzz/_internal/bot/untrusted_runner/environment.py index 10a20ac1240..4b08d3d16b5 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/environment.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/environment.py @@ -13,19 +13,19 @@ # limitations under the License. """Environment modification functions.""" +from collections.abc import Mapping +from collections.abc import MutableMapping import os import re +from typing import cast -try: - from clusterfuzz._internal.protos import untrusted_runner_pb2 +from clusterfuzz._internal.protos import untrusted_runner_pb2 +from clusterfuzz._internal.protos import untrusted_runner_pb2_grpc - from . import file_host - from . import host -except ImportError: - # TODO(ochang): Fix this. - pass +from . import file_host +from . import host -FORWARDED_ENVIRONMENT_VARIABLES = [ +FORWARDED_ENVIRONMENT_VARIABLES: list[re.Pattern[str]] = [ re.compile(pattern) for pattern in ( r'^AFL_.*', r'^APPLICATION_ID$', @@ -59,24 +59,24 @@ ) ] -REBASED_ENVIRONMENT_VARIABLES = { +REBASED_ENVIRONMENT_VARIABLES: set[str] = { 'FUZZER_DIR', } -def is_forwarded_environment_variable(environment_variable): +def is_forwarded_environment_variable(environment_variable: str) -> bool: """Return whether or not |environment_variable| should be forwarded.""" return any( pattern.match(environment_variable) for pattern in FORWARDED_ENVIRONMENT_VARIABLES) -def should_rebase_environment_value(environment_variable): +def should_rebase_environment_value(environment_variable: str) -> bool: """Return whether or not |environment_variable|'s value should be rebased.""" return environment_variable in REBASED_ENVIRONMENT_VARIABLES -def update_environment(env): +def update_environment(env: Mapping[str, str]) -> None: """Update worker's environment.""" processed_env = {} for key, value in env.items(): @@ -86,10 +86,12 @@ def update_environment(env): processed_env[key] = value request = untrusted_runner_pb2.UpdateEnvironmentRequest(env=processed_env) # pylint: disable=no-member - host.stub().UpdateEnvironment(request) + stub = cast(untrusted_runner_pb2_grpc.UntrustedRunnerStub, host.stub()) + stub.UpdateEnvironment(request) -def set_environment_vars(env, source_env): +def set_environment_vars(env: MutableMapping[str, str], + source_env: Mapping[str, str] | None) -> None: """Copy allowed environment variables from |source_env|.""" if not source_env: return @@ -104,9 +106,10 @@ def set_environment_vars(env, source_env): env[name] = value -def get_env_for_untrusted_process(overrides): +def get_env_for_untrusted_process( + overrides: Mapping[str, str] | None) -> dict[str, str]: """Return environment for running an untrusted process.""" - env = {} + env: dict[str, str] = {} if overrides is not None: set_environment_vars(env, overrides) else: @@ -114,7 +117,7 @@ def get_env_for_untrusted_process(overrides): return env -def forward_environment_variable(key, value): +def forward_environment_variable(key: str, value: str) -> None: """Forward the environment variable if needed.""" if not host.is_initialized(): return @@ -123,7 +126,8 @@ def forward_environment_variable(key, value): update_environment({key: value}) -def reset_environment(): +def reset_environment() -> None: """Reset environment variables.""" request = untrusted_runner_pb2.ResetEnvironmentRequest() # pylint: disable=no-member - host.stub().ResetEnvironment(request) + stub = cast(untrusted_runner_pb2_grpc.UntrustedRunnerStub, host.stub()) + stub.ResetEnvironment(request) diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/file_host.py b/src/clusterfuzz/_internal/bot/untrusted_runner/file_host.py index a87d057ff4c..fbea255bca8 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/file_host.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/file_host.py @@ -13,19 +13,30 @@ # limitations under the License. """File operations host (client).""" +from collections.abc import Sequence import os import shutil +from typing import cast +from typing import overload from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.protos import untrusted_runner_pb2 +from clusterfuzz._internal.protos import untrusted_runner_pb2_grpc from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import shell from . import file_utils from . import host +# pylint: disable=no-member -def is_directory_parent(path, directory): + +def _stub() -> untrusted_runner_pb2_grpc.UntrustedRunnerStub: + """Return the UntrustedRunnerStub.""" + return cast(untrusted_runner_pb2_grpc.UntrustedRunnerStub, host.stub()) + + +def is_directory_parent(path: str, directory: str) -> bool: """Check whether if |directory| is a parent of |path|.""" path = os.path.abspath(path) directory = os.path.abspath(directory) @@ -40,7 +51,22 @@ def is_directory_parent(path, directory): for i in range(len(directory_components))) -def _rebase(path, target_base, cur_base): +@overload +def _rebase(path: None, target_base: str, cur_base: str) -> None: + ... + + +@overload +def _rebase(path: str, target_base: str, cur_base: str) -> str: + ... + + +@overload +def _rebase(path: str | None, target_base: str, cur_base: str) -> str | None: + ... + + +def _rebase(path: str | None, target_base: str, cur_base: str) -> str | None: """Rebase a path.""" if not path: # Don't rebase if the path is None or empty string (in case of default @@ -61,71 +87,100 @@ def _rebase(path, target_base, cur_base): return os.path.join(target_base, rel_path) -def rebase_to_host_root(worker_path): +@overload +def rebase_to_host_root(worker_path: None) -> None: + ... + + +@overload +def rebase_to_host_root(worker_path: str) -> str: + ... + + +@overload +def rebase_to_host_root(worker_path: str | None) -> str | None: + ... + + +def rebase_to_host_root(worker_path: str | None) -> str | None: """Return corresponding host root given a worker CF path.""" return _rebase(worker_path, environment.get_value('ROOT_DIR'), environment.get_value('WORKER_ROOT_DIR')) -def rebase_to_worker_root(host_path): +@overload +def rebase_to_worker_root(host_path: None) -> None: + ... + + +@overload +def rebase_to_worker_root(host_path: str) -> str: + ... + + +@overload +def rebase_to_worker_root(host_path: str | None) -> str | None: + ... + + +def rebase_to_worker_root(host_path: str | None) -> str | None: """Return corresponding worker path given a host CF path.""" return _rebase(host_path, environment.get_value('WORKER_ROOT_DIR'), environment.get_value('ROOT_DIR')) -def create_directory(path, create_intermediates=False): +def create_directory(path: str, create_intermediates: bool = False) -> bool: """Create a directory.""" - request = untrusted_runner_pb2.CreateDirectoryRequest( # pylint: disable=no-member - path=path, - create_intermediates=create_intermediates) + request = untrusted_runner_pb2.CreateDirectoryRequest( + path=path, create_intermediates=create_intermediates) - response = host.stub().CreateDirectory(request) + response = _stub().CreateDirectory(request) return response.result -def remove_directory(path, recreate=False): +def remove_directory(path: str, recreate: bool = False) -> bool: """Remove a directory. If |recreate| is set, always creates the directory even if it did not exist.""" - request = untrusted_runner_pb2.RemoveDirectoryRequest( # pylint: disable=no-member + request = untrusted_runner_pb2.RemoveDirectoryRequest( path=path, recreate=recreate) - response = host.stub().RemoveDirectory(request) + response = _stub().RemoveDirectory(request) return response.result -def list_files(path, recursive=False): +def list_files(path: str, recursive: bool = False) -> Sequence[str]: """List files in the directory. Returns full file paths.""" - request = untrusted_runner_pb2.ListFilesRequest( # pylint: disable=no-member + request = untrusted_runner_pb2.ListFilesRequest( path=path, recursive=recursive) - response = host.stub().ListFiles(request) + response = _stub().ListFiles(request) return response.file_paths -def copy_file_to_worker(host_path, worker_path): +def copy_file_to_worker(host_path: str, worker_path: str) -> bool: """Copy file from host to worker. |worker_path| must be a full path (including the filename). Any directories will be created if needed.""" with open(host_path, 'rb') as f: request_iterator = file_utils.file_chunk_generator(f) metadata = [('path-bin', worker_path.encode('utf-8'))] - response = host.stub().CopyFileTo(request_iterator, metadata=metadata) + response = _stub().CopyFileTo(request_iterator, metadata=metadata) return response.result -def write_data_to_worker(data, worker_path): +def write_data_to_worker(data: bytes, worker_path: str) -> bool: """Write data to a file on the worker.""" request_iterator = file_utils.data_chunk_generator(data) metadata = [('path-bin', worker_path.encode('utf-8'))] - response = host.stub().CopyFileTo(request_iterator, metadata=metadata) + response = _stub().CopyFileTo(request_iterator, metadata=metadata) return response.result -def copy_file_from_worker(worker_path, host_path): +def copy_file_from_worker(worker_path: str, host_path: str) -> bool: """Copy file from worker to host.""" - request = untrusted_runner_pb2.CopyFileFromRequest(path=worker_path) # pylint: disable=no-member - response = host.stub().CopyFileFrom(request) + request = untrusted_runner_pb2.CopyFileFromRequest(path=worker_path) + response = _stub().CopyFileFrom(request) file_utils.write_chunks(host_path, response) metadata = dict(response.trailing_metadata()) if metadata.get('result') != 'ok': @@ -137,7 +192,9 @@ def copy_file_from_worker(worker_path, host_path): return True -def copy_directory_to_worker(host_directory, worker_directory, replace=False): +def copy_directory_to_worker(host_directory: str, + worker_directory: str, + replace: bool = False) -> bool: """Recursively copy a directory to the worker. Directories are created as needed. Unless |replace| is True, files already in |worker_path| will remain after this call.""" @@ -156,7 +213,9 @@ def copy_directory_to_worker(host_directory, worker_directory, replace=False): return True -def copy_directory_from_worker(worker_directory, host_directory, replace=False): +def copy_directory_from_worker(worker_directory: str, + host_directory: str, + replace: bool = False) -> bool: """Recursively copy a directory from the worker. Directories are created as needed. Unless |replace| is True, files already in |host_directory| will remain after this call.""" @@ -185,17 +244,17 @@ def copy_directory_from_worker(worker_directory, host_directory, replace=False): return True -def stat(path): +def stat(path: str) -> untrusted_runner_pb2.StatResponse | None: """stat() a path.""" - request = untrusted_runner_pb2.StatRequest(path=path) # pylint: disable=no-member - response = host.stub().Stat(request) + request = untrusted_runner_pb2.StatRequest(path=path) + response = _stub().Stat(request) if not response.result: return None return response -def clear_testcase_directories(): +def clear_testcase_directories() -> None: """Clear the testcases directories on the worker.""" remove_directory( rebase_to_worker_root(environment.get_value('FUZZ_INPUTS')), @@ -205,19 +264,19 @@ def clear_testcase_directories(): recreate=True) -def clear_build_urls_directory(): +def clear_build_urls_directory() -> None: """Clear the build urls directory on the worker.""" remove_directory( rebase_to_worker_root(environment.get_value('BUILD_URLS_DIR')), recreate=True) -def clear_temp_directory(): +def clear_temp_directory() -> None: """Clear the temp directory on the worker.""" remove_directory(environment.get_value('WORKER_BOT_TMPDIR'), recreate=True) -def push_testcases_to_worker(): +def push_testcases_to_worker() -> bool: """Push all testcases to the worker.""" local_testcases_directory = environment.get_value('FUZZ_INPUTS') worker_testcases_directory = rebase_to_worker_root(local_testcases_directory) @@ -225,7 +284,7 @@ def push_testcases_to_worker(): local_testcases_directory, worker_testcases_directory, replace=True) -def pull_testcases_from_worker(): +def pull_testcases_from_worker() -> bool: """Pull all testcases to the worker.""" local_testcases_directory = environment.get_value('FUZZ_INPUTS') worker_testcases_directory = rebase_to_worker_root(local_testcases_directory) @@ -233,8 +292,8 @@ def pull_testcases_from_worker(): worker_testcases_directory, local_testcases_directory, replace=True) -def get_fuzz_targets(path): +def get_fuzz_targets(path: str) -> Sequence[str]: """Get list of fuzz target paths.""" - request = untrusted_runner_pb2.GetFuzzTargetsRequest(path=path) # pylint: disable=no-member - response = host.stub().GetFuzzTargets(request) + request = untrusted_runner_pb2.GetFuzzTargetsRequest(path=path) + response = _stub().GetFuzzTargets(request) return response.fuzz_target_paths diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/file_impl.py b/src/clusterfuzz/_internal/bot/untrusted_runner/file_impl.py index 17cefff4489..3169019dd66 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/file_impl.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/file_impl.py @@ -13,7 +13,12 @@ # limitations under the License. """File operations implementations.""" +from collections.abc import Iterable +from collections.abc import Iterator import os +from typing import cast + +import grpc from clusterfuzz._internal.bot.fuzzers import utils as fuzzers_utils from clusterfuzz._internal.protos import untrusted_runner_pb2 @@ -21,24 +26,33 @@ from . import file_utils -# pylint: disable=no-member +# pylint: disable=no-member,unused-argument -def create_directory(request, _): +def create_directory( + request: untrusted_runner_pb2.CreateDirectoryRequest, + context: grpc.ServicerContext, +) -> untrusted_runner_pb2.CreateDirectoryResponse: """Create a directory.""" result = shell.create_directory(request.path, request.create_intermediates) return untrusted_runner_pb2.CreateDirectoryResponse(result=result) -def remove_directory(request, _): +def remove_directory( + request: untrusted_runner_pb2.RemoveDirectoryRequest, + context: grpc.ServicerContext, +) -> untrusted_runner_pb2.RemoveDirectoryResponse: """Remove a directory.""" result = shell.remove_directory(request.path, request.recreate) return untrusted_runner_pb2.RemoveDirectoryResponse(result=result) -def list_files(request, _): +def list_files( + request: untrusted_runner_pb2.ListFilesRequest, + context: grpc.ServicerContext, +) -> untrusted_runner_pb2.ListFilesResponse: """List files.""" - file_paths = [] + file_paths: list[str] = [] if request.recursive: for root, _, files in shell.walk(request.path): for filename in files: @@ -50,10 +64,13 @@ def list_files(request, _): return untrusted_runner_pb2.ListFilesResponse(file_paths=file_paths) -def copy_file_to_worker(request_iterator, context): +def copy_file_to_worker( + request_iterator: Iterable[untrusted_runner_pb2.FileChunk], + context: grpc.ServicerContext, +) -> untrusted_runner_pb2.CopyFileToResponse: """Copy file from host to worker.""" metadata = dict(context.invocation_metadata()) - path = metadata['path-bin'].decode('utf-8') + path = cast(bytes, metadata['path-bin']).decode('utf-8') # Create intermediate directories if needed. directory = os.path.dirname(path) @@ -71,7 +88,10 @@ def copy_file_to_worker(request_iterator, context): return untrusted_runner_pb2.CopyFileToResponse(result=True) -def copy_file_from_worker(request, context): +def copy_file_from_worker( + request: untrusted_runner_pb2.CopyFileFromRequest, + context: grpc.ServicerContext, +) -> Iterator[untrusted_runner_pb2.FileChunk]: """Copy file from worker to host.""" path = request.path if not os.path.isfile(path): @@ -83,7 +103,10 @@ def copy_file_from_worker(request, context): context.set_trailing_metadata([('result', 'ok')]) -def stat(request, _): +def stat( + request: untrusted_runner_pb2.StatRequest, + context: grpc.ServicerContext, +) -> untrusted_runner_pb2.StatResponse: """Stat a path.""" if not os.path.exists(request.path): return untrusted_runner_pb2.StatResponse(result=False) @@ -98,7 +121,10 @@ def stat(request, _): st_ctime=stat_result.st_ctime) -def get_fuzz_targets(request, _): +def get_fuzz_targets( + request: untrusted_runner_pb2.GetFuzzTargetsRequest, + context: grpc.ServicerContext, +) -> untrusted_runner_pb2.GetFuzzTargetsResponse: """Get list of fuzz targets.""" fuzz_target_paths = fuzzers_utils.get_fuzz_targets_local(request.path) return untrusted_runner_pb2.GetFuzzTargetsResponse( diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/file_utils.py b/src/clusterfuzz/_internal/bot/untrusted_runner/file_utils.py index 206ea408ff2..adf51844265 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/file_utils.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/file_utils.py @@ -13,30 +13,40 @@ # limitations under the License. """Helpers for file operations.""" +from collections.abc import Iterable +from collections.abc import Iterator +from typing import BinaryIO + from clusterfuzz._internal.protos import untrusted_runner_pb2 from . import config +# pylint: disable=no-member + -def file_chunk_generator(handle): +def file_chunk_generator( + handle: BinaryIO) -> Iterator[untrusted_runner_pb2.FileChunk]: """Yields chunks from handle.""" data = handle.read(config.FILE_TRANSFER_CHUNK_SIZE) while data: - yield untrusted_runner_pb2.FileChunk(data=data) # pylint: disable=no-member + yield untrusted_runner_pb2.FileChunk(data=data) data = handle.read(config.FILE_TRANSFER_CHUNK_SIZE) -def data_chunk_generator(data): +def data_chunk_generator( + data: bytes) -> Iterator[untrusted_runner_pb2.FileChunk]: """Yields chunks for data.""" index = 0 while index < len(data): cur_chunk = data[index:index + config.FILE_TRANSFER_CHUNK_SIZE] - yield untrusted_runner_pb2.FileChunk(data=cur_chunk) # pylint: disable=no-member + yield untrusted_runner_pb2.FileChunk(data=cur_chunk) index += config.FILE_TRANSFER_CHUNK_SIZE -def write_chunks(file_path, chunk_iterator): +def write_chunks( + file_path: str, + chunk_iterator: Iterable[untrusted_runner_pb2.FileChunk]) -> None: """Writes chunks to file.""" with open(file_path, 'wb') as f: for chunk in chunk_iterator: diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/host.py b/src/clusterfuzz/_internal/bot/untrusted_runner/host.py index 1328798039c..fe6f25d425a 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/host.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/host.py @@ -16,6 +16,11 @@ import sys import threading import time +from typing import Any +from typing import Callable +from typing import cast +from typing import NoReturn +from typing import TypeVar from google.cloud import ndb import grpc @@ -33,28 +38,39 @@ from . import config -WAIT_TLS_CERT_SECONDS = 60 -RPC_FAIL_WAIT_TIME = 10 +WAIT_TLS_CERT_SECONDS: int = 60 +RPC_FAIL_WAIT_TIME: int = 10 + +_F = TypeVar('_F', bound=Callable[..., Any]) class ChannelState: """The host's view of the channel state.""" # Channel isn't ready for sending RPCs. - NOT_READY = 0 + NOT_READY: int = 0 # Channel is ready for RPCS. - READY = 1 + READY: int = 1 # The host found that the worker is in an inconsistent state. That is, we # detected that either the worker has a different source version, or if it # restarted without us knowing. - INCONSISTENT = 2 + INCONSISTENT: int = 2 class HostState: """The state of the host.""" - def __init__(self): + channel: grpc.Channel | None + stub: 'UntrustedRunnerStub | None' + heartbeat_thread: threading.Thread | None + expect_shutdown: bool + worker_start_time: int | None + worker_bot_name: str | None + channel_condition: threading.Condition + channel_state: int + + def __init__(self) -> None: self.channel = None self.stub = None self.heartbeat_thread = None @@ -69,7 +85,7 @@ def __init__(self): self.channel_state = ChannelState.NOT_READY -_host_state = HostState() +_host_state: HostState = HostState() class UntrustedRunnerStub(untrusted_runner_pb2_grpc.UntrustedRunnerStub): @@ -78,7 +94,7 @@ class UntrustedRunnerStub(untrusted_runner_pb2_grpc.UntrustedRunnerStub): We override the generated stub because we need to wrap these RPC calls to add error handling/retry logic.""" - def __init__(self, channel): + def __init__(self, channel: grpc.Channel) -> None: super().__init__(channel) # Don't wrap GetStatus() because it's used during connection state changes. @@ -107,7 +123,7 @@ def __init__(self, channel): # pylint: enable=invalid-name -def _check_channel_state(wait_time): +def _check_channel_state(wait_time: float | int) -> int: """Check the channel's state.""" with _host_state.channel_condition: if (_host_state.channel_state in (ChannelState.READY, @@ -120,10 +136,10 @@ def _check_channel_state(wait_time): return _host_state.channel_state -def _wrap_call(func, num_retries=config.RPC_RETRY_ATTEMPTS): +def _wrap_call(func: _F, num_retries: int = config.RPC_RETRY_ATTEMPTS) -> _F: """Wrapper for stub calls to add error handling and retry logic.""" - def wrapped(*args, **kwargs): + def wrapped(*args: Any, **kwargs: Any) -> Any: """Wrapper for adding retry logic.""" for retry_attempt in range(num_retries + 1): # Wait for channel to (re)connect if necessary. @@ -166,12 +182,13 @@ def wrapped(*args, **kwargs): return None - return wrapped + return cast(_F, wrapped) -def _do_heartbeat(): +def _do_heartbeat() -> None: """Heartbeat thread.""" # grpc stubs and channels should be thread-safe. + assert _host_state.channel is not None heartbeat_stub = heartbeat_pb2_grpc.HeartbeatStub(_host_state.channel) while True: try: @@ -184,7 +201,7 @@ def _do_heartbeat(): time.sleep(config.HEARTBEAT_INTERVAL_SECONDS) -def _get_host_worker_assignment(): +def _get_host_worker_assignment() -> data_types.HostWorkerAssignment | None: """Get the host worker assignment for the current host.""" # This only needs to be called once before the host connects to the worker. # This is because the host->worker assignment algorithm should ensure that a @@ -199,7 +216,7 @@ def _get_host_worker_assignment(): return key.get() -def _get_root_cert(project_name): +def _get_root_cert(project_name: str) -> bytes | None: """Get the root TLS cert for connecting to the worker.""" key = ndb.Key(data_types.WorkerTlsCert, project_name) tls_cert = key.get() @@ -210,7 +227,7 @@ def _get_root_cert(project_name): return tls_cert.cert_contents -def _connect(): +def _connect() -> None: """Initial connect to the worker.""" worker_assignment = _get_host_worker_assignment() assert worker_assignment is not None @@ -260,7 +277,8 @@ def _connect(): _host_state.heartbeat_thread.start() -def _channel_connectivity_changed(connectivity): +def _channel_connectivity_changed( + connectivity: grpc.ChannelConnectivity) -> None: """Callback for channel connectivity changes.""" try: with _host_state.channel_condition: @@ -295,10 +313,12 @@ def _channel_connectivity_changed(connectivity): logs.info('Reconnecting to worker.') -def _check_state(): +def _check_state() -> bool: """Check that the worker's state is consistent with the host's knowledge.""" + runner_stub = stub() + assert runner_stub is not None try: - status = stub().GetStatus( + status = runner_stub.GetStatus( untrusted_runner_pb2.GetStatusRequest(), # pylint: disable=no-member timeout=config.GET_STATUS_TIMEOUT_SECONDS) except grpc.RpcError: @@ -325,21 +345,23 @@ def _check_state(): return True -def init(): +def init() -> None: """Initialize channel to untrusted instance.""" _connect() -def stub(): +def stub() -> UntrustedRunnerStub | None: """Return the UntrustedRunnerStub.""" return _host_state.stub -def update_worker(): +def update_worker() -> None: """Update untrusted worker.""" _host_state.expect_shutdown = True + runner_stub = stub() + assert runner_stub is not None try: - stub().UpdateSource( + runner_stub.UpdateSource( untrusted_runner_pb2.UpdateSourceRequest(), # pylint: disable=no-member timeout=config.UPDATE_SOURCE_TIMEOUT_SECONDS) except grpc.RpcError: @@ -347,7 +369,7 @@ def update_worker(): pass -def host_exit_no_return(return_code=1): +def host_exit_no_return(return_code: int = 1) -> NoReturn: """Called when there is a host error.""" if return_code: monitoring_metrics.HOST_ERROR_COUNT.increment({'return_code': return_code}) @@ -356,6 +378,7 @@ def host_exit_no_return(return_code=1): update_worker() # Prevent exceptions during shutdown. + assert _host_state.channel is not None _host_state.channel.unsubscribe(_channel_connectivity_changed) # This should bypass most exception handlers and avoid callers from catching @@ -364,6 +387,6 @@ def host_exit_no_return(return_code=1): raise untrusted.HostError(return_code) -def is_initialized(): +def is_initialized() -> bool: """Return whether or not the host is initialized.""" return _host_state.stub is not None diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/protobuf_utils.py b/src/clusterfuzz/_internal/bot/untrusted_runner/protobuf_utils.py index 1bc3a8184f3..7a5774b3352 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/protobuf_utils.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/protobuf_utils.py @@ -13,14 +13,33 @@ # limitations under the License. """Protobuf helpers.""" +from typing import Any +from typing import overload +from typing import TypeVar -def get_protobuf_field(result, buf, field_name): +from google.protobuf import message + +_T = TypeVar('_T') + + +def get_protobuf_field(result: dict[str, Any], buf: message.Message, + field_name: str) -> None: """Add protobuf field to dict, if the field exists.""" if buf.HasField(field_name): result[field_name] = getattr(buf, field_name) -def encode_utf8_if_unicode(data): +@overload +def encode_utf8_if_unicode(data: str) -> bytes: + ... + + +@overload +def encode_utf8_if_unicode(data: _T) -> _T: + ... + + +def encode_utf8_if_unicode(data: Any) -> Any: """Encode string as utf-8 if it's unicode.""" if isinstance(data, str): return data.encode('utf-8') diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/remote_process.py b/src/clusterfuzz/_internal/bot/untrusted_runner/remote_process.py index 3aaad4cd112..2f7b755e6f1 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/remote_process.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/remote_process.py @@ -13,6 +13,9 @@ # limitations under the License. """Remote process implementation.""" +from typing import Any +from typing import cast + from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.protos import untrusted_runner_pb2 from clusterfuzz._internal.system import new_process @@ -20,25 +23,31 @@ from . import protobuf_utils +# pylint: disable=no-member + -def process_result_to_proto(process_result): +def process_result_to_proto(process_result: new_process.ProcessResult, + ) -> untrusted_runner_pb2.ProcessResult: """Convert new_process.ProcessResult to proto.""" - process_result_proto = untrusted_runner_pb2.ProcessResult( # pylint: disable=no-member + process_result_proto = untrusted_runner_pb2.ProcessResult( return_code=process_result.return_code, output=process_result.output, time_executed=process_result.time_executed, timed_out=process_result.timed_out) - process_result_proto.command.extend(process_result.command) # pylint: disable=no-member + process_result_proto.command.extend(cast(list[str], process_result.command)) return process_result_proto -def run_and_wait(request, _): +def run_and_wait( + request: untrusted_runner_pb2.RunAndWaitRequest, + _: Any, +) -> untrusted_runner_pb2.RunAndWaitResponse: """Implementation of RunAndWait.""" process_runner = new_process.ProcessRunner(request.executable_path, request.default_args) - args = {} + args: dict[str, Any] = {} protobuf_utils.get_protobuf_field(args, request.popen_args, 'bufsize') protobuf_utils.get_protobuf_field(args, request.popen_args, 'executable') protobuf_utils.get_protobuf_field(args, request.popen_args, 'shell') @@ -58,13 +67,16 @@ def run_and_wait(request, _): logs.info('Running command: %s' % process_runner.get_command()) - return untrusted_runner_pb2.RunAndWaitResponse( # pylint: disable=no-member + return untrusted_runner_pb2.RunAndWaitResponse( result=process_result_to_proto(process_runner.run_and_wait(**args))) -def run_process(request, _): +def run_process( + request: untrusted_runner_pb2.RunProcessRequest, + _: Any, +) -> untrusted_runner_pb2.RunProcessResponse: """Implementation of RunProcess.""" - args = {} + args: dict[str, Any] = {} protobuf_utils.get_protobuf_field(args, request, 'cmdline') protobuf_utils.get_protobuf_field(args, request, 'current_working_directory') protobuf_utils.get_protobuf_field(args, request, 'timeout') @@ -80,9 +92,7 @@ def run_process(request, _): protobuf_utils.get_protobuf_field(args, request, 'ignore_children') return_code, execution_time, output = process_handler.run_process(**args) - response = untrusted_runner_pb2.RunProcessResponse( # pylint: disable=no-member - return_code=return_code, - execution_time=execution_time, - output=output) + response = untrusted_runner_pb2.RunProcessResponse( + return_code=return_code, execution_time=execution_time, output=output) return response diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/remote_process_host.py b/src/clusterfuzz/_internal/bot/untrusted_runner/remote_process_host.py index 78408a09d84..5ec7a56caca 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/remote_process_host.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/remote_process_host.py @@ -13,10 +13,15 @@ # limitations under the License. """Remote process host (client).""" +from collections.abc import Mapping +from collections.abc import Sequence import os import subprocess +from typing import Any +from typing import cast from clusterfuzz._internal.protos import untrusted_runner_pb2 +from clusterfuzz._internal.protos import untrusted_runner_pb2_grpc from clusterfuzz._internal.system import new_process from clusterfuzz._internal.system import process_handler @@ -26,22 +31,29 @@ # pylint:disable=no-member -def process_result_from_proto(process_result_proto): +def process_result_from_proto( + process_result_proto: untrusted_runner_pb2.ProcessResult, +) -> new_process.ProcessResult: """Convert ProcessResult proto to new_process.ProcessResult.""" return new_process.ProcessResult( - process_result_proto.command, process_result_proto.return_code, - process_result_proto.output, process_result_proto.time_executed, - process_result_proto.timed_out) - - -def run_process(cmdline, - current_working_directory=None, - timeout=process_handler.DEFAULT_TEST_TIMEOUT, - need_shell=False, - gestures=None, - env_copy=None, - testcase_run=True, - ignore_children=True): + list(process_result_proto.command), + process_result_proto.return_code, + process_result_proto.output, + process_result_proto.time_executed, + process_result_proto.timed_out, + ) + + +def run_process( + cmdline: str, + current_working_directory: str | None = None, + timeout: float = process_handler.DEFAULT_TEST_TIMEOUT, + need_shell: bool = False, + gestures: Sequence[str] | None = None, + env_copy: Mapping[str, str] | None = None, + testcase_run: bool = True, + ignore_children: bool = True, +) -> tuple[int | None, float | None, str]: """Remote version of process_handler.run_process.""" request = untrusted_runner_pb2.RunProcessRequest( cmdline=cmdline, @@ -54,7 +66,7 @@ def run_process(cmdline, if gestures: request.gestures.extend(gestures) - env = {} + env: dict[str, str] = {} # run_process's local behaviour is to apply the passed |env_copy| on top of # the current environment instead of replacing it completely (like with # subprocess). @@ -62,37 +74,56 @@ def run_process(cmdline, environment.set_environment_vars(env, env_copy) request.env_copy.update(env) - response = host.stub().RunProcess(request) + stub = cast(untrusted_runner_pb2_grpc.UntrustedRunnerStub, host.stub()) + response = stub.RunProcess(request) return response.return_code, response.execution_time, response.output class RemoteProcessRunner(new_process.ProcessRunner): """Remote child process.""" - def __init__(self, executable_path, default_args=None): + def __init__( + self, + executable_path: str, + default_args: Sequence[str] | None = None, + ) -> None: super().__init__(executable_path, default_args=default_args) - def run(self, **kwargs): # pylint: disable=arguments-differ + def run( + self, + additional_args: Sequence[str] | None = None, + max_stdout_len: int | None = None, + extra_env: dict[str, str] | None = None, + stdin: Any = subprocess.PIPE, + stdout: Any = subprocess.PIPE, + stderr: Any = subprocess.STDOUT, + **popen_args: Any, + ) -> new_process.ChildProcess: # TODO(ochang): This can be implemented, but isn't necessary yet. raise NotImplementedError - def run_and_wait(self, - additional_args=None, - timeout=None, - terminate_before_kill=False, - terminate_wait_time=None, - input_data=None, - max_stdout_len=None, - extra_env=None, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - **popen_args): + def run_and_wait( + self, + additional_args: Sequence[str] | None = None, + timeout: float | None = None, + terminate_before_kill: bool = False, + terminate_wait_time: float | None = None, + input_data: str | bytes | None = None, + max_stdout_len: int | None = None, + extra_env: dict[str, str] | None = None, + stdin: Any = subprocess.PIPE, + stdout: Any = subprocess.PIPE, + stderr: Any = subprocess.STDOUT, + **popen_args: Any, + ) -> new_process.ProcessResult: # pylint: disable=unused-argument - # pylint: disable=arguments-differ """Remote version of new_process.ProcessRunner.run_and_wait.""" assert stdout == subprocess.PIPE assert stderr == subprocess.STDOUT + if isinstance(input_data, str): + input_data = input_data.encode('utf-8') + request = untrusted_runner_pb2.RunAndWaitRequest( executable_path=self.executable_path, timeout=timeout, @@ -124,11 +155,13 @@ def run_and_wait(self, # variables if the caller passes e.g. os.environ.copy(). environment.set_environment_vars(request.popen_args.env, passed_env) - response = host.stub().RunAndWait(request) + stub = cast(untrusted_runner_pb2_grpc.UntrustedRunnerStub, host.stub()) + response = stub.RunAndWait(request) return process_result_from_proto(response.result) -def terminate_stale_application_instances(): +def terminate_stale_application_instances() -> None: """Terminate stale application instances.""" - host.stub().TerminateStaleApplicationInstances( + stub = cast(untrusted_runner_pb2_grpc.UntrustedRunnerStub, host.stub()) + stub.TerminateStaleApplicationInstances( untrusted_runner_pb2.TerminateStaleApplicationInstancesRequest()) diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/symbolize.py b/src/clusterfuzz/_internal/bot/untrusted_runner/symbolize.py index 279b5771c25..6b851bc551d 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/symbolize.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/symbolize.py @@ -16,11 +16,15 @@ from clusterfuzz._internal.crash_analysis.stack_parsing import stack_symbolizer from clusterfuzz._internal.protos import untrusted_runner_pb2 +# pylint: disable=no-member -def symbolize_stacktrace(request): + +def symbolize_stacktrace( + request: untrusted_runner_pb2.SymbolizeStacktraceRequest, +) -> untrusted_runner_pb2.SymbolizeStacktraceResponse: """Symbolize stacktrace.""" symbolized_stacktrace = stack_symbolizer.symbolize_stacktrace( request.unsymbolized_crash_stacktrace, request.enable_inline_frames) - return untrusted_runner_pb2.SymbolizeStacktraceResponse( # pylint: disable=no-member + return untrusted_runner_pb2.SymbolizeStacktraceResponse( symbolized_stacktrace=symbolized_stacktrace) diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/symbolize_host.py b/src/clusterfuzz/_internal/bot/untrusted_runner/symbolize_host.py index 88f8e2afbca..de4062a73d6 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/symbolize_host.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/symbolize_host.py @@ -13,19 +13,21 @@ # limitations under the License. """Stacktrace symbolization (host side).""" +from typing import cast + from clusterfuzz._internal.protos import untrusted_runner_pb2 +from clusterfuzz._internal.protos import untrusted_runner_pb2_grpc from . import host -from . import protobuf_utils -def symbolize_stacktrace(unsymbolized_crash_stacktrace, - enable_inline_frames=True): +def symbolize_stacktrace(unsymbolized_crash_stacktrace: str, + enable_inline_frames: bool = True) -> str: """Symbolize stacktrace.""" request = untrusted_runner_pb2.SymbolizeStacktraceRequest( # pylint: disable=no-member - unsymbolized_crash_stacktrace=protobuf_utils.encode_utf8_if_unicode( - unsymbolized_crash_stacktrace), + unsymbolized_crash_stacktrace=unsymbolized_crash_stacktrace, enable_inline_frames=enable_inline_frames) - response = host.stub().SymbolizeStacktrace(request) + stub = cast(untrusted_runner_pb2_grpc.UntrustedRunnerStub, host.stub()) + response = stub.SymbolizeStacktrace(request) return response.symbolized_stacktrace diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_host.py b/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_host.py index 1bf296a7b8d..5e9e3af7f84 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_host.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_host.py @@ -13,8 +13,14 @@ # limitations under the License. """Tasks host.""" +# pyright: reportAttributeAccessIssue=none + +from collections.abc import Mapping +from collections.abc import Sequence import datetime +from typing import cast +from google.protobuf import any_pb2 from google.protobuf import wrappers_pb2 import grpc @@ -23,6 +29,7 @@ from clusterfuzz._internal.bot.untrusted_runner import file_host from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.protos import untrusted_runner_pb2 +from clusterfuzz._internal.protos import untrusted_runner_pb2_grpc from clusterfuzz._internal.protos import uworker_msg_pb2 from clusterfuzz.fuzz import engine @@ -31,7 +38,13 @@ # pylint: disable=no-member -def _fuzz_target_to_proto(fuzz_target): +def _stub() -> untrusted_runner_pb2_grpc.UntrustedRunnerStub: + """Return the UntrustedRunnerStub.""" + return cast(untrusted_runner_pb2_grpc.UntrustedRunnerStub, host.stub()) + + +def _fuzz_target_to_proto(fuzz_target: data_types.FuzzTarget, + ) -> untrusted_runner_pb2.FuzzTarget: """Convert fuzz_target to protobuf.""" return untrusted_runner_pb2.FuzzTarget( engine=fuzz_target.engine, @@ -40,12 +53,15 @@ def _fuzz_target_to_proto(fuzz_target): ) -def _get_datetime(): +def _get_datetime() -> datetime.datetime: return datetime.datetime.utcnow() -def do_corpus_pruning(uworker_input, context, - revision) -> corpus_pruning_task.CorpusPruningResult: +def do_corpus_pruning( + uworker_input: uworker_msg_pb2.Input, + context: corpus_pruning_task.Context, + revision: int, +) -> corpus_pruning_task.CorpusPruningResult: """Do corpus pruning on untrusted worker.""" cross_pollinate_fuzzers = [ untrusted_runner_pb2.CrossPollinateFuzzer( @@ -61,7 +77,7 @@ def do_corpus_pruning(uworker_input, context, revision=revision, uworker_input=uworker_input) - response = host.stub().PruneCorpus(request) + response = _stub().PruneCorpus(request) project_qualified_name = context.fuzz_target.project_qualified_name() today_date = _get_datetime() @@ -111,27 +127,35 @@ def do_corpus_pruning(uworker_input, context, cross_pollination_stats=pollination_stats) -def _unpack_values(values): +def _unpack_values(values: Mapping[str, any_pb2.Any], + ) -> dict[str, float | int | str]: """Unpack protobuf values.""" - unpacked = {} + unpacked: dict[str, float | int | str] = {} for key, packed_value in values.items(): if packed_value.Is(wrappers_pb2.DoubleValue.DESCRIPTOR): - value = wrappers_pb2.DoubleValue() + double_value = wrappers_pb2.DoubleValue() + packed_value.Unpack(double_value) + unpacked[key] = double_value.value elif packed_value.Is(wrappers_pb2.Int64Value.DESCRIPTOR): - value = wrappers_pb2.Int64Value() + int64_value = wrappers_pb2.Int64Value() + packed_value.Unpack(int64_value) + unpacked[key] = int64_value.value elif packed_value.Is(wrappers_pb2.StringValue.DESCRIPTOR): - value = wrappers_pb2.StringValue() + string_value = wrappers_pb2.StringValue() + packed_value.Unpack(string_value) + unpacked[key] = string_value.value else: raise ValueError('Unknown stat type for ' + key) - packed_value.Unpack(value) - unpacked[key] = value.value - return unpacked -def engine_fuzz(engine_impl, target_name, sync_corpus_directory, - testcase_directory): +def engine_fuzz( + engine_impl: engine.Engine, + target_name: str, + sync_corpus_directory: str, + testcase_directory: str, +) -> tuple[engine.FuzzResult, dict[str, str], dict[str, float | int | str]]: """Run engine fuzzer on untrusted worker.""" request = untrusted_runner_pb2.EngineFuzzRequest( engine=engine_impl.name, @@ -140,7 +164,7 @@ def engine_fuzz(engine_impl, target_name, sync_corpus_directory, sync_corpus_directory), testcase_directory=file_host.rebase_to_worker_root(testcase_directory)) - response = host.stub().EngineFuzz(request) + response = _stub().EngineFuzz(request) crashes = [ engine.Crash( input_path=file_host.rebase_to_host_root(crash.input_path), @@ -163,8 +187,13 @@ def engine_fuzz(engine_impl, target_name, sync_corpus_directory, return result, dict(response.fuzzer_metadata), unpacked_strategies -def engine_reproduce(engine_impl, target_name, testcase_path, arguments, - timeout): +def engine_reproduce( + engine_impl: engine.Engine, + target_name: str, + testcase_path: str, + arguments: Sequence[str], + timeout: int | float, +) -> engine.ReproduceResult: """Run engine reproduce on untrusted worker.""" rebased_testcase_path = file_host.rebase_to_worker_root(testcase_path) file_host.copy_file_to_worker(testcase_path, rebased_testcase_path) @@ -177,7 +206,7 @@ def engine_reproduce(engine_impl, target_name, testcase_path, arguments, timeout=timeout) try: - response = host.stub().EngineReproduce(request) + response = _stub().EngineReproduce(request) except grpc.RpcError as e: if 'TargetNotFoundError' in repr(e): # Resurface the right exception. diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_impl.py b/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_impl.py index 3479dbd4fe8..22a4e1df7f1 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_impl.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/tasks_impl.py @@ -13,8 +13,13 @@ # limitations under the License. """Tasks RPC implementations.""" +# pyright: reportAttributeAccessIssue=none + +from collections.abc import Mapping + +from google.protobuf import any_pb2 from google.protobuf import wrappers_pb2 -from google.protobuf.any_pb2 import Any # pylint: disable=no-name-in-module +import grpc from clusterfuzz._internal.bot import testcase_manager from clusterfuzz._internal.bot.tasks.utasks import fuzz_task @@ -24,14 +29,14 @@ # pylint:disable=no-member -def _pack_values(values): +def _pack_values(values: Mapping[str, object] | None) -> dict[str, any_pb2.Any]: """Pack protobuf values.""" - packed = {} + packed: dict[str, any_pb2.Any] = {} if values is None: return packed for key, value in values.items(): - packed_value = Any() + packed_value = any_pb2.Any() if isinstance(value, float): packed_value.Pack(wrappers_pb2.DoubleValue(value=value)) elif isinstance(value, int): @@ -46,9 +51,13 @@ def _pack_values(values): return packed -def engine_fuzz(request, _): +def engine_fuzz( + request: untrusted_runner_pb2.EngineFuzzRequest, + _: grpc.ServicerContext, +) -> untrusted_runner_pb2.EngineFuzzResponse: """Run engine fuzzer.""" engine_impl = engine.get(request.engine) + assert engine_impl is not None result, fuzzer_metadata, strategies = fuzz_task.run_engine_fuzzer( engine_impl, request.target_name, request.sync_corpus_directory, request.testcase_directory) @@ -74,9 +83,13 @@ def engine_fuzz(request, _): strategies=packed_strategies) -def engine_reproduce(request, _): +def engine_reproduce( + request: untrusted_runner_pb2.EngineReproduceRequest, + _: grpc.ServicerContext, +) -> untrusted_runner_pb2.EngineReproduceResult: """Run engine reproduce.""" engine_impl = engine.get(request.engine) + assert engine_impl is not None result = testcase_manager.engine_reproduce(engine_impl, request.target_name, request.testcase_path, request.arguments, request.timeout) diff --git a/src/clusterfuzz/_internal/bot/untrusted_runner/untrusted.py b/src/clusterfuzz/_internal/bot/untrusted_runner/untrusted.py index 565a432c0b3..21137f3f34b 100644 --- a/src/clusterfuzz/_internal/bot/untrusted_runner/untrusted.py +++ b/src/clusterfuzz/_internal/bot/untrusted_runner/untrusted.py @@ -13,12 +13,20 @@ # limitations under the License. """Untrusted instance.""" +# pyright: reportAttributeAccessIssue=none + +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator from concurrent import futures import functools import os import threading import time import traceback +from typing import Any +from typing import cast +from typing import TypeVar import grpc @@ -42,13 +50,19 @@ SHUTDOWN_GRACE_SECONDS = 5 +_F = TypeVar('_F', bound=Callable[..., Any]) + # pylint: disable=no-member class WorkerState: """Worker's state.""" - def __init__(self): + server: grpc.Server | None + shutting_down: threading.Event + start_time: int | None + + def __init__(self) -> None: self.server = None self.shutting_down = threading.Event() self.start_time = None @@ -61,11 +75,11 @@ def __init__(self): _rpc_count = 0 -def wrap_servicer(func): +def wrap_servicer(func: _F) -> _F: """Wrap a servicer to add additional functionality.""" @functools.wraps(func) - def wrapper(self, request, context): # pylint: disable=unused-argument + def wrapper(self: Any, request: Any, context: grpc.ServicerContext) -> Any: # pylint: disable=unused-argument """Wrapper function.""" global _rpc_count @@ -92,7 +106,7 @@ def wrapper(self, request, context): # pylint: disable=unused-argument return result - return wrapper + return cast(_F, wrapper) class UntrustedRunnerServicer( @@ -100,102 +114,186 @@ class UntrustedRunnerServicer( """Untrusted runner implementation.""" @wrap_servicer - def GetStatus(self, request, context): # pylint: disable=unused-argument + def GetStatus( + self, + request: untrusted_runner_pb2.GetStatusRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.GetStatusResponse: # pylint: disable=unused-argument return untrusted_runner_pb2.GetStatusResponse( revision=utils.current_source_version(), start_time=_worker_state.start_time, bot_name=environment.get_value('BOT_NAME')) @wrap_servicer - def SetupRegularBuild(self, request, _): + def SetupRegularBuild( + self, + request: untrusted_runner_pb2.SetupRegularBuildRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.SetupBuildResponse: return build_setup.setup_regular_build(request) @wrap_servicer - def RunAndWait(self, request, context): + def RunAndWait( + self, + request: untrusted_runner_pb2.RunAndWaitRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.RunAndWaitResponse: return remote_process.run_and_wait(request, context) @wrap_servicer - def RunProcess(self, request, context): + def RunProcess( + self, + request: untrusted_runner_pb2.RunProcessRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.RunProcessResponse: return remote_process.run_process(request, context) @wrap_servicer - def CreateDirectory(self, request, context): + def CreateDirectory( + self, + request: untrusted_runner_pb2.CreateDirectoryRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.CreateDirectoryResponse: return file_impl.create_directory(request, context) @wrap_servicer - def RemoveDirectory(self, request, context): + def RemoveDirectory( + self, + request: untrusted_runner_pb2.RemoveDirectoryRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.RemoveDirectoryResponse: return file_impl.remove_directory(request, context) @wrap_servicer - def ListFiles(self, request, context): + def ListFiles( + self, + request: untrusted_runner_pb2.ListFilesRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.ListFilesResponse: return file_impl.list_files(request, context) @wrap_servicer - def CopyFileTo(self, request_iterator, context): + def CopyFileTo( + self, + request_iterator: Iterable[untrusted_runner_pb2.FileChunk], + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.CopyFileToResponse: return file_impl.copy_file_to_worker(request_iterator, context) @wrap_servicer - def CopyFileFrom(self, request, context): + def CopyFileFrom( + self, + request: untrusted_runner_pb2.CopyFileFromRequest, + context: grpc.ServicerContext, + ) -> Iterator[untrusted_runner_pb2.FileChunk]: return file_impl.copy_file_from_worker(request, context) @wrap_servicer - def Stat(self, request, context): + def Stat( + self, + request: untrusted_runner_pb2.StatRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.StatResponse: return file_impl.stat(request, context) @wrap_servicer - def UpdateEnvironment(self, request, _): + def UpdateEnvironment( + self, + request: untrusted_runner_pb2.UpdateEnvironmentRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.UpdateEnvironmentResponse: os.environ.update(request.env) return untrusted_runner_pb2.UpdateEnvironmentResponse() @wrap_servicer - def ResetEnvironment(self, _, context): # pylint: disable=unused-argument + def ResetEnvironment( + self, + request: untrusted_runner_pb2.ResetEnvironmentRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.ResetEnvironmentResponse: # pylint: disable=unused-argument environment.reset_environment() return untrusted_runner_pb2.ResetEnvironmentResponse() @wrap_servicer - def UpdateSource(self, request, context): # pylint: disable=unused-argument + def UpdateSource( + self, + request: untrusted_runner_pb2.UpdateSourceRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.UpdateSourceResponse: # pylint: disable=unused-argument # Exit and let run.py update source. _worker_state.shutting_down.set() return untrusted_runner_pb2.UpdateSourceResponse() @wrap_servicer - def SymbolizeStacktrace(self, request, _): + def SymbolizeStacktrace( + self, + request: untrusted_runner_pb2.SymbolizeStacktraceRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.SymbolizeStacktraceResponse: return symbolize.symbolize_stacktrace(request) @wrap_servicer - def TerminateStaleApplicationInstances(self, request, context): # pylint: disable=unused-argument + def TerminateStaleApplicationInstances( + self, + request: untrusted_runner_pb2.TerminateStaleApplicationInstancesRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.TerminateStaleApplicationInstancesResponse: # pylint: disable=unused-argument process_handler.terminate_stale_application_instances() return untrusted_runner_pb2.TerminateStaleApplicationInstancesResponse() @wrap_servicer - def GetFuzzTargets(self, request, context): + def GetFuzzTargets( + self, + request: untrusted_runner_pb2.GetFuzzTargetsRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.GetFuzzTargetsResponse: return file_impl.get_fuzz_targets(request, context) @wrap_servicer - def ProcessTestcase(self, request, context): - return tasks_impl.process_testcase(request, context) + def ProcessTestcase( + self, + request: untrusted_runner_pb2.ProcessTestcaseRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.EngineReproduceResult: + return tasks_impl.process_testcase(request, context) # type: ignore @wrap_servicer - def EngineFuzz(self, request, context): + def EngineFuzz( + self, + request: untrusted_runner_pb2.EngineFuzzRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.EngineFuzzResponse: return tasks_impl.engine_fuzz(request, context) @wrap_servicer - def EngineReproduce(self, request, context): + def EngineReproduce( + self, + request: untrusted_runner_pb2.EngineReproduceRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.EngineReproduceResult: return tasks_impl.engine_reproduce(request, context) @wrap_servicer - def PruneCorpus(self, request, context): - return tasks_impl.prune_corpus(request, context) + def PruneCorpus( + self, + request: untrusted_runner_pb2.PruneCorpusRequest, + context: grpc.ServicerContext, + ) -> untrusted_runner_pb2.PruneCorpusResponse: + return tasks_impl.prune_corpus(request, context) # type: ignore class HeartbeatServicer(heartbeat_pb2_grpc.HeartbeatServicer): """Heartbeat service (for keeping connections alive).""" - def Beat(self, _, context): # pylint: disable=unused-argument + def Beat( + self, + request: heartbeat_pb2.HeartbeatRequest, + context: grpc.ServicerContext, + ) -> heartbeat_pb2.HeartbeatResponse: # pylint: disable=unused-argument return heartbeat_pb2.HeartbeatResponse() -def _get_tls_cert_and_key(): +def _get_tls_cert_and_key() -> tuple[bytes, bytes]: """Get the TLS cert from instance metadata.""" # TODO(ochang): Implement a fake metadata server for testing. local_cert_location = environment.get_value('UNTRUSTED_TLS_CERT_FOR_TESTING') @@ -215,7 +313,7 @@ def _get_tls_cert_and_key(): return cert_contents, key_contents -def start_server(): +def start_server() -> None: """Start the server.""" # Check overall free disk space. If we are running too low, clear all # data directories like builds, fuzzers, data bundles, etc. @@ -225,20 +323,20 @@ def start_server(): assert cert_contents and key_contents server_credentials = grpc.ssl_server_credentials([(key_contents, cert_contents)]) - _worker_state.server = grpc.server( + server_instance = grpc.server( futures.ThreadPoolExecutor(max_workers=config.NUM_WORKER_THREADS), options=config.GRPC_OPTIONS) + _worker_state.server = server_instance untrusted_runner_pb2_grpc.add_UntrustedRunnerServicer_to_server( - UntrustedRunnerServicer(), _worker_state.server) + UntrustedRunnerServicer(), server_instance) heartbeat_pb2_grpc.add_HeartbeatServicer_to_server(HeartbeatServicer(), - _worker_state.server) + server_instance) - _worker_state.server.add_secure_port('[::]:%d' % config.PORT, - server_credentials) + server_instance.add_secure_port('[::]:%d' % config.PORT, server_credentials) _worker_state.start_time = int(time.time()) - _worker_state.server.start() + server_instance.start() logs.info('Server started.') @@ -246,7 +344,7 @@ def start_server(): _worker_state.shutting_down.wait() logs.info('Server shutting down.') - stopped = _worker_state.server.stop(SHUTDOWN_GRACE_SECONDS) + stopped = server_instance.stop(SHUTDOWN_GRACE_SECONDS) stopped.wait() # Prevent python GIL deadlocks on shutdown. See https://crbug.com/744680. @@ -254,6 +352,6 @@ def start_server(): os._exit(0) -def server(): +def server() -> grpc.Server | None: """Return the grpc.Server.""" return _worker_state.server diff --git a/src/clusterfuzz/_internal/bot/webserver/http_server.py b/src/clusterfuzz/_internal/bot/webserver/http_server.py index d7d6f1f78cb..443fb4be8f9 100644 --- a/src/clusterfuzz/_internal/bot/webserver/http_server.py +++ b/src/clusterfuzz/_internal/bot/webserver/http_server.py @@ -23,7 +23,7 @@ from clusterfuzz._internal.system import environment -def get_absolute_testcase_file(request_path): +def get_absolute_testcase_file(request_path: str) -> str | None: """Search the input directory and additional paths for the requested file.""" # Gather the list of search path directories. current_working_directory = os.getcwd() @@ -61,7 +61,7 @@ def get_absolute_testcase_file(request_path): return None -def guess_mime_type(filename): +def guess_mime_type(filename: str) -> str | None: """Guess mime type based of file extension.""" if not mimetypes.inited: mimetypes.init() @@ -72,10 +72,11 @@ def guess_mime_type(filename): class BotHTTPServer(http.server.HTTPServer): """Host the bot's test case directories over HTTP.""" - def __init__(self, server_address, handler_class): + def __init__(self, server_address: tuple[str, int], + handler_class: type[http.server.BaseHTTPRequestHandler]) -> None: http.server.HTTPServer.__init__(self, server_address, handler_class) - def _handle_request_noblock(self): + def _handle_request_noblock(self) -> None: """Process a single http request.""" try: request, client_address = self.get_request() @@ -91,7 +92,7 @@ def _handle_request_noblock(self): class RequestHandler(http.server.BaseHTTPRequestHandler): """Handler for get requests to test cases.""" - def do_GET(self): # pylint: disable=invalid-name + def do_GET(self) -> None: # pylint: disable=invalid-name """Handle a GET request.""" absolute_path = get_absolute_testcase_file(self.path) if not absolute_path: @@ -121,30 +122,30 @@ def do_GET(self): # pylint: disable=invalid-name self.end_headers() self.wfile.write(data) - def log_message(self, fmt, *args): # pylint: disable=arguments-differ + def log_message(self, format: str, *args: object) -> None: # pylint: disable=arguments-differ,redefined-builtin """Do not output a log entry to stderr for every request made.""" -def run_server(host, port): +def run_server(host: str, port: int) -> None: """Run the HTTP server on the given port.""" httpd = BotHTTPServer((host, port), RequestHandler) httpd.serve_forever() -def port_is_open(host, port): +def port_is_open(host: str, port: int) -> bool: socket_handle = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = socket_handle.connect_ex((host, port)) socket_handle.close() return result == 0 -def start_server_thread(host, port): +def start_server_thread(host: str, port: int) -> None: server = threading.Thread(target=run_server, args=(host, port)) server.daemon = True server.start() -def start(): +def start() -> None: """Initialize the HTTP server on the specified ports.""" http_host = 'localhost' http_port_1 = environment.get_value('HTTP_PORT_1', 8000) diff --git a/src/clusterfuzz/_internal/build_management/build_archive.py b/src/clusterfuzz/_internal/build_management/build_archive.py index 7fa22f6486d..e622518902e 100644 --- a/src/clusterfuzz/_internal/build_management/build_archive.py +++ b/src/clusterfuzz/_internal/build_management/build_archive.py @@ -14,13 +14,11 @@ """Build Archive manager.""" import abc +from collections.abc import Callable import json import os -from typing import BinaryIO -from typing import Callable -from typing import List -from typing import Optional -from typing import Union +from typing import cast +from typing import IO from typing_extensions import override @@ -29,12 +27,12 @@ # Extensions to exclude when unarchiving a fuzz target. Note that fuzz target # own files like seed corpus, options, etc are covered by its own regex. -FUZZ_TARGET_EXCLUDED_EXTENSIONS = [ +FUZZ_TARGET_EXCLUDED_EXTENSIONS: list[str] = [ 'exe', 'options', 'txt', 'zip', 'exe.pdb', 'par' ] # File prefixes to explicitly include when unarchiving a fuzz target. -FUZZ_TARGET_ALLOWLISTED_PREFIXES = [ +FUZZ_TARGET_ALLOWLISTED_PREFIXES: list[str] = [ 'afl-cmin', 'afl-fuzz', 'afl-showmap', @@ -57,7 +55,7 @@ # Manifest file present in Chrome build archives. Specifies the archive # schema version along with optional build metadata (e.g., fuzz targets). -CHROME_MANIFEST_FILENAME = 'clusterfuzz_manifest.json' +CHROME_MANIFEST_FILENAME: str = 'clusterfuzz_manifest.json' class BuildArchive(archive.ArchiveReader): @@ -67,26 +65,33 @@ class inherits from `archive.ArchiveReader`, so this also provides class archive management features. """ - def __init__(self, reader: archive.ArchiveReader): + _reader: archive.ArchiveReader + _fuzz_targets: dict[str, str] | None + + def __init__(self, reader: archive.ArchiveReader) -> None: self._reader = reader self._fuzz_targets = None - def list_members(self) -> List[archive.ArchiveMemberInfo]: + @override + def list_members(self) -> list[archive.ArchiveMemberInfo]: return self._reader.list_members() + @override def extract(self, member: str, - path: Union[str, os.PathLike], - trusted: bool = False) -> str: + path: str | os.PathLike[str], + trusted: bool = False) -> str | None: return self._reader.extract(member=member, path=path, trusted=trusted) - def open(self, member: str) -> BinaryIO: + @override + def open(self, member: str) -> IO[bytes] | None: return self._reader.open(member) + @override def close(self) -> None: self._reader.close() - def list_fuzz_targets(self) -> List[str]: + def list_fuzz_targets(self) -> list[str]: """Lists fuzzing targets in the archive. Guarantees that self._fuzz_targets is populated with a dict mapping @@ -107,7 +112,7 @@ def list_fuzz_targets(self) -> List[str]: return list(self._fuzz_targets.keys()) @abc.abstractmethod - def find_fuzz_targets(self) -> List[str]: + def find_fuzz_targets(self) -> list[str]: """Finds fuzzing targets in the archive. Returns: @@ -117,7 +122,7 @@ def find_fuzz_targets(self) -> List[str]: @abc.abstractmethod def get_target_dependencies( - self, fuzz_target: str) -> List[archive.ArchiveMemberInfo]: + self, fuzz_target: str) -> list[archive.ArchiveMemberInfo]: """Gets the target dependencies. This returns all the necessary files in order for the fuzzer to run correctly. @@ -130,7 +135,7 @@ def get_target_dependencies( raise NotImplementedError @abc.abstractmethod - def unpacked_size(self, fuzz_target: Optional[str] = None) -> int: + def unpacked_size(self, fuzz_target: str | None = None) -> int: """Gets the total extracted size for the files returned by `get_target_dependencies`. In other words this returns the necessary space needed in order for the fuzz_target to run correctly. @@ -147,8 +152,8 @@ def unpacked_size(self, fuzz_target: Optional[str] = None) -> int: @abc.abstractmethod def unpack(self, - build_dir: str, - fuzz_target: Optional[str] = None, + build_dir: str | os.PathLike[str], + fuzz_target: str | None = None, trusted: bool = False) -> bool: """Unpacks the build to build_dir. During this operation, all the fuzz_target dependencies will be computed in order to unpack all the @@ -167,7 +172,7 @@ class DefaultBuildArchive(BuildArchive): """Default class for handling builds. This should work with everything. """ - def get_path_for_target(self, fuzz_target: str) -> str: + def get_path_for_target(self, fuzz_target: str) -> str | None: """Returns the path in the archive of the fuzz_target if found. This is needed because target name normalization means we're losing information about the actual file reprensenting the fuzz_target. @@ -175,15 +180,17 @@ def get_path_for_target(self, fuzz_target: str) -> str: if self._fuzz_targets is None: self.list_fuzz_targets() + assert self._fuzz_targets is not None return self._fuzz_targets.get(fuzz_target) + @override def get_target_dependencies( - self, fuzz_target: str) -> List[archive.ArchiveMemberInfo]: + self, fuzz_target: str) -> list[archive.ArchiveMemberInfo]: allowlisted_names = tuple([fuzz_target] + FUZZ_TARGET_ALLOWLISTED_PREFIXES) blocklisted_extensions = tuple( '.' + extension for extension in FUZZ_TARGET_EXCLUDED_EXTENSIONS) - to_extract = [] + to_extract: list[archive.ArchiveMemberInfo] = [] for file in self.list_members(): filepath = file.name path_components = os.path.normpath(filepath).split(os.sep) @@ -206,7 +213,7 @@ def get_target_dependencies( return to_extract @override - def find_fuzz_targets(self) -> List[str]: + def find_fuzz_targets(self) -> list[str]: # Import here as this path is not available in App Engine context. from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils @@ -216,16 +223,18 @@ def find_fuzz_targets(self) -> List[str]: if fuzzer_utils.is_fuzz_target(member.name, self.open) ] - def unpacked_size(self, fuzz_target: Optional[str] = None) -> int: + @override + def unpacked_size(self, fuzz_target: str | None = None) -> int: if not fuzz_target: return self.extracted_size() files = self.get_target_dependencies(fuzz_target=fuzz_target) return sum(f.size_bytes for f in files) + @override def unpack(self, - build_dir: str, - fuzz_target: Optional[str] = None, + build_dir: str | os.PathLike[str], + fuzz_target: str | None = None, trusted: bool = False) -> bool: if not fuzz_target: return self.extract_all(build_dir, trusted=trusted) @@ -300,9 +309,13 @@ class ChromeBuildArchive(DefaultBuildArchive): # etc. for other deps """ + _manifest_fuzz_targets: list[str] | None + _archive_schema_version: int + _root_dir: str + def __init__(self, reader: archive.ArchiveReader, - default_archive_schema_version: int = 0): + default_archive_schema_version: int = 0) -> None: """Initializes a `ChromiumBuildArchive` with the given reader. Arguments: @@ -319,7 +332,9 @@ def __init__(self, self._archive_schema_version = default_archive_schema_version return - with self.open(manifest_path) as f: + manifest_file = self.open(manifest_path) + assert manifest_file is not None + with manifest_file as f: manifest = json.load(f) self._archive_schema_version = manifest.get('archive_schema_version') if self._archive_schema_version is None: @@ -334,6 +349,7 @@ def __init__(self, self._manifest_fuzz_targets = ( fuzzer_utils.extract_fuzz_targets_from_manifest(manifest)) + @override def root_dir(self) -> str: if not hasattr(self, '_root_dir'): self._root_dir = super().root_dir() # pylint: disable=attribute-defined-outside-init @@ -344,7 +360,7 @@ def archive_schema_version(self) -> int: return self._archive_schema_version @override - def find_fuzz_targets(self) -> List[str]: + def find_fuzz_targets(self) -> list[str]: if self._manifest_fuzz_targets is not None: return self._manifest_fuzz_targets return super().find_fuzz_targets() @@ -402,15 +418,15 @@ def _get_prefix_matcher(self, prefix: str) -> Callable[[str], bool]: def _get_filename_matcher(self, file: str) -> Callable[[str], bool]: return lambda f: os.path.basename(f) == file - def _match_files(self, matchers: List[Callable[[str], bool]] - ) -> List[archive.ArchiveMemberInfo]: - res = [] + def _match_files(self, matchers: list[Callable[[str], bool]] + ) -> list[archive.ArchiveMemberInfo]: + res: list[archive.ArchiveMemberInfo] = [] for member in self.list_members(): if any(matcher(member.name) for matcher in matchers): res.append(member) return res - def _get_common_files(self) -> List[str]: + def _get_common_files(self) -> list[str]: """Those files are always to be extracted. """ return [ @@ -419,8 +435,9 @@ def _get_common_files(self) -> List[str]: 'clusterfuzz_manifest.json', ] + @override def get_target_dependencies( - self, fuzz_target: str) -> List[archive.ArchiveMemberInfo]: + self, fuzz_target: str) -> list[archive.ArchiveMemberInfo]: target_path = self.get_path_for_target(fuzz_target) if not target_path: logs.warning(f'Target path not found for {fuzz_target}') @@ -437,9 +454,11 @@ def get_target_dependencies( logs.warning(f'runtime_deps file not found for {target_path}') return super().get_target_dependencies(fuzz_target) - res = [] - matchers = [] - with self.open(deps_file) as f: + res: list[archive.ArchiveMemberInfo] = [] + matchers: list[Callable[[str], bool]] = [] + deps_file_obj = self.open(deps_file) + assert deps_file_obj is not None + with deps_file_obj as f: deps = [ self.get_dependency_path(l.decode(), deps_file) for l in f.read().splitlines() @@ -486,7 +505,7 @@ def open_with_reader(reader: archive.ArchiveReader) -> BuildArchive: return DefaultBuildArchive(reader) -def open(archive_path: str) -> BuildArchive: # pylint: disable=redefined-builtin +def open(archive_path: str | os.PathLike[str]) -> BuildArchive: # pylint: disable=redefined-builtin """Opens the archive and gets the appropriate build archive based on the `archive_path`. The resulting object is usable as a normal archive reader, but provides additional feature related to build handling. @@ -514,7 +533,7 @@ def open_uri(uri: str) -> BuildArchive: Returns: The build archive. """ - reader = archive.ZipArchiveReader(archive.HttpZipFile(uri)) + reader = archive.ZipArchiveReader(cast(IO[bytes], archive.HttpZipFile(uri))) return open_with_reader(reader) diff --git a/src/clusterfuzz/_internal/build_management/build_manager.py b/src/clusterfuzz/_internal/build_management/build_manager.py index 102d5a4f68f..0321ec88686 100644 --- a/src/clusterfuzz/_internal/build_management/build_manager.py +++ b/src/clusterfuzz/_internal/build_management/build_manager.py @@ -13,7 +13,6 @@ # limitations under the License. """Build manager.""" -from collections import namedtuple import contextlib import datetime import os @@ -21,7 +20,16 @@ import shutil import subprocess import time +from typing import Any +from typing import cast +from typing import Dict +from typing import Generator +from typing import Iterable +from typing import List +from typing import NamedTuple from typing import Optional +from typing import Tuple +from typing import Union from clusterfuzz._internal.base import errors from clusterfuzz._internal.base import utils @@ -42,51 +50,55 @@ from clusterfuzz._internal.system import shell # The default environment variables for specifying build bucket paths. -DEFAULT_BUILD_BUCKET_PATH_ENV_VARS = ( +DEFAULT_BUILD_BUCKET_PATH_ENV_VARS: Tuple[str, ...] = ( 'RELEASE_BUILD_BUCKET_PATH', 'SYM_RELEASE_BUILD_BUCKET_PATH', 'SYM_DEBUG_BUILD_BUCKET_PATH', ) # File name for storing current build revision. -REVISION_FILE_NAME = 'REVISION' +REVISION_FILE_NAME: str = 'REVISION' # Various build type mapping strings. -BUILD_TYPE_SUBSTRINGS = [ +BUILD_TYPE_SUBSTRINGS: List[str] = [ '-beta', '-stable', '-debug', '-release', '-symbolized', '-extended_stable' ] # Build eviction constants. -MAX_EVICTED_BUILDS = 100 -MIN_FREE_DISK_SPACE_CHROMIUM = 10 * 1024 * 1024 * 1024 # 10 GB -MIN_FREE_DISK_SPACE_DEFAULT = 5 * 1024 * 1024 * 1024 # 5 GB -TIMESTAMP_FILE = '.timestamp' +MAX_EVICTED_BUILDS: int = 100 +MIN_FREE_DISK_SPACE_CHROMIUM: int = 10 * 1024 * 1024 * 1024 # 10 GB +MIN_FREE_DISK_SPACE_DEFAULT: int = 5 * 1024 * 1024 * 1024 # 5 GB +TIMESTAMP_FILE: str = '.timestamp' # Indicates if this is a partial build (due to selected files copied from fuzz # target). -PARTIAL_BUILD_FILE = '.partial_build' +PARTIAL_BUILD_FILE: str = '.partial_build' # Time for unpacking a build beyond which an error should be logged. -UNPACK_TIME_LIMIT = 60 * 20 +UNPACK_TIME_LIMIT: int = 60 * 20 -PATCHELF_SIZE_LIMIT = 1.5 * 1024 * 1024 * 1024 # 1.5 GiB +PATCHELF_SIZE_LIMIT: float = 1.5 * 1024 * 1024 * 1024 # 1.5 GiB -TARGETS_LIST_FILENAME = 'targets.list' +TARGETS_LIST_FILENAME: str = 'targets.list' -BuildUrls = namedtuple('BuildUrls', ['bucket_path', 'urls_list']) + +class BuildUrls(NamedTuple): + bucket_path: str + urls_list: List[str] class BuildManagerError(Exception): """Build manager exceptions.""" -def _base_build_dir(bucket_path): +def _base_build_dir(bucket_path: Optional[str]) -> str: """Get the base directory for a build.""" job_name = environment.get_value('JOB_NAME') return _get_build_directory(bucket_path, job_name) -def _make_space(requested_size, current_build_dir=None): +def _make_space(requested_size: int, + current_build_dir: Optional[str] = None) -> bool: """Try to make the requested number of bytes available by deleting builds.""" if utils.is_chromium(): min_free_disk_space = MIN_FREE_DISK_SPACE_CHROMIUM @@ -111,17 +123,19 @@ def _make_space(requested_size, current_build_dir=None): return False free_disk_space = shell.get_free_disk_space(builds_directory) + if free_disk_space is None: + return False result = requested_size + min_free_disk_space < free_disk_space if not result: logs.error(error_message) return result -def _evict_build(current_build_dir): +def _evict_build(current_build_dir: Optional[str]) -> bool: """Remove the least recently used build to make room.""" builds_directory = environment.get_value('BUILDS_DIR') - least_recently_used = None - least_recently_used_timestamp = None + least_recently_used: Optional[BaseBuild] = None + least_recently_used_timestamp: Optional[float] = None for build_directory in os.listdir(builds_directory): absolute_build_directory = os.path.abspath( @@ -129,7 +143,7 @@ def _evict_build(current_build_dir): if not os.path.isdir(absolute_build_directory): continue - if os.path.commonpath( + if current_build_dir and os.path.commonpath( [absolute_build_directory, os.path.abspath(current_build_dir)]) == absolute_build_directory: # Don't evict the build we're trying to extract. This could be a parent @@ -154,7 +168,7 @@ def _evict_build(current_build_dir): return True -def _handle_unrecoverable_error_on_windows(): +def _handle_unrecoverable_error_on_windows() -> None: """Handle non-recoverable error on Windows. This is usually either due to disk corruption or processes failing to terminate using regular methods. Force a restart for recovery.""" @@ -166,7 +180,7 @@ def _handle_unrecoverable_error_on_windows(): utils.restart_machine() -def _remove_scheme(bucket_path): +def _remove_scheme(bucket_path: str) -> str: """Remove scheme from the bucket path.""" if '://' not in bucket_path: raise BuildManagerError('Invalid bucket path: ' + bucket_path) @@ -174,7 +188,8 @@ def _remove_scheme(bucket_path): return bucket_path.split('://')[1] -def _get_build_directory(bucket_path, job_name): +def _get_build_directory(bucket_path: Optional[str], + job_name: Optional[str]) -> str: """Return the build directory based on bucket path and job name.""" builds_directory = environment.get_value('BUILDS_DIR') @@ -191,29 +206,31 @@ def _get_build_directory(bucket_path, job_name): file_pattern_hash = utils.string_hash(file_pattern) job_directory = f'{bucket_path}_{file_pattern_hash}' else: - job_directory = job_name + job_directory = job_name or '' return os.path.join(builds_directory, job_directory) -def set_random_fuzz_target_for_fuzzing_if_needed(fuzz_targets, target_weights): +def set_random_fuzz_target_for_fuzzing_if_needed( + fuzz_targets: Iterable[str], + target_weights: Dict[str, float]) -> Optional[str]: """Sets a random fuzz target for fuzzing.""" if not environment.is_engine_fuzzer_job(): return None - fuzz_targets = list(fuzz_targets) - if not fuzz_targets: + fuzz_targets_list = list(fuzz_targets) + if not fuzz_targets_list: logs.error('No fuzz targets found. Unable to pick random one.') return None - fuzz_target = fuzzer_selection.select_fuzz_target(fuzz_targets, + fuzz_target = fuzzer_selection.select_fuzz_target(fuzz_targets_list, target_weights) logs.info(f'Picked fuzz target {fuzz_target} for fuzzing.') return fuzz_target -def _setup_build_directories(base_build_dir): +def _setup_build_directories(base_build_dir: str) -> None: """Set up build directories for a job.""" # Create the root build directory for this job. shell.create_directory(base_build_dir, create_intermediates=True) @@ -231,8 +248,9 @@ def _setup_build_directories(base_build_dir): shell.create_directory(build_directory) -def set_environment_vars(search_directories, app_path='APP_PATH', - env_prefix=''): +def set_environment_vars(search_directories: Iterable[str], + app_path: str = 'APP_PATH', + env_prefix: str = '') -> None: """Set build-related environment variables (APP_PATH, APP_DIR etc) by walking through the build directory.""" app_name = environment.get_value(env_prefix + 'APP_NAME') @@ -259,7 +277,7 @@ def set_environment_vars(search_directories, app_path='APP_PATH', f'Use default LLVM symbolizer: {use_default_llvm_symbolizer}', ])) - def set_env_var(name, value): + def set_env_var(name: str, value: Any) -> None: full_name = env_prefix + name logs.info(f'Setting environment variable: {full_name} = {value}') environment.set_value(full_name, value) @@ -299,7 +317,8 @@ def set_env_var(name, value): logs.error(f'Could not find app {app_name!r} in search directories.') -def _emit_job_build_retrieval_metric(start_time, step, build_type): +def _emit_job_build_retrieval_metric(start_time: float, step: str, + build_type: str) -> None: elapsed_minutes = (time.time() - start_time) / 60 monitoring_metrics.JOB_BUILD_RETRIEVAL_TIME.add( elapsed_minutes, { @@ -313,17 +332,17 @@ def _emit_job_build_retrieval_metric(start_time, step, build_type): class BaseBuild: """Represents a build.""" - def __init__(self, base_build_dir): - self.base_build_dir = base_build_dir + def __init__(self, base_build_dir: str) -> None: + self.base_build_dir: str = base_build_dir - def last_used_time(self): + def last_used_time(self) -> float: """Return the last used time for the build.""" timestamp_file_path = os.path.join(self.base_build_dir, TIMESTAMP_FILE) timestamp = utils.read_data_from_file(timestamp_file_path, eval_data=True) return timestamp or 0 - def delete(self): + def delete(self) -> None: """Delete this build.""" shell.remove_directory(self.base_build_dir) @@ -344,7 +363,8 @@ def _read_schema_version_from_manifest(build_dir: str) -> int: return 0 -def _patch_rpath(binary_path, instrumented_library_paths): +def _patch_rpath(binary_path: str, + instrumented_library_paths: List[str]) -> None: """Patch rpaths of a binary to point to instrumented libraries""" rpaths = get_rpaths(binary_path) # Discard all RPATHs that aren't relative to build. @@ -357,7 +377,7 @@ def _patch_rpath(binary_path, instrumented_library_paths): set_rpaths(binary_path, rpaths) -def _patch_rpaths(build_dir: str, app_path_env: str): +def _patch_rpaths(build_dir: str, app_path_env: str) -> None: """Patch rpaths of builds to point to instrumented libraries.""" instrumented_library_paths = environment.get_instrumented_libraries_paths() if not instrumented_library_paths: @@ -386,39 +406,39 @@ class Build(BaseBuild): """Represent a build type at a particular revision.""" def __init__(self, - base_build_dir, - revision, - build_prefix='', - fuzz_target=None): + base_build_dir: str, + revision: Any, + build_prefix: Optional[str] = '', + fuzz_target: Optional[str] = None) -> None: super().__init__(base_build_dir) - self.revision = revision - self.build_prefix = build_prefix - self.env_prefix = build_prefix + '_' if build_prefix else '' + self.revision: Any = revision + self.build_prefix: Optional[str] = build_prefix + self.env_prefix: str = build_prefix + '_' if build_prefix else '' # This is used by users of the class to learn the fuzz targets in the build. - self._fuzz_targets = None - self._unpack_everything = environment.get_value( + self._fuzz_targets: Optional[List[str]] = None + self._unpack_everything: bool = environment.get_value( 'UNPACK_ALL_FUZZ_TARGETS_AND_FILES', default_value=False) # This is used by users of the class to instruct the class which fuzz # target to unpack. - self.fuzz_target = fuzz_target + self.fuzz_target: Optional[str] = fuzz_target # Every fetched build is a release one, except when SymbolizedBuild # explicitly downloads a debug build - self._build_type = 'release' + self._build_type: str = 'release' - def _reset_cwd(self): + def _reset_cwd(self) -> None: """Reset current working directory. Needed to clean up build without hitting dir-in-use exception on Windows.""" root_directory = environment.get_value('ROOT_DIR') os.chdir(root_directory) - def _delete_partial_build_file(self): + def _delete_partial_build_file(self) -> None: """Deletes partial build file (if present). This is needed to make sure we clean up build directory if the previous build was partial.""" partial_build_file_path = os.path.join(self.build_dir, PARTIAL_BUILD_FILE) if os.path.exists(partial_build_file_path): self.delete() - def _pre_setup(self): + def _pre_setup(self) -> None: """Common pre-setup.""" self._reset_cwd() shell.clear_temp_directory() @@ -432,7 +452,7 @@ def _pre_setup(self): environment.set_value(self.env_prefix + 'APP_PATH', '') environment.set_value(self.env_prefix + 'APP_PATH_DEBUG', '') - def _post_setup_success(self, update_revision=True): + def _post_setup_success(self, update_revision: bool = True) -> None: """Common post-setup.""" if update_revision: self._write_revision() @@ -443,8 +463,9 @@ def _post_setup_success(self, update_revision=True): utils.write_data_to_file(time.time(), timestamp_file_path) @contextlib.contextmanager - def _download_and_open_build_archive(self, base_build_dir: str, - build_dir: str, build_url: str): + def _download_and_open_build_archive( + self, base_build_dir: str, build_dir: str, + build_url: str) -> Generator[build_archive.BuildArchive, None, None]: """Downloads the build archive at `build_url` and opens it. Args: @@ -482,8 +503,10 @@ def _download_and_open_build_archive(self, base_build_dir: str, finally: shell.remove_file(build_local_archive) - def _open_build_archive(self, base_build_dir: str, build_dir: str, - build_url: str, http_build_url: Optional[str]): + def _open_build_archive( + self, base_build_dir: str, build_dir: str, build_url: str, + http_build_url: Optional[str] + ) -> contextlib.AbstractContextManager[build_archive.BuildArchive]: """Gets a handle on a build archive for the current build. Depending on the provided parameters, this function might download the build archive into the build directory or directly use remote HTTP archive. @@ -518,10 +541,10 @@ def _open_build_archive(self, base_build_dir: str, build_dir: str, return build_archive.open_uri(http_build_url) def _unpack_build(self, - base_build_dir, - build_dir, - build_url, - http_build_url=None): + base_build_dir: str, + build_dir: str, + build_url: str, + http_build_url: Optional[str] = None) -> bool: """Unpacks a build from a build url into the build directory.""" # Track time taken to unpack builds so that it doesn't silently regress. start_time = time.time() @@ -605,7 +628,7 @@ def _unpack_build(self, return True - def _get_fuzz_targets_from_dir(self, build_dir): + def _get_fuzz_targets_from_dir(self, build_dir: str) -> Iterable[str]: """Get iterator of fuzz targets from build dir.""" # Import here as this path is not available in App Engine context. from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils @@ -613,18 +636,18 @@ def _get_fuzz_targets_from_dir(self, build_dir): for path in fuzzer_utils.get_fuzz_targets(build_dir): yield fuzzer_utils.normalize_target_name(path) - def setup(self): + def setup(self) -> bool: """Set up the build on disk, and set all the necessary environment variables. Should return whether or not build setup succeeded.""" raise NotImplementedError @property - def build_dir(self): + def build_dir(self) -> str: """The build directory. Usually a subdirectory of base_build_dir.""" raise NotImplementedError @property - def is_discovery(self): + def is_discovery(self) -> bool: """Return True if this is a discovery run. A discovery run is an engine fuzzer job (e.g., LibFuzzer) with no fuzz @@ -634,7 +657,7 @@ def is_discovery(self): not self.build_prefix) @property - def fuzz_targets(self): + def fuzz_targets(self) -> Optional[List[str]]: if not self._fuzz_targets and self._unpack_everything: # we can lazily compute that when unpacking the whole archive, since we # know all the fuzzers will be in the build directory. @@ -644,7 +667,7 @@ def fuzz_targets(self): self._build_type) return self._fuzz_targets - def exists(self): + def exists(self) -> bool: """Check if build already exists.""" revision_file = os.path.join(self.build_dir, REVISION_FILE_NAME) if os.path.exists(revision_file): @@ -661,20 +684,20 @@ def exists(self): return False - def delete(self): + def delete(self) -> None: """Delete this build.""" # This overrides BaseBuild.delete (which deletes the entire base build # directory) to delete this specific build. shell.remove_directory(self.build_dir) - def _write_revision(self): + def _write_revision(self) -> None: revision_file = os.path.join(self.build_dir, REVISION_FILE_NAME) revisions.write_revision_to_revision_file(revision_file, self.revision) def _setup_application_path(self, - build_dir=None, - app_path='APP_PATH', - build_update=False): + build_dir: Optional[str] = None, + app_path: str = 'APP_PATH', + build_update: bool = False) -> None: """Sets up APP_PATH environment variables for revision build.""" logs.info('Setup application path.') @@ -733,12 +756,12 @@ class RegularBuild(Build): """Represents a regular build.""" def __init__(self, - base_build_dir, - revision, - build_url, - build_prefix='', - fuzz_target=None, - http_build_url=None): + base_build_dir: str, + revision: Union[int, str], + build_url: str, + build_prefix: Optional[str] = '', + fuzz_target: Optional[str] = None, + http_build_url: Optional[str] = None) -> None: """RegularBuild constructor. See Build constructor for other parameters. Args: @@ -749,21 +772,22 @@ def __init__(self, """ super().__init__( base_build_dir, revision, build_prefix, fuzz_target=fuzz_target) - self.build_url = build_url - self.http_build_url = http_build_url + self.build_url: str = build_url + self.http_build_url: Optional[str] = http_build_url if build_prefix: - self.build_dir_name = build_prefix.lower() + self.build_dir_name: str = build_prefix.lower() else: - self.build_dir_name = 'revisions' + self.build_dir_name: str = 'revisions' - self._build_dir = os.path.join(self.base_build_dir, self.build_dir_name) + self._build_dir: str = os.path.join(self.base_build_dir, + self.build_dir_name) @property - def build_dir(self): + def build_dir(self) -> str: return self._build_dir - def setup(self): + def setup(self) -> bool: """Sets up build with a particular revision.""" self._pre_setup() environment.set_value(self.env_prefix + 'BUILD_URL', self.build_url) @@ -775,7 +799,7 @@ def setup(self): self.build_url, self.http_build_url): return False - logs.info('Retrieved build r%d.' % self.revision) + logs.info('Retrieved build r%s.' % str(self.revision)) else: # We have the revision required locally, no more work to do, other than # setting application path environment variables. @@ -796,7 +820,7 @@ def setup(self): class SplitTargetBuild(RegularBuild): """Represents a split target build.""" - def setup(self, *args, **kwargs): + def setup(self, *args: Any, **kwargs: Any) -> bool: result = super().setup(*args, **kwargs) self._fuzz_targets = list(_split_target_build_list_targets()) return result @@ -805,12 +829,12 @@ def setup(self, *args, **kwargs): class FuchsiaBuild(RegularBuild): """Represents a Fuchsia build.""" - def _get_fuzz_targets_from_dir(self, build_dir): + def _get_fuzz_targets_from_dir(self, build_dir: str) -> Iterable[str]: """A running instance is required to enumerate targets so this is a no-op.""" return [] - def setup(self): + def setup(self) -> bool: """Fuchsia build setup.""" # Prevent App Engine import issues. from clusterfuzz._internal.platforms import fuchsia @@ -844,21 +868,22 @@ def setup(self): class SymbolizedBuild(Build): """Symbolized build.""" - def __init__(self, base_build_dir, revision, release_build_url, - debug_build_url): + def __init__(self, base_build_dir: str, revision: int | str | None, + release_build_url: Optional[str], + debug_build_url: Optional[str]) -> None: super().__init__(base_build_dir, revision) - self._build_dir = os.path.join(self.base_build_dir, 'symbolized') - self.release_build_dir = os.path.join(self.build_dir, 'release') - self.debug_build_dir = os.path.join(self.build_dir, 'debug') + self._build_dir: str = os.path.join(self.base_build_dir, 'symbolized') + self.release_build_dir: str = os.path.join(self.build_dir, 'release') + self.debug_build_dir: str = os.path.join(self.build_dir, 'debug') - self.release_build_url = release_build_url - self.debug_build_url = debug_build_url + self.release_build_url: Optional[str] = release_build_url + self.debug_build_url: Optional[str] = debug_build_url @property - def build_dir(self): + def build_dir(self) -> str: return self._build_dir - def _unpack_builds(self): + def _unpack_builds(self) -> bool: """Download and unpack builds.""" if not shell.remove_directory(self.build_dir, recreate=True): logs.error('Unable to clear symbolized build directory.') @@ -883,16 +908,16 @@ def _unpack_builds(self): return True - def setup(self): + def setup(self) -> bool: self._pre_setup() - logs.info('Retrieving symbolized build r%d.' % self.revision) + logs.info('Retrieving symbolized build r%s.' % str(self.revision)) build_update = not self.exists() if build_update: if not self._unpack_builds(): return False - logs.info('Retrieved symbolized build r%d.' % self.revision) + logs.info('Retrieved symbolized build r%s.' % str(self.revision)) else: logs.info('Build already exists.') @@ -916,18 +941,19 @@ def setup(self): class CustomBuild(Build): """Custom binary.""" - def __init__(self, base_build_dir, custom_binary_key, custom_binary_filename, - custom_binary_revision): + def __init__(self, base_build_dir: str, custom_binary_key: str, + custom_binary_filename: str, + custom_binary_revision: Any) -> None: super().__init__(base_build_dir, custom_binary_revision) - self.custom_binary_key = custom_binary_key - self.custom_binary_filename = custom_binary_filename - self._build_dir = os.path.join(self.base_build_dir, 'custom') + self.custom_binary_key: str = custom_binary_key + self.custom_binary_filename: str = custom_binary_filename + self._build_dir: str = os.path.join(self.base_build_dir, 'custom') @property - def build_dir(self): + def build_dir(self) -> str: return self._build_dir - def _unpack_custom_build(self): + def _unpack_custom_build(self) -> bool: """Unpack the custom build.""" if not shell.remove_directory(self.build_dir, recreate=True): logs.error('Unable to clear custom binary directory.') @@ -988,7 +1014,7 @@ def _unpack_custom_build(self): self._build_type) return True - def setup(self): + def setup(self) -> bool: """Set up the custom binary for a particular job.""" self._pre_setup() @@ -1016,11 +1042,12 @@ def setup(self): return True -def _sort_build_urls_by_revision(build_urls, bucket_path, reverse): +def _sort_build_urls_by_revision(build_urls: Iterable[str], bucket_path: str, + reverse: bool) -> List[str]: """Return a sorted list of build url by revision.""" base_url = os.path.dirname(bucket_path) file_pattern = os.path.basename(bucket_path) - filename_by_revision_dict = {} + filename_by_revision_dict: Dict[str, str] = {} _, base_path = storage.get_bucket_name_and_path(base_url) base_path_with_seperator = base_path + '/' if base_path else '' @@ -1060,7 +1087,8 @@ def _sort_build_urls_by_revision(build_urls, bucket_path, reverse): return sorted_build_urls -def get_build_urls_list(bucket_path, reverse=True): +def get_build_urls_list(bucket_path: Optional[str], + reverse: bool = True) -> List[str]: """Returns a sorted list of build urls from a bucket path.""" if not bucket_path: return [] @@ -1093,7 +1121,7 @@ def get_build_urls_list(bucket_path, reverse=True): return _sort_build_urls_by_revision(build_urls, bucket_path, reverse) -def get_primary_bucket_path(): +def get_primary_bucket_path() -> str: """Get the main bucket path for the current job.""" release_build_bucket_path = environment.get_value('RELEASE_BUILD_BUCKET_PATH') if release_build_bucket_path: @@ -1114,7 +1142,10 @@ def get_primary_bucket_path(): 'needs to be defined.') -def get_revisions_list(bucket_path, bad_revisions, testcase=None): +def get_revisions_list( + bucket_path: str, + bad_revisions: Iterable[int], + testcase: Optional[data_types.Testcase] = None) -> Optional[List[int]]: """Returns a sorted ascending list of revisions from a bucket path, excluding bad build revisions. Testcase crash revision is not excluded from the list even if it appears in the bad_revisions list.""" @@ -1146,7 +1177,7 @@ def get_revisions_list(bucket_path, bad_revisions, testcase=None): return revision_list -def get_job_bad_revisions(): +def get_job_bad_revisions() -> List[int]: job_type = environment.get_value('JOB_NAME') bad_builds = list( @@ -1157,12 +1188,12 @@ def get_job_bad_revisions(): return [build.revision for build in bad_builds] -def _base_fuzz_target_name(target_name): +def _base_fuzz_target_name(target_name: str) -> str: """Get the base fuzz target name "X" from "X@Y".""" return target_name.split('@')[0] -def _get_targets_list(bucket_path): +def _get_targets_list(bucket_path: str) -> Optional[List[str]]: """Get the target list for a given fuzz target bucket path. This is done by reading the targets.list file, which contains a list of the currently active fuzz targets.""" @@ -1181,12 +1212,15 @@ def _get_targets_list(bucket_path): return [t for t in targets if _base_fuzz_target_name(t) in listed_targets] -def _full_fuzz_target_path(bucket_path, fuzz_target): +def _full_fuzz_target_path(bucket_path: str, fuzz_target: str) -> str: """Get the full fuzz target bucket path.""" return bucket_path.replace('%TARGET%', _base_fuzz_target_name(fuzz_target)) -def _setup_split_targets_build(bucket_path, fuzz_target, revision=None): +def _setup_split_targets_build( + bucket_path: str, + fuzz_target: Optional[str], + revision: Optional[Union[int, str]] = None) -> Optional[RegularBuild]: """Set up targets build.""" bucket_path = environment.get_value('FUZZ_TARGET_BUILD_BUCKET_PATH') if not fuzz_target: @@ -1206,7 +1240,7 @@ def _setup_split_targets_build(bucket_path, fuzz_target, revision=None): revision, bucket_path=fuzz_target_bucket_path, fuzz_target=fuzz_target) -def _get_latest_revision(bucket_paths): +def _get_latest_revision(bucket_paths: Iterable[str]) -> Optional[int]: """Get the latest revision.""" build_urls = [] for bucket_path in bucket_paths: @@ -1241,7 +1275,7 @@ def _get_latest_revision(bucket_paths): return None -def _emit_build_age_metric(gcs_path): +def _emit_build_age_metric(gcs_path: str) -> None: """Emits a metric to track the age of a build.""" try: last_update_time = storage.get(gcs_path).get('updated') @@ -1267,8 +1301,10 @@ def _emit_build_age_metric(gcs_path): logs.error(f'Failed to emit build age metric for {gcs_path}: {e}') -def _emit_build_revision_metric(revision): +def _emit_build_revision_metric(revision: int | None) -> None: """Emits a gauge metric to track the build revision.""" + if revision is None: + return monitoring_metrics.JOB_BUILD_REVISION.set( revision, labels={ @@ -1279,7 +1315,7 @@ def _emit_build_revision_metric(revision): def _get_build_url(bucket_path: Optional[str], revision: int, - job_type: Optional[str]): + job_type: Optional[str]) -> Optional[str]: """Returns the GCS url for a build, given a bucket path and revision""" build_urls = get_build_urls_list(bucket_path) if not build_urls: @@ -1293,7 +1329,7 @@ def _get_build_url(bucket_path: Optional[str], revision: int, return build_url -def _get_build_bucket_paths(): +def _get_build_bucket_paths() -> List[str]: """Returns gcs bucket endpoints that contain the build of interest.""" bucket_paths = [] for env_var in DEFAULT_BUILD_BUCKET_PATH_ENV_VARS: @@ -1305,7 +1341,9 @@ def _get_build_bucket_paths(): return bucket_paths -def setup_trunk_build(fuzz_target, build_prefix=None): +def setup_trunk_build(fuzz_target: Optional[str], + build_prefix: Optional[str] = None + ) -> Optional[RegularBuild]: """Sets up latest trunk build.""" bucket_paths = _get_build_bucket_paths() if not bucket_paths: @@ -1328,10 +1366,11 @@ def setup_trunk_build(fuzz_target, build_prefix=None): return build -def setup_regular_build(revision, - bucket_path=None, - build_prefix='', - fuzz_target=None) -> Optional[RegularBuild]: +def setup_regular_build( + revision: Any, + bucket_path: Optional[str] = None, + build_prefix: Optional[str] = '', + fuzz_target: Optional[str] = None) -> Optional[RegularBuild]: """Sets up build with a particular revision.""" if not bucket_path: # Bucket path can be customized, otherwise get it from the default env var. @@ -1401,12 +1440,18 @@ def setup_regular_build(revision, return result -def setup_symbolized_builds(revision): - """Set up symbolized release and debug build.""" - sym_release_build_bucket_path = environment.get_value( - 'SYM_RELEASE_BUILD_BUCKET_PATH') - sym_debug_build_bucket_path = environment.get_value( - 'SYM_DEBUG_BUILD_BUCKET_PATH') +def setup_symbolized_builds( + revision: int | None = None) -> SymbolizedBuild | None: + """Sets up both release and debug symbolized builds.""" + sym_release_build_bucket_path = get_bucket_path( + 'RELEASE_BUILD_BUCKET_PATH') or get_bucket_path('BUILD_BUCKET_PATH') + sym_debug_build_bucket_path = get_bucket_path('SYM_DEBUG_BUILD_BUCKET_PATH') + + # Fuzzing build might not be symbolized, so try using a symbolized build + # if available. + sym_release_build_bucket_path = ( + get_bucket_path('SYM_RELEASE_BUILD_BUCKET_PATH') or + sym_release_build_bucket_path) sym_release_build_urls = get_build_urls_list(sym_release_build_bucket_path) sym_debug_build_urls = get_build_urls_list(sym_debug_build_bucket_path) @@ -1427,7 +1472,7 @@ def setup_symbolized_builds(revision): build_class = SymbolizedBuild if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import build_setup_host - build_class = build_setup_host.RemoteSymbolizedBuild # pylint: disable=no-member + build_class = build_setup_host.RemoteSymbolizedBuild # type: ignore # pylint: disable=no-member build = build_class(base_build_dir, revision, sym_release_build_url, sym_debug_build_url) @@ -1437,7 +1482,7 @@ def setup_symbolized_builds(revision): return None -def setup_custom_binary(): +def setup_custom_binary() -> Union[CustomBuild, bool, None]: """Set up the custom binary for a particular job.""" job_name = environment.get_value('JOB_NAME') # Verify that this is really a custom binary job. @@ -1457,7 +1502,8 @@ def setup_custom_binary(): return None -def setup_build(revision=0, fuzz_target=None): +def setup_build(revision: int | None = 0, + fuzz_target: Optional[str] = None) -> Any: """Set up a custom or regular build based on revision.""" result = _setup_build(revision, fuzz_target) if fuzz_target: @@ -1467,7 +1513,7 @@ def setup_build(revision=0, fuzz_target=None): return result -def _setup_build(revision, fuzz_target): +def _setup_build(revision: int | None, fuzz_target: Optional[str]) -> Any: """Helper for setup_build, so setup_build can be sure to set FUZZ_TARGET on successful execution of this function.""" # For custom binaries we always use the latest version. Revision is ignored. @@ -1491,19 +1537,21 @@ def _setup_build(revision, fuzz_target): return setup_trunk_build(fuzz_target=fuzz_target) -def is_custom_binary(): +def is_custom_binary() -> bool: """Determine if this is a custom binary.""" return bool(environment.get_value('CUSTOM_BINARY')) -def has_symbolized_builds(): +def has_symbolized_builds() -> bool: """Return a bool on if job type has either a release or debug build for stack symbolization.""" - return (environment.get_value('SYM_RELEASE_BUILD_BUCKET_PATH') or - environment.get_value('SYM_DEBUG_BUILD_BUCKET_PATH')) + return cast( + bool, + environment.get_value('SYM_RELEASE_BUILD_BUCKET_PATH') or + environment.get_value('SYM_DEBUG_BUILD_BUCKET_PATH')) -def _set_rpaths_chrpath(binary_path, rpaths): +def _set_rpaths_chrpath(binary_path: str, rpaths: Iterable[str]) -> None: """Set rpaths using chrpath.""" chrpath = environment.get_default_tool_path('chrpath') if not chrpath: @@ -1513,7 +1561,7 @@ def _set_rpaths_chrpath(binary_path, rpaths): [chrpath, '-r', ':'.join(rpaths), binary_path], stderr=subprocess.PIPE) -def _set_rpaths_patchelf(binary_path, rpaths): +def _set_rpaths_patchelf(binary_path: str, rpaths: Iterable[str]) -> None: """Set rpaths using patchelf.""" patchelf = shutil.which('patchelf') if not patchelf: @@ -1524,7 +1572,7 @@ def _set_rpaths_patchelf(binary_path, rpaths): stderr=subprocess.PIPE) -def set_rpaths(binary_path, rpaths): +def set_rpaths(binary_path: str, rpaths: Iterable[str]) -> None: """Set rpath of a binary.""" # Patchelf handles rpath patching much better, and allows e.g. extending the # length of the rpath. However, it loads the entire binary into memory so @@ -1536,7 +1584,7 @@ def set_rpaths(binary_path, rpaths): _set_rpaths_patchelf(binary_path, rpaths) -def get_rpaths(binary_path): +def get_rpaths(binary_path: str) -> List[str]: """Get rpath of a binary.""" chrpath = environment.get_default_tool_path('chrpath') if not chrpath: @@ -1560,12 +1608,13 @@ def get_rpaths(binary_path): return [] -def _pick_random_fuzz_target_for_standard_build(target_weights): +def _pick_random_fuzz_target_for_standard_build( + target_weights: Dict[str, float]) -> Optional[str]: return set_random_fuzz_target_for_fuzzing_if_needed(target_weights.keys(), target_weights) -def _split_target_build_list_targets(): +def _split_target_build_list_targets() -> List[str]: bucket_path = environment.get_value('FUZZ_TARGET_BUILD_BUCKET_PATH') targets_list = _get_targets_list(bucket_path) if not targets_list: @@ -1574,7 +1623,8 @@ def _split_target_build_list_targets(): return targets_list -def _pick_random_fuzz_target_for_split_build(target_weights): +def _pick_random_fuzz_target_for_split_build( + target_weights: Dict[str, float]) -> str: targets_list = _split_target_build_list_targets() fuzz_target = set_random_fuzz_target_for_fuzzing_if_needed( targets_list, target_weights) @@ -1585,13 +1635,13 @@ def _pick_random_fuzz_target_for_split_build(target_weights): return fuzz_target -def pick_random_fuzz_target(target_weights): +def pick_random_fuzz_target(target_weights: Dict[str, float]) -> Optional[str]: if environment.get_value('FUZZ_TARGET_BUILD_BUCKET_PATH'): return _pick_random_fuzz_target_for_split_build(target_weights) return _pick_random_fuzz_target_for_standard_build(target_weights) -def check_app_path(app_path='APP_PATH') -> bool: +def check_app_path(app_path: str = 'APP_PATH') -> bool: """Check if APP_PATH is properly set.""" # If APP_NAME is not set (e.g. for grey box jobs), then we don't need # APP_PATH. @@ -1605,7 +1655,7 @@ def check_app_path(app_path='APP_PATH') -> bool: return bool(app_path_value) -def get_bucket_path(name): +def get_bucket_path(name: str) -> Optional[str]: """Return build bucket path, applying any set overrides.""" bucket_path = environment.get_value(name) bucket_path = overrides.check_and_apply_overrides( diff --git a/src/clusterfuzz/_internal/build_management/overrides.py b/src/clusterfuzz/_internal/build_management/overrides.py index 7495f82e8df..521821b4bd6 100644 --- a/src/clusterfuzz/_internal/build_management/overrides.py +++ b/src/clusterfuzz/_internal/build_management/overrides.py @@ -15,21 +15,29 @@ import json import os +from typing import Any +from typing import Dict +from typing import Optional from clusterfuzz._internal.google_cloud_utils import storage from clusterfuzz._internal.system import environment -PLATFORM_ID_URLS_FILENAME = 'config.json' -PLATFORM_ID_TO_BUILD_PATH_KEY = 'build_paths' -PLATFORM_ID_TO_REV_PATH_KEY = 'revisions_paths' +PLATFORM_ID_URLS_FILENAME: str = 'config.json' +PLATFORM_ID_TO_BUILD_PATH_KEY: str = 'build_paths' +PLATFORM_ID_TO_REV_PATH_KEY: str = 'revisions_paths' -OVERRIDE_PATH_NOT_FOUND_ERROR = ('Could not find override path from config, ' - 'config_key: {}, config_url: {}, ' - 'platform_id = {}.') -OVERRIDE_CONFIG_NOT_READ_ERROR = 'Could not import config file. config_url: {}' +OVERRIDE_PATH_NOT_FOUND_ERROR: str = ( + 'Could not find override path from config, ' + 'config_key: {}, config_url: {}, ' + 'platform_id = {}.') +OVERRIDE_CONFIG_NOT_READ_ERROR: str = ( + 'Could not import config file. config_url: {}') -def check_and_apply_overrides(curr_path, config_key, platform_id=None): +def check_and_apply_overrides( + curr_path: Optional[str], + config_key: str, + platform_id: Optional[str] = None) -> Optional[str]: """Check if the given file points to a config, if so, use that to override any given paths""" if not curr_path: @@ -43,7 +51,8 @@ def check_and_apply_overrides(curr_path, config_key, platform_id=None): return curr_path -def _apply_platform_id_overrides(platform_id, config_url, config_key): +def _apply_platform_id_overrides(platform_id: str, config_url: str, + config_key: str) -> str: """read the `bucket_path`, parse as JSON, and map based on platform_id.""" config_dict = _get_config_dict(config_url) path = _get_path_from_config(config_dict, config_key, platform_id) @@ -54,7 +63,7 @@ def _apply_platform_id_overrides(platform_id, config_url, config_key): return path -def _get_config_dict(url): +def _get_config_dict(url: str) -> Dict[str, Any]: """Read configs from a json and return them as a dict""" url_data = storage.read_data(url) if not url_data: @@ -62,7 +71,8 @@ def _get_config_dict(url): return json.loads(url_data) -def _get_path_from_config(config_dict, config_key, platform_id): +def _get_path_from_config(config_dict: Dict[str, Any], config_key: str, + platform_id: str) -> Optional[str]: """Return True if a path override is present and return the override.""" if config_key not in config_dict: return None diff --git a/src/clusterfuzz/_internal/build_management/revisions.py b/src/clusterfuzz/_internal/build_management/revisions.py index 6465761a235..9eeb35f103f 100644 --- a/src/clusterfuzz/_internal/build_management/revisions.py +++ b/src/clusterfuzz/_internal/build_management/revisions.py @@ -19,6 +19,7 @@ import os import re import time +from typing import Any import urllib.parse from clusterfuzz._internal.base import memoize @@ -46,7 +47,9 @@ r'Cr-Branched-From:.*(?:master|main)@\{#(\d+)\}') -def _add_components_from_dict(deps_dict, vars_dict, revisions_dict): +def _add_components_from_dict( + deps_dict: dict[str, Any] | None, vars_dict: dict[str, Any], + revisions_dict: dict[str, dict[str, Any]]) -> None: """Add components from a dict representing a DEPS file.""" if not deps_dict: # If the dictionary is None, bail out early. @@ -73,7 +76,8 @@ def _add_components_from_dict(deps_dict, vars_dict, revisions_dict): } -def _clank_revision_file_to_revisions_dict(content): +def _clank_revision_file_to_revisions_dict( + content: str) -> dict[str, dict[str, Any]] | None: """Parse Clank revision file and return revisions dict.""" component_revision_mappings = {} for line in content.splitlines(): @@ -110,7 +114,7 @@ def _clank_revision_file_to_revisions_dict(content): return revisions_dict -def _get_component_display_name(name, default=None): +def _get_component_display_name(name: str, default: str | None = None) -> str: """Display name for a component.""" if default and name in ['', 'default', '/src']: return default.capitalize() @@ -124,7 +128,7 @@ def _get_component_display_name(name, default=None): return names[name_index].capitalize() -def _get_display_revision(component_revision_dict): +def _get_display_revision(component_revision_dict: dict[str, Any]) -> Any: """Return display revision for a component revision dict.""" if 'commit_pos' in component_revision_dict: return component_revision_dict['commit_pos'] @@ -132,7 +136,8 @@ def _get_display_revision(component_revision_dict): return component_revision_dict['rev'] or '' -def _get_link_text(start_component_revision_dict, end_component_revision_dict): +def _get_link_text(start_component_revision_dict: dict[str, Any], + end_component_revision_dict: dict[str, Any]) -> str: """Return link text given a start and end revision. This is used in cases when revision url is not available.""" start_revision = _get_display_revision(start_component_revision_dict) @@ -144,7 +149,8 @@ def _get_link_text(start_component_revision_dict, end_component_revision_dict): return '%s:%s' % (start_revision, end_revision) -def _get_link_url(start_component_revision_dict, end_component_revision_dict): +def _get_link_url(start_component_revision_dict: dict[str, Any], + end_component_revision_dict: dict[str, Any]) -> str | None: """Return link text given a start and end revision. This is used in cases when revision url is not available.""" url = start_component_revision_dict['url'] @@ -166,12 +172,12 @@ def _get_link_url(start_component_revision_dict, end_component_revision_dict): end_revision) -def _get_revision(component_revision_dict): +def _get_revision(component_revision_dict: dict[str, Any]) -> Any: """Return revision for a component revision dict.""" return component_revision_dict['rev'] -def _get_url_content(url): +def _get_url_content(url: str) -> str | None: """Read a potentially base64-encoded resource from the given URL.""" if url.startswith(storage.GS_PREFIX): # Fetch a GCS path with authentication. @@ -192,34 +198,36 @@ def _get_url_content(url): # cross-site scripting attacks. If the requested url contains |format=text|, # then the output is base64 encoded. So, decode it first. if url_content and url.endswith('format=text'): - url_content = base64.b64decode(url_content) + url_content = base64.b64decode(url_content).decode('utf-8') return url_content -def _git_url_for_chromium_repository(repository): +def _git_url_for_chromium_repository(repository: str) -> str: """Return git url for a chromium repository.""" return '%s/%s.git' % (CHROMIUM_GIT_ROOT_URL, repository) -def _is_clank(url): +def _is_clank(url: str) -> bool: """Return bool on whether this is a clank url or not.""" # FIXME: Need a better way to check for this. return '/chrome-test-builds/android' in url -def _is_deps(url): +def _is_deps(url: str) -> bool: """Return bool on whether this is a DEPS url or not.""" return urllib.parse.urlparse(url).path.endswith('/DEPS') -def _src_map_to_revisions_dict(src_map, project_name): +def _src_map_to_revisions_dict(src_map: dict[str, Any], project_name: str | None + ) -> dict[str, dict[str, Any]]: """Convert src map contents to revisions dict.""" revisions_dict = {} for key in src_map: # Only add keys that have both url and rev attributes. - if 'url' in src_map[key] and 'rev' in src_map[key]: + if isinstance(src_map[key], + dict) and 'url' in src_map[key] and 'rev' in src_map[key]: revisions_dict[key] = { 'name': _get_component_display_name(key, project_name), 'rev': src_map[key]['rev'], @@ -231,7 +239,8 @@ def _src_map_to_revisions_dict(src_map, project_name): @memoize.wrap(memoize.FifoOnDisk(DISK_CACHE_SIZE)) @memoize.wrap(memoize.Memcache(60 * 60 * 24 * 30)) # 30 day TTL -def _git_commit_position_to_git_hash_for_chromium(revision, repository): +def _git_commit_position_to_git_hash_for_chromium( + revision: str | int, repository: str) -> str | None: """Return git hash for a git commit position using cr-rev.appspot.com.""" request_variables = { 'number': revision, @@ -256,7 +265,7 @@ def _git_commit_position_to_git_hash_for_chromium(revision, repository): return result_dict['git_sha'] -def _to_dict(contents): +def _to_dict(contents: str) -> dict[str, Any] | None: """Parse |contents| as a dict, returning None on failure or if it's not a dict.""" try: @@ -270,9 +279,9 @@ def _to_dict(contents): return None -def deps_to_revisions_dict(content): +def deps_to_revisions_dict(content: str) -> dict[str, dict[str, Any]] | None: """Parses DEPS content and returns a dictionary of revision variables.""" - local_context = {} + local_context: dict[str, Any] = {} global_context = { 'Var': lambda x: local_context.get('vars', {}).get(x), 'Str': str, @@ -280,7 +289,7 @@ def deps_to_revisions_dict(content): # pylint: disable=exec-used exec(content, global_context, local_context) - revisions_dict = {} + revisions_dict: dict[str, dict[str, Any]] = {} vars_dict = local_context.get('vars', {}) deps_dict = local_context.get('deps') @@ -300,7 +309,8 @@ def deps_to_revisions_dict(content): return revisions_dict -def get_components_list(component_revisions_dict, job_type): +def get_components_list(component_revisions_dict: dict[str, dict[str, Any]], + job_type: str | None) -> list[str]: """Return a prioritized order of components based on job type.""" components = sorted(component_revisions_dict.keys()) @@ -308,12 +318,17 @@ def get_components_list(component_revisions_dict, job_type): # Components prioritization only applies to non-chromium projects. return components - project_name = data_handler.get_project_name(job_type) + if job_type is None: + project_name = utils.default_project_name() + main_repo = None + else: + project_name = data_handler.get_project_name(job_type) + main_repo = data_handler.get_main_repo(job_type) + if not project_name: # No project name found in job environment, return list as-is. return components - main_repo = data_handler.get_main_repo(job_type) project_src = '/src/' + project_name for component in components.copy(): if component_revisions_dict[component]['url'] == main_repo: @@ -335,7 +350,8 @@ def get_components_list(component_revisions_dict, job_type): return components -def _get_revision_vars_url_format(job_type, platform_id=None): +def _get_revision_vars_url_format(job_type: str | None, + platform_id: str | None = None) -> str | None: """Return REVISION_VARS_URL from job environment if available. Otherwise, default to one set in project.yaml. For custom binary jobs, this is not applicable.""" @@ -357,7 +373,10 @@ def _get_revision_vars_url_format(job_type, platform_id=None): @memoize.wrap(memoize.FifoOnDisk(DISK_CACHE_SIZE)) @memoize.wrap(memoize.Memcache(60 * 60 * 24 * 30)) # 30 day TTL -def get_component_revisions_dict(revision, job_type, platform_id=None): +def get_component_revisions_dict( + revision: str | int | None, + job_type: str | None, + platform_id: str | None = None) -> dict[str, dict[str, Any]] | None: """Retrieve revision vars dict.""" if revision == 0 or revision == '0' or revision is None: # Return empty dict for zero start revision. @@ -368,11 +387,13 @@ def get_component_revisions_dict(revision, job_type, platform_id=None): if not revision_vars_url_format: return None - project_name = data_handler.get_project_name(job_type) - revisions_dict = {} + project_name = ( + data_handler.get_project_name(job_type) + if job_type else utils.default_project_name()) + revisions_dict: dict[str, dict[str, Any]] = {} if utils.is_chromium(): - component = data_handler.get_component_name(job_type) + component = data_handler.get_component_name(job_type) if job_type else '' repository = data_handler.get_repository_for_component(component) if repository and not _is_clank(revision_vars_url_format): revision_hash = _git_commit_position_to_git_hash_for_chromium( @@ -414,12 +435,14 @@ def get_component_revisions_dict(revision, job_type, platform_id=None): return _clank_revision_file_to_revisions_dict(url_content) # Default case: parse content as yaml. - revisions_dict = _to_dict(url_content) - if revisions_dict is None: + parsed_dict = _to_dict(url_content) + if parsed_dict is None: logs.error( 'Failed to parse component revisions from %s.' % revision_vars_url) return None + revisions_dict = parsed_dict + # Parse as per source map format. if revision_vars_url.endswith(SOURCE_MAP_EXTENSION): revisions_dict = _src_map_to_revisions_dict(revisions_dict, project_name) @@ -427,15 +450,17 @@ def get_component_revisions_dict(revision, job_type, platform_id=None): return revisions_dict -def get_component_list(revision, job_type): +def get_component_list(revision: str | int | None, + job_type: str | None) -> list[dict[str, Any]]: """Gets mapped revisions for a given revision.""" return get_component_range_list(revision, revision, job_type) -def get_component_range_list(start_revision, - end_revision, - job_type, - platform_id=None): +def get_component_range_list( + start_revision: str | int | None, + end_revision: str | int | None, + job_type: str | None, + platform_id: str | None = None) -> list[dict[str, Any]]: """Gets revision variable ranges for a changeset range.""" start_component_revisions_dict = get_component_revisions_dict( start_revision, job_type, platform_id=platform_id) @@ -450,7 +475,7 @@ def get_component_range_list(start_revision, end_component_revisions_dict is None): return [] - component_revisions = [] + component_revisions: list[dict[str, Any]] = [] keys = get_components_list(end_component_revisions_dict, job_type) for key in keys: if not start_component_revisions_dict: @@ -486,9 +511,10 @@ def get_component_range_list(start_revision, return component_revisions -def get_start_and_end_revision(revision_range): +def get_start_and_end_revision(revision_range: str | None) -> list[int]: """Return start and end revision for a regression range.""" try: + assert revision_range is not None revision_range_list = revision_range.split(':') start_revision = int(revision_range_list[0]) end_revision = int(revision_range_list[1]) @@ -498,7 +524,8 @@ def get_start_and_end_revision(revision_range): return [start_revision, end_revision] -def format_revision_list(revisions, use_html=True): +def format_revision_list(revisions: list[dict[str, Any]], + use_html: bool = True) -> str: """Converts component revision list to html.""" result = '' for revision in revisions: @@ -519,7 +546,7 @@ def format_revision_list(revisions, use_html=True): return result -def convert_revision_to_integer(revision): +def convert_revision_to_integer(revision: str) -> int: """Returns an integer that represents the given revision.""" # If revision is only decimal digits, like '249055', then do a simple # conversion. @@ -544,9 +571,10 @@ def convert_revision_to_integer(revision): raise ValueError(error) -def find_build_url(bucket_path, build_url_list, revision): +def find_build_url(bucket_path: str | None, build_url_list: list[str] | None, + revision: int | None) -> str | None: """Returns the build url associated with a revision.""" - if not build_url_list: + if not bucket_path or not build_url_list or revision is None: return None revision_pattern = revision_pattern_from_build_bucket_path(bucket_path) @@ -562,7 +590,8 @@ def find_build_url(bucket_path, build_url_list, revision): return None -def find_min_revision_index(revisions_list, revision): +def find_min_revision_index(revisions_list: list[Any], + revision: Any) -> int | None: """Find the min index for bisection. Find largest revision <= the given revision.""" # bisect_left partitions |revisions_list| into 2 such that: @@ -579,7 +608,8 @@ def find_min_revision_index(revisions_list, revision): return None -def find_max_revision_index(revisions_list, revision): +def find_max_revision_index(revisions_list: list[Any], + revision: Any) -> int | None: """Find the max index for bisection. Find smallest revision >= the given revision.""" index = bisect.bisect_left(revisions_list, revision) @@ -590,7 +620,7 @@ def find_max_revision_index(revisions_list, revision): return None -def get_first_revision_in_list(revision_list): +def get_first_revision_in_list(revision_list: list[Any]) -> Any: """Gets the first revision in list greater than or equal to MIN_REVISION.""" first_revision = revision_list[0] @@ -608,12 +638,15 @@ def get_first_revision_in_list(revision_list): return first_revision -def get_last_revision_in_list(revision_list): +def get_last_revision_in_list(revision_list: list[Any]) -> Any: """Gets the last revision in list.""" return revision_list[-1] -def get_real_revision(revision, job_type, display=False, platform_id=None): +def get_real_revision(revision: str | int | None, + job_type: str | None, + display: bool = False, + platform_id: str | None = None) -> Any: """Convert the revision number into a real revision hash (e.g. git hash).""" if revision is None: # Bail early when caller passes revision from a non-existent attribute. @@ -631,7 +664,7 @@ def get_real_revision(revision, job_type, display=False, platform_id=None): return helper(component_revisions_dict[key]) -def needs_update(revision_file, revision): +def needs_update(revision_file: str, revision: str | int) -> bool: """Check a revision file against the provided revision to see if an update is required.""" failure_wait_interval = environment.get_value('FAIL_WAIT') @@ -676,7 +709,8 @@ def needs_update(revision_file, revision): return False -def write_revision_to_revision_file(revision_file, revision): +def write_revision_to_revision_file(revision_file: str, + revision: str | int) -> None: """Writes a revision to the revision file.""" try: with open(revision_file, 'wb') as file_handle: @@ -685,14 +719,14 @@ def write_revision_to_revision_file(revision_file, revision): logs.error("Could not save revision to revision file '%s'" % revision_file) -def revision_pattern_from_build_bucket_path(bucket_path): +def revision_pattern_from_build_bucket_path(bucket_path: str) -> str: """Get the revision pattern from a build bucket path.""" return '.*?' + os.path.basename(bucket_path) @memoize.wrap(memoize.FifoOnDisk(DISK_CACHE_SIZE)) @memoize.wrap(memoize.Memcache(60 * 60 * 24 * 30)) # 30 day TTL -def revision_to_branched_from(uri, revision): +def revision_to_branched_from(uri: str, revision: str | int) -> str | None: """Interrogates git code review server to find the branch-from revision of a component.""" full_uri = "%s/+/%s?format=JSON" % (uri, revision) @@ -700,6 +734,7 @@ def revision_to_branched_from(uri, revision): # gerrit intentionally returns nonsense in the first line. # See 'cross site script inclusion here: # https://gerrit-review.googlesource.com/Documentation/rest-api.html + assert url_content is not None url_content = '\n'.join(url_content.splitlines()[1:]) result = _to_dict(url_content) if not result: diff --git a/src/clusterfuzz/_internal/build_management/source_mapper.py b/src/clusterfuzz/_internal/build_management/source_mapper.py index 0443b79c1aa..899d989da24 100644 --- a/src/clusterfuzz/_internal/build_management/source_mapper.py +++ b/src/clusterfuzz/_internal/build_management/source_mapper.py @@ -14,29 +14,40 @@ """Helper functions for fetch source links.""" import re +from typing import Any +from typing import ClassVar +from typing import Dict +from typing import List +from typing import Optional +from typing import Type +from typing import Union from clusterfuzz._internal.base import utils from clusterfuzz._internal.config import local_config -DRIVE_LETTER_REGEX = re.compile(r'^[a-zA-Z]:\\') -RANGE_LIMIT = 10000 -SOURCE_START_ID = 'src/' -SOURCE_STRIP_REGEX = re.compile(r'^[/]?src[/]?') -STACK_FRAME_PATH_LINE_REGEX = re.compile( +DRIVE_LETTER_REGEX: re.Pattern[str] = re.compile(r'^[a-zA-Z]:\\') +RANGE_LIMIT: int = 10000 +SOURCE_START_ID: str = 'src/' +SOURCE_STRIP_REGEX: re.Pattern[str] = re.compile(r'^[/]?src[/]?') +STACK_FRAME_PATH_LINE_REGEX: re.Pattern[str] = re.compile( r'(?<=\[|\(|\s)([a-zA-Z/.][^\s]*?)\s*(:|@)\s*(\d+)(?=\]$|\)$|:\d+$|$)') -JAVA_STACK_FRAME_REGEX = re.compile( +JAVA_STACK_FRAME_REGEX: re.Pattern[str] = re.compile( r'at (?P[a-zA-Z0-9\.\/]+)\.(?P[a-zA-Z0-9\.\/]+)\([a-zA-Z0-9]+' r'\.java\:(?P\d+)\)') class ComponentPath: + """Component path info.""" - def __init__(self, source=None, relative_path=None, display_path=None): + def __init__(self, + source: Optional[str] = None, + relative_path: Optional[str] = None, + display_path: Optional[str] = None) -> None: self.source = source self.relative_path = relative_path self.display_path = display_path - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return (self.source == other.source and self.relative_path == other.relative_path and self.display_path == other.display_path) @@ -44,28 +55,32 @@ def __eq__(self, other): class VCSViewer: """Base viewer class.""" - VCS_URL_REGEX = None - VCS_REVISION_SUB = None - VCS_REVISION_DIFF_SUB = None - VCS_REVISION_PATH_LINE_SUB = None + VCS_URL_REGEX: ClassVar[Optional[re.Pattern[str]]] = None + VCS_REVISION_SUB: ClassVar[Optional[str]] = None + VCS_REVISION_DIFF_SUB: ClassVar[Optional[str]] = None + VCS_REVISION_PATH_LINE_SUB: ClassVar[Optional[str]] = None - def __init__(self, url): + def __init__(self, url: str) -> None: self.url = url - def get_mapped_url(self, repl, **kwargs): + def get_mapped_url(self, repl: str, **kwargs: Any) -> str: """Return mapped url given a url map and arguments.""" + assert self.VCS_URL_REGEX is not None mapped_url = self.VCS_URL_REGEX.sub(repl, self.url) mapped_url = mapped_url.format(**kwargs) return mapped_url - def get_source_url_for_revision(self, revision): + def get_source_url_for_revision(self, + revision: Union[int, str]) -> Optional[str]: """Return source revision url given a url and revision.""" if not self.VCS_REVISION_SUB: return None return self.get_mapped_url(self.VCS_REVISION_SUB, revision=revision) - def get_source_url_for_revision_diff(self, start_revision, end_revision): + def get_source_url_for_revision_diff( + self, start_revision: Union[int, str], + end_revision: Union[int, str]) -> Optional[str]: """Return source revision diff url given a url and revision.""" if not self.VCS_REVISION_DIFF_SUB: return None @@ -76,7 +91,9 @@ def get_source_url_for_revision_diff(self, start_revision, end_revision): end_revision=end_revision, range_limit=RANGE_LIMIT) - def get_source_url_for_revision_path_and_line(self, revision, path, line): + def get_source_url_for_revision_path_and_line( + self, revision: Union[int, str], path: str, + line: Union[int, str]) -> Optional[str]: """Return source revision url given a url, revision, path and line.""" if not self.VCS_REVISION_PATH_LINE_SUB: return None @@ -144,7 +161,7 @@ class MercurialVCS(VCSViewer): VCS_REVISION_PATH_LINE_SUB = r'\1/file/{revision}/{path}#l{line}' -VCS_LIST = [ +VCS_LIST: List[Type[VCSViewer]] = [ FreeDesktopVCS, GitHubVCS, GitLabVCS, @@ -155,7 +172,8 @@ class MercurialVCS(VCSViewer): ] -def get_component_source_and_relative_path(path, revisions_dict): +def get_component_source_and_relative_path( + path: str, revisions_dict: Optional[Dict[str, Any]]) -> ComponentPath: """Get component source and relative path given a revisions dictionary and path.""" if not revisions_dict: @@ -189,28 +207,29 @@ def get_component_source_and_relative_path(path, revisions_dict): normalized_path) -def get_vcs_viewer_for_url(url): +def get_vcs_viewer_for_url(url: str) -> Optional[VCSViewer]: """Return a VCS instance given an input url.""" for vcs in VCS_LIST: + assert vcs.VCS_URL_REGEX is not None if vcs.VCS_URL_REGEX.match(url): return vcs(url) return None -def should_linkify_java_stack_frames(): +def should_linkify_java_stack_frames() -> Any: return local_config.Config(local_config.PROJECT_PATH).get('linkify_java') class StackFrameLinkifier: """Class for converting stackframes to clickable links to the source code.""" - def __init__(self, revisions_dict): + def __init__(self, revisions_dict: Dict[str, Any]) -> None: self.revisions_dict = revisions_dict # Make this a class so we're not requerying the config so often. self.should_linkify_java_stack_frames = should_linkify_java_stack_frames() - def convert_java_stack_frame(self, stack_frame): + def convert_java_stack_frame(self, stack_frame: str) -> str: """Converts a Java |stack_frame| to a more C-like one so the rest of our magic works on it. Returns |stack_frame| if not Java.""" if not self.should_linkify_java_stack_frames: @@ -225,7 +244,7 @@ def convert_java_stack_frame(self, stack_frame): path = path.replace('.', '/') return f' in {method} {path}.java:{line}' - def linkify_stack_frame(self, stack_frame): + def linkify_stack_frame(self, stack_frame: str) -> str: """Linkify a stack frame with source links to its repo.""" stack_frame = self.convert_java_stack_frame(stack_frame) match = STACK_FRAME_PATH_LINE_REGEX.search(stack_frame) @@ -242,6 +261,8 @@ def linkify_stack_frame(self, stack_frame): # Can't find any matching component source in revisions dict, bail out. return stack_frame + assert component_path.relative_path is not None + source_dict = self.revisions_dict[component_path.source] repo_url = source_dict['url'] revision = source_dict['rev'] @@ -262,7 +283,7 @@ def linkify_stack_frame(self, stack_frame): return linkified_stack_frame -def normalize_source_path(path): +def normalize_source_path(path: str) -> Optional[str]: """Normalizes source path for comparison with component sources.""" # Account for ../../ at start of path due to working directory # out// at time of build generation (chromium only). diff --git a/src/clusterfuzz/_internal/chrome/build_info.py b/src/clusterfuzz/_internal/chrome/build_info.py index 3745037c1e4..5965b4b43e4 100644 --- a/src/clusterfuzz/_internal/chrome/build_info.py +++ b/src/clusterfuzz/_internal/chrome/build_info.py @@ -14,29 +14,31 @@ """Utilities for fetching build info from OmahaProxy.""" import json +from typing import Any from clusterfuzz._internal.base import utils from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -BUILD_INFO_PATTERN = ('([a-z]+),([a-z]+),([0-9.]+),' - '[^,]*,[^,]*,[^,]*,[^,]*,[^,]*,' - '([0-9a-f]+),.*') -BUILD_INFO_URL_CD = ('https://chromiumdash.appspot.com/fetch_releases?' - 'num=1&platform={platform}') +BUILD_INFO_PATTERN: str = ('([a-z]+),([a-z]+),([0-9.]+),' + '[^,]*,[^,]*,[^,]*,[^,]*,[^,]*,' + '([0-9a-f]+),.*') +BUILD_INFO_URL_CD: str = ('https://chromiumdash.appspot.com/fetch_releases?' + 'num=1&platform={platform}') class BuildInfo: """BuildInfo holds build metadata pulled from OmahaProxy.""" - def __init__(self, platform, build_type, version, revision): + def __init__(self, platform: str, build_type: str, version: str, + revision: str) -> None: self.platform = platform self.build_type = build_type self.version = version self.revision = revision -def _convert_platform_to_chromiumdash_platform(platform): +def _convert_platform_to_chromiumdash_platform(platform: str) -> str: """Converts platform to Chromium Dash platform. Note that Windows in Chromium Dash is win64 and we only want win32.""" platform_lower = platform.lower() @@ -45,7 +47,8 @@ def _convert_platform_to_chromiumdash_platform(platform): return platform_lower.capitalize() -def _fetch_releases_from_chromiumdash(platform, channel=None): +def _fetch_releases_from_chromiumdash( + platform: str, channel: str | None = None) -> list[dict[str, Any]]: """Makes a Call to chromiumdash's fetch_releases api, and returns its json array response.""" chromiumdash_platform = _convert_platform_to_chromiumdash_platform(platform) @@ -70,14 +73,14 @@ def _fetch_releases_from_chromiumdash(platform, channel=None): return build_info_json -def get_production_builds_info_from_cd(platform): +def get_production_builds_info_from_cd(platform: str) -> list[BuildInfo]: """Gets the build information from Chromium Dash for production builds. Omits platforms containing digits, namely, win64. Omits channels containing underscore, namely, canary_asan. Platform is e.g. ANDROID, LINUX, MAC, WINDOWS. """ - builds_metadata = [] + builds_metadata: list[BuildInfo] = [] build_info_json = _fetch_releases_from_chromiumdash(platform) for info in build_info_json: build_type = info['channel'].lower() @@ -100,7 +103,7 @@ def get_production_builds_info_from_cd(platform): return builds_metadata -def get_release_milestone(build_type, platform): +def get_release_milestone(build_type: str, platform: str) -> int | None: """Return milestone for a particular release.""" if build_type == 'head': actual_build_type = 'canary' @@ -122,12 +125,13 @@ def get_release_milestone(build_type, platform): return None -def get_build_to_revision_mappings(platform=None): +def get_build_to_revision_mappings( + platform: str | None = None) -> dict[str, dict[str, str]]: """Gets the build information.""" if not platform: platform = environment.platform() - result = {} + result: dict[str, dict[str, str]] = {} build_info_json = _fetch_releases_from_chromiumdash(platform) for info in build_info_json: diff --git a/src/clusterfuzz/_internal/common/testcase_utils.py b/src/clusterfuzz/_internal/common/testcase_utils.py index 4314dcd55fa..42155da6250 100644 --- a/src/clusterfuzz/_internal/common/testcase_utils.py +++ b/src/clusterfuzz/_internal/common/testcase_utils.py @@ -13,10 +13,6 @@ # limitations under the License. """Holds helpers for reuse across different tasks.""" -import datetime -import os -from typing import Optional - from clusterfuzz._internal.datastore import data_handler from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.metrics import logs @@ -30,7 +26,8 @@ TESTCASE_TRIAGE_DURATION_ISSUE_UPDATED_STEP = 'issue_updated' -def emit_testcase_triage_duration_metric(testcase_id: int, step: str): +def emit_testcase_triage_duration_metric(testcase_id: int | str, + step: str) -> None: """Finds out if a testcase is fuzzer generated or manually uploaded, and emits the TESTCASE_TRIAGE_DURATION metric.""" testcase = data_handler.get_testcase_by_id(testcase_id) @@ -47,12 +44,13 @@ def emit_testcase_triage_duration_metric(testcase_id: int, step: str): from_fuzzer = not get_testcase_upload_metadata(testcase_id) - if not testcase.get_age_in_seconds(): + age_in_seconds = testcase.get_age_in_seconds() + if not age_in_seconds: logs.warning(f'No timestamp associated to testcase {testcase_id},' ' failed to emit TESTCASE_TRIAGE_DURATION metric.') return - testcase_age_in_hours = testcase.get_age_in_seconds() / (60 * 60) + testcase_age_in_hours = age_in_seconds / (60 * 60) logs.info('Emiting TESTCASE_TRIAGE_DURATION metric for testcase ' f'{testcase_id} (age = {testcase_age_in_hours} hours.) ' @@ -68,6 +66,6 @@ def emit_testcase_triage_duration_metric(testcase_id: int, step: str): def get_testcase_upload_metadata( - testcase_id) -> Optional[data_types.TestcaseUploadMetadata]: + testcase_id: int | str) -> data_types.TestcaseUploadMetadata | None: return data_types.TestcaseUploadMetadata.query( data_types.TestcaseUploadMetadata.testcase_id == int(testcase_id)).get() diff --git a/src/clusterfuzz/_internal/config/db_config.py b/src/clusterfuzz/_internal/config/db_config.py index d1a101a4c5d..47e46358634 100644 --- a/src/clusterfuzz/_internal/config/db_config.py +++ b/src/clusterfuzz/_internal/config/db_config.py @@ -14,6 +14,7 @@ """Get values from the global configuration.""" import base64 +from typing import Any from clusterfuzz._internal.datastore import data_types @@ -24,12 +25,12 @@ BASE64_MARKER = 'base64;' -def get(): +def get() -> data_types.Config | None: """Return configuration data.""" return data_types.Config.query().get() -def get_value(key): +def get_value(key: str) -> Any: """Return a configuration key value.""" config = get() if not config: @@ -44,7 +45,7 @@ def get_value(key): return value -def get_value_for_job(data, target_job_type): +def get_value_for_job(data: str, target_job_type: str) -> str: """Parses a value for a particular job type. If job type is not found, return the default value.""" # All data is in a single line, just return that. @@ -60,7 +61,7 @@ def get_value_for_job(data, target_job_type): return result -def set_value(key, value): +def set_value(key: str, value: Any) -> None: """Sets a configuration key value and commits change.""" config = get() if not config: diff --git a/src/clusterfuzz/_internal/config/local_config.py b/src/clusterfuzz/_internal/config/local_config.py index 87a252fb503..f71186ea130 100644 --- a/src/clusterfuzz/_internal/config/local_config.py +++ b/src/clusterfuzz/_internal/config/local_config.py @@ -14,6 +14,9 @@ """Get values / settings from local configuration.""" import os +from typing import Any +from typing import cast +from typing import Sequence import yaml @@ -37,7 +40,7 @@ EXTERNAL_TESTCASE_UPLOADER_PATH = 'external_testcase_reader.config' -def _load_yaml_file(yaml_file_path): +def _load_yaml_file(yaml_file_path: str) -> Any: """Load yaml file and return parsed contents.""" with open(yaml_file_path) as f: try: @@ -46,8 +49,9 @@ def _load_yaml_file(yaml_file_path): raise errors.ConfigParseError(yaml_file_path) -def _find_key_in_yaml_file(yaml_file_path, search_keys, full_key_name, - value_is_relative_path): +def _find_key_in_yaml_file(yaml_file_path: str, search_keys: Sequence[str], + full_key_name: str, + value_is_relative_path: bool) -> Any: """Find a key in a yaml file.""" if not os.path.isfile(yaml_file_path): return None @@ -78,7 +82,8 @@ def _find_key_in_yaml_file(yaml_file_path, search_keys, full_key_name, return result -def _get_key_location(search_path, full_key_name): +def _get_key_location(search_path: str, + full_key_name: str) -> tuple[str, str, list[str]]: """Get the path of the the yaml file and the key components given a full key name.""" key_parts = full_key_name.split(SEPARATOR) @@ -107,7 +112,7 @@ def _get_key_location(search_path, full_key_name): return dir_path, '', [] -def _validate_root(search_path, root): +def _validate_root(search_path: str, root: str | None) -> bool: """Validate that a root is valid.""" if root is None: return True @@ -125,7 +130,8 @@ def _validate_root(search_path, root): yaml_path, search_keys, root, value_is_relative_path=False) is not None) -def _search_key(search_path, full_key_name, value_is_relative_path): +def _search_key(search_path: str, full_key_name: str, + value_is_relative_path: bool) -> Any: """Search the key in a search path.""" directory, filename, search_keys = _get_key_location(search_path, full_key_name) @@ -139,7 +145,9 @@ def _search_key(search_path, full_key_name, value_is_relative_path): class Config: """Config class helper.""" - def __init__(self, root=None, *args, **kwargs): # pylint: disable=keyword-arg-before-vararg + # pylint: disable=keyword-arg-before-vararg + def __init__(self, root: str | None = None, *args: Any, + **kwargs: Any) -> None: self._root = root.format(*args, **kwargs) if root is not None else None self._config_dir = environment.get_config_directory() self._cache = memoize.FifoInMemory(CACHE_SIZE) @@ -152,7 +160,7 @@ def __init__(self, root=None, *args, **kwargs): # pylint: disable=keyword-arg-b if not _validate_root(self._config_dir, self._root): raise errors.BadConfigError(self._config_dir) - def sub_config(self, path): + def sub_config(self, path: str) -> 'Config': """Return a new config with a new sub-root.""" if self._root: new_root = self._root + SEPARATOR + path @@ -161,14 +169,16 @@ def sub_config(self, path): return Config(root=new_root) - def _get_helper(self, key_name='', default=None, - value_is_relative_path=False): + def _get_helper(self, + key_name: str | None = '', + default: Any = None, + value_is_relative_path: bool = False) -> Any: """Helper for get and get_absolute_functions.""" if self._root: key_name = self._root + SEPARATOR + key_name if key_name else self._root if not key_name: - raise errors.InvalidConfigKey(key_name) + raise errors.InvalidConfigKey(cast(str, key_name)) cache_key_name = self._cache.get_key(self._get_helper, (key_name, value_is_relative_path), {}) @@ -183,11 +193,12 @@ def _get_helper(self, key_name='', default=None, self._cache.put(cache_key_name, value) return value - def get(self, key_name='', default=None): + def get(self, key_name: str | None = '', default: Any = None) -> Any: """Get key value using a key name.""" return self._get_helper(key_name, default=default) - def get_absolute_path(self, key_name='', default=None): + def get_absolute_path(self, key_name: str | None = '', + default: Any = None) -> Any: """Get absolute path of key value using a key name.""" return self._get_helper( key_name, default=default, value_is_relative_path=True) @@ -196,10 +207,10 @@ def get_absolute_path(self, key_name='', default=None): class ProjectConfig(Config): """Project Config class helper.""" - def __init__(self): + def __init__(self) -> None: super().__init__(PROJECT_PATH) - def set_environment(self): + def set_environment(self) -> None: """Sets environment vars from project config.""" env_variable_values = self.get('env') if not env_variable_values: @@ -216,54 +227,54 @@ def set_environment(self): class AuthConfig(Config): """Authentication config.""" - def __init__(self): + def __init__(self) -> None: super().__init__(GAE_AUTH_PATH) class GAEConfig(Config): """GAE config.""" - def __init__(self): + def __init__(self) -> None: super().__init__(GAE_CONFIG_PATH) class MonitoringRegionsConfig(Config): """Monitoring regions config.""" - def __init__(self): + def __init__(self) -> None: super().__init__(MONITORING_REGIONS_PATH) class IssueTrackerConfig(Config): """Issue tracker config.""" - def __init__(self): + def __init__(self) -> None: super().__init__(ISSUE_TRACKERS_PATH) class BatchConfig(Config): """Batch config.""" - def __init__(self): + def __init__(self) -> None: super().__init__(BATCH_PATH) class GCEClustersConfig(Config): """Issue tracker config.""" - def __init__(self): + def __init__(self) -> None: super().__init__(GCE_CLUSTERS_PATH) class SwarmingConfig(Config): """Swarming config.""" - def __init__(self): + def __init__(self) -> None: super().__init__(SWARMING_PATH) class ExternalTestcaseReaderConfig(Config): """External testcase reader config.""" - def __init__(self): + def __init__(self) -> None: super().__init__(EXTERNAL_TESTCASE_UPLOADER_PATH) diff --git a/src/clusterfuzz/_internal/crash_analysis/crash_analyzer.py b/src/clusterfuzz/_internal/crash_analysis/crash_analyzer.py index d3354904549..5a199e9aeca 100644 --- a/src/clusterfuzz/_internal/crash_analysis/crash_analyzer.py +++ b/src/clusterfuzz/_internal/crash_analysis/crash_analyzer.py @@ -13,25 +13,27 @@ # limitations under the License. """Functions for helping in crash analysis.""" +from collections.abc import Sequence import re +from typing import cast from clusterfuzz._internal.config import local_config from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -ASSERT_CRASH_ADDRESSES = [ +ASSERT_CRASH_ADDRESSES: list[int] = [ 0x0000bbadbeef, 0x0000fbadbeef, 0x00001f75b7dd, 0x0000977537dd, 0x00009f7537dd, ] -GENERIC_CRASH_TYPES = [ +GENERIC_CRASH_TYPES: list[str] = [ 'Null-dereference', 'Null-dereference READ', 'Null-dereference WRITE', 'READ', 'UNKNOWN', 'UNKNOWN READ', 'UNKNOWN WRITE', 'WRITE', 'Uncaught exception' ] -CRASH_TYPES_NON_SECURITY = [ +CRASH_TYPES_NON_SECURITY: list[str] = [ 'Stack-overflow', 'Fatal-signal', 'Missing-library', @@ -49,14 +51,14 @@ # Release SECURITY_CHECK in Blink shouldn't be marked as a security bug 'Security CHECK failure', ] -SIGNAL_SIGNATURES_NOT_SECURITY = [ +SIGNAL_SIGNATURES_NOT_SECURITY: list[str] = [ 'Sanitizer: ABRT', 'Sanitizer: BUS', 'Sanitizer: FPE', 'Sanitizer: ILL', 'Sanitizer: breakpoint', ] -STACKTRACE_TOOL_MARKERS = [ +STACKTRACE_TOOL_MARKERS: list[str] = [ 'AddressSanitizer', 'ASAN:', 'CFI: Most likely a control flow integrity violation;', @@ -68,7 +70,7 @@ 'UndefinedBehaviorSanitizer', 'UndefinedSanitizer', ] -STACKTRACE_END_MARKERS = [ +STACKTRACE_END_MARKERS: list[str] = [ 'ABORTING', 'END MEMORY TOOL REPORT', 'End of process memory map.', @@ -79,8 +81,8 @@ '\nExiting', 'minidump has been written', ] -UBSAN_RUNTIME_ERROR = ' runtime error: ' -UBSAN_CRASH_TYPES_NON_SECURITY = [ +UBSAN_RUNTIME_ERROR: str = ' runtime error: ' +UBSAN_CRASH_TYPES_NON_SECURITY: list[str] = [ 'Divide-by-zero', 'Float-cast-overflow', 'Implicit-conversion', @@ -106,54 +108,55 @@ 'Assumption-violation', 'Upcast-of-null-pointer', ] -UBSAN_CRASH_TYPES_SECURITY = [ +UBSAN_CRASH_TYPES_SECURITY: list[str] = [ 'Bad-cast', 'Index-out-of-bounds', 'Incorrect-function-pointer-type', 'Non-positive-vla-bound-value', 'Object-size', ] -GOLANG_CRASH_TYPES_NON_SECURITY = [ +GOLANG_CRASH_TYPES_NON_SECURITY: list[str] = [ 'Index out of range', 'Integer divide by zero', 'Makeslice: len out of range', 'Slice bounds out of range', 'Stack overflow', ] -EXTRA_SANITIZERS_SECURITY = [ +EXTRA_SANITIZERS_SECURITY: list[str] = [ 'Arbitrary DNS resolution', 'Arbitrary file open', 'Command injection', 'PySecSan', ] -EXTERNAL_TOOL_SECURITY = [ +EXTERNAL_TOOL_SECURITY: list[str] = [ 'Wycheproof error', ] -EXPERIMENTAL_CRASH_TYPES = EXTRA_SANITIZERS_SECURITY +EXPERIMENTAL_CRASH_TYPES: list[str] = EXTRA_SANITIZERS_SECURITY # Default page size of 4KB. -NULL_DEREFERENCE_BOUNDARY = 0x1000 +NULL_DEREFERENCE_BOUNDARY: int = 0x1000 -CHROMIUM_MIRACLEPTR_REGEX = re.compile(r'.*MiraclePtr Status:.+') +CHROMIUM_MIRACLEPTR_REGEX: re.Pattern[str] = re.compile( + r'.*MiraclePtr Status:.+') -MIRACLEPTR_STATUS = { +MIRACLEPTR_STATUS: dict[str, str] = { 'PROTECTED': 'MiraclePtr-Protected', 'MANUAL ANALYSIS REQUIRED': 'MiraclePtr-ManualAnalysisRequired', 'NOT PROTECTED': 'MiraclePtr-NotProtected' } -def address_to_integer(address): +def address_to_integer(address: str | None) -> int: """Attempt to convert an address from a string (hex) to an integer.""" try: - return int(address, 16) + return int(cast(str, address), 16) except: return 0 -def has_marker(stacktrace, marker_list): +def has_marker(stacktrace: str, marker_list: Sequence[str]) -> bool: """Return true if the stacktrace has atleast one marker in the marker list.""" for marker in marker_list: if marker in stacktrace: @@ -162,7 +165,7 @@ def has_marker(stacktrace, marker_list): return False -def ignore_stacktrace(crash_stacktrace): +def ignore_stacktrace(crash_stacktrace: str) -> bool: """Return whether the stacktrace needs to be ignored.""" # Filter crash based on search exclude pattern specified in job definition. search_excludes = environment.get_value('SEARCH_EXCLUDES') @@ -183,19 +186,19 @@ def ignore_stacktrace(crash_stacktrace): return False -def is_crash(return_code, console_output): +def is_crash(return_code: int | None, console_output: str) -> bool: """Analyze the return code and console output to see if this was a crash.""" if not return_code: return False crash_signature = environment.get_value('CRASH_SIGNATURE') if crash_signature: - return re.search(crash_signature, console_output) + return cast(bool, re.search(crash_signature, console_output)) return True -def is_check_failure_crash(stacktrace): +def is_check_failure_crash(stacktrace: str) -> bool: """Return true if it a CHECK failure crash.""" # Android-specific exception patterns. if environment.is_android(): @@ -227,7 +230,7 @@ def is_check_failure_crash(stacktrace): return False -def is_memory_tool_crash(stacktrace): +def is_memory_tool_crash(stacktrace: str) -> bool: """Return true if it is a memory debugging tool crash.""" # Job-specific generic checks. crash_signature = environment.get_value('CRASH_SIGNATURE') @@ -256,17 +259,17 @@ def is_memory_tool_crash(stacktrace): return False -def is_null_dereference(int_address): +def is_null_dereference(int_address: int) -> bool: """Check to see if this is a null dereference crash address.""" return int_address < NULL_DEREFERENCE_BOUNDARY -def is_assert_crash_address(int_address): +def is_assert_crash_address(int_address: int) -> bool: """Check to see if this is an ASSERT crash based on the address.""" return int_address in ASSERT_CRASH_ADDRESSES -def has_signal_for_non_security_bug_type(stacktrace): +def has_signal_for_non_security_bug_type(stacktrace: str) -> bool: """Checks if any signal which means not security bug presented.""" if re.search(r'^[ \t]+#0[ \t]+0x[0-9a-f]+[ \t]+in gsignal ', stacktrace, re.MULTILINE): @@ -279,7 +282,7 @@ def has_signal_for_non_security_bug_type(stacktrace): return False -def check_miracleptr_status(stacktrace): +def check_miracleptr_status(stacktrace: str) -> str | None: """Look for MiraclePtr status string and return the appropriate label.""" for line in stacktrace.split('\n'): if CHROMIUM_MIRACLEPTR_REGEX.match(line): @@ -292,7 +295,8 @@ def check_miracleptr_status(stacktrace): return None -def is_security_issue(crash_stacktrace, crash_type, crash_address): +def is_security_issue(crash_stacktrace: str, crash_type: str, + crash_address: str | None) -> bool: """Based on unsymbolized crash parameters, determine whether it has security consequences or not.""" # MiraclePtr protected crashes are not security issues. @@ -326,7 +330,7 @@ def is_security_issue(crash_stacktrace, crash_type, crash_address): # DCHECKs from CHECKs. checks_have_security_implication = environment.get_value( 'CHECKS_HAVE_SECURITY_IMPLICATION', False) - return checks_have_security_implication + return cast(bool, checks_have_security_implication) # Debug CHECK failure should be marked with security implications. if crash_type in ('Security DCHECK failure', 'DCHECK failure'): @@ -364,7 +368,7 @@ def is_security_issue(crash_stacktrace, crash_type, crash_address): if crash_type == 'ASSERT' or 'ASSERTION FAILED' in crash_stacktrace: asserts_have_security_implication = environment.get_value( 'ASSERTS_HAVE_SECURITY_IMPLICATION', True) - return asserts_have_security_implication + return cast(bool, asserts_have_security_implication) # Kernel Failures are security bugs if crash_type.startswith('Kernel failure'): @@ -405,7 +409,7 @@ def is_security_issue(crash_stacktrace, crash_type, crash_address): return False -def has_ubsan_error(stacktrace): +def has_ubsan_error(stacktrace: str) -> bool: """Return a bool whether the process output contains UBSan errors that should be handled as crashes. Suppressions file alone does not provide granular control, e.g. to ignore left shift of negative value which can cause false @@ -438,6 +442,6 @@ def has_ubsan_error(stacktrace): return False -def is_experimental_crash(crash_type): +def is_experimental_crash(crash_type: str | None) -> bool: """Return whether or not the crash type is experimental.""" - return crash_type in EXPERIMENTAL_CRASH_TYPES + return bool(crash_type and crash_type in EXPERIMENTAL_CRASH_TYPES) diff --git a/src/clusterfuzz/_internal/crash_analysis/crash_comparer.py b/src/clusterfuzz/_internal/crash_analysis/crash_comparer.py index 31d25e9a80e..09329742d84 100644 --- a/src/clusterfuzz/_internal/crash_analysis/crash_comparer.py +++ b/src/clusterfuzz/_internal/crash_analysis/crash_comparer.py @@ -15,8 +15,11 @@ # pylint: disable=consider-using-enumerate +from collections.abc import Sequence +from typing import cast -def _levenshtein_distance(string_1, string_2): + +def _levenshtein_distance(string_1: str, string_2: str) -> int: """"Levenshtein_distance calculation: Iterative with two matrix rows, based on Wikipedia article and code by Christopher P. Matthews.""" if string_1 == string_2: @@ -27,7 +30,7 @@ def _levenshtein_distance(string_1, string_2): return len(string_1) v0 = list(range(len(string_2) + 1)) - v1 = [None] * (len(string_2) + 1) + v1: list[int] = cast(list[int], [None] * (len(string_2) + 1)) for i in range(len(string_1)): v1[0] = i + 1 @@ -42,7 +45,7 @@ def _levenshtein_distance(string_1, string_2): return v1[len(string_2)] -def _similarity_ratio(string_1, string_2): +def _similarity_ratio(string_1: str, string_2: str) -> float: """Return a ratio on how similar two strings are.""" length_sum = len(string_1) + len(string_2) if length_sum == 0: @@ -52,7 +55,8 @@ def _similarity_ratio(string_1, string_2): 1.0 * length_sum) -def longest_common_subsequence(first_frames, second_frames): +def longest_common_subsequence(first_frames: Sequence[str], + second_frames: Sequence[str]) -> int: """Count number of frames which are the same (taking into account order).""" first_len = len(first_frames) second_len = len(second_frames) @@ -75,17 +79,17 @@ class CrashComparer: SAME_FRAMES_THRESHOLD = 2 def __init__(self, - crash_state_1, - crash_state_2, - compare_threshold=None, - same_frames_threshold=None): + crash_state_1: str | None, + crash_state_2: str | None, + compare_threshold: float | None = None, + same_frames_threshold: int | None = None) -> None: self.crash_state_1 = crash_state_1 self.crash_state_2 = crash_state_2 self.compare_threshold = compare_threshold or self.COMPARE_THRESHOLD self.same_frames_threshold = ( same_frames_threshold or self.SAME_FRAMES_THRESHOLD) - def is_similar(self): + def is_similar(self) -> bool: """Return a bool for whether the two crash results are similar.""" # If one of the crash state is empty, it can't match anything. if not self.crash_state_1 or not self.crash_state_2: diff --git a/src/clusterfuzz/_internal/crash_analysis/crash_result.py b/src/clusterfuzz/_internal/crash_analysis/crash_result.py index 2e9dfe57549..e81ebbca8d8 100644 --- a/src/clusterfuzz/_internal/crash_analysis/crash_result.py +++ b/src/clusterfuzz/_internal/crash_analysis/crash_result.py @@ -24,18 +24,25 @@ class CrashResult: """Represents a crash result from a test run.""" - def __init__(self, return_code, crash_time, output, unexpected_crash=False): - self.return_code = return_code - self.crash_time = crash_time - self.output = utils.decode_to_unicode(output) if output else 'No output!' + def __init__( + self, + return_code: int | None, + crash_time: float | None, + output: str | bytes | None, + unexpected_crash: bool = False, + ) -> None: + self.return_code: int | None = return_code + self.crash_time: float | None = crash_time + self.output: str = ( + utils.decode_to_unicode(output) if output else 'No output!') # For crashes against an expected state, this indicates whether if there # was a crash that didn't match. - self.unexpected_crash = unexpected_crash + self.unexpected_crash: bool = unexpected_crash - self._symbolized_crash_data = None - self._unsymbolized_crash_data = None + self._symbolized_crash_data: CrashInfo | None = None + self._unsymbolized_crash_data: CrashInfo | None = None - def get_crash_time(self): + def get_crash_time(self) -> float | None: """Return the crash time.""" return self.crash_time @@ -57,7 +64,7 @@ def get_unsymbolized_data(self) -> CrashInfo: self.output, symbolize_flag=False) return self._unsymbolized_crash_data - def get_state(self, symbolized=True): + def get_state(self, symbolized: bool = True) -> str: """Return the crash state.""" if symbolized: state = self.get_symbolized_data() @@ -66,7 +73,7 @@ def get_state(self, symbolized=True): return state.crash_state - def get_stacktrace(self, symbolized=True): + def get_stacktrace(self, symbolized: bool = True) -> str: """Return the crash stacktrace.""" if symbolized: state = self.get_symbolized_data() @@ -75,13 +82,13 @@ def get_stacktrace(self, symbolized=True): return state.crash_stacktrace - def get_type(self): + def get_type(self) -> str: """Return the crash type.""" # It does not matter whether we use symbolized or unsymbolized data. state = self.get_unsymbolized_data() return state.crash_type - def is_crash(self, ignore_state=False): + def is_crash(self, ignore_state: bool = False) -> bool: """Return True if this result was a crash.""" crashed = crash_analyzer.is_crash(self.return_code, self.output) if not crashed: @@ -92,12 +99,12 @@ def is_crash(self, ignore_state=False): return False return True - def should_ignore(self): + def should_ignore(self) -> bool: """Return True if this crash should be ignored.""" state = self.get_symbolized_data() return crash_analyzer.ignore_stacktrace(state.crash_stacktrace) - def is_security_issue(self): + def is_security_issue(self) -> bool: """Return True if this crash is a security issue.""" state = self.get_unsymbolized_data() return crash_analyzer.is_security_issue( diff --git a/src/clusterfuzz/_internal/crash_analysis/severity_analyzer.py b/src/clusterfuzz/_internal/crash_analysis/severity_analyzer.py index ac7b063f9e4..b4dabbfb365 100644 --- a/src/clusterfuzz/_internal/crash_analysis/severity_analyzer.py +++ b/src/clusterfuzz/_internal/crash_analysis/severity_analyzer.py @@ -22,30 +22,32 @@ from clusterfuzz._internal.system import environment # These should be generic within ClusterFuzz. -LOW_SEVERITY_CRASH_TYPES = [ +LOW_SEVERITY_CRASH_TYPES: list[str] = [ 'V8 sandbox violation', ] -MEDIUM_SEVERITY_CRASH_TYPES = [ +MEDIUM_SEVERITY_CRASH_TYPES: list[str] = [ 'Container-overflow', 'Heap-buffer-overflow', 'Incorrect-function-pointer-type', 'Index-out-of-bounds', 'Memcpy-param-overlap', 'Non-positive-vla-bound-value', 'Object-size', 'Stack-buffer-overflow', 'UNKNOWN', 'Use-of-uninitialized-value' ] -HIGH_SEVERITY_CRASH_TYPES = [ +HIGH_SEVERITY_CRASH_TYPES: list[str] = [ 'Bad-cast', 'Heap-double-free', 'Heap-use-after-free', 'Security DCHECK failure', 'Use-after-poison' ] -SEVERITY_ORDER = [ +SEVERITY_ORDER: list[int] = [ SecuritySeverity.LOW, SecuritySeverity.MEDIUM, SecuritySeverity.HIGH, SecuritySeverity.CRITICAL ] -def _modify_severity(severity, - delta, - min_severity=SecuritySeverity.LOW, - max_severity=SecuritySeverity.CRITICAL): +def _modify_severity( + severity: int, + delta: int, + min_severity: int = SecuritySeverity.LOW, + max_severity: int = SecuritySeverity.CRITICAL, +) -> int: """Increase/decrease the given |severity| by |delta|.""" min_index = SEVERITY_ORDER.index(min_severity) max_index = SEVERITY_ORDER.index(max_severity) @@ -63,7 +65,7 @@ def _modify_severity(severity, return SEVERITY_ORDER[severity_index] -def get_analyzer(name): +def get_analyzer(name: str) -> 'SeverityAnalyzerSanitizer | None': """Return an analyzer for the given |name|.""" if name == 'sanitizer_generic': return SeverityAnalyzerSanitizer() @@ -74,9 +76,16 @@ def get_analyzer(name): return None -def get_security_severity(crash_type, crash_output, job_name, - requires_gestures): +def get_security_severity( + crash_type: str | None, + crash_output: str | None, + job_name: str | None, + requires_gestures: bool = False, +) -> int | None: """Convenience function to get the security severity of a crash.""" + if not crash_type or not crash_output or not job_name: + return None + analyzer = None severity_analyzer_name = environment.get_value('SECURITY_SEVERITY_ANALYZER') @@ -103,7 +112,12 @@ def get_security_severity(crash_type, crash_output, job_name, class SeverityAnalyzerSanitizer: """Generic ASan severity analyzer.""" - def analyze(self, crash_type, crash_output, requires_gestures): + def analyze( + self, + crash_type: str, + crash_output: str, + requires_gestures: bool, + ) -> int | None: """Return a security severity based on the ASan crash output.""" manual_severity_match = re.search( @@ -141,15 +155,20 @@ def analyze(self, crash_type, crash_output, requires_gestures): class SeverityAnalyzerSanitizerChrome(SeverityAnalyzerSanitizer): """Chrome specific severity analyzer.""" - PROCESS_TYPE_EXCEPTIONS = [ + PROCESS_TYPE_EXCEPTIONS: list[str] = [ 'Use-of-uninitialized-value', ] - def __init__(self, is_compromised_renderer): + def __init__(self, is_compromised_renderer: bool) -> None: SeverityAnalyzerSanitizer.__init__(self) - self.is_compromised_renderer = is_compromised_renderer - - def analyze(self, crash_type, crash_output, requires_gestures): + self.is_compromised_renderer: bool = is_compromised_renderer + + def analyze( + self, + crash_type: str, + crash_output: str, + requires_gestures: bool, + ) -> int | None: """Return a security severity based on the ASan crash output.""" # Base severity. severity = SeverityAnalyzerSanitizer.analyze(self, crash_type, crash_output, @@ -166,10 +185,10 @@ def analyze(self, crash_type, crash_output, requires_gestures): return severity @staticmethod - def _find_process_type(crash_output): + def _find_process_type(crash_output: str) -> str | None: """Return the process type of the process that crashed.""" # TODO(ochang): Support Android. - process_type = None + process_type: str | None = None # This is best effort, and won't work in cases where we have weird stacks # (or V8). Since we only care right now if this is a browser process (where # this should be rare from an uncompromised renderer), it shouldn't matter @@ -198,9 +217,9 @@ def _find_process_type(crash_output): return process_type -def severity_to_string(severity): +def severity_to_string(severity: int | None) -> str: """Convert a severity value to a human-readable string.""" - severity_map = { + severity_map: dict[int | None, str] = { SecuritySeverity.CRITICAL: 'Critical', SecuritySeverity.HIGH: 'High', SecuritySeverity.MEDIUM: 'Medium', @@ -211,9 +230,9 @@ def severity_to_string(severity): return severity_map.get(severity, '') -def string_to_severity(severity): +def string_to_severity(severity: str) -> int: """Convert a string value to a severity value.""" - severity_map = { + severity_map: dict[str, int] = { 'critical': SecuritySeverity.CRITICAL, 'high': SecuritySeverity.HIGH, 'medium': SecuritySeverity.MEDIUM, diff --git a/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_analyzer.py b/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_analyzer.py index 6cf6b6c865d..71c424c721b 100644 --- a/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_analyzer.py +++ b/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_analyzer.py @@ -13,6 +13,8 @@ # limitations under the License. """Stack analyzer module.""" +from typing import cast + from clusterfuzz import stacktraces from clusterfuzz._internal.config import local_config from clusterfuzz._internal.platforms.android import \ @@ -20,10 +22,11 @@ from clusterfuzz._internal.platforms.linux.lkl import kernel_utils as lkl_kernel from clusterfuzz._internal.system import environment -MAX_REDZONE_SIZE_FOR_OOMS_AND_HANGS = 64 +MAX_REDZONE_SIZE_FOR_OOMS_AND_HANGS: int = 64 -def linkify_kernel_or_lkl_stacktrace_if_needed(crash_info): +def linkify_kernel_or_lkl_stacktrace_if_needed( + crash_info: stacktraces.CrashInfo) -> None: """Linkify Android Kernel or lkl stacktrace.""" kernel_prefix = '' kernel_hash = '' @@ -41,7 +44,9 @@ def linkify_kernel_or_lkl_stacktrace_if_needed(crash_info): _linkify_android_kernel_stacktrace(crash_info, kernel_prefix, kernel_hash) -def _linkify_android_kernel_stacktrace(crash_info, kernel_prefix, kernel_hash): +def _linkify_android_kernel_stacktrace(crash_info: stacktraces.CrashInfo, + kernel_prefix: str, + kernel_hash: str) -> None: """Linkify Android Kernel or lkl stacktrace.""" temp_crash_stacktrace = '' for line in crash_info.crash_stacktrace.splitlines(): @@ -51,11 +56,12 @@ def _linkify_android_kernel_stacktrace(crash_info, kernel_prefix, kernel_hash): crash_info.crash_stacktrace = temp_crash_stacktrace -def get_crash_data(crash_data, - symbolize_flag=True, - fuzz_target=None, - already_symbolized=False, - detect_ooms_and_hangs=None) -> stacktraces.CrashInfo: +def get_crash_data( + crash_data: str, + symbolize_flag: bool = True, + fuzz_target: str | None = None, + already_symbolized: bool = False, + detect_ooms_and_hangs: bool | None = None) -> stacktraces.CrashInfo: """Get crash parameters from crash data. Crash parameters include crash type, address, state and stacktrace. If the stacktrace is not already symbolized, we will try to symbolize it @@ -94,8 +100,8 @@ def get_crash_data(crash_data, redzone_size = environment.get_value('REDZONE') if detect_ooms_and_hangs is None: report_value = environment.get_value('REPORT_OOMS_AND_HANGS') - detect_ooms_and_hangs = ( - report_value == 'FORCE' or report_value and + detect_ooms_and_hangs = cast( + bool, report_value == 'FORCE' or report_value and (not redzone_size or redzone_size <= MAX_REDZONE_SIZE_FOR_OOMS_AND_HANGS)) diff --git a/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_parser.py b/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_parser.py index 80bda902e09..52898032042 100644 --- a/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_parser.py +++ b/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_parser.py @@ -14,136 +14,139 @@ """Stack parser module.""" import inspect +import re +from typing import Any from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.protos import process_state_pb2 -def unsigned_to_signed(address): +def unsigned_to_signed(address: int) -> int: """Convert unsigned address to signed int64 (as defined in the proto).""" return (address - 2**64) if address >= 2**63 else address -def format_address_to_dec(address, base=16): +def format_address_to_dec(address: object, base: int = 16) -> int | None: """Addresses may be formatted as decimal, hex string with 0x or 0X prefix, or without any prefix. Convert to decimal int.""" if address is None: return None - address = str(address).replace('`', '').strip() - if not address: + address_str = str(address).replace('`', '').strip() + if not address_str: return None # This is required for Chrome Win and Mac stacks, which mix decimal and hex. try_bases = [base, 16] if base != 16 else [base] for base_try in try_bases: try: - address = int(address, base_try) - return address + return int(address_str, base_try) except Exception: continue logs.warning('Error formatting address %s to decimal int64 in bases %s.' % - (str(address), str(try_bases))) + (str(address_str), str(try_bases))) return None class StackFrameStructure: """IR for fields a stackframe may conptain/expect.""" - def __init__(self, - address=None, - function_name=None, - function_base=None, - function_offset=None, - filename=None, - fileline=None, - module_name=None, - module_base=None, - module_offset=None): - self._address = address - self._function_name = function_name - self._function_base = function_base - self._function_offset = function_offset - self._filename = filename - self._fileline = fileline - self._module_name = module_name - self._module_base = module_base - self._module_offset = module_offset + def __init__( + self, + address: Any = None, + function_name: Any = None, + function_base: Any = None, + function_offset: Any = None, + filename: Any = None, + fileline: Any = None, + module_name: Any = None, + module_base: Any = None, + module_offset: Any = None, + ) -> None: + self._address: Any = address + self._function_name: Any = function_name + self._function_base: Any = function_base + self._function_offset: Any = function_offset + self._filename: Any = filename + self._fileline: Any = fileline + self._module_name: Any = module_name + self._module_base: Any = module_base + self._module_offset: Any = module_offset @property - def address(self): + def address(self) -> Any: return self._address @address.setter - def address(self, address): + def address(self, address: Any) -> None: self._address = address @property - def function_name(self): + def function_name(self) -> Any: return self._function_name @function_name.setter - def function_name(self, function_name): + def function_name(self, function_name: Any) -> None: self._function_name = function_name @property - def function_base(self): + def function_base(self) -> Any: return self._function_base @function_base.setter - def function_base(self, function_base): + def function_base(self, function_base: Any) -> None: self._function_base = function_base @property - def function_offset(self): + def function_offset(self) -> Any: return self._function_offset @function_offset.setter - def function_offset(self, function_offset): + def function_offset(self, function_offset: Any) -> None: self._function_offset = function_offset @property - def filename(self): + def filename(self) -> Any: return self._filename @filename.setter - def filename(self, filename): + def filename(self, filename: Any) -> None: self._filename = filename @property - def fileline(self): + def fileline(self) -> Any: return self._fileline @fileline.setter - def fileline(self, fileline): + def fileline(self, fileline: Any) -> None: self._fileline = fileline @property - def module_name(self): + def module_name(self) -> Any: return self._module_name @module_name.setter - def module_name(self, module_name): + def module_name(self, module_name: Any) -> None: self._module_name = module_name @property - def module_base(self): + def module_base(self) -> Any: return self._module_base @module_base.setter - def module_base(self, module_base): + def module_base(self, module_base: Any) -> None: self._module_base = module_base @property - def module_offset(self): + def module_offset(self) -> Any: return self._module_offset @module_offset.setter - def module_offset(self, module_offset): + def module_offset(self, module_offset: Any) -> None: self._module_offset = module_offset - def to_proto(self): + def to_proto(self) -> process_state_pb2.StackFrame: # pylint: disable=no-member """Convert StackFrame to process_state.proto format for upload to crash/.""" frame_proto = process_state_pb2.StackFrame() # pylint: disable=no-member if self.address is not None: @@ -168,17 +171,21 @@ def to_proto(self): class StackFrame(StackFrameStructure): """IR for canonicalizing stackframe strings.""" - def __init__(self, - address=None, - function_name=None, - function_base=None, - function_offset=None, - filename=None, - fileline=None, - module_name=None, - module_base=None, - module_offset=None, - base=16): + base: int + + def __init__( + self, + address: object = None, + function_name: object = None, + function_base: object = None, + function_offset: object = None, + filename: object = None, + fileline: object = None, + module_name: object = None, + module_base: object = None, + module_offset: object = None, + base: int = 16, + ) -> None: super().__init__( address=format_address_to_dec(address), function_name=function_name, @@ -191,9 +198,9 @@ def __init__(self, module_offset=format_address_to_dec(module_offset)) # Base for converting addresses set in frame. Most will be in hex. - self._base = base + self._base: int = base - def __setattr__(self, field_name, field_value): + def __setattr__(self, field_name: str, field_value: Any) -> None: """Set attributes, performing conversions as needed for address fields.""" if field_name == 'base': self._base = field_value @@ -213,8 +220,8 @@ def __setattr__(self, field_name, field_value): super().__setattr__(field_name, field_value) - def __str__(self): - s = [] + def __str__(self) -> str: + s: list[str] = [] for name, member in inspect.getmembers(StackFrame): if not isinstance(member, property): continue @@ -226,51 +233,56 @@ class StackFrameSpec(StackFrameStructure): """Representation paralleling that of StackFrames for pulling out the correct groups in a *_STACK_FRAME_REGEX match.""" - def __init__(self, - address=None, - function_name=None, - function_base=None, - function_offset=None, - filename=None, - fileline=None, - module_name=None, - module_base=None, - module_offset=None, - base=16): + def __init__( + self, + address: Any = None, + function_name: Any = None, + function_base: Any = None, + function_offset: Any = None, + filename: Any = None, + fileline: Any = None, + module_name: Any = None, + module_base: Any = None, + module_offset: Any = None, + base: int = 16, + ) -> None: """Specify a stackframe format. Each field should be an index into a match. See comments inline *_STACK_FRAME_REGEX for appropriate indices.""" - address = address if address is not None else [] - function_name = function_name if function_name is not None else [] - function_base = function_base if function_base is not None else [] - function_offset = function_offset if function_offset is not None else [] - filename = filename if filename is not None else [] - fileline = fileline if fileline is not None else [] - module_name = module_name if module_name is not None else [] - module_base = module_base if module_base is not None else [] - module_offset = module_offset if module_offset is not None else [] + address_val = address if address is not None else [] + function_name_val = function_name if function_name is not None else [] + function_base_val = function_base if function_base is not None else [] + function_offset_val = function_offset if function_offset is not None else [] + filename_val = filename if filename is not None else [] + fileline_val = fileline if fileline is not None else [] + module_name_val = module_name if module_name is not None else [] + module_base_val = module_base if module_base is not None else [] + module_offset_val = module_offset if module_offset is not None else [] super().__init__( - address=address, - function_name=function_name, - function_base=function_base, - function_offset=function_offset, - filename=filename, - fileline=fileline, - module_name=module_name, - module_base=module_base, - module_offset=module_offset) + address=address_val, + function_name=function_name_val, + function_base=function_base_val, + function_offset=function_offset_val, + filename=filename_val, + fileline=fileline_val, + module_name=module_name_val, + module_base=module_base_val, + module_offset=module_offset_val) # Base for converting addresses processed by this spec. Most will be in hex. - self._base = base + self._base: int = base @property - def base(self): + def base(self) -> int: return self._base @base.setter - def base(self, base): + def base(self, base: int) -> None: self._base = base - def parse_stack_frame(self, frame_match): + def parse_stack_frame( + self, + frame_match: re.Match[str] | None, + ) -> StackFrame | None: """Given a match and stackframe specification, populate a stackframe with information from the match.""" if frame_match is None: diff --git a/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_symbolizer.py b/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_symbolizer.py index 01e5c29e668..91c43235d76 100644 --- a/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_symbolizer.py +++ b/src/clusterfuzz/_internal/crash_analysis/stack_parsing/stack_symbolizer.py @@ -29,11 +29,15 @@ # pylint: disable-all import collections +from collections.abc import Iterable import os import re import subprocess import sys import threading +from typing import Any +from typing import Callable +from typing import cast from clusterfuzz._internal.base import utils from clusterfuzz._internal.google_cloud_utils import storage @@ -52,12 +56,13 @@ import termios except ImportError: # Applies only on unix platforms. - pass + pty = None # type: ignore + termios = None # type: ignore -stack_inlining = 'false' -llvm_symbolizer_path = '' -pipes = [] -symbolizers = {} +stack_inlining: str = 'false' +llvm_symbolizer_path: str | None = '' +pipes: list[subprocess.Popen] = [] +symbolizers: dict[str, 'ChainSymbolizer'] = {} # 0 0x7f6e35cf2e45 (/blah/foo.so+0x11fe45) STACK_TRACE_LINE_REGEX = re.compile( @@ -74,15 +79,15 @@ class LineBuffered: """Disable buffering on a file object.""" - def __init__(self, stream): + def __init__(self, stream: Any) -> None: self.stream = stream - def write(self, data): + def write(self, data: str) -> None: self.stream.write(data) if '\n' in data: self.stream.flush() - def __getattr__(self, attr): + def __getattr__(self, attr: str) -> Any: return getattr(self.stream, attr) @@ -100,7 +105,7 @@ def __getattr__(self, attr): # We're assuming that there're no more than two nested bundles in the binary # path. Only one of these bundles may be a framework and frameworks cannot # contain other bundles. -def chrome_dsym_hints(binary): +def chrome_dsym_hints(binary: str) -> list[str]: """Construct a path to the .dSYM bundle for the given binary. There are three possible cases for binary location in Chromium: 1. The binary is a standalone executable or dynamic library in the product @@ -113,8 +118,8 @@ def chrome_dsym_hints(binary): The first case is handled by llvm-symbolizer, so we only need to construct .dSYM paths for .app bundles and frameworks.""" path_parts = binary.split(os.path.sep) - app_positions = [] - framework_positions = [] + app_positions: list[int] = [] + framework_positions: list[int] = [] for index, part in enumerate(path_parts): if part.endswith('.app'): app_positions.append(index) @@ -142,7 +147,7 @@ def chrome_dsym_hints(binary): return [result] -def disable_buffering(): +def disable_buffering() -> None: """Make this process and child processes stdout unbuffered.""" os.environ['PYTHONUNBUFFERED'] = '1' @@ -151,10 +156,10 @@ def disable_buffering(): # See https://github.com/google/clusterfuzz/issues/234 for why. # Since sys.stdout is a C++ object, it's impossible to do sys.stdout.write = # lambda... - sys.stdout = LineBuffered(sys.stdout) + sys.stdout = cast(Any, LineBuffered(sys.stdout)) -def fix_filename(file_name): +def fix_filename(file_name: str) -> str: """Clean up the filename, nulls out tool specific ones.""" file_name = re.sub('.*asan_[a-z_]*.cc:[0-9]*', '_asan_rtl_', file_name) file_name = re.sub('.*crtstuff.c:0', '', file_name) @@ -167,7 +172,7 @@ def fix_filename(file_name): return os.path.normpath(file_name) -def fix_function_name(function_name): +def fix_function_name(function_name: str) -> str: """Clean up function name.""" if function_name.startswith('??'): return '' @@ -175,7 +180,8 @@ def fix_function_name(function_name): return function_name -def get_stack_frame(binary, addr, function_name, file_name): +def get_stack_frame(binary: str, addr: str, function_name: str, + file_name: str) -> str: """Return a stack frame entry.""" # Cleanup file and function name. file_name = fix_filename(file_name) @@ -196,7 +202,7 @@ def get_stack_frame(binary, addr, function_name, file_name): return '%s in %s %s' % (addr, function_name, file_name) -def is_valid_arch(s): +def is_valid_arch(s: str) -> bool: """Check if this is a valid supported architecture.""" return s in [ "i386", "x86_64", "x86_64h", "arm", "armv6", "armv7", "armv7s", "armv7k", @@ -204,7 +210,7 @@ def is_valid_arch(s): ] -def guess_arch(address): +def guess_arch(address: str) -> str: """Guess which architecture we're running on (32/64). 10 = len('0x') + 8 hex digits.""" if len(address) > 10: @@ -215,10 +221,10 @@ def guess_arch(address): class Symbolizer: - def __init__(self): + def __init__(self) -> None: pass - def symbolize(self, addr, binary, offset): + def symbolize(self, addr: str, binary: str, offset: str) -> list[str] | None: """Symbolize the given address (pair of binary and offset). Overridden in subclasses. @@ -236,18 +242,25 @@ def symbolize(self, addr, binary, offset): class LLVMSymbolizer(Symbolizer): - def __init__(self, symbolizer_path, default_arch, system, dsym_hints=[]): + def __init__( + self, + symbolizer_path: str, + default_arch: str, + system: str, + dsym_hints: Iterable[str] = [], + ) -> None: super().__init__() - self.symbolizer_path = symbolizer_path - self.default_arch = default_arch - self.system = system - self.dsym_hints = dsym_hints - self.stderr_buffer = collections.deque(maxlen=100) - self.pipe = self.open_llvm_symbolizer() - self.crashed = False - self.last_successful_frame = None - - def open_llvm_symbolizer(self): + self.symbolizer_path: str = symbolizer_path + self.default_arch: str = default_arch + self.system: str = system + self.dsym_hints: Iterable[str] = dsym_hints + self.stderr_buffer: collections.deque[str] = collections.deque(maxlen=100) + self.stderr_thread: threading.Thread | None = None + self.pipe: subprocess.Popen | None = self.open_llvm_symbolizer() + self.crashed: bool = False + self.last_successful_frame: str | None = None + + def open_llvm_symbolizer(self) -> subprocess.Popen | None: if not os.path.exists(self.symbolizer_path): logs.info(f'os.path() does not have {self.symbolizer_path}') return None @@ -284,7 +297,9 @@ def open_llvm_symbolizer(self): stdout=subprocess.PIPE, stderr=subprocess.PIPE) - def read_stderr(p, buffer): + def read_stderr(p: subprocess.Popen, + buffer: collections.deque[str]) -> None: + assert p.stderr is not None for line in iter(p.stderr.readline, b''): buffer.append(line.decode('utf-8', errors='replace')) @@ -298,7 +313,7 @@ def read_stderr(p, buffer): logs.info(f'created pipe - {pipe}') return pipe - def symbolize(self, addr, binary, offset): + def symbolize(self, addr: str, binary: str, offset: str) -> list[str] | None: """Overrides Symbolizer.symbolize.""" if self.crashed: return None @@ -306,8 +321,12 @@ def symbolize(self, addr, binary, offset): if not binary.strip(): return ['%s in' % addr] - result = [] + result: list[str] | None = [] + symbolizer_input = '' try: + assert self.pipe is not None + assert self.pipe.stdin is not None + assert self.pipe.stdout is not None symbolizer_input = '"%s" %s' % (binary, offset) self.pipe.stdin.write(symbolizer_input.encode('utf-8') + b'\n') self.pipe.stdin.flush() @@ -326,8 +345,10 @@ def symbolize(self, addr, binary, offset): except Exception as e: self.crashed = True + assert self.pipe is not None return_code = self.pipe.poll() if return_code is not None: + assert self.stderr_thread is not None self.stderr_thread.join(timeout=0.1) stderr_content = ''.join(self.stderr_buffer).strip() stderr_msg = f' Stderr: {stderr_content}' if stderr_content else '' @@ -344,25 +365,30 @@ def symbolize(self, addr, binary, offset): return result -def LLVMSymbolizerFactory(system, default_arch, dsym_hints=[]): +def LLVMSymbolizerFactory( + system: str, + default_arch: str, + dsym_hints: Iterable[str] = [], +) -> LLVMSymbolizer: + assert llvm_symbolizer_path is not None return LLVMSymbolizer(llvm_symbolizer_path, default_arch, system, dsym_hints) class Addr2LineSymbolizer(Symbolizer): - def __init__(self, binary): + def __init__(self, binary: str) -> None: super().__init__() - self.binary = binary - self.pipe = self.open_addr2line() + self.binary: str = binary + self.pipe: subprocess.Popen = self.open_addr2line() - def open_addr2line(self): + def open_addr2line(self) -> subprocess.Popen: cmd = ['addr2line', '--demangle', '-f', '-e', self.binary] pipe = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) global pipes pipes.append(pipe) return pipe - def symbolize(self, addr, binary, offset): + def symbolize(self, addr: str, binary: str, offset: str) -> list[str] | None: """Overrides Symbolizer.symbolize.""" if self.binary != binary: return None @@ -371,6 +397,8 @@ def symbolize(self, addr, binary, offset): try: symbolizer_input = str(offset).encode('utf-8') + assert self.pipe.stdin is not None + assert self.pipe.stdout is not None self.pipe.stdin.write(symbolizer_input + b'\n') self.pipe.stdin.flush() function_name = self.pipe.stdout.readline().rstrip().decode('utf-8') @@ -390,7 +418,9 @@ class UnbufferedLineConverter: Uses pty to trick the child into providing unbuffered output. """ - def __init__(self, args, close_stderr=False): + def __init__(self, args: list[str], close_stderr: bool = False) -> None: + assert pty is not None + assert termios is not None pid, fd = pty.fork() if pid == 0: # We're the child. Transfer control to command. @@ -407,31 +437,33 @@ def __init__(self, args, close_stderr=False): self.r = os.fdopen(fd, 'r', 1) self.w = os.fdopen(os.dup(fd), 'w', 1) - def convert(self, line): + def convert(self, line: str) -> str: self.w.write(line + '\n') return self.readline() - def readline(self): + def readline(self) -> str: return self.r.readline().rstrip() class DarwinSymbolizer(Symbolizer): - def __init__(self, addr, binary, arch): + def __init__(self, addr: str, binary: str, arch: str) -> None: super().__init__() - self.binary = binary - self.arch = arch + self.binary: str = binary + self.arch: str = arch + self.atos: UnbufferedLineConverter | None = None self.open_atos() - def open_atos(self): + def open_atos(self) -> None: cmdline = ['atos', '-o', self.binary, '-arch', self.arch] self.atos = UnbufferedLineConverter(cmdline, close_stderr=True) - def symbolize(self, addr, binary, offset): + def symbolize(self, addr: str, binary: str, offset: str) -> list[str] | None: """Overrides Symbolizer.symbolize.""" if self.binary != binary: return None + assert self.atos is not None try: atos_line = self.atos.convert('0x%x' % int(offset, 16)) while 'got symbolicator for' in atos_line: @@ -456,11 +488,11 @@ def symbolize(self, addr, binary, offset): # to the next symbolizer in chain. class ChainSymbolizer(Symbolizer): - def __init__(self, symbolizer_list): + def __init__(self, symbolizer_list: list[Symbolizer | None]) -> None: super().__init__() - self.symbolizer_list = symbolizer_list + self.symbolizer_list: list[Symbolizer | None] = symbolizer_list - def symbolize(self, addr, binary, offset): + def symbolize(self, addr: str, binary: str, offset: str) -> list[str] | None: """Overrides Symbolizer.symbolize.""" for symbolizer in self.symbolizer_list: if symbolizer: @@ -469,31 +501,41 @@ def symbolize(self, addr, binary, offset): return result return None - def append_symbolizer(self, symbolizer): + def append_symbolizer(self, symbolizer: Symbolizer | None) -> None: self.symbolizer_list.append(symbolizer) -def SystemSymbolizerFactory(system, addr, binary, arch): +def SystemSymbolizerFactory(system: str, addr: str, binary: str, + arch: str) -> Symbolizer | None: if system == 'darwin': return DarwinSymbolizer(addr, binary, arch) elif system.startswith('linux'): return Addr2LineSymbolizer(binary) + return None class SymbolizationLoop: - def __init__(self, binary_path_filter=None, dsym_hint_producer=None): + def __init__( + self, + binary_path_filter: Callable[[str], str] | None = None, + dsym_hint_producer: Callable[[str], list[str]] | None = None, + ) -> None: # Used by clients who may want to supply a different binary name. # E.g. in Chrome several binaries may share a single .dSYM. - self.binary_path_filter = binary_path_filter - self.dsym_hint_producer = dsym_hint_producer - self.system = sys.platform - self.llvm_symbolizers = {} - self.last_llvm_symbolizer = None - self.dsym_hints = set() - self.lkl_binary_name = None - - def symbolize_address(self, addr, binary, offset, arch): + self.binary_path_filter: Callable[[str], str] | None = binary_path_filter + self.dsym_hint_producer: Callable[[str], + list[str]] | None = dsym_hint_producer + self.system: str = sys.platform + self.llvm_symbolizers: dict[str, LLVMSymbolizer] = {} + self.last_llvm_symbolizer: LLVMSymbolizer | None = None + self.dsym_hints: set[str] = set() + self.lkl_binary_name: str | None = None + self.frame_no: int = 0 + self.current_line: str = '' + + def symbolize_address(self, addr: str, binary: str, offset: str, + arch: str) -> list[str]: # On non-Darwin (i.e. on platforms without .dSYM debug info) always use # a single symbolizer binary. # On Darwin, if the dsym hint producer is present: @@ -505,7 +547,7 @@ def symbolize_address(self, addr, binary, offset, arch): # if so, reuse |last_llvm_symbolizer| which has the full set of hints; # 3. otherwise create a new symbolizer and pass all currently known # .dSYM hints to it. - if not binary in self.llvm_symbolizers: + if binary not in self.llvm_symbolizers: use_new_symbolizer = True if self.system == 'darwin' and self.dsym_hint_producer: dsym_hints_for_binary = set(self.dsym_hint_producer(binary)) @@ -521,7 +563,7 @@ def symbolize_address(self, addr, binary, offset, arch): # Use the chain of symbolizers: # LLVM symbolizer -> addr2line/atos # (fall back to next symbolizer if the previous one fails). - if not binary in symbolizers: + if binary not in symbolizers: symbolizers[binary] = ChainSymbolizer([self.llvm_symbolizers[binary]]) result = symbolizers[binary].symbolize(addr, binary, offset) if result is None: @@ -533,7 +575,9 @@ def symbolize_address(self, addr, binary, offset, arch): assert result return result - def _line_parser(self, line): + def _line_parser( + self, line: str + ) -> tuple[str | None, str | None, str | None, str | None, str | None]: """Parses line for frameno_str, addr, binary, offset, arch.""" match = STACK_TRACE_LINE_REGEX.match(line) if match: @@ -553,7 +597,9 @@ def _line_parser(self, line): return None, None, None, None, None - def _lkl_line_parser(self, line): + def _lkl_line_parser( + self, line: str + ) -> tuple[str | None, str | None, str | None, str | None, str | None]: """Parses line for frameno_str, addr, binary, offset, arch.""" match = STACK_TRACE_LINE_REGEX_LKL.match(line) if match: @@ -567,7 +613,8 @@ def _lkl_line_parser(self, line): return None, None, None, None, None - def _trusty_line_parser(self, line): + def _trusty_line_parser( + self, line: str) -> tuple[str | None, str | None, str | None]: """Parses line for memory_space, offset, and addr.""" match = STACK_TRACE_LINE_TRUSTY.match(line) if match: @@ -576,7 +623,7 @@ def _trusty_line_parser(self, line): return None, None, None - def _extract_trusty_app_name(self, stacktrace): + def _extract_trusty_app_name(self, stacktrace: str) -> str: """Returns the name of the crashed Trusted App.""" #(app: keymaster) match = re.compile(r'\(app:\s(\w+)\)').search(stacktrace) @@ -585,7 +632,7 @@ def _extract_trusty_app_name(self, stacktrace): return '' - def _extract_trusty_bid(self, stacktrace): + def _extract_trusty_bid(self, stacktrace: str) -> str: """Returns the build of the crashed Trusted App.""" #, Build: 1234567 match = re.compile(r',\sBuild:\s(\d+),\sBuilt:').search(stacktrace) @@ -594,15 +641,17 @@ def _extract_trusty_bid(self, stacktrace): return '' - def _close_pipes(self): + def _close_pipes(self) -> None: """Closes any open pipes.""" for pipe in pipes: try: + assert pipe.stdin is not None pipe.stdin.close() except Exception as e: logs.warning(f'Failed to close symbolizer stdin pipe: {e}') try: + assert pipe.stdout is not None pipe.stdout.close() except Exception as e: logs.warning(f'Failed to close symbolizer stdout pipe: {e}') @@ -612,7 +661,8 @@ def _close_pipes(self): except ProcessLookupError: pass - def _can_process_trusty_stack_trace(self, trusty_app, trusty_bid) -> bool: + def _can_process_trusty_stack_trace(self, trusty_app: str, + trusty_bid: str) -> bool: """Checks whether the trusty stacktrace can be processed. Uworker bots can't access DB nor GCS, which are required to process the @@ -631,7 +681,8 @@ def _can_process_trusty_stack_trace(self, trusty_app, trusty_bid) -> bool: trusty_app and trusty_bid and (symbols_dir or not environment.is_uworker())) - def process_trusty_stacktrace(self, unsymbolized_crash_stacktrace): + def process_trusty_stacktrace(self, + unsymbolized_crash_stacktrace: str) -> str: """Adds debug line information to a Trusted App stacktrace.""" trusty_app = self._extract_trusty_app_name(unsymbolized_crash_stacktrace) trusty_bid = self._extract_trusty_bid(unsymbolized_crash_stacktrace) @@ -648,21 +699,25 @@ def process_trusty_stacktrace(self, unsymbolized_crash_stacktrace): trusty_binary = f'{symbols_dir}/{trusty_app}.syms.elf' symbolized_stacktrace = '' - unsymbolized_crash_stacktrace_lines = \ - unsymbolized_crash_stacktrace.splitlines() + unsymbolized_crash_stacktrace_lines = ( + unsymbolized_crash_stacktrace.splitlines()) for line in unsymbolized_crash_stacktrace_lines: line = line.strip() memory_space, offset, addr = self._trusty_line_parser(line) if memory_space == 'kSP': + assert addr is not None addr2line = Addr2LineSymbolizer(kernel_binary) - symbolized_stacktrace += '{}+{}: {}\n'.format( - memory_space, offset, - addr2line.symbolize(addr, kernel_binary, addr)[0]) + sym = addr2line.symbolize(addr, kernel_binary, addr) + assert sym is not None + symbolized_stacktrace += '{}+{}: {}\n'.format(memory_space, offset, + sym[0]) elif memory_space == 'uSP': + assert addr is not None addr2line = Addr2LineSymbolizer(trusty_binary) - symbolized_stacktrace += '{}+{}: {}\n'.format( - memory_space, offset, - addr2line.symbolize(addr, trusty_binary, addr)[0]) + sym = addr2line.symbolize(addr, trusty_binary, addr) + assert sym is not None + symbolized_stacktrace += '{}+{}: {}\n'.format(memory_space, offset, + sym[0]) else: symbolized_stacktrace += '%s\n' % line @@ -670,12 +725,12 @@ def process_trusty_stacktrace(self, unsymbolized_crash_stacktrace): return symbolized_stacktrace - def process_stacktrace(self, unsymbolized_crash_stacktrace): + def process_stacktrace(self, unsymbolized_crash_stacktrace: str) -> str: """Symbolizes a crash stacktrace.""" self.frame_no = 0 symbolized_crash_stacktrace = '' - unsymbolized_crash_stacktrace_lines = \ - unsymbolized_crash_stacktrace.splitlines() + unsymbolized_crash_stacktrace_lines = ( + unsymbolized_crash_stacktrace.splitlines()) if lkl.is_lkl_stack_trace(unsymbolized_crash_stacktrace): line_parser = self._lkl_line_parser self.lkl_binary_name = lkl.get_lkl_binary_name( @@ -700,11 +755,12 @@ def process_stacktrace(self, unsymbolized_crash_stacktrace): original_binary = binary if self.binary_path_filter: binary = self.binary_path_filter(binary) - symbolized_line = self.symbolize_address(addr, binary, offset, arch) + symbolized_line = self.symbolize_address( + cast(str, addr), binary, offset, cast(str, arch)) if not symbolized_line: if original_binary != binary: - symbolized_line = self.symbolize_address(addr, original_binary, - offset, arch) + symbolized_line = self.symbolize_address( + cast(str, addr), original_binary, offset, cast(str, arch)) if not symbolized_line: symbolized_crash_stacktrace += '%s\n' % self.current_line @@ -719,7 +775,7 @@ def process_stacktrace(self, unsymbolized_crash_stacktrace): return symbolized_crash_stacktrace -def filter_binary_path(binary_path): +def filter_binary_path(binary_path: str) -> str: """Filters binary path to provide a local copy.""" if environment.is_android() or environment.is_lkl_job(): return symbols_downloader.filter_binary_path(binary_path) @@ -733,6 +789,7 @@ def filter_binary_path(binary_path): # Linux. Ensure that the binary is always looked for in the chroot and not # in system directories. build_dir = environment.get_value('BUILD_DIR') + assert build_dir is not None if not binary_path.startswith(build_dir): # Fixup path so |binary_path| points to a binary in the chroot (probably # a system library). @@ -743,8 +800,10 @@ def filter_binary_path(binary_path): return binary_path -def symbolize_stacktrace(unsymbolized_crash_stacktrace, - enable_inline_frames=True): +def symbolize_stacktrace( + unsymbolized_crash_stacktrace: str, + enable_inline_frames: bool = True, +) -> str: """Symbolize a crash stacktrace.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import symbolize_host diff --git a/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py b/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py index 3e4cfdc6a4d..86814a1432b 100644 --- a/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py +++ b/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py @@ -14,12 +14,15 @@ """Cron job to aggregate fuzzer stats onto a daily_stats BigQuery table.""" import argparse +from collections.abc import Sequence from concurrent.futures import as_completed from concurrent.futures import ThreadPoolExecutor import datetime import io import json import time +from typing import Any +from typing import cast from googleapiclient.errors import HttpError from googleapiclient.http import MediaIoBaseUpload @@ -37,11 +40,11 @@ # Limit worker count to 4 concurrent threads to prevent exhaustion of # project-wide queued interactive queries quota (1,000 maximum). See # https://cloud.google.com/bigquery/quotas#query_jobs -NUM_THREADS = 4 -NUM_RETRIES = 2 -RETRY_SLEEP_TIME = 5 +NUM_THREADS: int = 4 +NUM_RETRIES: int = 2 +RETRY_SLEEP_TIME: int = 5 -DAILY_STATS_SCHEMA = { +DAILY_STATS_SCHEMA: dict[str, Any] = { 'fields': [{ 'name': 'fuzzer_name', 'type': 'STRING', @@ -81,7 +84,7 @@ 'clusterfuzz._internal.cron.aggregate_fuzzer_stats._execute_insert_request', backoff=2, exception_types=[httplib2.HttpLib2Error]) -def _execute_insert_request(request): +def _execute_insert_request(request: Any) -> None: """Executes a table/dataset insert request, retrying on transport errors.""" try: request.execute() @@ -96,7 +99,7 @@ def _execute_insert_request(request): raise -def _create_dataset_if_needed(bigquery_client, dataset_id): +def _create_dataset_if_needed(bigquery_client: Any, dataset_id: str) -> None: """Writes a dataset for the `dataset_id`. No-op if it already exists""" project_id = utils.get_application_id() dataset_body = { @@ -111,7 +114,8 @@ def _create_dataset_if_needed(bigquery_client, dataset_id): _execute_insert_request(dataset_insert) -def _create_table_if_needed(bigquery_client, dataset_id, table_id, schema): +def _create_table_if_needed(bigquery_client: Any, dataset_id: str, + table_id: str, schema: dict[str, Any]) -> None: """Writes a table for the `table_id`. No-op if it already exists""" project_id = utils.get_application_id() table_body = { @@ -132,7 +136,8 @@ def _create_table_if_needed(bigquery_client, dataset_id, table_id, schema): _execute_insert_request(table_insert) -def _poll_completion(bigquery_client, project_id, job_id): +def _poll_completion(bigquery_client: Any, project_id: str | None, + job_id: str) -> dict[str, Any]: """Poll bigquery for job completion.""" response = bigquery_client.jobs().get( projectId=project_id, jobId=job_id).execute(num_retries=2) @@ -144,7 +149,8 @@ def _poll_completion(bigquery_client, project_id, job_id): return response -def _query_fuzzer_stats(fuzzer_name, project_id, target_date_str): +def _query_fuzzer_stats(fuzzer_name: str, project_id: str | None, + target_date_str: str) -> list[dict[str, Any]]: """Queries single fuzzer stats for the given target date.""" dataset_id = fuzzer_stats.dataset_name(fuzzer_name) table_id = 'JobRun' @@ -197,12 +203,14 @@ def _query_fuzzer_stats(fuzzer_name, project_id, target_date_str): return [] -def _gather_all_stats(fuzzers, project_id, target_date_str): +def _gather_all_stats(fuzzers: Sequence[data_types.Fuzzer], + project_id: str | None, + target_date_str: str) -> list[dict[str, Any]]: """Gathers fuzzer statistics concurrently using a thread pool.""" all_rows = [] with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor: future_to_fuzzer = { - executor.submit(_query_fuzzer_stats, fuzzer.name, project_id, + executor.submit(_query_fuzzer_stats, cast(str, fuzzer.name), project_id, target_date_str): fuzzer for fuzzer in fuzzers } @@ -218,8 +226,9 @@ def _gather_all_stats(fuzzers, project_id, target_date_str): return all_rows -def _persist_daily_stats(all_rows, bigquery_client, project_id, - date_partition_str): +def _persist_daily_stats(all_rows: list[dict[str, Any]], bigquery_client: Any, + project_id: str | None, + date_partition_str: str) -> None: """Writes gathered row statistics to destination table.""" if not all_rows: logs.error(f'No data to write to daily_stats on {date_partition_str}') @@ -270,7 +279,7 @@ def _persist_daily_stats(all_rows, bigquery_client, project_id, logs.error('Failed to execute batch load job in BigQuery', exception=e) -def main(argv=None): +def main(argv: list[str] | None = None) -> bool: """Main entry point for the aggregate_fuzzer_stats cron job.""" parser = argparse.ArgumentParser(prog='aggregate_fuzzer_stats') parser.add_argument( diff --git a/src/clusterfuzz/_internal/cron/backup.py b/src/clusterfuzz/_internal/cron/backup.py index fb589a4755a..372239ca91b 100644 --- a/src/clusterfuzz/_internal/cron/backup.py +++ b/src/clusterfuzz/_internal/cron/backup.py @@ -17,6 +17,7 @@ send them to the backup URL.""" import datetime +from typing import Any from google.cloud import ndb from googleapiclient import discovery @@ -31,15 +32,15 @@ # CrashStatistic is excluded because the number of records is too high and # can be rebuilt from BigQuery dataset. -EXCLUDED_MODELS = {'CrashStatistic', 'CrashStatisticJobHistory'} +EXCLUDED_MODELS: set[str] = {'CrashStatistic', 'CrashStatisticJobHistory'} -def _datastore_client(): +def _datastore_client() -> Any: """Returns an api client for datastore.""" return discovery.build('datastore', 'v1') -def main(): +def main() -> bool: """Backups all entities in a datastore bucket.""" backup_bucket = local_config.Config( local_config.PROJECT_PATH).get('backup.bucket') diff --git a/src/clusterfuzz/_internal/cron/batch_fuzzer_jobs.py b/src/clusterfuzz/_internal/cron/batch_fuzzer_jobs.py index 0d702d7ca1b..d3b1edc14ff 100644 --- a/src/clusterfuzz/_internal/cron/batch_fuzzer_jobs.py +++ b/src/clusterfuzz/_internal/cron/batch_fuzzer_jobs.py @@ -13,25 +13,28 @@ # limitations under the License. """A cron task that batches FuzzerJobs.""" +from typing import Any +from typing import cast + from google.cloud import ndb from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.metrics import logs -FUZZER_JOB_BATCH_SIZE = 4000 +FUZZER_JOB_BATCH_SIZE: int = 4000 -def batch_fuzzer_jobs(): +def batch_fuzzer_jobs() -> None: """Batches FuzzerJobs for reduced Datastore read ops by bots.""" platforms = [ - item.platform for item in data_types.FuzzerJob.query( + cast(str, item.platform) for item in data_types.FuzzerJob.query( projection=[data_types.FuzzerJob.platform], distinct=True) ] for platform in platforms: fuzzer_jobs = list( data_types.FuzzerJob.query(data_types.FuzzerJob.platform == platform)) - fuzzer_jobs.sort(key=lambda item: item.job) + fuzzer_jobs.sort(key=lambda item: cast(str, item.job)) batches_to_remove = { b.key for b in data_types.FuzzerJobs.query( @@ -45,7 +48,7 @@ def batch_fuzzer_jobs(): batched = data_types.FuzzerJobs(id=key_id, platform=platform) batched.platform = platform - batched.fuzzer_jobs = fuzzer_jobs[i:end] + batched.fuzzer_jobs = cast(Any, fuzzer_jobs[i:end]) batched.put() batch_count += 1 @@ -53,10 +56,10 @@ def batch_fuzzer_jobs(): # Removes additional batches if number reduced. if batches_to_remove: - ndb.delete_multi(batches_to_remove) + ndb.delete_multi(cast(Any, batches_to_remove)) -def main(): +def main() -> bool: """Batches FuzzerJobs.""" batch_fuzzer_jobs() logs.info('Batch fuzzer jobs succeeded.') diff --git a/src/clusterfuzz/_internal/cron/build_crash_stats.py b/src/clusterfuzz/_internal/cron/build_crash_stats.py index 5dc0dd074ac..628a8f6aa8a 100644 --- a/src/clusterfuzz/_internal/cron/build_crash_stats.py +++ b/src/clusterfuzz/_internal/cron/build_crash_stats.py @@ -40,15 +40,16 @@ # We give it a few minutes. # # We add one hour because our hour spans from 0 minutes to 59 minutes. -BIGQUERY_INSERTION_DELAY = datetime.timedelta(hours=1, minutes=2) +BIGQUERY_INSERTION_DELAY: datetime.timedelta = datetime.timedelta( + hours=1, minutes=2) # The template for job ids. Running on a already-run hour is fine to certain # degree. We de-dup when reading. -JOB_ID_TEMPLATE = 'build_crash_stats_test_{unique_number}' +JOB_ID_TEMPLATE: str = 'build_crash_stats_test_{unique_number}' -TIMEOUT = 2 * 60 +TIMEOUT: int = 2 * 60 -SQL = """ +SQL: str = """ SELECT COUNT(*) as count, crash_type, crash_state, security_flag, parent_fuzzer_name, fuzzer_name, @@ -75,7 +76,7 @@ class TooEarlyError(Exception): """The end hour is too early according to BIGQUERY_INSERTION_DELAY.""" -def get_start_hour(): +def get_start_hour() -> int: """Gets the start hour from the first crash.""" client = big_query.Client() @@ -91,7 +92,7 @@ def get_start_hour(): return 0 -def get_last_successful_hour_or_start_hour(): +def get_last_successful_hour_or_start_hour() -> int: """Gets the last hour that ran successfully or the start hour.""" last_hour = crash_stats.get_last_successful_hour() if last_hour: @@ -100,7 +101,7 @@ def get_last_successful_hour_or_start_hour(): return get_start_hour() -def get_next_end_hour(): +def get_next_end_hour() -> int: """Gets the next end hour. If it's too early to compute data for the next end hour, return None.""" last_successful_hour = get_last_successful_hour_or_start_hour() @@ -117,7 +118,7 @@ def get_next_end_hour(): return next_end_hour -def make_request(client, job_id, end_hour): +def make_request(client: big_query.Client, job_id: str, end_hour: int) -> None: """Makes a request to BigQuery to build crash stats.""" table_id = ( 'crash_stats$%s' % crash_stats.get_datetime(end_hour).strftime('%Y%m%d')) @@ -131,7 +132,7 @@ def make_request(client, job_id, end_hour): dataset_id='main', table_id=table_id, job_id=job_id, query=sql) -def build(end_hour): +def build(end_hour: int) -> None: """Builds crash stats for the end hour.""" logging.info('Started building crash stats for %s.', crash_stats.get_datetime(end_hour)) @@ -155,7 +156,7 @@ def build(end_hour): raise Exception('Building crash stats exceeded %d seconds.' % TIMEOUT) # pylint: disable=broad-exception-raised -def build_if_needed(): +def build_if_needed() -> int | None: """Gets the next end hour and decide whether to execute build(). If build() succeeds, then record the next end hour.""" try: @@ -174,7 +175,7 @@ def build_if_needed(): return None -def main(): +def main() -> bool: """Builds crash stats from data_types.CrashsStats2.""" end_hour = build_if_needed() logging.info('OK (end_hour=%s)', end_hour) diff --git a/src/clusterfuzz/_internal/cron/chrome_tests_syncer.py b/src/clusterfuzz/_internal/cron/chrome_tests_syncer.py index c01833a535a..2d89a4ddbe8 100644 --- a/src/clusterfuzz/_internal/cron/chrome_tests_syncer.py +++ b/src/clusterfuzz/_internal/cron/chrome_tests_syncer.py @@ -37,18 +37,20 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import shell -ENGINE_FUZZER_NAMES = ['afl', 'centipede', 'googlefuzztest', 'libFuzzer'] -MAX_TESTCASE_DIRECTORY_SIZE = 10 * 1024 * 1024 # in bytes. -MAX_TESTCASES = 25000 -TESTCASES_REPORT_INTERVAL = 2500 -STORED_TESTCASES_LIST = [] +ENGINE_FUZZER_NAMES: list[str] = [ + 'afl', 'centipede', 'googlefuzztest', 'libFuzzer' +] +MAX_TESTCASE_DIRECTORY_SIZE: int = 10 * 1024 * 1024 # in bytes. +MAX_TESTCASES: int = 25000 +TESTCASES_REPORT_INTERVAL: int = 2500 +STORED_TESTCASES_LIST: list[int | str] = [] -NUMBER_OF_FUZZILLI_ARCHIVES = 9 -NUMBER_OF_FUZZILLI_IMPORT_ARCHIVES = 9 -NUMBER_OF_FUZZILLI_DIFF_FUZZ_ARCHIVES = 5 +NUMBER_OF_FUZZILLI_ARCHIVES: int = 9 +NUMBER_OF_FUZZILLI_IMPORT_ARCHIVES: int = 9 +NUMBER_OF_FUZZILLI_DIFF_FUZZ_ARCHIVES: int = 5 -def unpack_crash_testcases(crash_testcases_directory): +def unpack_crash_testcases(crash_testcases_directory: str) -> None: """Unpacks the old crash testcases in the provided directory.""" count = 0 # Make sure that it is a unique crash testcase. Ignore duplicates, @@ -80,11 +82,12 @@ def unpack_crash_testcases(crash_testcases_directory): # Existing IPC testcases are un-interesting and unused in further # mutations. Due to size bloat, ignoring these for now. - if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION): + if (testcase.absolute_path or + '').endswith(testcase_manager.IPCDUMP_EXTENSION): continue # Ignore testcases that are archives (e.g. Langfuzz fuzzer tests). - if archive.get_archive_type(testcase.absolute_path): + if archive.get_archive_type(testcase.absolute_path or ''): continue # Skip in-process fuzzer testcases, since these are only applicable to @@ -129,7 +132,8 @@ def unpack_crash_testcases(crash_testcases_directory): shell.remove_empty_directories(crash_testcases_directory) -def clone_git_repository(tests_directory, name, repo_url): +def clone_git_repository(tests_directory: str, name: str, + repo_url: str) -> None: """Clone a git repo.""" logs.info('Syncing %s tests.' % name) @@ -144,8 +148,8 @@ def clone_git_repository(tests_directory, name, repo_url): raise RuntimeError(f'Unable to checkout {name} tests.') -def create_symbolic_link(tests_directory, source_subdirectory, - target_subdirectory): +def create_symbolic_link(tests_directory: str, source_subdirectory: str, + target_subdirectory: str) -> None: """Create symbolic link.""" source_directory = os.path.join(tests_directory, source_subdirectory) target_directory = os.path.join(tests_directory, target_subdirectory) @@ -165,8 +169,9 @@ def create_symbolic_link(tests_directory, source_subdirectory, subprocess.check_call(['ln', '-s', source_directory, target_directory]) -def create_gecko_tests_directory(tests_directory, gecko_checkout_subdirectory, - gecko_tests_subdirectory): +def create_gecko_tests_directory(tests_directory: str, + gecko_checkout_subdirectory: str, + gecko_tests_subdirectory: str) -> None: """Create Gecko tests directory from a Gecko source checkout using links.""" gecko_checkout_directory = os.path.join(tests_directory, gecko_checkout_subdirectory) @@ -194,7 +199,7 @@ def create_gecko_tests_directory(tests_directory, gecko_checkout_subdirectory, target_subdirectory) -def create_fuzzilli_tests_directory(tests_directory): +def create_fuzzilli_tests_directory(tests_directory: str) -> None: """Create Fuzzilli tests directory from the autozilli GCS archives.""" logs.info('Syncing fuzzilli tests.') fuzzilli_tests_directory = os.path.join(tests_directory, 'fuzzilli') @@ -210,10 +215,12 @@ def create_fuzzilli_tests_directory(tests_directory): process_fuzzilli_archive(fuzzilli_tests_directory, f'diff-fuzz-{i}') -def process_fuzzilli_archive(fuzzilli_tests_directory, archive_suffix): +def process_fuzzilli_archive(fuzzilli_tests_directory: str, + archive_suffix: str | int) -> None: """Download, extract and copy one archive from Fuzzilli.""" - def filter_members(member, path): + def filter_members(member: tarfile.TarInfo, + path: str) -> tarfile.TarInfo | None: # We only need JS files and the settings.json from the archive. if member.name.endswith('fzil') or member.name.startswith('fuzzdir/stats'): return None @@ -236,7 +243,7 @@ def filter_members(member, path): shell.remove_file(local_archive) -def rename_testcase_files(directory): +def rename_testcase_files(directory: str) -> None: """Rename files with the 'fuzz-' prefix to avoid picking them up for fuzzing without modifying them with a fuzzer. """ @@ -257,7 +264,7 @@ def rename_testcase_files(directory): raise RuntimeError(f'Failed to rename testcase {file_path}') from e -def clean_up_filenames(tests_directory, test_folders): +def clean_up_filenames(tests_directory: str, test_folders: list[str]) -> None: """Rename files with the 'fuzz-' prefix in all test folders.""" for folder in test_folders: logs.info(f'Renaming testcase files in {folder}') @@ -265,7 +272,7 @@ def clean_up_filenames(tests_directory, test_folders): def sync_tests(tests_archive_bucket: str, tests_archive_name: str, - tests_directory: str): + tests_directory: str) -> None: """Main sync routine.""" shell.create_directory(tests_directory) @@ -356,7 +363,7 @@ def sync_tests(tests_archive_bucket: str, tests_archive_name: str, monitoring_metrics.CHROME_TEST_SYNCER_SUCCESS.increment() -def main(): +def main() -> bool: # Make sure environment is correctly configured. logs.configure('run_bot') environment.set_bot_environment() diff --git a/src/clusterfuzz/_internal/cron/cleanup.py b/src/clusterfuzz/_internal/cron/cleanup.py index 13a054559b9..d001de7f862 100644 --- a/src/clusterfuzz/_internal/cron/cleanup.py +++ b/src/clusterfuzz/_internal/cron/cleanup.py @@ -13,9 +13,10 @@ # limitations under the License. """Cleanup task for cleaning up unneeded testcases.""" -import collections import datetime import json +from typing import Any +from typing import NamedTuple from googleapiclient.errors import HttpError @@ -60,10 +61,18 @@ VRP_UPLOAD_COMPONENT_ID = 1600865 CHROMIUM_COMPONENT_ID = 1363614 -ProjectMap = collections.namedtuple('ProjectMap', 'jobs platforms') +class ProjectMap(NamedTuple): + jobs: set[str] + platforms: set[str] -def _get_predator_result_item(testcase, key, default=None): + +TopCrashesMap = dict[str, dict[str, list[dict[str, Any]]]] + + +def _get_predator_result_item(testcase: data_types.Testcase, + key: str, + default: Any = None) -> Any: """Return the suspected components for a test case.""" predator_result = testcase.get_metadata('predator_result') if not predator_result: @@ -72,7 +81,9 @@ def _get_predator_result_item(testcase, key, default=None): return predator_result['result'].get(key, default) -def _append_generic_incorrect_comment(comment, policy, issue, suffix): +def _append_generic_incorrect_comment( + comment: str, policy: issue_tracker_policy.IssueTrackerPolicy, issue: Any, + suffix: str) -> str: """Get the generic incorrect comment.""" wrong_label = policy.label('wrong') if not wrong_label: @@ -82,7 +93,8 @@ def _append_generic_incorrect_comment(comment, policy, issue, suffix): label_text=issue.issue_tracker.label_text(wrong_label)) + suffix -def _emit_issue_closing_event(testcase, issue, closing_reason): +def _emit_issue_closing_event(testcase: data_types.Testcase, issue: Any, + closing_reason: str) -> None: """Helper function to emit issue closing events.""" issue_tracker_name = data_handler.get_issue_tracker_name(testcase.job_type) events.emit( @@ -93,7 +105,7 @@ def _emit_issue_closing_event(testcase, issue, closing_reason): closing_reason=closing_reason)) -def job_platform_to_real_platform(job_platform): +def job_platform_to_real_platform(job_platform: str) -> str: """Get real platform from job platform.""" for platform in data_types.PLATFORMS: if platform in job_platform: @@ -102,7 +114,7 @@ def job_platform_to_real_platform(job_platform): raise ValueError('Unknown platform: ' + job_platform) -def cleanup_reports_metadata(): +def cleanup_reports_metadata() -> None: """Delete ReportMetadata for uploaded reports.""" uploaded_reports = ndb_utils.get_all_from_query( data_types.ReportMetadata.query( @@ -111,9 +123,11 @@ def cleanup_reports_metadata(): ndb_utils.delete_multi(uploaded_reports) -def _cleanup_testcases_and_issues(testcase, jobs, - top_crashes_by_project_and_platform_map, - empty_issue_tracker_policy): +def _cleanup_testcases_and_issues( + testcase: data_types.Testcase, jobs: list[str], + top_crashes_by_project_and_platform_map: TopCrashesMap, + empty_issue_tracker_policy: issue_tracker_policy.IssueTrackerPolicy +) -> None: """Clean up unneeded open testcase and its associated issues.""" testcase_id = testcase.key.id() logs.info(f'Processing testcase {testcase_id}.') @@ -165,7 +179,7 @@ def _cleanup_testcases_and_issues(testcase, jobs, logs.error(f'Failed to process testcase {testcase_id}.') -def cleanup_testcases_and_issues(): +def cleanup_testcases_and_issues() -> None: """Clean up unneeded open testcases and their associated issues.""" logs.info('Getting all job type names.') jobs = data_handler.get_all_job_type_names() @@ -200,7 +214,7 @@ def cleanup_testcases_and_issues(): utils.python_gc() -def cleanup_unused_fuzz_targets_and_jobs(): +def cleanup_unused_fuzz_targets_and_jobs() -> None: """Clean up unused FuzzTarget and FuzzTargetJob entities.""" last_run_cutoff = utils.utcnow() - datetime.timedelta( days=FUZZ_TARGET_UNUSED_THRESHOLD) @@ -232,12 +246,15 @@ def cleanup_unused_fuzz_targets_and_jobs(): f'{len(valid_target_jobs)} valid FuzzTargetJob entities remain.') -def get_jobs_and_platforms_for_project(): +def get_jobs_and_platforms_for_project() -> dict[str, ProjectMap]: """Return a map of projects to jobs and platforms map to use for picking top crashes.""" all_jobs = ndb_utils.get_all_from_model(data_types.Job) - projects_to_jobs_and_platforms = {} + projects_to_jobs_and_platforms: dict[str, ProjectMap] = {} for job in all_jobs: + if not job.project or not job.name or not job.platform: + continue + job_environment = job.get_environment() # Skip experimental jobs. @@ -265,7 +282,8 @@ def get_jobs_and_platforms_for_project(): @memoize.wrap(memoize.Memcache(12 * 60 * 60)) def _get_crash_occurrence_platforms_from_crash_parameters( - crash_type, crash_state, security_flag, project_name, lookbehind_days): + crash_type: str | None, crash_state: str | None, security_flag: bool | None, + project_name: str | None, lookbehind_days: int) -> list[str]: """Get platforms from crash stats based on crash parameters.""" last_hour = crash_stats.get_last_successful_hour() if not last_hour: @@ -296,7 +314,8 @@ def _get_crash_occurrence_platforms_from_crash_parameters( return list(platforms) -def get_platforms_from_testcase_variants(testcase): +def get_platforms_from_testcase_variants( + testcase: data_types.Testcase) -> set[str]: """Get platforms from crash stats based on crash parameters.""" variant_query = data_types.TestcaseVariant.query( data_types.TestcaseVariant.testcase_id == testcase.key.id()) @@ -308,7 +327,8 @@ def get_platforms_from_testcase_variants(testcase): return platforms -def get_crash_occurrence_platforms(testcase, lookbehind_days=1): +def get_crash_occurrence_platforms(testcase: data_types.Testcase, + lookbehind_days: int = 1) -> set[str]: """Get platforms from crash stats for a testcase.""" return set( _get_crash_occurrence_platforms_from_crash_parameters( @@ -316,7 +336,8 @@ def get_crash_occurrence_platforms(testcase, lookbehind_days=1): testcase.project_name, lookbehind_days)) -def get_top_crashes_for_all_projects_and_platforms(limit=TOP_CRASHES_LIMIT): +def get_top_crashes_for_all_projects_and_platforms( + limit: int = TOP_CRASHES_LIMIT) -> TopCrashesMap: """Return top crashes for all projects and platforms.""" last_hour = crash_stats.get_last_successful_hour() if not last_hour: @@ -324,7 +345,7 @@ def get_top_crashes_for_all_projects_and_platforms(limit=TOP_CRASHES_LIMIT): return {} projects_to_jobs_and_platforms = get_jobs_and_platforms_for_project() - top_crashes_by_project_and_platform_map = {} + top_crashes_by_project_and_platform_map: TopCrashesMap = {} for project_name, project_map in projects_to_jobs_and_platforms.items(): top_crashes_by_project_and_platform_map[project_name] = {} @@ -362,7 +383,9 @@ def get_top_crashes_for_all_projects_and_platforms(limit=TOP_CRASHES_LIMIT): return top_crashes_by_project_and_platform_map -def get_top_crash_platforms(testcase, top_crashes_by_project_and_platform_map): +def get_top_crash_platforms( + testcase: data_types.Testcase, + top_crashes_by_project_and_platform_map: TopCrashesMap) -> list[str]: """Return list of platforms where this testcase is a top crasher.""" if testcase.project_name not in top_crashes_by_project_and_platform_map: return [] @@ -388,7 +411,8 @@ def get_top_crash_platforms(testcase, top_crashes_by_project_and_platform_map): return sorted(list(top_crash_platforms)) -def delete_unreproducible_testcase_with_no_issue(testcase): +def delete_unreproducible_testcase_with_no_issue( + testcase: data_types.Testcase) -> None: """Delete an unreproducible testcase if it has no associated issue and has been open for a certain time interval.""" # Make sure that this testcase is an unreproducible bug. If not, bail out. @@ -423,7 +447,8 @@ def delete_unreproducible_testcase_with_no_issue(testcase): CLEANUP_UNREPRODUCIBLE_NO_ISSUE)) -def mark_duplicate_testcase_as_closed_with_no_issue(testcase): +def mark_duplicate_testcase_as_closed_with_no_issue( + testcase: data_types.Testcase) -> None: """Closes a duplicate testcase if it has no associated issue and has been open for a certain time interval.""" # Make sure that this testcase is a duplicate bug. If not, bail out. @@ -450,7 +475,9 @@ def mark_duplicate_testcase_as_closed_with_no_issue(testcase): rejection_reason=events.RejectionReason.CLEANUP_DUPLICATE_NO_ISSUE)) -def mark_issue_as_closed_if_testcase_is_fixed(policy, testcase, issue): +def mark_issue_as_closed_if_testcase_is_fixed( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Mark an issue as fixed if all of its associated reproducible testcase are fixed.""" verified_label = policy.label('verified') @@ -552,7 +579,8 @@ def mark_issue_as_closed_if_testcase_is_fixed(policy, testcase, issue): raise e -def mark_unreproducible_testcase_as_fixed_if_issue_is_closed(testcase, issue): +def mark_unreproducible_testcase_as_fixed_if_issue_is_closed( + testcase: data_types.Testcase, issue: Any) -> None: """Mark an unreproducible testcase as fixed if the associated issue is closed.""" # If the testcase is already closed, no more work to do. @@ -584,7 +612,8 @@ def mark_unreproducible_testcase_as_fixed_if_issue_is_closed(testcase, issue): def mark_unreproducible_testcase_and_issue_as_closed_after_deadline( - policy, testcase, issue): + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Closes an unreproducible testcase and its associated issue after a certain time period.""" # If the testcase is already closed, no more work to do. @@ -681,7 +710,9 @@ def mark_unreproducible_testcase_and_issue_as_closed_after_deadline( events.ClosingReason.TESTCASE_UNREPRO) -def mark_na_testcase_issues_as_wontfix(policy, testcase, issue): +def mark_na_testcase_issues_as_wontfix( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Mark issues for testcases with fixed == 'NA' as fixed.""" # Check for for closed, NA testcases. if testcase.open or testcase.fixed != 'NA': @@ -728,7 +759,8 @@ def mark_na_testcase_issues_as_wontfix(policy, testcase, issue): events.ClosingReason.TESTCASE_INVALID) -def mark_testcase_as_triaged_if_needed(testcase, issue): +def mark_testcase_as_triaged_if_needed(testcase: data_types.Testcase, + issue: Any) -> None: """Mark testcase as triage complete if both testcase and associated issue are closed.""" # Check if testcase is open. If yes, bail out. @@ -739,14 +771,16 @@ def mark_testcase_as_triaged_if_needed(testcase, issue): if issue: # Get latest issue object to ensure our update went through. issue = issue_tracker_utils.get_issue_for_testcase(testcase) - if issue.is_open: + if issue and issue.is_open: return testcase.triaged = True testcase.put() -def mark_testcase_as_closed_if_issue_is_closed(policy, testcase, issue): +def mark_testcase_as_closed_if_issue_is_closed( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Mark testcase as closed if the associated issue is closed.""" # If the testcase is already closed, no more work to do. if not testcase.open: @@ -781,7 +815,8 @@ def mark_testcase_as_closed_if_issue_is_closed(policy, testcase, issue): rejection_reason=events.RejectionReason.CLEANUP_ISSUE_CLOSED)) -def mark_testcase_as_closed_if_job_is_invalid(testcase, jobs): +def mark_testcase_as_closed_if_job_is_invalid(testcase: data_types.Testcase, + jobs: list[str]) -> None: """Mark testcase as closed if the associated job type does not exist.""" # If the testcase is already closed, no more work to do. if not testcase.open: @@ -801,7 +836,9 @@ def mark_testcase_as_closed_if_job_is_invalid(testcase, jobs): rejection_reason=events.RejectionReason.CLEANUP_INVALID_JOB)) -def notify_closed_issue_if_testcase_is_open(policy, testcase, issue): +def notify_closed_issue_if_testcase_is_open( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Notify closed issue if associated testcase is still open after a certain time period.""" needs_feedback_label = policy.label('needs_feedback') @@ -875,7 +912,9 @@ def notify_closed_issue_if_testcase_is_open(policy, testcase, issue): logs.info(f'Notified closed issue for open testcase {testcase.key.id()}.') -def notify_issue_if_testcase_is_invalid(policy, testcase, issue): +def notify_issue_if_testcase_is_invalid( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Leave comments on associated issues when test cases are no longer valid.""" invalid_fuzzer_label = policy.label('invalid_fuzzer') if not invalid_fuzzer_label: @@ -909,7 +948,8 @@ def notify_issue_if_testcase_is_invalid(policy, testcase, issue): f'invalid testcase {testcase.key.id()}.') -def _send_email_to_uploader(testcase_id, to_email, content): +def _send_email_to_uploader(testcase_id: int | str, to_email: str, + content: str) -> None: """Send email to uploader when all the testcase tasks are finished.""" subject = f'Your testcase upload {testcase_id} analysis is complete.' content_with_footer = (f'{content.strip()}\n\n' @@ -920,7 +960,7 @@ def _send_email_to_uploader(testcase_id, to_email, content): mail.send(to_email, subject, html_content) -def _get_severity_from_labels(security_severity_label, labels): +def _get_severity_from_labels(security_severity_label: str, labels: Any) -> Any: """Get the severity from the label list.""" pattern = issue_filer.get_label_pattern(security_severity_label) for label in labels: @@ -931,7 +971,9 @@ def _get_severity_from_labels(security_severity_label, labels): return data_types.SecuritySeverity.MISSING -def _update_issue_security_severity_and_get_comment(policy, testcase, issue): +def _update_issue_security_severity_and_get_comment( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> str: """Apply a new security severity label if none exists on issue already and return a comment on this addition. If a label already exists and does not match security severity label on issue, then just return a comment on @@ -966,7 +1008,9 @@ def _update_issue_security_severity_and_get_comment(policy, testcase, issue): def _update_issue_when_uploaded_testcase_is_processed( - policy, testcase, issue, description, update_bug_summary, notify): + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any, description: str, + update_bug_summary: bool | None, notify: bool) -> None: """Add issue comment when uploaded testcase is processed.""" if update_bug_summary and testcase.is_crash(): issue.title = data_handler.get_issue_summary(testcase) @@ -982,13 +1026,19 @@ def _update_issue_when_uploaded_testcase_is_processed( # Testcase is a data_types.Testcase testcase_id = testcase.key.id() + if not testcase_id: + return testcase_utils.emit_testcase_triage_duration_metric( testcase_id, testcase_utils.TESTCASE_TRIAGE_DURATION_ISSUE_UPDATED_STEP) -def notify_uploader_when_testcase_is_processed(policy, testcase, issue): +def notify_uploader_when_testcase_is_processed( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Notify uploader by email when all the testcase tasks are finished.""" testcase_id = testcase.key.id() + if not testcase_id: + return # Check if this is a user upload. If not, bail out. upload_metadata = data_types.TestcaseUploadMetadata.query( @@ -1033,7 +1083,8 @@ def notify_uploader_when_testcase_is_processed(policy, testcase, issue): data_handler.create_notification_entry(testcase_id, to_email) -def update_os_labels(policy, testcase, issue): +def update_os_labels(policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Add OS labels to issue.""" os_label = policy.label('os') if not os_label: @@ -1056,8 +1107,10 @@ def update_os_labels(policy, testcase, issue): logs.info(f'Updated labels of issue {issue.id}.', labels=issue.labels) -def update_fuzz_blocker_label(policy, testcase, issue, - top_crashes_by_project_and_platform_map): +def update_fuzz_blocker_label( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any, + top_crashes_by_project_and_platform_map: TopCrashesMap) -> None: """Add top crash label to issue.""" fuzz_blocker_label = policy.label('fuzz_blocker') if not fuzz_blocker_label: @@ -1100,15 +1153,18 @@ def update_fuzz_blocker_label(policy, testcase, issue, issue.save(new_comment=update_message, notify=True) -def _should_update_component_id(issue, component_id): +def _should_update_component_id(issue: Any, + component_id: int | str | None) -> bool: """Check whether updating the issue component id is needed.""" if not hasattr(issue, 'component_id'): return False - return component_id and issue.component_id != component_id + return bool(component_id and issue.component_id != component_id) -def update_component_labels_and_id(policy, testcase, issue): +def update_component_labels_and_id( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Add components to the issue if needed.""" if not issue: return @@ -1187,7 +1243,7 @@ def update_component_labels_and_id(policy, testcase, issue): issue.save(new_comment=issue_comment, notify=True) -def _sanitize_ccs_list(ccs_list): +def _sanitize_ccs_list(ccs_list: list[str]) -> list[str]: """Remove and log all entries with trailing comments. Eg: Do not add "xyz@test.com #{LAST_RESORT_SUGGESTION}". @@ -1201,7 +1257,9 @@ def _sanitize_ccs_list(ccs_list): return ret_list -def update_issue_ccs_from_owners_file(policy, testcase, issue): +def update_issue_ccs_from_owners_file( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Add cc to an issue based on owners list from owners file. This is currently applicable to fuzz targets only.""" auto_cc_label = policy.label('auto_cc_from_owners') @@ -1269,7 +1327,9 @@ def update_issue_ccs_from_owners_file(policy, testcase, issue): issue.save(new_comment=issue_comment, notify=True) -def update_issue_labels_for_flaky_testcase(policy, testcase, issue): +def update_issue_labels_for_flaky_testcase( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, issue: Any) -> None: """Update issue reproducibility label when testcase becomes flaky or unreproducible.""" if not issue or not issue.is_open: @@ -1304,10 +1364,11 @@ def update_issue_labels_for_flaky_testcase(policy, testcase, issue): issue.save(new_comment=comment) -def update_issue_owner_and_ccs_from_predator_results(policy, - testcase, - issue, - only_allow_ccs=False): +def update_issue_owner_and_ccs_from_predator_results( + policy: issue_tracker_policy.IssueTrackerPolicy, + testcase: data_types.Testcase, + issue: Any, + only_allow_ccs: bool = False) -> None: """Assign the issue to an appropriate owner if possible.""" logs.info(f'{update_issue_owner_and_ccs_from_predator_results}') if not issue or not issue.is_open: @@ -1454,7 +1515,7 @@ def update_issue_owner_and_ccs_from_predator_results(policy, policy, testcase, issue, only_allow_ccs=True) -def cleanup_unused_heartbeats(): +def cleanup_unused_heartbeats() -> None: """Clean up unused heartbeat entities.""" cutoff_time = utils.utcnow() - datetime.timedelta( days=UNUSED_HEARTBEAT_THRESHOLD) @@ -1467,7 +1528,7 @@ def cleanup_unused_heartbeats(): @logs.cron_log_context() -def main(): +def main() -> bool: """Cleaning up unneeded testcases""" cleanup_testcases_and_issues() cleanup_reports_metadata() diff --git a/src/clusterfuzz/_internal/cron/corpus_backup.py b/src/clusterfuzz/_internal/cron/corpus_backup.py index 270dc4fea10..1efaee9649f 100644 --- a/src/clusterfuzz/_internal/cron/corpus_backup.py +++ b/src/clusterfuzz/_internal/cron/corpus_backup.py @@ -15,6 +15,7 @@ import datetime import os +from typing import cast from clusterfuzz._internal.base import utils from clusterfuzz._internal.datastore import data_types @@ -25,7 +26,7 @@ from clusterfuzz._internal.metrics import logs -def _set_public_acl_if_needed(url): +def _set_public_acl_if_needed(url: str) -> bool: """Sets public ACL on the object with given URL, if it's not public yet.""" if storage.get_acl(url, 'allUsers'): logs.info('%s is already marked public, skipping.' % url) @@ -38,8 +39,9 @@ def _set_public_acl_if_needed(url): return True -def _make_corpus_backup_public(target, corpus_fuzzer_name_override, - corpus_backup_bucket_name): +def _make_corpus_backup_public(target: data_types.FuzzTarget, + corpus_fuzzer_name_override: str | None, + corpus_backup_bucket_name: str) -> None: """Identifies old corpus backups and makes them public.""" corpus_backup_date = utils.utcnow().date() - datetime.timedelta( days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS) @@ -47,6 +49,7 @@ def _make_corpus_backup_public(target, corpus_fuzzer_name_override, corpus_backup_url = corpus_manager.gcs_url_for_backup_file( corpus_backup_bucket_name, corpus_fuzzer_name_override or target.engine, target.project_qualified_name(), corpus_backup_date) + corpus_backup_url = cast(str, corpus_backup_url) if not storage.get(corpus_backup_url): logs.warning('Failed to find corpus backup %s.' % corpus_backup_url) @@ -71,7 +74,7 @@ def _make_corpus_backup_public(target, corpus_fuzzer_name_override, logs.info('Corpus backup %s is now marked public.' % corpus_backup_url) -def main(): +def main() -> bool: """Makes corpuses older than 90 days public.""" jobs = ndb_utils.get_all_from_model(data_types.Job) default_backup_bucket = utils.default_backup_bucket() diff --git a/src/clusterfuzz/_internal/cron/external_testcase_reader.py b/src/clusterfuzz/_internal/cron/external_testcase_reader.py index 4a23ac5a3cf..fd94435ded3 100644 --- a/src/clusterfuzz/_internal/cron/external_testcase_reader.py +++ b/src/clusterfuzz/_internal/cron/external_testcase_reader.py @@ -15,6 +15,8 @@ import datetime import re +from typing import Any +from typing import cast from google.cloud import storage import requests @@ -26,14 +28,14 @@ from clusterfuzz._internal.issue_management.google_issue_tracker import \ issue_tracker -ACCEPTED_FILETYPES = [ +ACCEPTED_FILETYPES: list[str] = [ 'text/javascript', 'application/pdf', 'text/html', 'application/zip' ] -ISSUETRACKER_ACCEPTED_STATE = 'ACCEPTED' -ISSUETRACKER_WONTFIX_STATE = 'NOT_REPRODUCIBLE' +ISSUETRACKER_ACCEPTED_STATE: str = 'ACCEPTED' +ISSUETRACKER_WONTFIX_STATE: str = 'NOT_REPRODUCIBLE' -def get_vrp_uploaders(config): +def get_vrp_uploaders(config: Any) -> list[str]: """Checks whether the given reporter has permission to upload.""" storage_client = storage.Client() bucket = storage_client.bucket(config.get('vrp-uploaders-bucket')) @@ -42,8 +44,9 @@ def get_vrp_uploaders(config): return members -def close_issue_if_invalid(upload_request, attachment_info, description, - vrp_uploaders): +def close_issue_if_invalid(upload_request: issue_tracker.Issue, + attachment_info: Any, description: str | None, + vrp_uploaders: list[str]) -> bool: """Closes any invalid upload requests with a helpful message.""" comment_message = ( 'Hello, this issue is automatically closed. Please file a new bug after' @@ -96,9 +99,10 @@ def close_issue_if_invalid(upload_request, attachment_info, description, return invalid -def close_issue_if_not_reproducible(issue, config): +def close_issue_if_not_reproducible(issue: issue_tracker.Issue, + config: Any) -> bool: if issue.status == ISSUETRACKER_ACCEPTED_STATE and filed_n_days_ago( - issue.created_time, config): + cast(str, issue.created_time), config): comment_message = ('Clusterfuzz failed to reproduce - ' 'please check testcase details for more info.') issue.status = ISSUETRACKER_WONTFIX_STATE @@ -107,14 +111,15 @@ def close_issue_if_not_reproducible(issue, config): return False -def filed_n_days_ago(issue_created_time_string, config): +def filed_n_days_ago(issue_created_time_string: str, config: Any) -> bool: created_time = datetime.datetime.strptime(issue_created_time_string, '%Y-%m-%dT%H:%M:%S.%fZ') return datetime.datetime.now() - created_time > datetime.timedelta( days=config.get('submitted-buffer-days')) -def submit_testcase(issue_id, file, filename, filetype, cmds): +def submit_testcase(issue_id: int | str, file: bytes | str, filename: str, + filetype: str, cmds: str | None) -> requests.Response: """Uploads the given testcase file to Clusterfuzz.""" if filetype == 'text/javascript': job = 'linux_asan_d8_dbg' @@ -152,7 +157,7 @@ def submit_testcase(issue_id, file, filename, filetype, cmds): 'https://clusterfuzz.com/upload-testcase/upload', data=data, timeout=10) -def handle_testcases(tracker, config): +def handle_testcases(tracker: issue_tracker.IssueTracker, config: Any) -> None: """Fetches and submits testcases from bugs or closes unnecessary bugs.""" # Handle bugs that were already submitted and still open. @@ -171,32 +176,35 @@ def handle_testcases(tracker, config): keywords=[], query_filters=['componentid:1600865', 'status:new'], only_open=True) - if len(issues) == 0: + if len(issues) == 0: # type: ignore return vrp_uploaders = get_vrp_uploaders(config) # Rudimentary rate limiting - # Process only a certain number of bugs per reporter for each job run. - reporters_map = {} + reporters_map: dict[str, int] = {} for issue in issues: attachment_metadata = tracker.get_attachment_metadata(issue.id) commandline_flags = tracker.get_description(issue.id) - if reporters_map.get(issue.reporter, + if reporters_map.get(cast(str, issue.reporter), 0) > config.get('max-report-count-per-run'): continue - reporters_map[issue.reporter] = reporters_map.get(issue.reporter, 1) + 1 + reporters_map[cast( + str, + issue.reporter)] = reporters_map.get(cast(str, issue.reporter), 1) + 1 if close_issue_if_invalid(issue, attachment_metadata, commandline_flags, vrp_uploaders): helpers.log('Closing issue {issue_id} as it is invalid', issue.id) continue # Submit valid testcases. - attachment_metadata = attachment_metadata[0] + attachment_metadata = attachment_metadata[0] # type: ignore attachment = tracker.get_attachment( attachment_metadata['attachmentDataRef']['resourceName']) - submit_testcase(issue.id, attachment, attachment_metadata['filename'], + submit_testcase(issue.id, cast(bytes | str, + attachment), attachment_metadata['filename'], attachment_metadata['contentType'], commandline_flags) comment_message = 'Testcase submitted to clusterfuzz' issue.status = ISSUETRACKER_ACCEPTED_STATE @@ -205,7 +213,7 @@ def handle_testcases(tracker, config): helpers.log('Submitted testcase file for issue {issue_id}', issue.id) -def main(): +def main() -> None: tracker = issue_tracker.IssueTracker('chromium', None, {'default_component_id': 1363614}) handle_testcases(tracker, local_config.ExternalTestcaseReaderConfig()) diff --git a/src/clusterfuzz/_internal/cron/fuzz_strategy_selection.py b/src/clusterfuzz/_internal/cron/fuzz_strategy_selection.py index dec1d278843..d34eea6745c 100644 --- a/src/clusterfuzz/_internal/cron/fuzz_strategy_selection.py +++ b/src/clusterfuzz/_internal/cron/fuzz_strategy_selection.py @@ -21,7 +21,8 @@ metric to be for edges, crash, features, or units. Currently based on new edges.""" -from collections import namedtuple +from typing import Any +from typing import NamedTuple from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.datastore import ndb_utils @@ -36,19 +37,24 @@ # Maintain a list of strategies to include in query for each fuzzing engine. # Keep this strategy order for strategy combination tracking as strategy # combinations are tracked as strings. -libfuzzer_query_strategy_list = [ +libfuzzer_query_strategy_list: list[strategy.Strategy] = [ strategy_tuple for strategy_tuple in strategy.LIBFUZZER_STRATEGY_LIST if not strategy_tuple.manually_enable ] -afl_query_strategy_list = [ +afl_query_strategy_list: list[strategy.Strategy] = [ strategy_tuple for strategy_tuple in strategy.AFL_STRATEGY_LIST if not strategy_tuple.manually_enable ] + # A tuple of engine name and corresponding strategies to include in multi-armed # bandit query. -Engine = namedtuple('Engine', 'name query_strategy_list performance_metric') +class Engine(NamedTuple): + name: str + query_strategy_list: list[strategy.Strategy] + performance_metric: str + LIBFUZZER_ENGINE = Engine( name='libFuzzer', @@ -125,7 +131,8 @@ """ -def _query_multi_armed_bandit_probabilities(engine): +def _query_multi_armed_bandit_probabilities( + engine: Engine) -> list[dict[str, Any]]: """Get query results. Queries above BANDIT_PROBABILITY_QUERY and yields results @@ -149,7 +156,8 @@ def _query_multi_armed_bandit_probabilities(engine): return client.query(query=formatted_query).rows -def _store_probabilities_in_bigquery(engine, data): +def _store_probabilities_in_bigquery(engine: Engine, + data: list[dict[str, Any]]) -> None: """Update a bigquery table containing the daily updated probability distribution over strategies.""" bigquery_data = [] @@ -173,7 +181,7 @@ def _store_probabilities_in_bigquery(engine, data): 'BigQuery.') -def _query_and_upload_strategy_probabilities(engine): +def _query_and_upload_strategy_probabilities(engine: Engine) -> None: """Uploads queried data into datastore. Calls query functions and uploads query results @@ -203,7 +211,7 @@ def _query_and_upload_strategy_probabilities(engine): engine.name)) -def main(): +def main() -> bool: """Periodically update fuzz strategy bandit probabilities based on a performance metric (currently based on new_edges).""" for engine in ENGINE_LIST: diff --git a/src/clusterfuzz/_internal/cron/fuzzer_and_job_weights.py b/src/clusterfuzz/_internal/cron/fuzzer_and_job_weights.py index dd4cbea9f4e..0412a8ead58 100644 --- a/src/clusterfuzz/_internal/cron/fuzzer_and_job_weights.py +++ b/src/clusterfuzz/_internal/cron/fuzzer_and_job_weights.py @@ -13,8 +13,15 @@ # limitations under the License. """Manage automatic weight adjustments.""" -import collections import datetime +from typing import Any +from typing import Callable +from typing import cast +from typing import Dict +from typing import List +from typing import NamedTuple +from typing import Set +from typing import Tuple from google.cloud import ndb @@ -27,20 +34,27 @@ from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -QuerySpecification = collections.namedtuple( - 'QuerySpecification', ['query_format', 'formatter', 'reason']) -SpecificationMatch = collections.namedtuple('SpecificationMatch', - ['new_weight', 'reason']) +class QuerySpecification(NamedTuple): + query_format: str + formatter: Callable[[str, str], str] + reason: str -DEFAULT_MULTIPLIER = 30.0 # Used for blackbox and jobs that are not yet run. -DEFAULT_SANITIZER_WEIGHT = 0.1 -DEFAULT_ENGINE_WEIGHT = 1.0 -SANITIZER_BASE_WEIGHT = 0.1 +class SpecificationMatch(NamedTuple): + new_weight: float + reason: str + + +# Used for blackbox and jobs that are not yet run. +DEFAULT_MULTIPLIER: float = 30.0 +DEFAULT_SANITIZER_WEIGHT: float = 0.1 +DEFAULT_ENGINE_WEIGHT: float = 1.0 + +SANITIZER_BASE_WEIGHT: float = 0.1 # TODO(ochang): architecture weights. -SANITIZER_WEIGHTS = { +SANITIZER_WEIGHTS: Dict[str, float] = { 'ASAN': 5 * SANITIZER_BASE_WEIGHT, 'CFI': 1 * SANITIZER_BASE_WEIGHT, 'MSAN': 2 * SANITIZER_BASE_WEIGHT, @@ -48,7 +62,7 @@ 'UBSAN': 1 * SANITIZER_BASE_WEIGHT, } -ENGINE_WEIGHTS = { +ENGINE_WEIGHTS: Dict[str, float] = { 'afl': 1.0, 'centipede': 1.0, 'googlefuzztest': 0.5, @@ -58,7 +72,7 @@ # Formatters for query specifications. -def _past_day_formatter(query_format, dataset): +def _past_day_formatter(query_format: str, dataset: str) -> str: """Simple formatter to get stats for the past day.""" end_time = utils.utcnow().date() start_time = end_time - datetime.timedelta(days=1) @@ -66,14 +80,14 @@ def _past_day_formatter(query_format, dataset): dataset=dataset, start_time=start_time, end_time=end_time) -def _new_fuzzer_formatter(query_format, dataset): +def _new_fuzzer_formatter(query_format: str, dataset: str) -> str: """Prepare a query to check for new fuzzers from the past week.""" now = utils.utcnow().date() cutoff_time = now - datetime.timedelta(days=7) return query_format.format(dataset=dataset, cutoff_time=cutoff_time) -def _coverage_formatter(query_format, dataset): +def _coverage_formatter(query_format: str, dataset: str) -> str: """Prepare a query to check for changes in coverage week over week.""" end_date = utils.utcnow().date() - datetime.timedelta(days=1) middle_date = end_date - datetime.timedelta(days=7) @@ -87,7 +101,7 @@ def _coverage_formatter(query_format, dataset): # Most of our queries should simply average a field name to get a ratio showing # how often some behavior occurs. -GENERIC_QUERY_FORMAT = """ +GENERIC_QUERY_FORMAT: str = """ SELECT fuzzer, job, @@ -150,7 +164,7 @@ def _coverage_formatter(query_format, dataset): # New fuzzers/jobs should run much more frequently than others. In this case, we # test the fraction of days for which we have no stats for this fuzzer/job pair # and increase if it's nonzero. -NEW_FUZZER_FORMAT = """ +NEW_FUZZER_FORMAT: str = """ SELECT fuzzer, job, @@ -171,7 +185,7 @@ def _coverage_formatter(query_format, dataset): reason='new fuzzer') # Format to query for fuzzers with minimal change in week to week coverage. -COVERAGE_UNCHANGED_FORMAT = """ +COVERAGE_UNCHANGED_FORMAT: str = """ SELECT recent.fuzzer AS fuzzer, recent.job AS job, @@ -223,7 +237,7 @@ def _coverage_formatter(query_format, dataset): reason='coverage flat over past 2 weeks') # Mappings for which specifications to use for which -LIBFUZZER_SPECIFICATIONS = [ +LIBFUZZER_SPECIFICATIONS: List[QuerySpecification] = [ COVERAGE_UNCHANGED_SPECIFICATION, CRASH_SPECIFICATION, NEW_FUZZER_SPECIFICATION, @@ -233,7 +247,7 @@ def _coverage_formatter(query_format, dataset): TIMEOUT_SPECIFICATION, ] -AFL_SPECIFICATIONS = [ +AFL_SPECIFICATIONS: List[QuerySpecification] = [ CRASH_SPECIFICATION, NEW_FUZZER_SPECIFICATION, STARTUP_CRASH_SPECIFICATION, @@ -243,12 +257,13 @@ def _coverage_formatter(query_format, dataset): new_weight=1.0, reason='no longer matches any weight adjustment rules') -def _query_helper(client, query): +def _query_helper(client: big_query.Client, query: str) -> List[Dict[str, Any]]: """Helper function to get fuzzer stats.""" return client.query(query=query).rows -def _update_match(matches, fuzzer, job, match): +def _update_match(matches: Dict[Tuple[str, str], SpecificationMatch], + fuzzer: str, job: str, match: SpecificationMatch) -> None: """Update the weight for a fuzzer/job.""" key = (fuzzer, job) old_match = matches.get(key, RESTORE_DEFAULT_MATCH) @@ -269,7 +284,8 @@ def _update_match(matches, fuzzer, job, match): matches[key] = match -def update_weight_for_target(fuzz_target_name, job, match): +def update_weight_for_target(fuzz_target_name: str, job: str, + match: SpecificationMatch) -> None: """Set the weight for a particular target.""" target_job = data_handler.get_fuzz_target_job(fuzz_target_name, job) @@ -285,8 +301,13 @@ def update_weight_for_target(fuzz_target_name, job, match): target_job.put() -def update_matches_for_specification(specification, client, engine, matches, - run_set): +def update_matches_for_specification( + specification: QuerySpecification, + client: big_query.Client, + engine: str, + matches: Dict[Tuple[str, str], SpecificationMatch], + run_set: Set[Tuple[str, str]], +) -> None: """Run a query and adjust weights based on a given query specification.""" query = specification.formatter(specification.query_format, fuzzer_stats.dataset_name(engine)) @@ -306,10 +327,14 @@ def update_matches_for_specification(specification, client, engine, matches, _update_match(matches, fuzzer, job, match) -def update_target_weights_for_engine(client, engine, specifications): +def update_target_weights_for_engine( + client: big_query.Client, + engine: str, + specifications: List[QuerySpecification], +) -> None: """Update all fuzz target weights for the specified engine.""" - matches = {} - run_set = set() + matches: Dict[Tuple[str, str], SpecificationMatch] = {} + run_set: Set[Tuple[str, str]] = set() # All fuzzers with non-default weights must be tracked with a special # specification. This ensures that they will be restored to normal weight @@ -319,8 +344,8 @@ def update_target_weights_for_engine(client, engine, specifications): data_types.FuzzTargetJob.weight != 1.0) for target_job in target_jobs: - matches[(target_job.fuzz_target_name, - target_job.job)] = RESTORE_DEFAULT_MATCH + matches[(cast(str, target_job.fuzz_target_name), + cast(str, target_job.job))] = RESTORE_DEFAULT_MATCH for match in specifications: update_matches_for_specification(match, client, engine, matches, run_set) @@ -336,9 +361,9 @@ def update_target_weights_for_engine(client, engine, specifications): logs.info('Weight adjustments complete for engine %s.' % engine) -def store_current_weights_in_bigquery(): +def store_current_weights_in_bigquery() -> None: """Update a bigquery table containing the daily stats.""" - rows = [] + rows: List[big_query.Insert] = [] target_jobs = ndb_utils.get_all_from_model(data_types.FuzzTargetJob) for target_job in target_jobs: row = { @@ -352,16 +377,17 @@ def store_current_weights_in_bigquery(): client.insert(rows) -def update_job_weight(job_name, multiplier): +def update_job_weight(job_name: str, multiplier: float) -> None: """Update a job weight.""" tool_name = environment.get_memory_tool_name(job_name) - multiplier *= SANITIZER_WEIGHTS.get(tool_name, DEFAULT_SANITIZER_WEIGHT) + multiplier *= SANITIZER_WEIGHTS.get( + cast(Any, tool_name), DEFAULT_SANITIZER_WEIGHT) engine = environment.get_engine_for_job(job_name) - multiplier *= ENGINE_WEIGHTS.get(engine, DEFAULT_ENGINE_WEIGHT) + multiplier *= ENGINE_WEIGHTS.get(cast(Any, engine), DEFAULT_ENGINE_WEIGHT) query = data_types.FuzzerJob.query(data_types.FuzzerJob.job == job_name) - changed_weights = [] + changed_weights: List[data_types.FuzzerJob] = [] for fuzzer_job in query: if fuzzer_job.multiplier != multiplier: fuzzer_job.multiplier = multiplier @@ -371,7 +397,7 @@ def update_job_weight(job_name, multiplier): ndb_utils.put_multi(changed_weights) -def update_job_weights(): +def update_job_weights() -> None: """Update job weights.""" for job in data_types.Job.query(): multiplier = DEFAULT_MULTIPLIER @@ -382,10 +408,10 @@ def update_job_weights(): if targets_count and targets_count.count: multiplier = targets_count.count - update_job_weight(job.name, multiplier) + update_job_weight(cast(str, job.name), cast(float, multiplier)) -def main(): +def main() -> bool: """Periodically update fuzz target weights based on performance.""" client = big_query.Client() update_target_weights_for_engine(client, 'libFuzzer', diff --git a/src/clusterfuzz/_internal/cron/fuzzer_coverage.py b/src/clusterfuzz/_internal/cron/fuzzer_coverage.py index 57a4d84aa41..e6d44cc8525 100644 --- a/src/clusterfuzz/_internal/cron/fuzzer_coverage.py +++ b/src/clusterfuzz/_internal/cron/fuzzer_coverage.py @@ -16,6 +16,8 @@ import datetime import json import os +from typing import Any +from typing import cast from clusterfuzz._internal.config import local_config from clusterfuzz._internal.datastore import data_handler @@ -25,17 +27,17 @@ from clusterfuzz._internal.metrics import logs -def _latest_report_info_dir(bucket): +def _latest_report_info_dir(bucket: str) -> str: """Returns a GCS URL to the latest report info for the given bucket.""" return f'gs://{bucket}/latest_report_info/' -def _basename(gcs_path): +def _basename(gcs_path: str) -> str: """Returns the basename for the given path without file extension.""" return os.path.splitext(os.path.basename(gcs_path))[0] -def _read_json(url): +def _read_json(url: str) -> Any: """Returns a JSON obejct loaded from the given GCS url.""" data = storage.read_data(url) @@ -49,7 +51,9 @@ def _read_json(url): return result -def _coverage_information(summary_path, name, report_info): +def _coverage_information( + summary_path: str, name: str, + report_info: dict[str, Any]) -> data_types.CoverageInformation | None: """Returns a CoverageInformation entity with coverage stats populated.""" date = datetime.datetime.strptime( report_info['report_date'], @@ -58,6 +62,7 @@ def _coverage_information(summary_path, name, report_info): # |name| can be either a project qualified fuzz target name or a project name. cov_info = data_handler.get_coverage_information( name, date, create_if_needed=True) + cov_info = cast(data_types.CoverageInformation, cov_info) cov_info.fuzzer = name cov_info.date = date @@ -83,7 +88,9 @@ def _coverage_information(summary_path, name, report_info): return None -def _process_fuzzer_stats(fuzzer, project_info, project_name, bucket): +def _process_fuzzer_stats(fuzzer: str, project_info: dict[str, Any], + project_name: str, + bucket: str) -> data_types.CoverageInformation | None: """Processes coverage stats for a single fuzz target.""" fuzzer_name = data_types.fuzz_target_project_qualified_name( project_name, _basename(fuzzer)) @@ -93,7 +100,8 @@ def _process_fuzzer_stats(fuzzer, project_info, project_name, bucket): return _coverage_information(fuzzer_info_path, fuzzer_name, project_info) -def _process_project_stats(project_info, project_name): +def _process_project_stats(project_info: dict[str, Any], project_name: str + ) -> data_types.CoverageInformation | None: """Processes coverage stats for a single project.""" summary_path = project_info['report_summary_path'] logs.info('Processing total stats for %s project (%s).' % (project_name, @@ -101,7 +109,8 @@ def _process_project_stats(project_info, project_name): return _coverage_information(summary_path, project_name, project_info) -def _process_project(project_name, latest_project_info_url, bucket): +def _process_project(project_name: str, latest_project_info_url: str, + bucket: str) -> None: """Collects coverage information for all fuzz targets in the given project and the total stats for the project.""" logs.info('Processing coverage for %s project.' % project_name) @@ -112,7 +121,7 @@ def _process_project(project_name, latest_project_info_url, bucket): # Iterate through report_info['fuzzer_stats_dir'] and prepare # CoverageInformation entities for invididual fuzz targets. - entities = [] + entities: list[data_types.CoverageInformation] = [] for fuzzer in storage.list_blobs( report_info['fuzzer_stats_dir'], recursive=False): fuzzer_stats = _process_fuzzer_stats(fuzzer, report_info, project_name, @@ -130,7 +139,7 @@ def _process_project(project_name, latest_project_info_url, bucket): ndb_utils.put_multi(entities) -def collect_fuzzer_coverage(bucket): +def collect_fuzzer_coverage(bucket: str) -> None: """Actual implementation of the fuzzer coverage task.""" url = _latest_report_info_dir(bucket) for latest_project_report_info_path in storage.list_blobs( @@ -142,7 +151,7 @@ def collect_fuzzer_coverage(bucket): _process_project(project, latest_project_info_url, bucket) -def main(): +def main() -> bool: """Collects the latest code coverage stats and links to reports.""" # The task is supposed to be super reliable and never fail. If anything goes # wrong, we just fail with the exception going straight into StackDriver. diff --git a/src/clusterfuzz/_internal/cron/group_leader.py b/src/clusterfuzz/_internal/cron/group_leader.py index ced3f1ca02d..dc80278b45c 100644 --- a/src/clusterfuzz/_internal/cron/group_leader.py +++ b/src/clusterfuzz/_internal/cron/group_leader.py @@ -14,12 +14,19 @@ """The module chooses the leader given a group of testcases. It needs to be here because it is used by group_task.py and attribute_builder.py.""" +from collections.abc import Callable +from collections.abc import Iterable import itertools +from typing import Any +from typing import TypeVar from clusterfuzz._internal.system import environment +T = TypeVar('T') -def find_index(items, condition_fn): + +def find_index(items: Iterable[T], + condition_fn: Callable[[T], bool]) -> int | None: """Return the index of the first item whose condition_fn is True.""" for index, item in enumerate(items): if condition_fn(item): @@ -27,23 +34,24 @@ def find_index(items, condition_fn): return None -def is_reproducible(item): +def is_reproducible(item: Any) -> bool: """Return True if the testcase is reproducible by checking the one_time_crasher_flag.""" return not item.one_time_crasher_flag -def has_issue(item): +def has_issue(item: Any) -> bool: """Return True if the testcase has an issue.""" return bool(item.issue_id) -def is_reproducible_and_has_issue(item): +def is_reproducible_and_has_issue(item: Any) -> bool: """Return True if the testcase is reproducible and has an issue.""" return is_reproducible(item) and has_issue(item) -def choose(testcase_map, group_map=None): +def choose(testcase_map: dict[Any, Any], + group_map: dict[Any, Any] | None = None) -> None: """Choose one leader for each group. We choose the highest quality testcase to be the leader. @@ -51,12 +59,12 @@ def choose(testcase_map, group_map=None): testcase_map: a dict of (testcase_id, testcase). A dict contains testcases from multiple groups. """ - scores = {} + scores: dict[Any, int] = {} - def _key_func(testcase): + def _key_func(testcase: Any) -> Any: return testcase.group_id - def _get_score(testcase_id): + def _get_score(testcase_id: Any) -> int: return scores[testcase_id] testcases = sorted([v for _, v in testcase_map.items()], key=_key_func) @@ -64,9 +72,9 @@ def _get_score(testcase_id): if group_id == 0: # group_id=0 means there's no group. continue - items = sorted(items, reverse=True, key=lambda t: t.timestamp) + group_items = sorted(items, reverse=True, key=lambda t: t.timestamp) - for item in items: + for item in group_items: item.is_leader = False item_score = 0 if item.security_flag: @@ -77,17 +85,19 @@ def _get_score(testcase_id): item_score += 1 scores[item.id] = item_score - items = sorted(items, reverse=True, key=lambda t: _get_score(t.id)) + group_items = sorted( + group_items, reverse=True, key=lambda t: _get_score(t.id)) - leader_index = find_index(items, is_reproducible_and_has_issue) + leader_index = find_index(group_items, is_reproducible_and_has_issue) if leader_index is None: - leader_index = find_index(items, has_issue) + leader_index = find_index(group_items, has_issue) if leader_index is None: - leader_index = find_index(items, is_reproducible) + leader_index = find_index(group_items, is_reproducible) if leader_index is None: leader_index = 0 - items[leader_index].is_leader = True + group_items[leader_index].is_leader = True if group_map: - group_map[items[leader_index].group_id].leader_id = items[leader_index].id + group_map[group_items[leader_index].group_id].leader_id = group_items[ + leader_index].id diff --git a/src/clusterfuzz/_internal/cron/grouper.py b/src/clusterfuzz/_internal/cron/grouper.py index f9bc298072a..dc600972ddd 100644 --- a/src/clusterfuzz/_internal/cron/grouper.py +++ b/src/clusterfuzz/_internal/cron/grouper.py @@ -14,7 +14,10 @@ """Grouper for grouping similar looking testcases.""" import collections +import datetime import re +from typing import Any +from typing import cast from clusterfuzz._internal.base import errors from clusterfuzz._internal.config import local_config @@ -29,28 +32,29 @@ from . import cleanup from . import group_leader -FORWARDED_ATTRIBUTES = ('crash_state', 'crash_type', 'group_id', - 'one_time_crasher_flag', 'project_name', - 'security_flag', 'timestamp', 'job_type', 'fuzzer_name') +FORWARDED_ATTRIBUTES: tuple[str, ...] = ('crash_state', 'crash_type', + 'group_id', 'one_time_crasher_flag', + 'project_name', 'security_flag', + 'timestamp', 'job_type', 'fuzzer_name') -VARIANT_CRASHES_IGNORE = re.compile( +VARIANT_CRASHES_IGNORE: re.Pattern[str] = re.compile( r'^(Out-of-memory|Timeout|Missing-library|Data race|GPU failure)') -VARIANT_STATES_IGNORE = re.compile(r'^NULL$') +VARIANT_STATES_IGNORE: re.Pattern[str] = re.compile(r'^NULL$') -VARIANT_THRESHOLD_PERCENTAGE = 0.2 -VARIANT_MIN_THRESHOLD = 5 -VARIANT_MAX_THRESHOLD = 10 +VARIANT_THRESHOLD_PERCENTAGE: float = 0.2 +VARIANT_MIN_THRESHOLD: int = 5 +VARIANT_MAX_THRESHOLD: int = 10 -TOP_CRASHES_LIMIT = 10 +TOP_CRASHES_LIMIT: int = 10 -DELETE_TESTCASES_FROM_GROUPING = local_config.ProjectConfig().get( +DELETE_TESTCASES_FROM_GROUPING: bool = local_config.ProjectConfig().get( 'deduplication.delete_testcases_from_grouping', True) -GROUP_MAX_TESTCASE_LIMIT = local_config.ProjectConfig().get( +GROUP_MAX_TESTCASE_LIMIT: Any = local_config.ProjectConfig().get( 'deduplication.group_max_limit', 25) -CRASH_COMPARER_THRESHOLD = local_config.ProjectConfig().get( +CRASH_COMPARER_THRESHOLD: Any = local_config.ProjectConfig().get( 'deduplication.crash_comparer_threshold') -CRASH_COMPARER_SAME_FRAMES = local_config.ProjectConfig().get( +CRASH_COMPARER_SAME_FRAMES: Any = local_config.ProjectConfig().get( 'deduplication.crash_comparer_same_frames') @@ -59,12 +63,25 @@ class TestcaseAttributes: __slots__ = ('id', 'is_leader', 'issue_id') + FORWARDED_ATTRIBUTES - def __init__(self, testcase_id): + id: int + is_leader: bool + issue_id: int | None + crash_state: str + crash_type: str + group_id: int + one_time_crasher_flag: bool + project_name: str + security_flag: bool + timestamp: datetime.datetime | None + job_type: str + fuzzer_name: str + + def __init__(self, testcase_id: int): self.id = testcase_id self.is_leader = True self.issue_id = None - def get_metadata(self, key=None, default=None): + def get_metadata(self, key: str | None = None, default: Any = None) -> Any: """Retrieve class attributes.""" # This method is useful so that an object calling it gets a similar result # independently of its class being a Testcase or TestcaseAttributes. @@ -78,7 +95,7 @@ def _emit_grouping_event(moved_testcase: int, prev_group_id: int, similar_testcase: int | None, reason: str, - group_merge: bool = False): + group_merge: bool = False) -> None: """Helper for emitting a testcase grouping event.""" # If this is due to a group merge, we have to use the grouping reason as the # reason for the merge itself. @@ -165,21 +182,23 @@ def combine_testcases_into_group( 'moved testcases: ' + ', '.join(moved_testcase_ids)) -def _get_new_group_id(): +def _get_new_group_id() -> int: """Get a new group id for testcase grouping.""" new_group = data_types.TestcaseGroup() new_group.put() - return new_group.key.id() + return cast(int, new_group.key.id()) -def is_same_variant(variant1, variant2): +def is_same_variant(variant1: Any, variant2: Any) -> bool: """Checks for the testcase variants equality.""" return (variant1.crash_type == variant2.crash_type and variant1.crash_state == variant2.crash_state and variant1.security_flag == variant2.security_flag) -def matches_top_crash(testcase, top_crashes_by_project_and_platform): +def matches_top_crash(testcase: TestcaseAttributes, + top_crashes_by_project_and_platform: dict[str, dict[ + str, list[dict[str, Any]]]]) -> bool: """Returns whether or not a testcase is a top crash.""" if testcase.project_name not in top_crashes_by_project_and_platform: return False @@ -196,9 +215,9 @@ def matches_top_crash(testcase, top_crashes_by_project_and_platform): return False -def _check_variant_grouping_candidate(testcase_1: TestcaseAttributes, - testcase_2: TestcaseAttributes, - grouping_candidates: dict[str, list]): +def _check_variant_grouping_candidate( + testcase_1: TestcaseAttributes, testcase_2: TestcaseAttributes, + grouping_candidates: dict[str, list[tuple[int, int]]]) -> None: """Check if a pair of testcases is a candidate for variant-based grouping. If the pair is a valid candidate for variant grouping, their IDs are appended @@ -256,7 +275,8 @@ def _check_variant_grouping_candidate(testcase_1: TestcaseAttributes, grouping_candidates[current_project].append((testcase_1_id, testcase_2_id)) -def _group_testcases_based_on_variants(testcase_map): +def _group_testcases_based_on_variants( + testcase_map: dict[int, TestcaseAttributes]) -> None: """Group testcases that are associated based on variant analysis.""" # Skip this if the project is configured so (like Google3). enable = local_config.ProjectConfig().get('deduplication.variant', True) @@ -264,8 +284,9 @@ def _group_testcases_based_on_variants(testcase_map): return logs.info('Grouping based on variant analysis.') - grouping_candidates = collections.defaultdict(list) - project_num_testcases = collections.defaultdict(int) + grouping_candidates: dict[str, list[tuple[int, int]]] = ( + collections.defaultdict(list)) + project_num_testcases: dict[str, int] = collections.defaultdict(int) # Phase 1: collect all grouping candidates. for testcase_1_id, testcase_1 in testcase_map.items(): # Count the number of testcases for each project. @@ -284,9 +305,9 @@ def _group_testcases_based_on_variants(testcase_map): # Phase 2: check for the anomalous candidates # i.e. candiates matched with many testcases. for project, candidate_list in grouping_candidates.items(): - project_ignore_testcases = set() + project_ignore_testcases: set[int] = set() # Count the number of times a testcase is matched for grouping. - project_counter = collections.defaultdict(int) + project_counter: dict[int, int] = collections.defaultdict(int) for candidate_tuple in candidate_list: for testcase_id in candidate_tuple: project_counter[testcase_id] += 1 @@ -325,7 +346,8 @@ def _group_testcases_based_on_variants(testcase_map): events.GroupingReason.IDENTICAL_VARIANT) -def _group_testcases_with_same_issues(testcase_map): +def _group_testcases_with_same_issues( + testcase_map: dict[int, TestcaseAttributes]) -> None: """Group testcases that are associated with same underlying issue.""" logs.info('Grouping based on same issues.') for testcase_1_id, testcase_1 in testcase_map.items(): @@ -356,7 +378,8 @@ def _group_testcases_with_same_issues(testcase_map): events.GroupingReason.SAME_ISSUE) -def _compare_testcases_crash_states(testcase_1, testcase_2) -> bool: +def _compare_testcases_crash_states(testcase_1: TestcaseAttributes, + testcase_2: TestcaseAttributes) -> bool: """Apply crash comparison rules for a pair of testcases.""" # Rule: Follow different comparison rules when crash types is one of the # ones that have unique crash state (custom ones specifically). @@ -383,7 +406,8 @@ def _compare_testcases_crash_states(testcase_1, testcase_2) -> bool: crash_comparer_type.is_similar()) -def _group_testcases_with_similar_states(testcase_map): +def _group_testcases_with_similar_states( + testcase_map: dict[int, TestcaseAttributes]) -> None: """Group testcases with similar looking crash states.""" logs.info('Grouping based on similar states.') for testcase_1_id, testcase_1 in testcase_map.items(): @@ -413,7 +437,9 @@ def _group_testcases_with_similar_states(testcase_map): events.GroupingReason.SIMILAR_CRASH) -def _has_testcase_with_same_params(testcase, testcase_map): +def _has_testcase_with_same_params( + testcase: data_types.Testcase, + testcase_map: dict[int, TestcaseAttributes]) -> bool: """Return a bool whether there is another testcase with same params.""" for other_testcase_id in testcase_map: # yapf: disable @@ -433,17 +459,19 @@ def _has_testcase_with_same_params(testcase, testcase_map): return False -def _increment_group_overflow_metric(group_overflow, testcase): +def _increment_group_overflow_metric(group_overflow: dict[tuple[str, str], int], + testcase: data_types.Testcase) -> None: """Increments the counter for group overflow metric.""" job = testcase.job_type fuzzer = testcase.fuzzer_name - id_tuple = (job, fuzzer) + id_tuple = (cast(str, job), cast(str, fuzzer)) if id_tuple not in group_overflow: group_overflow[id_tuple] = 0 group_overflow[id_tuple] += 1 -def _emit_group_overflow_metric(group_overflow): +def _emit_group_overflow_metric( + group_overflow: dict[tuple[str, str], int]) -> None: """Emits the testcase group overflow count metric.""" for (job, fuzzer) in group_overflow: monitoring_metrics.TESTCASE_GROUP_OVERFLOW_COUNT.set( @@ -453,9 +481,10 @@ def _emit_group_overflow_metric(group_overflow): }) -def _shrink_large_groups_if_needed(testcase_map): +def _shrink_large_groups_if_needed( + testcase_map: dict[int, TestcaseAttributes]) -> None: """Shrinks groups that exceed a particular limit.""" - group_overflow = {} + group_overflow: dict[tuple[str, str], int] = {} if isinstance(GROUP_MAX_TESTCASE_LIMIT, int): group_max_testcase_limit = int(GROUP_MAX_TESTCASE_LIMIT) else: @@ -463,7 +492,7 @@ def _shrink_large_groups_if_needed(testcase_map): f'{GROUP_MAX_TESTCASE_LIMIT}. Defaulting to 25.') group_max_testcase_limit = 25 - def _key_func(testcase): + def _key_func(testcase: TestcaseAttributes) -> int: weight = 0 if not testcase.one_time_crasher_flag: weight |= 2**1 @@ -471,7 +500,7 @@ def _key_func(testcase): weight |= 2**2 return weight - group_id_with_testcases_map = {} + group_id_with_testcases_map: dict[int, list[TestcaseAttributes]] = {} for testcase in testcase_map.values(): if not testcase.group_id: continue @@ -522,9 +551,11 @@ def _key_func(testcase): _emit_group_overflow_metric(group_overflow) -def _get_testcase_attributes(testcase, testcase_map, cached_issue_map): +def _get_testcase_attributes( + testcase: data_types.Testcase, testcase_map: dict[int, TestcaseAttributes], + cached_issue_map: dict[str, dict[int, int]]) -> None: """Retrieve testcase attributes for grouping and add it to the map.""" - testcase_id = testcase.key.id() + testcase_id = cast(int, testcase.key.id()) # Remove duplicates early on to avoid large groups. if (not testcase.bug_information and not testcase.uploader_email and _has_testcase_with_same_params(testcase, testcase_map)): @@ -554,8 +585,9 @@ def _get_testcase_attributes(testcase, testcase_map, cached_issue_map): project_name = testcase.project_name if (project_name in cached_issue_map and - issue_id in cached_issue_map[project_name]): - testcase_attributes.issue_id = cached_issue_map[project_name][issue_id] + issue_id in cached_issue_map[cast(str, project_name)]): + testcase_attributes.issue_id = cached_issue_map[cast( + str, project_name)][issue_id] else: try: issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase( @@ -577,8 +609,8 @@ def _get_testcase_attributes(testcase, testcase_map, cached_issue_map): # Determine the original issue id traversing the list of duplicates. try: issue = issue_tracker.get_original_issue(issue_id) - original_issue_id = int(issue.id) - except: + original_issue_id = int(cast(Any, issue).id) + except: # pylint: disable=bare-except # If we are unable to access the issue, then we can't determine # the original issue id. Assume that it is the same as issue id. logs.error( @@ -587,17 +619,18 @@ def _get_testcase_attributes(testcase, testcase_map, cached_issue_map): return if project_name not in cached_issue_map: - cached_issue_map[project_name] = {} - cached_issue_map[project_name][issue_id] = original_issue_id - cached_issue_map[project_name][original_issue_id] = original_issue_id + cached_issue_map[cast(str, project_name)] = {} + cached_issue_map[cast(str, project_name)][issue_id] = original_issue_id + cached_issue_map[cast( + str, project_name)][original_issue_id] = original_issue_id testcase_attributes.issue_id = original_issue_id -def group_testcases(): +def group_testcases() -> None: """Group testcases based on rules like same bug numbers, similar crash states, etc.""" - testcase_map = {} - cached_issue_map = {} + testcase_map: dict[int, TestcaseAttributes] = {} + cached_issue_map: dict[str, dict[int, int]] = {} for testcase_id in data_handler.get_open_testcase_id_iterator(): try: @@ -684,7 +717,7 @@ def group_testcases(): @logs.cron_log_context() -def main(): +def main() -> bool: """Group testcases. This will be useful to run grouper as a standalone cron in dev/staging @@ -694,7 +727,7 @@ def main(): logs.info('Grouping testcases.') group_testcases() logs.info('Grouping done.') - except: + except: # pylint: disable=bare-except logs.error('Error occurred while grouping test cases.') return False diff --git a/src/clusterfuzz/_internal/cron/helpers/tasks_scheduler.py b/src/clusterfuzz/_internal/cron/helpers/tasks_scheduler.py index 64978db5b0e..bfa1dd4890f 100644 --- a/src/clusterfuzz/_internal/cron/helpers/tasks_scheduler.py +++ b/src/clusterfuzz/_internal/cron/helpers/tasks_scheduler.py @@ -20,13 +20,13 @@ from clusterfuzz._internal.metrics import logs -def schedule(task): +def schedule(task: str) -> None: """Creates tasks for open reproducible testcases.""" if task == 'impact' and not utils.is_chromium(): logs.warning('Not creating impact tasks outside of Chrome.') return - testcase_ids = [] + testcase_ids: list[int | str] = [] for status in ['Processed', 'Duplicate']: for testcase in data_types.Testcase.query( ndb_utils.is_true(data_types.Testcase.open), diff --git a/src/clusterfuzz/_internal/cron/job_exporter.py b/src/clusterfuzz/_internal/cron/job_exporter.py index aedbe3ec400..4d421fb1bf4 100644 --- a/src/clusterfuzz/_internal/cron/job_exporter.py +++ b/src/clusterfuzz/_internal/cron/job_exporter.py @@ -16,6 +16,8 @@ import os import re +from typing import Any +from typing import cast from google.cloud import ndb from google.protobuf import any_pb2 @@ -29,7 +31,7 @@ from clusterfuzz._internal.google_cloud_utils import storage from clusterfuzz._internal.metrics import logs -target_entities = [ +target_entities: list[tuple[type[ndb.Model], list[str], str]] = [ (data_types.Fuzzer, ['blobstore_key', 'sample_testcase'], 'fuzzer'), (data_types.Job, ['custom_binary_key'], 'job'), (data_types.DataBundle, [], 'databundle'), @@ -40,10 +42,10 @@ class RSyncClient: """Interface that defines the rsync contract.""" - def __init__(self): + def __init__(self) -> None: pass - def rsync(self, source: str, target: str): + def rsync(self, source: str, target: str) -> bool: # type: ignore pass @@ -51,10 +53,10 @@ class GCloudCLIRSync(RSyncClient): """RSyncClient implementation that delegates to the gcloud cli. Unsuitable for unit testing.""" - def __init__(self): + def __init__(self) -> None: self._runner = gsutil.GSUtilRunner() - def rsync(self, source: str, target: str): + def rsync(self, source: str, target: str) -> bool: """Rsyncs a source to a target destination. Returns True if successful, False if there was any failure. Considers successful any gsutil execution with a 0 return code.""" @@ -67,10 +69,10 @@ class StorageRSync(RSyncClient): """RSyncClient implementation for unit testing, meant for use with GCS emulator.""" - def __init__(self): + def __init__(self) -> None: pass - def rsync(self, source: str, target: str): + def rsync(self, source: str, target: str) -> bool: """Lists all files under the source path, and uploads them to the target path. Since list_blobs returns fully qualified names, the source prefix is trimmed @@ -107,12 +109,12 @@ class EntityMigrator: """Serializes entities to GCS, and imports them back.""" def __init__(self, - target_cls: ndb.Model, + target_cls: type[ndb.Model], blobstore_keys: list[str], entity_type: str, rsync_client: RSyncClient, export_bucket: str, - env_string_substitutions: dict[str, str] | None = None): + env_string_substitutions: dict[str, str] | None = None) -> None: self._target_cls = target_cls self.blobstore_keys = blobstore_keys self._entity_type = entity_type @@ -120,24 +122,28 @@ def __init__(self, self._export_bucket = export_bucket self._env_string_substitutions = env_string_substitutions or {} - def _serialize(self, entity) -> bytes: + def _serialize(self, entity: ndb.Model) -> bytes: return uworker_io.entity_to_protobuf(entity).SerializeToString() def _deserialize(self, proto_as_str: bytes) -> ndb.Model: - deserialized_any = any_pb2.Any() # pylint: disable=no-member + # pylint: disable=no-member + deserialized_any = ( + any_pb2.Any() # pyright: ignore[reportAttributeAccessIssue] + ) # Parse the bytes into the Any message deserialized_any.ParseFromString(proto_as_str) return uworker_io.entity_from_protobuf(deserialized_any, self._target_cls) - def _serialize_entity_to_gcs(self, entity: ndb.Model, upload_path: str): + def _serialize_entity_to_gcs(self, entity: ndb.Model, + upload_path: str) -> None: entity_as_bytes = self._serialize(entity) storage.write_data(entity_as_bytes, upload_path) - def _deserialize_entity_from_gcs(self, download_path: str): + def _deserialize_entity_from_gcs(self, download_path: str) -> ndb.Model: entity_as_bytes = storage.read_data(download_path) - return self._deserialize(entity_as_bytes) + return self._deserialize(cast(bytes, entity_as_bytes)) - def _export_blobs(self, entity: ndb.Model, bucket_prefix: str): + def _export_blobs(self, entity: ndb.Model, bucket_prefix: str) -> None: """Exports blobs for an entity, if applicable (fuzzer/job only).""" for blobstore_key in self.blobstore_keys: blob_id = getattr(entity, blobstore_key, None) @@ -145,13 +151,13 @@ def _export_blobs(self, entity: ndb.Model, bucket_prefix: str): blob_gcs_path = blobs.get_gcs_path(blob_id) if not storage.get(blob_gcs_path): logs.warning(f'{blobstore_key} with id {blob_id} not present ' - f'for {entity.name}, skipping.') + f'for {cast(Any, entity).name}, skipping.') continue blob_destination_path = f'{bucket_prefix}/{blobstore_key}' storage.copy_blob(blob_gcs_path, blob_destination_path) def _export_data_bundle_contents_if_applicable(self, entity: ndb.Model, - bucket_prefix: str): + bucket_prefix: str) -> None: """Uploads data bundle proto and rsyncs the respective bucket contents.""" if not isinstance(entity, data_types.DataBundle): logs.info( @@ -173,7 +179,7 @@ def _export_data_bundle_contents_if_applicable(self, entity: ndb.Model, f'Failed to rsync {source_location} to {target_location}.') def _export_entity(self, entity: ndb.Model, entity_bucket_prefix: str, - entity_name: str): + entity_name: str) -> None: """Exports entity as protobuf and its respective blobs to GCS.""" # Entitites get their name from the 'name' field in datastore bucket_prefix = f'{entity_bucket_prefix}/{entity_name}' @@ -182,16 +188,17 @@ def _export_entity(self, entity: ndb.Model, entity_bucket_prefix: str, self._export_blobs(entity, bucket_prefix) self._export_data_bundle_contents_if_applicable(entity, bucket_prefix) - def _export_entity_names(self, entities: set[str], entity_bucket_prefix: str): + def _export_entity_names(self, entities: set[str], + entity_bucket_prefix: str) -> None: """Writes entity name list to GCS.""" entity_list = '\n'.join(entities) storage.write_data( entity_list.encode('utf-8'), f'{entity_bucket_prefix}/entities') - def export_entities(self): + def export_entities(self) -> None: """Exports individual entities of a certain type, and populates a list the individual names of entities for future importing.""" - entity_names = set() + entity_names: set[str] = set() entity_bucket_prefix = f'gs://{self._export_bucket}/{self._entity_type}' for entity in self._target_cls.query(): entity_name = getattr(entity, 'name', None) @@ -202,10 +209,10 @@ def export_entities(self): self._export_entity_names(entity_names, entity_bucket_prefix) def _import_blobs(self, entity: ndb.Model, entity_name: str, - entity_location: str): + entity_location: str) -> dict[str, str | None]: """Copies exported blobs to a new blob id, and returns a map with the new blob ids.""" - new_blob_ids = {} + new_blob_ids: dict[str, str | None] = {} for blobstore_key in self.blobstore_keys: source_blob_location = f'{entity_location}/{blobstore_key}' if not getattr(entity, blobstore_key, None): @@ -225,7 +232,8 @@ def _import_blobs(self, entity: ndb.Model, entity_name: str, new_blob_ids[blobstore_key] = new_blob_id return new_blob_ids - def _substitute_environment_string(self, env_string: str | None): + def _substitute_environment_string(self, + env_string: str | None) -> str | None: """Performs raw text substitution in an environment string, given a substitution dictionary.""" if not env_string: @@ -237,7 +245,7 @@ def _substitute_environment_string(self, env_string: str | None): return env_string def _import_data_bundle_contents(self, source_location: str, - bundle_name: str): + bundle_name: str) -> str: """Imports data bundle contents from the export bucket to the new data bundle bucket in the target project. Skips if the contents are absent during export, and throws an exception if the rsync @@ -258,7 +266,7 @@ def _import_data_bundle_contents(self, source_location: str, f'to {target_location}.') return new_bundle_bucket - def _persist_entity(self, entity: ndb.Model): + def _persist_entity(self, entity: ndb.Model) -> None: """A raw deserialization and put() call will cause an exception, since the project from which the entity was serialized will mistmatch the project to which we are writing it to datastore. This forces creation of a new @@ -268,7 +276,7 @@ def _persist_entity(self, entity: ndb.Model): setattr(entity_to_persist, key, value) entity_to_persist.put() - def _import_entity(self, entity_name: str, entity_location: str): + def _import_entity(self, entity_name: str, entity_location: str) -> None: """Imports entity into datastore, blobs, databundle contents and substitutes environment strings, if applicable.""" entity_to_import = self._deserialize_entity_from_gcs( @@ -298,7 +306,7 @@ def _import_entity(self, entity_name: str, entity_location: str): # Do not assume that name is a primary key, avoid having two # different keys with the same name. preexisting_entities = list( - self._target_cls.query(self._target_cls.name == entity_name)) + self._target_cls.query(cast(Any, self._target_cls).name == entity_name)) logs.info(f'Found {len(preexisting_entities)} of type {self._entity_type}' f' and name {entity_name}, deleting.') for preexisting_entity in preexisting_entities: @@ -306,27 +314,27 @@ def _import_entity(self, entity_name: str, entity_location: str): self._persist_entity(entity_to_import) - def import_entities(self): + def import_entities(self) -> None: """Iterates over all entitiy names declared in the last export, and imports its contents.""" entity_bucket_prefix = f'gs://{self._export_bucket}/{self._entity_type}' entity_list_location = f'{entity_bucket_prefix}/entities' if not storage.get(entity_list_location): raise ValueError(f'Missing entity list in {entity_list_location}') - entities_to_sync = storage.read_data(entity_list_location) - if not entities_to_sync: - entities_to_sync = [] + entities_to_sync_bytes = storage.read_data(entity_list_location) + if not entities_to_sync_bytes: + entities_to_sync: list[str] = [] else: - entities_to_sync = entities_to_sync.decode('utf-8').split('\n') + entities_to_sync = entities_to_sync_bytes.decode('utf-8').split('\n') for entity in self._target_cls.query(): - if entity.name not in entities_to_sync: + if cast(Any, entity).name not in entities_to_sync: entity.key.delete() for entity_name in entities_to_sync: entity_location = f'{entity_bucket_prefix}/{entity_name}' self._import_entity(entity_name, entity_location) -def main(): +def main() -> bool: """Exports datastore entities and respective blobs.""" export_bucket = os.getenv('EXPORT_BUCKET', None) operation_mode = os.getenv('OPERATION_MODE', None) diff --git a/src/clusterfuzz/_internal/cron/libs/mail.py b/src/clusterfuzz/_internal/cron/libs/mail.py index 4d8a3b8e669..30021644f9a 100644 --- a/src/clusterfuzz/_internal/cron/libs/mail.py +++ b/src/clusterfuzz/_internal/cron/libs/mail.py @@ -23,7 +23,7 @@ from clusterfuzz._internal.metrics import logs -def send(to_email, subject, html_content): +def send(to_email: str, subject: str, html_content: str) -> None: """Send email.""" sendgrid_api_key = db_config.get_value('sendgrid_api_key') if not sendgrid_api_key: diff --git a/src/clusterfuzz/_internal/cron/load_bigquery_stats.py b/src/clusterfuzz/_internal/cron/load_bigquery_stats.py index 62ac411c6c5..5bd583b07a9 100644 --- a/src/clusterfuzz/_internal/cron/load_bigquery_stats.py +++ b/src/clusterfuzz/_internal/cron/load_bigquery_stats.py @@ -19,6 +19,7 @@ import random import string import time +from typing import Any from googleapiclient.errors import HttpError import httplib2 @@ -30,23 +31,26 @@ from clusterfuzz._internal.metrics import fuzzer_stats_schema from clusterfuzz._internal.metrics import logs -STATS_KINDS = [fuzzer_stats.JobRun, fuzzer_stats.TestcaseRun] +STATS_KINDS: list[type[fuzzer_stats.JobRun | fuzzer_stats.TestcaseRun]] = [ + fuzzer_stats.JobRun, + fuzzer_stats.TestcaseRun, +] -NUM_THREADS = 4 -NUM_RETRIES = 2 -RETRY_SLEEP_TIME = 5 -POLL_INTERVAL = 5 +NUM_THREADS: int = 4 +NUM_RETRIES: int = 2 +RETRY_SLEEP_TIME: int = 5 +POLL_INTERVAL: int = 5 # Ignore the repeated uppercase digits. -_HEX_DIGITS = string.hexdigits[:-6] +_HEX_DIGITS: str = string.hexdigits[:-6] -def _utc_now(): +def _utc_now() -> datetime.datetime: """Return datetime.datetime.utcnow().""" return datetime.datetime.utcnow() -def _execute_insert_request(request): +def _execute_insert_request(request: Any) -> bool: """Executes a table/dataset insert request, retrying on transport errors.""" for i in range(NUM_RETRIES + 1): try: @@ -68,7 +72,7 @@ def _execute_insert_request(request): return False -def _create_dataset_if_needed(bigquery, dataset_id): +def _create_dataset_if_needed(bigquery: Any, dataset_id: str) -> bool: """Create a new dataset if necessary.""" project_id = utils.get_application_id() dataset_body = { @@ -83,10 +87,11 @@ def _create_dataset_if_needed(bigquery, dataset_id): return _execute_insert_request(dataset_insert) -def _create_table_if_needed(bigquery, dataset_id, table_id, schema): +def _create_table_if_needed(bigquery: Any, dataset_id: str, table_id: str, + schema: Any) -> bool: """Create a new table if needed.""" project_id = utils.get_application_id() - table_body = { + table_body: dict[str, Any] = { 'tableReference': { 'datasetId': dataset_id, 'projectId': project_id, @@ -105,7 +110,8 @@ def _create_table_if_needed(bigquery, dataset_id, table_id, schema): return _execute_insert_request(table_insert) -def _poll_completion(bigquery, project_id, job_id): +def _poll_completion(bigquery: Any, project_id: str | None, + job_id: str) -> dict[str, Any]: """Poll for completion.""" response = bigquery.jobs().get( projectId=project_id, jobId=job_id).execute(num_retries=NUM_RETRIES) @@ -117,7 +123,7 @@ def _poll_completion(bigquery, project_id, job_id): return response -def _load_data(fuzzer, target_date): +def _load_data(fuzzer: str, target_date: datetime.date) -> None: """Load stats into BigQuery.""" bigquery = big_query.get_api_client() project_id = utils.get_application_id() @@ -145,6 +151,8 @@ def _load_data(fuzzer, target_date): continue gcs_path = fuzzer_stats.get_gcs_stats_path(kind_name, fuzzer, timestamp) + if not gcs_path: + continue # Shard loads by prefix to avoid causing BigQuery to run out of memory. first_write = True for prefix in _HEX_DIGITS: @@ -193,7 +201,7 @@ def _load_data(fuzzer, target_date): logs.error(f'Failed to load: {str(e)}') -def main(argv=None): +def main(argv: list[str] | None = None) -> bool: """Load bigquery stats from GCS.""" parser = argparse.ArgumentParser(prog='load_bigquery_stats') parser.add_argument( @@ -221,6 +229,8 @@ def main(argv=None): # Retrieve list of fuzzers before iterating them, since the query can expire # as we create the load jobs. for fuzzer in list(data_types.Fuzzer.query()): + if not fuzzer.name: + continue logs.info('Loading stats to BigQuery for %s.' % fuzzer.name) thread_pool.submit(_load_data, fuzzer.name, target_date) diff --git a/src/clusterfuzz/_internal/cron/oss_fuzz_apply_ccs.py b/src/clusterfuzz/_internal/cron/oss_fuzz_apply_ccs.py index fa856cd0f7e..57dffd2e1bb 100644 --- a/src/clusterfuzz/_internal/cron/oss_fuzz_apply_ccs.py +++ b/src/clusterfuzz/_internal/cron/oss_fuzz_apply_ccs.py @@ -15,6 +15,8 @@ import logging +from google.cloud import ndb # pylint: disable=no-name-in-module + from clusterfuzz._internal.base import external_users from clusterfuzz._internal.base import memoize from clusterfuzz._internal.datastore import data_types @@ -22,7 +24,7 @@ from clusterfuzz._internal.issue_management import issue_tracker_utils -def get_open_testcases_with_bugs(): +def get_open_testcases_with_bugs() -> ndb.Query: """Return iterator to open testcases with bugs.""" return data_types.Testcase.query( ndb_utils.is_true(data_types.Testcase.open), @@ -32,13 +34,13 @@ def get_open_testcases_with_bugs(): @memoize.wrap(memoize.FifoInMemory(256)) -def cc_users_for_job(job_type, security_flag): +def cc_users_for_job(job_type: str | None, security_flag: bool) -> list[str]: """Return users to CC for a job.""" # Memoized per cron run. return external_users.cc_users_for_job(job_type, security_flag) -def main(): +def main() -> bool: """Cron handler for adding new CC's to oss-fuzz bugs.""" some_issue_failed = False for testcase in get_open_testcases_with_bugs(): diff --git a/src/clusterfuzz/_internal/cron/oss_fuzz_build_status.py b/src/clusterfuzz/_internal/cron/oss_fuzz_build_status.py index 38147fdb44e..3c921e717b7 100644 --- a/src/clusterfuzz/_internal/cron/oss_fuzz_build_status.py +++ b/src/clusterfuzz/_internal/cron/oss_fuzz_build_status.py @@ -16,6 +16,8 @@ import datetime import json import re +from typing import Any +from typing import cast from google.cloud import ndb import requests @@ -24,6 +26,7 @@ from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.issue_management import issue_tracker_policy from clusterfuzz._internal.issue_management import issue_tracker_utils +from clusterfuzz._internal.issue_management.issue_tracker import IssueTracker from clusterfuzz._internal.metrics import logs HTTP_GET_TIMEOUT_SECS = 30 @@ -37,7 +40,7 @@ MAIN_BUILD_TYPE = FUZZING_BUILD_TYPE # It's important not to use a dict, so that fuzzing builds get processed first. -BUILD_STATUS_MAPPINGS = [ +BUILD_STATUS_MAPPINGS: list[tuple[str, str]] = [ (FUZZING_BUILD_TYPE, FUZZING_STATUS_URL), (COVERAGE_BUILD_TYPE, COVERAGE_STATUS_URL), ] @@ -61,7 +64,7 @@ class OssFuzzBuildStatusError(Exception): """Exceptions for the build status cron.""" -def _get_issue_body(project_name, build_id, build_type): +def _get_issue_body(project_name: str, build_id: str, build_type: str) -> str: """Return the issue body for filing new bugs.""" template = ('The last {num_builds} builds for {project} have been failing.\n' 'Build log: {log_link}\n' @@ -82,17 +85,18 @@ def _get_issue_body(project_name, build_id, build_type): build_type=build_type) -def _get_oss_fuzz_project(project_name): +def _get_oss_fuzz_project( + project_name: str) -> data_types.OssFuzzProject | None: """Return the OssFuzzProject entity for the given project.""" return ndb.Key(data_types.OssFuzzProject, project_name).get() -def _get_build_link(build_id): +def _get_build_link(build_id: str) -> str: """Return a link to the build log.""" return BUCKET_URL + '/log-' + build_id + '.txt' -def _get_ndb_key(project_name, build_type): +def _get_ndb_key(project_name: str, build_type: str) -> str: """Constructs a Key literal for build failure entities.""" if build_type == MAIN_BUILD_TYPE: return project_name @@ -101,7 +105,8 @@ def _get_ndb_key(project_name, build_type): return '%s-%s' % (project_name, build_type) -def create_build_failure(project_name, failure, build_type): +def create_build_failure(project_name: str, failure: dict[str, Any], + build_type: str) -> data_types.OssFuzzBuildFailure: """Create new build failure.""" return data_types.OssFuzzBuildFailure( id=_get_ndb_key(project_name, build_type), @@ -110,19 +115,20 @@ def create_build_failure(project_name, failure, build_type): build_type=build_type) -def get_build_failure(project_name, build_type): +def get_build_failure(project_name: str, + build_type: str) -> data_types.OssFuzzBuildFailure | None: """Return the last build failure for the project.""" key = ndb.Key(data_types.OssFuzzBuildFailure, _get_ndb_key(project_name, build_type)) return key.get() -def close_build_failure(build_failure): +def close_build_failure(build_failure: data_types.OssFuzzBuildFailure) -> None: """Delete the build failure.""" build_failure.key.delete() -def get_build_time(build): +def get_build_time(build: dict[str, Any]) -> datetime.datetime | None: """Return a datetime for when the build was done.""" # Strip the nanosecond precision from the timestamp, since it's not # supported by Python. @@ -136,7 +142,9 @@ def get_build_time(build): stripped_timestamp.group(0), TIMESTAMP_FORMAT) -def file_bug(issue_tracker, policy, project_name, build_id, ccs, build_type): +def file_bug(issue_tracker: IssueTracker, + policy: issue_tracker_policy.IssueTrackerPolicy, project_name: str, + build_id: str, ccs: Any, build_type: str) -> str: """File a new bug for a build failure.""" logs.info('Filing bug for new build failure (project=%s, build_type=%s, ' 'build_id=%s).' % (project_name, build_type, build_id)) @@ -154,19 +162,22 @@ def file_bug(issue_tracker, policy, project_name, build_id, ccs, build_type): return str(issue.id) -def close_bug(issue_tracker, policy, issue_id, project_name): +def close_bug(issue_tracker: IssueTracker, + policy: issue_tracker_policy.IssueTrackerPolicy, issue_id: str, + project_name: str) -> None: """Close a build failure bug.""" logs.info('Closing build failure bug (project=%s, issue_id=%s).' % (project_name, issue_id)) issue = issue_tracker.get_original_issue(issue_id) - issue.status = policy.status('verified') - issue.save( + cast(Any, issue).status = policy.status('verified') + cast(Any, issue).save( new_comment='The latest build has succeeded, closing this issue.', notify=True) -def send_reminder(issue_tracker, issue_id, build_id): +def send_reminder(issue_tracker: IssueTracker, issue_id: str, + build_id: str) -> None: """Send a reminder about the build still failing.""" issue = issue_tracker.get_original_issue(issue_id) @@ -175,10 +186,11 @@ def send_reminder(issue_tracker, issue_id, build_id): 'remains productive.\n' 'Latest build log: {log_link}\n') comment = comment.format(log_link=_get_build_link(build_id)) - issue.save(new_comment=comment, notify=True) + cast(Any, issue).save(new_comment=comment, notify=True) -def _close_fixed_builds(projects, build_type): +def _close_fixed_builds(projects: list[dict[str, Any]], + build_type: str) -> None: """Close bugs for fixed builds.""" issue_tracker = issue_tracker_utils.get_issue_tracker() if not issue_tracker: @@ -200,7 +212,8 @@ def _close_fixed_builds(projects, build_type): if not build['success']: continue - if build_failure.last_checked_timestamp >= get_build_time(build): + if build_failure.last_checked_timestamp >= cast( # type: ignore[operator] + Any, get_build_time(build)): logs.error('Latest successful build time for %s in %s config is ' 'older than or equal to last failure time.' % (project_name, build_type)) @@ -212,7 +225,7 @@ def _close_fixed_builds(projects, build_type): close_build_failure(build_failure) -def _process_failures(projects, build_type): +def _process_failures(projects: list[dict[str, Any]], build_type: str) -> None: """Process failures.""" issue_tracker = issue_tracker_utils.get_issue_tracker() if not issue_tracker: @@ -243,14 +256,14 @@ def _process_failures(projects, build_type): build_time = get_build_time(build) if build_failure: - if build_time <= build_failure.last_checked_timestamp: + if build_time <= cast(Any, build_failure.last_checked_timestamp): # No updates. continue else: build_failure = create_build_failure(project_name, build, build_type) build_failure.last_checked_timestamp = build_time - build_failure.consecutive_failures += 1 + build_failure.consecutive_failures += 1 # type: ignore[operator] if build_failure.consecutive_failures >= MIN_CONSECUTIVE_BUILD_FAILURES: if build_failure.issue_id is None: oss_fuzz_project = _get_oss_fuzz_project(project_name) @@ -264,12 +277,14 @@ def _process_failures(projects, build_type): oss_fuzz_project.ccs, build_type) elif (build_failure.consecutive_failures - MIN_CONSECUTIVE_BUILD_FAILURES) % REMINDER_INTERVAL == 0: - send_reminder(issue_tracker, build_failure.issue_id, build['build_id']) + send_reminder(issue_tracker, cast(str, build_failure.issue_id), + build['build_id']) build_failure.put() -def _check_last_get_build_time(projects, build_type): +def _check_last_get_build_time(projects: list[dict[str, Any]], + build_type: str) -> None: """Check that builds are up to date.""" for project in projects: project_name = project['name'] @@ -278,14 +293,15 @@ def _check_last_get_build_time(projects, build_type): continue build = builds[0] - time_since_last_build = utils.utcnow() - get_build_time(build) + time_since_last_build = utils.utcnow() - cast(datetime.datetime, + get_build_time(build)) if time_since_last_build >= NO_BUILDS_THRESHOLD: # Something likely went wrong with the build infrastructure, log errors. logs.error('%s has not been built in %s config for %d days.' % (project_name, build_type, time_since_last_build.days)) -def main(): +def main() -> bool: """Build status checker.""" for build_type, status_url in BUILD_STATUS_MAPPINGS: try: diff --git a/src/clusterfuzz/_internal/cron/oss_fuzz_cc_groups.py b/src/clusterfuzz/_internal/cron/oss_fuzz_cc_groups.py index afcc8dbb2a2..1deabe49f3d 100644 --- a/src/clusterfuzz/_internal/cron/oss_fuzz_cc_groups.py +++ b/src/clusterfuzz/_internal/cron/oss_fuzz_cc_groups.py @@ -13,6 +13,8 @@ # limitations under the License. """Cron to sync OSS-Fuzz projects groups used as CC in the issue tracker.""" +from typing import cast + from clusterfuzz._internal.base import retry from clusterfuzz._internal.base import utils from clusterfuzz._internal.datastore import data_types @@ -25,7 +27,7 @@ _API_DELAY = 3 # 3s delay to avoid cloud identity api rate limits. -def normalize_email_for_group(email): +def normalize_email_for_group(email: str) -> str | None: """Normalize an email address for google groups.""" email = email.strip().lower() @@ -54,7 +56,7 @@ def normalize_email_for_group(email): delay=_API_DELAY, function='cron.oss_fuzz_cc_groups._add_member_with_retry', retry_on_false=True) -def _add_member_with_retry(group_id, member): +def _add_member_with_retry(group_id: str, member: str) -> bool: """Add a member to a group with retry.""" return google_groups.add_member_to_group(group_id, member) @@ -64,13 +66,14 @@ def _add_member_with_retry(group_id, member): delay=_API_DELAY, function='cron.oss_fuzz_cc_groups._delete_member_with_retry', retry_on_false=True) -def _delete_member_with_retry(group_id, member, membership_name): +def _delete_member_with_retry(group_id: str, member: str, + membership_name: str) -> bool: """Delete a member from a group with retry.""" return google_groups.delete_google_group_membership(group_id, member, membership_name) -def sync_project_cc_group(project_name: str, ccs: list[str]): +def sync_project_cc_group(project_name: str, ccs: list[str]) -> None: """Sync the project's google group used for CCing in the issue tracker.""" group_name = f'{project_name}{_CC_GROUP_SUFFIX}' @@ -122,14 +125,14 @@ def sync_project_cc_group(project_name: str, ccs: list[str]): _delete_member_with_retry(group_id, member, membership_name) -def main(): +def main() -> bool: """Sync OSS-Fuzz projects groups used to CC owners in the issue tracker.""" logs.info('OSS-Fuzz CC groups sync started.') for project in ndb_utils.get_all_from_model(data_types.OssFuzzProject): project_name = project.name ccs = project.ccs - sync_project_cc_group(project_name, ccs) + sync_project_cc_group(cast(str, project_name), cast(list[str], ccs)) logs.info('OSS-Fuzz CC groups sync succeeded.') return True diff --git a/src/clusterfuzz/_internal/cron/predator_pull.py b/src/clusterfuzz/_internal/cron/predator_pull.py index aa23bdc1dda..654edeb7766 100644 --- a/src/clusterfuzz/_internal/cron/predator_pull.py +++ b/src/clusterfuzz/_internal/cron/predator_pull.py @@ -14,6 +14,7 @@ """Process results from Predator and update test cases accordingly.""" import json +from typing import cast from clusterfuzz._internal.base import errors from clusterfuzz._internal.config import db_config @@ -22,7 +23,7 @@ from clusterfuzz._internal.metrics import logs -def main(): +def main() -> bool: """Periodically gathers new results from Predator requests.""" subscription = db_config.get_value('predator_result_topic') if not subscription: @@ -32,7 +33,7 @@ def main(): client = pubsub.PubSubClient() messages = client.pull_from_subscription(subscription, acknowledge=True) for message in messages: - message = json.loads(message.data) + message = json.loads(cast(str | bytes, message.data)) testcase_id = message['crash_identifiers'] try: testcase = data_handler.get_testcase_by_id(testcase_id) diff --git a/src/clusterfuzz/_internal/cron/project_setup.py b/src/clusterfuzz/_internal/cron/project_setup.py index f60e9e47c98..76559792998 100644 --- a/src/clusterfuzz/_internal/cron/project_setup.py +++ b/src/clusterfuzz/_internal/cron/project_setup.py @@ -14,10 +14,18 @@ """Handler used for setting up oss-fuzz and android jobs.""" import base64 -import collections import copy import json import re +from typing import Any +from typing import cast +from typing import Dict +from typing import Iterable +from typing import List +from typing import NamedTuple +from typing import Optional +from typing import Set +from typing import Tuple from google.cloud import ndb import requests @@ -38,42 +46,49 @@ from . import service_accounts -BUILD_BUCKET_PATH_TEMPLATE = ( +BUILD_BUCKET_PATH_TEMPLATE: str = ( 'gs://%BUCKET%/%PROJECT%/%PROJECT%-%SANITIZER%-([0-9]+).zip') -BACKUPS_LIFECYCLE = storage.generate_life_cycle_config('Delete', age=100) -LOGS_LIFECYCLE = storage.generate_life_cycle_config('Delete', age=14) -QUARANTINE_LIFECYCLE = storage.generate_life_cycle_config('Delete', age=90) +BACKUPS_LIFECYCLE: Dict[str, Any] = storage.generate_life_cycle_config( + 'Delete', age=100) +LOGS_LIFECYCLE: Dict[str, Any] = storage.generate_life_cycle_config( + 'Delete', age=14) +QUARANTINE_LIFECYCLE: Dict[str, Any] = storage.generate_life_cycle_config( + 'Delete', age=90) -JOB_TEMPLATE = ('{build_type} = {build_bucket_path}\n' - 'PROJECT_NAME = {project_name}\n' - 'SUMMARY_PREFIX = {project_name}\n' - 'MANAGED = True\n' - 'DISK_SIZE_GB = {disk_size_gb}\n') +JOB_TEMPLATE: str = ('{build_type} = {build_bucket_path}\n' + 'PROJECT_NAME = {project_name}\n' + 'SUMMARY_PREFIX = {project_name}\n' + 'MANAGED = True\n' + 'DISK_SIZE_GB = {disk_size_gb}\n') -OBJECT_VIEWER_IAM_ROLE = 'roles/storage.objectViewer' -OBJECT_ADMIN_IAM_ROLE = 'roles/storage.objectAdmin' +OBJECT_VIEWER_IAM_ROLE: str = 'roles/storage.objectViewer' +OBJECT_ADMIN_IAM_ROLE: str = 'roles/storage.objectAdmin' -VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$') +VALID_PROJECT_NAME_REGEX: re.Pattern = re.compile(r'^[a-zA-Z0-9_-]+$') -REVISION_URL = ('https://commondatastorage.googleapis.com/' - '{bucket}/{project}/{project}-{sanitizer}-%s.srcmap.json') +REVISION_URL: str = ('https://commondatastorage.googleapis.com/' + '{bucket}/{project}/{project}-{sanitizer}-%s.srcmap.json') -REQUEST_TIMEOUT = 60 +REQUEST_TIMEOUT: int = 60 -ALLOWED_VIEW_RESTRICTIONS = ['none', 'security', 'all'] +ALLOWED_VIEW_RESTRICTIONS: List[str] = ['none', 'security', 'all'] -PUBSUB_PLATFORMS = ['linux'] +PUBSUB_PLATFORMS: List[str] = ['linux'] -MEMORY_UNSAFE_LANGUAGES = {'c++', 'c'} -OSS_FUZZ_DEFAULT_PROJECT_CPU_WEIGHT = 1.0 -OSS_FUZZ_MEMORY_SAFE_LANGUAGE_PROJECT_WEIGHT = 0.2 +MEMORY_UNSAFE_LANGUAGES: Set[str] = {'c++', 'c'} +OSS_FUZZ_DEFAULT_PROJECT_CPU_WEIGHT: float = 1.0 +OSS_FUZZ_MEMORY_SAFE_LANGUAGE_PROJECT_WEIGHT: float = 0.2 -SetupResult = collections.namedtuple('SetupResult', 'project_names job_names') -HTTP_TIMEOUT_SECONDS = 30 +class SetupResult(NamedTuple): + project_names: List[str] + job_names: List[str] -PROJECTS_USING_SUBQUEUES = {'android'} + +HTTP_TIMEOUT_SECONDS: int = 30 + +PROJECTS_USING_SUBQUEUES: Set[str] = {'android'} class ProjectSetupError(Exception): @@ -83,14 +98,22 @@ class ProjectSetupError(Exception): class JobInfo: """Job information.""" + prefix: str + engine: str + memory_tool: str + architecture: str + cf_job_templates: List[str] + experimental: bool + minimize_job_override: Optional['JobInfo'] + def __init__(self, - prefix, - engine, - memory_tool, - cf_job_templates, - architecture='x86_64', - experimental=False, - minimize_job_override=None): + prefix: str, + engine: str, + memory_tool: str, + cf_job_templates: List[str], + architecture: str = 'x86_64', + experimental: bool = False, + minimize_job_override: Optional['JobInfo'] = None) -> None: self.prefix = prefix self.engine = engine self.memory_tool = memory_tool @@ -99,7 +122,7 @@ def __init__(self, self.experimental = experimental self.minimize_job_override = minimize_job_override - def job_name(self, project_name, config_suffix): + def job_name(self, project_name: str, config_suffix: str) -> str: return ( self.prefix + data_types.normalized_name(project_name) + config_suffix) @@ -156,7 +179,7 @@ def job_name(self, project_name, config_suffix): 'none', ['libfuzzer', 'prune'], architecture='i386') -JOB_MAP = { +JOB_MAP: Dict[str, Dict[str, Dict[str, JobInfo]]] = { 'libfuzzer': { 'x86_64': { 'address': LIBFUZZER_ASAN_JOB, @@ -207,18 +230,18 @@ def job_name(self, project_name, config_suffix): }, } -DEFAULT_ARCHITECTURES = ['x86_64'] -DEFAULT_SANITIZERS = ['address', 'undefined'] -DEFAULT_ENGINES = ['libfuzzer', 'afl', 'honggfuzz'] +DEFAULT_ARCHITECTURES: List[str] = ['x86_64'] +DEFAULT_SANITIZERS: List[str] = ['address', 'undefined'] +DEFAULT_ENGINES: List[str] = ['libfuzzer', 'afl', 'honggfuzz'] -def _to_experimental_job(job_info): +def _to_experimental_job(job_info: JobInfo) -> JobInfo: job_info = copy.copy(job_info) job_info.experimental = True return job_info -def get_github_url(url): +def get_github_url(url: str) -> Any: """Return contents of URL.""" github_credentials = db_config.get_value('github_credentials') if not github_credentials: @@ -235,7 +258,8 @@ def get_github_url(url): return json.loads(response.text) -def find_github_item_url(github_json, name): +def find_github_item_url(github_json: Dict[str, Any], + name: str) -> Optional[str]: """Get url of a blob/tree from a github json response.""" for item in github_json['tree']: if item['path'] == name: @@ -244,7 +268,7 @@ def find_github_item_url(github_json, name): return None -def get_oss_fuzz_projects(): +def get_oss_fuzz_projects() -> List[Tuple[str, Dict[str, Any]]]: """Return list of projects for oss-fuzz.""" ossfuzz_tree_url = ('https://api.github.com/repos/google/oss-fuzz/' 'git/trees/master') @@ -267,7 +291,8 @@ def get_oss_fuzz_projects(): continue projects_yaml = get_github_url(project_yaml_url) - info = yaml.safe_load(base64.b64decode(projects_yaml['content'])) + info = cast(Dict[str, Any], + yaml.safe_load(base64.b64decode(projects_yaml['content']))) has_dockerfile = ( find_github_item_url(item_json, 'Dockerfile') or 'dockerfile' in info) @@ -279,7 +304,7 @@ def get_oss_fuzz_projects(): return projects -def get_projects_from_gcs(gcs_url): +def get_projects_from_gcs(gcs_url: str) -> List[Tuple[str, Dict[str, Any]]]: """Get projects from GCS path.""" try: data = json.loads(storage.read_data(gcs_url)) @@ -289,7 +314,8 @@ def get_projects_from_gcs(gcs_url): return [(project['name'], project) for project in data['projects']] -def _process_sanitizers_field(sanitizers): +def _process_sanitizers_field( + sanitizers: Any) -> Optional[Dict[str, Dict[str, Any]]]: """Pre-process sanitizers field into a map from sanitizer name -> dict of options.""" processed_sanitizers = {} @@ -314,7 +340,7 @@ def _process_sanitizers_field(sanitizers): return processed_sanitizers -def get_jobs_for_project(project, info): +def get_jobs_for_project(project: str, info: Dict[str, Any]) -> List[JobInfo]: """Return jobs for the project.""" sanitizers = _process_sanitizers_field( info.get('sanitizers', DEFAULT_SANITIZERS)) @@ -348,7 +374,7 @@ def get_jobs_for_project(project, info): return jobs -def convert_googlemail_to_gmail(email): +def convert_googlemail_to_gmail(email: str) -> str: """Convert @googlemail.com to @gmail.com.""" # TODO(ochang): Investiate if we can/need to do this in general, and not just # for cloud storage bucket IAMs. @@ -358,7 +384,9 @@ def convert_googlemail_to_gmail(email): return email -def _add_users_to_bucket(info, client, bucket_name, iam_policy): +def _add_users_to_bucket( + info: Dict[str, Any], client: Any, bucket_name: str, + iam_policy: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Add user account to bucket.""" ccs = sorted( ['user:' + convert_googlemail_to_gmail(cc) for cc in ccs_from_info(info)]) @@ -378,13 +406,16 @@ def _add_users_to_bucket(info, client, bucket_name, iam_policy): if len(filtered_members) != len(binding['members']): # Remove old members. binding['members'] = filtered_members - iam_policy = storage.set_bucket_iam_policy(client, bucket_name, - iam_policy) + modified_policy = storage.set_bucket_iam_policy(client, bucket_name, + iam_policy) + if modified_policy: + iam_policy = modified_policy # We might have no binding either from start or after filtering members above. # Create a new one in those cases. binding = storage.get_or_create_bucket_iam_binding(iam_policy, OBJECT_VIEWER_IAM_ROLE) + assert binding is not None for cc in ccs: if cc in binding['members']: @@ -397,9 +428,12 @@ def _add_users_to_bucket(info, client, bucket_name, iam_policy): client, iam_policy, OBJECT_VIEWER_IAM_ROLE, bucket_name, cc) if modified_iam_policy: iam_policy = modified_iam_policy - binding = storage.get_bucket_iam_binding(iam_policy, - OBJECT_VIEWER_IAM_ROLE) + new_binding = storage.get_bucket_iam_binding(iam_policy, + OBJECT_VIEWER_IAM_ROLE) + if new_binding: + binding = new_binding + assert binding is not None if not binding['members']: # Check that the final binding has members. Empty bindings are not valid. storage.remove_bucket_iam_binding(iam_policy, OBJECT_VIEWER_IAM_ROLE) @@ -407,12 +441,14 @@ def _add_users_to_bucket(info, client, bucket_name, iam_policy): return iam_policy -def _set_bucket_service_account(service_account, client, bucket_name, - iam_policy): +def _set_bucket_service_account( + service_account: Dict[str, Any], client: Any, bucket_name: str, + iam_policy: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Set service account for a bucket.""" # Add service account as objectAdmin. binding = storage.get_or_create_bucket_iam_binding(iam_policy, OBJECT_ADMIN_IAM_ROLE) + assert binding is not None members = ['serviceAccount:' + service_account['email']] if members == binding['members']: @@ -423,23 +459,29 @@ def _set_bucket_service_account(service_account, client, bucket_name, return storage.set_bucket_iam_policy(client, bucket_name, iam_policy) -def add_bucket_iams(info, client, bucket_name, service_account): +def add_bucket_iams(info: Dict[str, Any], client: Any, bucket_name: str, + service_account: Dict[str, Any]) -> None: """Add CC'ed users to storage bucket IAM.""" iam_policy = storage.get_bucket_iam_policy(client, bucket_name) if not iam_policy: return iam_policy = _add_users_to_bucket(info, client, bucket_name, iam_policy) - _set_bucket_service_account(service_account, client, bucket_name, iam_policy) + if iam_policy: + _set_bucket_service_account(service_account, client, bucket_name, + iam_policy) -def add_service_account_to_bucket(client, bucket_name, service_account, role): +def add_service_account_to_bucket(client: Any, bucket_name: str, + service_account: Dict[str, Any], + role: str) -> None: """Add service account to the gcr.io images bucket.""" iam_policy = storage.get_bucket_iam_policy(client, bucket_name) if not iam_policy: return binding = storage.get_or_create_bucket_iam_binding(iam_policy, role) + assert binding is not None member = 'serviceAccount:' + service_account['email'] if member in binding['members']: @@ -450,15 +492,15 @@ def add_service_account_to_bucket(client, bucket_name, service_account, role): storage.set_bucket_iam_policy(client, bucket_name, iam_policy) -def has_maintainer(info): +def has_maintainer(info: Dict[str, Any]) -> Any: """Return whether or not a project has at least one maintainer.""" return info.get('primary_contact') or info.get('auto_ccs') -def ccs_from_info(info): +def ccs_from_info(info: Dict[str, Any]) -> List[str]: """Get list of CC's from project info.""" - def _get_ccs(field_name, allow_list=True): + def _get_ccs(field_name: str, allow_list: bool = True) -> List[str]: """Return list of emails to cc given a field name.""" if field_name not in info: return [] @@ -481,7 +523,7 @@ def _get_ccs(field_name, allow_list=True): return [utils.normalize_email(cc) for cc in ccs] -def update_fuzzer_jobs(job_names): +def update_fuzzer_jobs(job_names: Iterable[str]) -> None: """Update fuzzer job mappings.""" to_delete = {} @@ -494,20 +536,21 @@ def update_fuzzer_jobs(job_names): if not utils.string_is_true(job_environment.get('MANAGED', 'False')): continue - if job.name in job_names: + if cast(str, job.name) in job_names: continue logs.info(f'Deleting job {job.name}') print(f'Deleting job {job.name}') - to_delete[job.name] = job.key + to_delete[cast(str, job.name)] = job.key # Clean up job references from all fuzzer entities for fuzzer in data_types.Fuzzer.query(): # modified = False + fuzzer_jobs = fuzzer.jobs for job_name in to_delete: try: - fuzzer.jobs.remove(job_name) + fuzzer_jobs.remove(job_name) # modified = True except ValueError: pass @@ -520,12 +563,12 @@ def update_fuzzer_jobs(job_names): ndb_utils.delete_multi(to_delete.values()) -def cleanup_old_projects_settings(project_names): +def cleanup_old_projects_settings(project_names: Iterable[str]) -> None: """Delete old projects that are no longer used or disabled.""" to_delete = [] for project in data_types.OssFuzzProject.query(): - if project.name not in project_names: + if cast(str, project.name) not in project_names: logs.info(f'Deleting project {project.name}.') to_delete.append(project.key) @@ -533,7 +576,8 @@ def cleanup_old_projects_settings(project_names): ndb_utils.delete_multi(to_delete) -def create_project_settings(project, info, service_account): +def create_project_settings(project: str, info: Dict[str, Any], + service_account: Dict[str, Any]) -> None: """Setup settings for ClusterFuzz (such as CPU distribution).""" key = ndb.Key(data_types.OssFuzzProject, project) oss_fuzz_project = key.get() @@ -577,20 +621,20 @@ def create_project_settings(project, info, service_account): base_os_version=base_os_version).put() -def _create_pubsub_topic(name, client): +def _create_pubsub_topic(name: str, client: pubsub.PubSubClient) -> None: """Create a pubsub topic and subscription if needed.""" application_id = utils.get_application_id() - topic_name = pubsub.topic_name(application_id, name) + topic_name = pubsub.topic_name(cast(str, application_id), name) if client.get_topic(topic_name) is None: client.create_topic(topic_name) - subscription_name = pubsub.subscription_name(application_id, name) + subscription_name = pubsub.subscription_name(cast(str, application_id), name) if client.get_subscription(subscription_name) is None: client.create_subscription(subscription_name, topic_name) -def create_pubsub_topics_for_untrusted(project): +def create_pubsub_topics_for_untrusted(project: str) -> None: """Create pubsub topics from untrusted sources for tasks.""" client = pubsub.PubSubClient() for platform in PUBSUB_PLATFORMS: @@ -598,7 +642,7 @@ def create_pubsub_topics_for_untrusted(project): _create_pubsub_topic(name, client) -def create_pubsub_topics_for_queue_id(platform): +def create_pubsub_topics_for_queue_id(platform: str) -> None: """Create pubsub topics from project configs for tasks.""" platform, queue_id = platform.split(tasks.SUBQUEUE_IDENTIFIER) name = untrusted.queue_name(platform, queue_id) @@ -606,7 +650,7 @@ def create_pubsub_topics_for_queue_id(platform): _create_pubsub_topic(name, client) -def cleanup_pubsub_topics(project_names): +def cleanup_pubsub_topics(project_names: Iterable[str]) -> None: """Delete old pubsub topics and subscriptions.""" client = pubsub.PubSubClient() application_id = utils.get_application_id() @@ -619,7 +663,8 @@ def cleanup_pubsub_topics(project_names): pubsub_config = local_config.Config('pubsub.queues') unmanaged_queues = [queue['name'] for queue in pubsub_config.get('resources')] - for topic in client.list_topics(pubsub.project_name(application_id)): + for topic in client.list_topics( + pubsub.project_name(cast(str, application_id))): _, name = pubsub.parse_name(topic) if (not name.startswith(tasks.JOBS_PREFIX) and @@ -642,19 +687,32 @@ def cleanup_pubsub_topics(project_names): class ProjectSetup: """Project setup.""" + _build_type: Optional[str] + _config_suffix: str + _external_config: Any + _build_bucket_path_template: str + _revision_url_template: str + _segregate_projects: bool + _experimental_sanitizers: Optional[List[str]] + _engine_build_buckets: Optional[Dict[str, Any]] + _fuzzer_entities: Optional[Dict[str, Any]] + _add_info_labels: bool + _add_revision_mappings: bool + _additional_vars: Optional[Dict[str, Any]] + def __init__(self, - build_bucket_path_template, - revision_url_template, - build_type, - config_suffix='', - external_config=None, - segregate_projects=False, - experimental_sanitizers=None, - engine_build_buckets=None, - fuzzer_entities=None, - add_info_labels=False, - add_revision_mappings=False, - additional_vars=None): + build_bucket_path_template: str, + revision_url_template: str, + build_type: Optional[str], + config_suffix: str = '', + external_config: Any = None, + segregate_projects: bool = False, + experimental_sanitizers: Optional[List[str]] = None, + engine_build_buckets: Optional[Dict[str, Any]] = None, + fuzzer_entities: Optional[Dict[str, Any]] = None, + add_info_labels: bool = False, + add_revision_mappings: bool = False, + additional_vars: Optional[Dict[str, Any]] = None) -> None: self._build_type = build_type self._config_suffix = config_suffix self._external_config = external_config @@ -668,38 +726,40 @@ def __init__(self, self._add_revision_mappings = add_revision_mappings self._additional_vars = additional_vars - def _get_build_bucket(self, engine, architecture): + def _get_build_bucket(self, engine: str, architecture: str) -> str: """Return the bucket for the given |engine| and |architecture|.""" if architecture != 'x86_64': engine += '-' + architecture - bucket = self._engine_build_buckets.get(engine) + bucket = cast(Dict[str, Any], self._engine_build_buckets).get(engine) if not bucket: raise ProjectSetupError('Invalid fuzzing engine ' + engine) return bucket - def _deployment_bucket_name(self): + def _deployment_bucket_name(self) -> str: """Deployment bucket name.""" return f'{utils.get_application_id()}-deployment' - def _backup_bucket_name(self, project_name): + def _backup_bucket_name(self, project_name: str) -> str: """Return the backup_bucket_name.""" return project_name + '-backup.' + data_handler.bucket_domain_suffix() - def _corpus_bucket_name(self, project_name): + def _corpus_bucket_name(self, project_name: str) -> str: """Return the corpus_bucket_name.""" return project_name + '-corpus.' + data_handler.bucket_domain_suffix() - def _quarantine_bucket_name(self, project_name): + def _quarantine_bucket_name(self, project_name: str) -> str: """Return the quarantine_bucket_name.""" return project_name + '-quarantine.' + data_handler.bucket_domain_suffix() - def _logs_bucket_name(self, project_name): + def _logs_bucket_name(self, project_name: str) -> str: """Return the logs bucket name.""" return project_name + '-logs.' + data_handler.bucket_domain_suffix() - def _create_service_accounts_and_buckets(self, project, info): + def _create_service_accounts_and_buckets( + self, project: str, + info: Dict[str, Any]) -> Tuple[Dict[str, Any], str, str, str, str]: """Create per-project service account and buckets.""" service_account, exists = service_accounts.get_or_create_service_account( project) @@ -732,10 +792,8 @@ def _create_service_accounts_and_buckets(self, project, info): # Grant the service account read access to deployment bucket. add_service_account_to_bucket(client, self._deployment_bucket_name(), service_account, OBJECT_VIEWER_IAM_ROLE) - data_bundles = { - fuzzer_entity.get().data_bundle_name - for fuzzer_entity in self._fuzzer_entities.values() - } + data_bundles = {(fuzzer_entity.get()).data_bundle_name for fuzzer_entity in + cast(Dict[str, Any], self._fuzzer_entities).values()} for data_bundle in data_bundles: if not data_bundle: continue @@ -749,8 +807,9 @@ def _create_service_accounts_and_buckets(self, project, info): return (service_account, backup_bucket_name, corpus_bucket_name, logs_bucket_name, quarantine_bucket_name) - def _get_build_bucket_path(self, project_name, info, engine, memory_tool, - architecture): + def _get_build_bucket_path(self, project_name: str, info: Dict[str, Any], + engine: str, memory_tool: str, + architecture: str) -> str: """Returns the build bucket path for the |project|, |engine|, |memory_tool|, and |architecture|.""" build_path = info.get('build_path') @@ -764,15 +823,18 @@ def _get_build_bucket_path(self, project_name, info, engine, memory_tool, build_path = build_path.replace('%SANITIZER%', memory_tool) return build_path - def _get_base_project_name(self, project): + def _get_base_project_name(self, project: str) -> str: """Returns the base project if subqueues are being used.""" for base_project in PROJECTS_USING_SUBQUEUES: if project.startswith(base_project): return base_project return project - def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name, - logs_bucket_name, backup_bucket_name): + def _sync_job(self, project: str, info: Dict[str, Any], + corpus_bucket_name: Optional[str], + quarantine_bucket_name: Optional[str], + logs_bucket_name: Optional[str], + backup_bucket_name: Optional[str]) -> List[str]: """Sync the config with ClusterFuzz.""" # Create/update ClusterFuzz jobs. job_names = [] @@ -788,7 +850,10 @@ def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name, # Get fuzzers specified for this project fuzzers = info.get('fuzzers', []) else: - fuzzer_entity = self._fuzzer_entities.get(template.engine).get() + fuzzer_entity = cast( + Any, + cast(Dict[str, Any], + self._fuzzer_entities).get(template.engine)).get() if not fuzzer_entity: raise ProjectSetupError('Invalid fuzzing engine ' + template.engine) fuzzers = [] @@ -941,7 +1006,8 @@ def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name, 'LINUX').upper() + ':' + queue_id.upper() job.platform = platform if platform.startswith('ANDROID'): - job.templates.append('android') + job_templates = job.templates + job_templates.append('android') create_pubsub_topics_for_queue_id(job.platform) @@ -970,7 +1036,7 @@ def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name, print(project, job_names) return job_names - def sync_user_permissions(self, project, info): + def sync_user_permissions(self, project: str, info: Dict[str, Any]) -> None: """Sync permissions of project based on project.yaml.""" ccs = ccs_from_info(info) @@ -1006,10 +1072,11 @@ def sync_user_permissions(self, project, info): is_prefix=False, auto_cc=data_types.AutoCCType.ALL).put() - def set_up(self, projects): + def set_up(self, projects: List[Tuple[str, Dict[str, Any]]]) -> SetupResult: """Do project setup. Return a list of all the project names that were set up.""" job_names = [] + service_account: Dict[str, Any] = {} for project, info in projects: logs.info(f'Syncing configs for {project}.') @@ -1048,7 +1115,9 @@ def set_up(self, projects): return SetupResult(enabled_projects, job_names) -def cleanup_stale_projects(project_names, job_names, segregate_projects): +def cleanup_stale_projects(project_names: Iterable[str], + job_names: Iterable[str], + segregate_projects: bool) -> None: """Clean up stale projects.""" update_fuzzer_jobs(job_names) cleanup_old_projects_settings(project_names) @@ -1057,7 +1126,7 @@ def cleanup_stale_projects(project_names, job_names, segregate_projects): cleanup_pubsub_topics(project_names) -def main(): +def main() -> bool: """Setup ClusterFuzz jobs for projects.""" libfuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == 'libFuzzer').get() diff --git a/src/clusterfuzz/_internal/cron/retry_stuck_tasks.py b/src/clusterfuzz/_internal/cron/retry_stuck_tasks.py index 240e86d8ef7..d80f1f85c56 100644 --- a/src/clusterfuzz/_internal/cron/retry_stuck_tasks.py +++ b/src/clusterfuzz/_internal/cron/retry_stuck_tasks.py @@ -25,7 +25,6 @@ import datetime from enum import StrEnum import os -from typing import cast from typing import NamedTuple from google.cloud import ndb # pylint: disable=no-name-in-module @@ -37,8 +36,8 @@ from clusterfuzz._internal.datastore import ndb_utils from clusterfuzz._internal.metrics import logs -RETRY_ATTEMPT_COUNT_KEY = 'retry_stuck_task_attempt_count' -RETRY_LAST_ATTEMPT_TIME_KEY = 'retry_stuck_task_last_attempt_time' +RETRY_ATTEMPT_COUNT_KEY: str = 'retry_stuck_task_attempt_count' +RETRY_LAST_ATTEMPT_TIME_KEY: str = 'retry_stuck_task_last_attempt_time' class StuckReason(StrEnum): @@ -176,8 +175,7 @@ def _get_testcase_id(testcase: data_types.Testcase) -> str: Returns: The string representation of the testcase ID. """ - key = cast(ndb.Key, testcase.key) - return str(key.id()) + return str(testcase.key.id()) def _is_job_valid(testcase: data_types.Testcase) -> bool: @@ -381,7 +379,8 @@ def filter_and_categorize_candidates( return categorized -def _log_verbose_summary(total_from_query: int, results: CategorizedTestcases): +def _log_verbose_summary(total_from_query: int, + results: CategorizedTestcases) -> None: """Prints a detailed, verbose summary of the filtering phase. This function provides a clear, human-readable summary of the script's @@ -409,7 +408,7 @@ def _log_verbose_summary(total_from_query: int, results: CategorizedTestcases): @logs.cron_log_context() -def main(args: list[str] | None = None): +def main(args: list[str] | None = None) -> None: """Finds, filters, and restarts stuck testcases in a three-phase process. The workflow is designed for clarity, safety, and observability: diff --git a/src/clusterfuzz/_internal/cron/schedule_corpus_pruning.py b/src/clusterfuzz/_internal/cron/schedule_corpus_pruning.py index 206a4e6b690..a8f6e133b70 100644 --- a/src/clusterfuzz/_internal/cron/schedule_corpus_pruning.py +++ b/src/clusterfuzz/_internal/cron/schedule_corpus_pruning.py @@ -14,6 +14,8 @@ """Schedule corpus pruning tasks.""" import time +from typing import cast +from typing import Iterator from clusterfuzz._internal.base import tasks from clusterfuzz._internal.base import utils @@ -23,7 +25,7 @@ from clusterfuzz._internal.metrics import logs -def get_tasks_to_schedule(): +def get_tasks_to_schedule() -> Iterator[tuple[str, str, str]]: """Return (task_target, job_name, queue_name) arguments to schedule a task.""" for job in ndb_utils.get_all_from_query(data_types.Job.query()): if not utils.string_is_true(job.get_environment().get('CORPUS_PRUNE')): @@ -31,11 +33,11 @@ def get_tasks_to_schedule(): queue_name = tasks.queue_for_job(job.name) for target_job in fuzz_target_utils.get_fuzz_target_jobs(job=job.name): - task_target = target_job.fuzz_target_name + task_target = cast(str, target_job.fuzz_target_name) yield (task_target, job.name, queue_name) -def main(): +def main() -> bool: """Schedule corpus pruning tasks.""" for task_target, job_name, queue_name in get_tasks_to_schedule(): logs.info(f'Adding corpus pruning task {task_target}.') diff --git a/src/clusterfuzz/_internal/cron/schedule_fuzz.py b/src/clusterfuzz/_internal/cron/schedule_fuzz.py index abd34bdd9ea..dd1d4b8b4c3 100644 --- a/src/clusterfuzz/_internal/cron/schedule_fuzz.py +++ b/src/clusterfuzz/_internal/cron/schedule_fuzz.py @@ -18,7 +18,9 @@ import collections import random import time +from typing import cast +from google.auth import credentials as google_auth_credentials from google.cloud import monitoring_v3 from clusterfuzz._internal import swarming @@ -37,7 +39,11 @@ @memoize.wrap(memoize.InMemory(60)) -def get_queue_size(creds, project_id, subscription_id): +def get_queue_size( + creds: google_auth_credentials.Credentials, + project_id: str, + subscription_id: str, +) -> int: """Returns the size of the queue (unacked messages).""" # TODO(metzman): Not all of these are fuzz_tasks. Deal with that. metric = 'pubsub.googleapis.com/subscription/num_undelivered_messages' @@ -80,19 +86,25 @@ class FuzzTaskCandidate: Something like this would probably not be needed if we were using SQL and could use joins.""" + job: str + project: str + fuzzer: str | None + weight: float | None + base_os_version: str | None + def __init__(self, - job, - project, - fuzzer=None, - weight=None, - base_os_version=None): + job: str, + project: str, + fuzzer: str | None = None, + weight: float | None = None, + base_os_version: str | None = None) -> None: self.job = job self.project = project self.fuzzer = fuzzer self.weight = weight self.base_os_version = base_os_version - def copy(self): + def copy(self) -> 'FuzzTaskCandidate': return FuzzTaskCandidate( job=self.job, project=self.project, @@ -113,7 +125,7 @@ def get_fuzz_tasks(self, num_tasks: int) -> list[tasks.Task]: logs.info(f'Got {len(projects)} projects.') total_cpu_weight = sum(project.cpu_weight for project in projects) - project_weights = {} + project_weights: dict[str, float] = {} for project in projects: project_weight = project.cpu_weight / total_cpu_weight project_weights[project.name] = project_weight @@ -123,7 +135,7 @@ def get_fuzz_tasks(self, num_tasks: int) -> list[tasks.Task]: # Then get FuzzTaskCandidate weights. logs.info('Getting jobs.') # TODO(metzman): Handle cases where jobs are fuzzed by multiple fuzzers. - candidates_by_job = {} + candidates_by_job: dict[str, FuzzTaskCandidate] = {} for job in ndb_utils.get_all_from_query(data_types.Job.query()): project = projects_by_name.get(job.project) base_os_version = None @@ -132,11 +144,14 @@ def get_fuzz_tasks(self, num_tasks: int) -> list[tasks.Task]: elif job.base_os_version: base_os_version = job.base_os_version - candidates_by_job[job.name] = FuzzTaskCandidate( - job=job.name, project=job.project, base_os_version=base_os_version) + candidates_by_job[cast(str, job.name)] = FuzzTaskCandidate( + job=cast(str, job.name), + project=cast(str, job.project), + base_os_version=base_os_version) - fuzzer_job_weight_by_project = collections.defaultdict(int) - fuzz_task_candidates = [] + fuzzer_job_weight_by_project: dict[str, float] = collections.defaultdict( + float) + fuzz_task_candidates: list[FuzzTaskCandidate] = [] fuzzer_job_query = ndb_utils.get_all_from_query( data_types.FuzzerJob.query()) @@ -155,17 +170,17 @@ def get_fuzz_tasks(self, num_tasks: int) -> list[tasks.Task]: if project_weight is None: logs.info(f'No project weight for {fuzz_task_candidate.project}.' 'Not scheduling.') - fuzz_task_candidate.weight = 0 + fuzz_task_candidate.weight = 0.0 continue total_project_weight = fuzzer_job_weight_by_project[ fuzz_task_candidate.project] - fuzz_task_candidate.weight = ( - fuzz_task_candidate.weight / total_project_weight) * project_weight + fuzz_task_candidate.weight = (cast(float, fuzz_task_candidate.weight) / + total_project_weight) * project_weight # Prepare lists for random.choice - weights = [] + weights: list[float] = [] for fuzz_task_candidate in fuzz_task_candidates: - weights.append(fuzz_task_candidate.weight) + weights.append(cast(float, fuzz_task_candidate.weight)) logs.info(f'Scheduling {num_tasks} fuzz tasks for OSS-Fuzz.') @@ -189,7 +204,7 @@ class ChromeFuzzTaskProvider(BaseFuzzTaskProvider): _candidates: list[FuzzTaskCandidate] - def __init__(self, jobs: list[data_types.Job]): + def __init__(self, jobs: list[data_types.Job]) -> None: self._candidates = _create_candidates_from_jobs(jobs) def get_fuzz_tasks(self, num_tasks: int) -> list[tasks.Task]: @@ -202,7 +217,8 @@ def get_fuzz_tasks(self, num_tasks: int) -> list[tasks.Task]: if not self._candidates: return [] - choices = random.choices(self._candidates, weights=weights, k=num_tasks) + choices = random.choices( + self._candidates, weights=cast(list[float], weights), k=num_tasks) fuzz_tasks = [ tasks.Task( 'fuzz', @@ -219,7 +235,7 @@ def _get_jobs_for_platforms(platforms: list[str]) -> list[data_types.Job]: return list(data_types.Job.query(data_types.Job.platform.IN(platforms))) -def _get_swarming_jobs(): +def _get_swarming_jobs() -> list[data_types.Job]: """Returns all jobs that have swarming environment variables.""" jobs = _get_jobs_for_platforms(['ANDROID', 'LINUX', 'ANDROID_EMULATOR']) return [ @@ -230,7 +246,7 @@ def _get_swarming_jobs(): def _remaining_queue_capacity(queue: PubSubTaskQueue) -> int: """Returns the remaining capacity of the given queue.""" - project = utils.get_application_id() + project = cast(str, utils.get_application_id()) creds = credentials.get_default()[0] preprocess_queue_size = get_queue_size(creds, project, queue.name) @@ -243,7 +259,7 @@ def _remaining_queue_capacity(queue: PubSubTaskQueue) -> int: return num_tasks -def _fill_queue(queue: PubSubTaskQueue, provider: BaseFuzzTaskProvider): +def _fill_queue(queue: PubSubTaskQueue, provider: BaseFuzzTaskProvider) -> None: """Fills the given queue with tasks from the provider.""" start = time.time() num_tasks = _remaining_queue_capacity(queue) @@ -276,13 +292,13 @@ def _create_candidates_from_jobs( fuzzer_job_query = ndb_utils.get_all_from_query( data_types.FuzzerJob.query( data_types.FuzzerJob.job.IN(list(jobs_by_name.keys())))) - fuzz_task_candidates = [] + fuzz_task_candidates: list[FuzzTaskCandidate] = [] for fuzzer_job in fuzzer_job_query: job = jobs_by_name[fuzzer_job.job] fuzz_task_candidate = FuzzTaskCandidate( - job=job.name, - project=job.project, + job=cast(str, job.name), + project=cast(str, job.project), base_os_version=job.base_os_version, fuzzer=fuzzer_job.fuzzer, weight=fuzzer_job.actual_weight, @@ -292,7 +308,7 @@ def _create_candidates_from_jobs( return fuzz_task_candidates -def schedule_chrome_fuzz_tasks(): +def schedule_chrome_fuzz_tasks() -> None: """Schedules fuzz tasks for Chrome.""" default_jobs = _get_jobs_for_platforms(['LINUX']) default_provider = ChromeFuzzTaskProvider(default_jobs) @@ -306,7 +322,7 @@ def schedule_chrome_fuzz_tasks(): _fill_queue(SWARMING_PREPROCESS_QUEUE, swarming_provider) -def schedule_fuzz_tasks(): +def schedule_fuzz_tasks() -> None: """Schedules fuzz tasks based on deployment type.""" if utils.is_oss_fuzz(): _fill_queue(PREPROCESS_QUEUE, OssfuzzFuzzTaskProvider()) @@ -314,5 +330,5 @@ def schedule_fuzz_tasks(): schedule_chrome_fuzz_tasks() -def main(): +def main() -> None: schedule_fuzz_tasks() diff --git a/src/clusterfuzz/_internal/cron/schedule_impact_tasks.py b/src/clusterfuzz/_internal/cron/schedule_impact_tasks.py index 907332e5d7f..15ac265a175 100644 --- a/src/clusterfuzz/_internal/cron/schedule_impact_tasks.py +++ b/src/clusterfuzz/_internal/cron/schedule_impact_tasks.py @@ -16,7 +16,7 @@ from clusterfuzz._internal.cron.helpers import tasks_scheduler -def main(): +def main() -> bool: """Creates impact tasks.""" task = 'impact' tasks_scheduler.schedule(task) diff --git a/src/clusterfuzz/_internal/cron/schedule_progression_tasks.py b/src/clusterfuzz/_internal/cron/schedule_progression_tasks.py index 8e4eec5e2c0..b932a5c09b1 100644 --- a/src/clusterfuzz/_internal/cron/schedule_progression_tasks.py +++ b/src/clusterfuzz/_internal/cron/schedule_progression_tasks.py @@ -17,7 +17,7 @@ from clusterfuzz._internal.system import environment -def main(): +def main() -> bool: """Creates progression tasks.""" environment.set_bot_environment() task = 'progression' diff --git a/src/clusterfuzz/_internal/cron/service_accounts.py b/src/clusterfuzz/_internal/cron/service_accounts.py index dfe8d623938..15009c5142d 100644 --- a/src/clusterfuzz/_internal/cron/service_accounts.py +++ b/src/clusterfuzz/_internal/cron/service_accounts.py @@ -13,30 +13,33 @@ # limitations under the License. """Service account creation/role helpers.""" import logging +from typing import Any +from typing import cast -import googleapiclient +import googleapiclient.discovery +import googleapiclient.errors from clusterfuzz._internal.base import utils -_ACCOUNT_PREFIX = 'bot-' -_MIN_LEN = 6 -_MAX_LEN = 30 -_HASH_PREFIX_LEN = _MAX_LEN - len(_ACCOUNT_PREFIX) +_ACCOUNT_PREFIX: str = 'bot-' +_MIN_LEN: int = 6 +_MAX_LEN: int = 30 +_HASH_PREFIX_LEN: int = _MAX_LEN - len(_ACCOUNT_PREFIX) # pylint: disable=no-member -def _create_client(service_name, version='v1'): +def _create_client(service_name: str, version: str = 'v1') -> Any: """Create a googleapiclient client.""" return googleapiclient.discovery.build(service_name, version) -def _service_account_email(project_id, service_account_id): +def _service_account_email(project_id: str, service_account_id: str) -> str: """Return full service account email.""" return '%s@%s.iam.gserviceaccount.com' % (service_account_id, project_id) -def _service_account_id(project): +def _service_account_id(project: str) -> str: """Return service account ID for project.""" # From # cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts/create: @@ -63,7 +66,8 @@ def _service_account_id(project): return account_id -def get_service_account(iam, project_id, service_account_id): +def get_service_account(iam: Any, project_id: str, + service_account_id: str) -> dict[str, Any] | None: """Try to get a service account. Returns None if it does not exist.""" try: request = iam.projects().serviceAccounts().get( @@ -78,10 +82,10 @@ def get_service_account(iam, project_id, service_account_id): raise -def get_or_create_service_account(project): +def get_or_create_service_account(project: str) -> tuple[dict[str, Any], bool]: """Get or create service account for the project.""" iam = _create_client('iam') - project_id = utils.get_application_id() + project_id = cast(str, utils.get_application_id()) service_account_id = _service_account_id(project) service_account = get_service_account(iam, project_id, service_account_id) @@ -102,7 +106,8 @@ def get_or_create_service_account(project): return request.execute(), False -def _get_or_insert_iam_binding(policy, role): +def _get_or_insert_iam_binding(policy: dict[str, Any], + role: str) -> dict[str, Any]: """Return the binding corresponding to the given role. Creates the binding if needed.""" existing_binding = next( @@ -120,7 +125,8 @@ def _get_or_insert_iam_binding(policy, role): return new_binding -def _add_service_account_role(policy, role, service_account): +def _add_service_account_role(policy: dict[str, Any], role: str, + service_account: str) -> bool: """Add a role to a service account. Returns whether or not changes were made.""" binding = _get_or_insert_iam_binding(policy, role) @@ -133,9 +139,9 @@ def _add_service_account_role(policy, role, service_account): return False -def set_service_account_roles(service_account): +def set_service_account_roles(service_account: dict[str, Any]) -> None: """Set roles for service account.""" - project_id = utils.get_application_id() + project_id = cast(str, utils.get_application_id()) resource_manager = _create_client('cloudresourcemanager') request = resource_manager.projects().getIamPolicy( diff --git a/src/clusterfuzz/_internal/cron/sync_admins.py b/src/clusterfuzz/_internal/cron/sync_admins.py index 8f60b1f5179..df1ba0eee0f 100644 --- a/src/clusterfuzz/_internal/cron/sync_admins.py +++ b/src/clusterfuzz/_internal/cron/sync_admins.py @@ -13,6 +13,11 @@ # limitations under the License. """Cron to sync admin users.""" +from collections.abc import Collection +from collections.abc import Container +from typing import Any +from typing import cast + from googleapiclient import discovery from clusterfuzz._internal.base import utils @@ -21,10 +26,10 @@ from clusterfuzz._internal.metrics import logs -def get_emails_from_bindings(iam_policy, principal_type, - allowed_roles) -> set[str]: +def get_emails_from_bindings(iam_policy: dict[str, Any], principal_type: str, + allowed_roles: Container[str]) -> set[str]: """Returns emails that should be admins, given constraints.""" - admins = set() + admins: set[str] = set() assert principal_type in ['user', 'serviceAccount'] @@ -40,7 +45,7 @@ def get_emails_from_bindings(iam_policy, principal_type, return admins -def admins_from_iam_policy(iam_policy): +def admins_from_iam_policy(iam_policy: dict[str, Any]) -> set[str]: """Gets a set of admins from the IAM policy.""" # Per # https://cloud.google.com/appengine/docs/standard/python/users/adminusers, An @@ -63,18 +68,18 @@ def admins_from_iam_policy(iam_policy): return user_admins -def update_admins(new_admins): +def update_admins(new_admins: Collection[str]) -> None: """Update list of admins.""" existing_admins = ndb_utils.get_all_from_model(data_types.Admin) to_remove = [] existing_admin_emails = set() for admin in existing_admins: - if admin.email not in new_admins: - logs.info('Removing admin ' + admin.email) + if cast(str, admin.email) not in new_admins: + logs.info('Removing admin ' + cast(str, admin.email)) to_remove.append(admin.key) - existing_admin_emails.add(admin.email) + existing_admin_emails.add(cast(str, admin.email)) ndb_utils.delete_multi(to_remove) @@ -87,7 +92,7 @@ def update_admins(new_admins): ndb_utils.put_multi(to_add) -def main(): +def main() -> bool: """Admin user syncing cron.""" resource_manager = discovery.build('cloudresourcemanager', 'v1') project_id = utils.get_application_id() diff --git a/src/clusterfuzz/_internal/cron/triage.py b/src/clusterfuzz/_internal/cron/triage.py index 5b55e80ea58..5905d385d37 100644 --- a/src/clusterfuzz/_internal/cron/triage.py +++ b/src/clusterfuzz/_internal/cron/triage.py @@ -17,6 +17,7 @@ import datetime import itertools import json +from typing import Iterable from clusterfuzz._internal.base import dates from clusterfuzz._internal.base import errors @@ -29,6 +30,7 @@ from clusterfuzz._internal.issue_management import issue_filer from clusterfuzz._internal.issue_management import issue_tracker_policy from clusterfuzz._internal.issue_management import issue_tracker_utils +from clusterfuzz._internal.issue_management.issue_tracker import IssueTracker from clusterfuzz._internal.metrics import crash_stats from clusterfuzz._internal.metrics import events from clusterfuzz._internal.metrics import logs @@ -36,28 +38,34 @@ from . import grouper -MAX_BUGS_PER_PROJECT_PER_24HRS_DEFAULT = 100 -UNREPRODUCIBLE_CRASH_IGNORE_CRASH_TYPES = [ +MAX_BUGS_PER_PROJECT_PER_24HRS_DEFAULT: int = 100 +UNREPRODUCIBLE_CRASH_IGNORE_CRASH_TYPES: list[str] = [ 'Out-of-memory', 'Stack-overflow', 'Timeout' ] -TRIAGE_MESSAGE_KEY = 'triage_message' +TRIAGE_MESSAGE_KEY: str = 'triage_message' -def _add_triage_message(testcase, message): +def _add_triage_message(testcase: data_types.Testcase, message: str) -> None: """Add a triage message.""" if testcase.get_metadata(TRIAGE_MESSAGE_KEY) == message: # Message already exists, skip update. return # Re-fetch testcase to get latest entity and avoid race condition in updates. - testcase = data_handler.get_testcase_by_id(testcase.key.id()) - testcase.set_metadata(TRIAGE_MESSAGE_KEY, message) + testcase_id = testcase.key.id() + assert testcase_id is not None + fetched_testcase = data_handler.get_testcase_by_id(testcase_id) + assert fetched_testcase is not None + fetched_testcase.set_metadata(TRIAGE_MESSAGE_KEY, message) -def _create_filed_bug_metadata(testcase): +def _create_filed_bug_metadata(testcase: data_types.Testcase) -> None: """Create a dummy bug entry for a test case.""" metadata = data_types.FiledBug() metadata.timestamp = datetime.datetime.utcnow() - metadata.testcase_id = testcase.key.id() + testcase_id = testcase.key.id() + assert testcase_id is not None + metadata.testcase_id = int(testcase_id) + assert testcase.bug_information is not None metadata.bug_information = int(testcase.bug_information) metadata.group_id = testcase.group_id metadata.crash_type = testcase.crash_type @@ -69,9 +77,9 @@ def _create_filed_bug_metadata(testcase): metadata.put() -def _get_excluded_jobs(): +def _get_excluded_jobs() -> list[str]: """Return list of jobs excluded from bug filing.""" - excluded_jobs = [] + excluded_jobs: list[str] = [] jobs = ndb_utils.get_all_from_model(data_types.Job) for job in jobs: @@ -79,12 +87,13 @@ def _get_excluded_jobs(): # Exclude experimental jobs. if utils.string_is_true(job_environment.get('EXPERIMENTAL')): + assert job.name is not None excluded_jobs.append(job.name) return excluded_jobs -def _is_bug_filed(testcase): +def _is_bug_filed(testcase: data_types.Testcase) -> bool: """Indicate if the bug is already filed.""" # Check if the testcase is already associated with a bug. if testcase.bug_information: @@ -99,9 +108,9 @@ def _is_bug_filed(testcase): return False -def _is_blocking_progress_android(testcase): +def _is_blocking_progress_android(testcase: data_types.Testcase) -> bool: """Checks the crash frequency if it is reported on libfuzzer""" - if testcase.job_type.startswith('libfuzzer'): + if (testcase.job_type or '').startswith('libfuzzer'): # Get crash statistics data on this unreproducible crash for last X days. last_hour = crash_stats.get_last_successful_hour() if not last_hour: @@ -146,14 +155,14 @@ def _is_blocking_progress_android(testcase): return False -def is_crash_important_android(testcase): +def is_crash_important_android(testcase: data_types.Testcase) -> bool: """"Indicate if the android crash is important to file.""" if _is_blocking_progress_android(testcase): return True return False -def _is_crash_important(testcase): +def _is_crash_important(testcase: data_types.Testcase) -> bool: """Indicate if the crash is important to file.""" if not testcase.one_time_crasher_flag: # A reproducible crash is an important crash. @@ -222,10 +231,11 @@ def _is_crash_important(testcase): data_types.FILE_UNREPRODUCIBLE_TESTCASE_MIN_CRASH_THRESHOLD) -def _check_and_update_similar_bug(testcase, issue_tracker): +def _check_and_update_similar_bug(testcase: data_types.Testcase, + issue_tracker: IssueTracker) -> bool: """Get list of similar open issues and ones that were recently closed.""" # Get similar testcases from the same group. - similar_testcases_from_group = [] + similar_testcases_from_group: Iterable[data_types.Testcase] = [] if testcase.group_id: group_query = data_types.Testcase.query( data_types.Testcase.group_id == testcase.group_id) @@ -241,9 +251,10 @@ def _check_and_update_similar_bug(testcase, issue_tracker): data_types.Testcase.project_name == testcase.project_name, data_types.Testcase.status == 'Processed') - similar_testcases_from_query = ndb_utils.get_all_from_query( - same_crash_params_query, - batch_size=data_types.TESTCASE_ENTITY_QUERY_LIMIT // 2) + similar_testcases_from_query: Iterable[ + data_types.Testcase] = ndb_utils.get_all_from_query( + same_crash_params_query, + batch_size=data_types.TESTCASE_ENTITY_QUERY_LIMIT // 2) for similar_testcase in itertools.chain(similar_testcases_from_group, similar_testcases_from_query): # Exclude ourself from comparison. @@ -280,7 +291,7 @@ def _check_and_update_similar_bug(testcase, issue_tracker): # file another one. policy = issue_tracker_policy.get(issue_tracker.project) ignore_label = policy.label('ignore') - if ignore_label in issue.labels: + if ignore_label and ignore_label in issue.labels: _add_triage_message( testcase, ('Skipping filing a bug since similar testcase ({testcase_id}) in ' @@ -316,7 +327,8 @@ def _check_and_update_similar_bug(testcase, issue_tracker): return False -def _emit_bug_filing_from_testcase_elapsed_time_metric(testcase): +def _emit_bug_filing_from_testcase_elapsed_time_metric( + testcase: data_types.Testcase) -> None: testcase_age = testcase.get_age_in_seconds() if not testcase_age: return @@ -328,7 +340,8 @@ def _emit_bug_filing_from_testcase_elapsed_time_metric(testcase): }) -def _file_issue(testcase, issue_tracker, throttler): +def _file_issue(testcase: data_types.Testcase, issue_tracker: IssueTracker, + throttler: 'Throttler') -> bool: """File an issue for the testcase.""" logs.info(f'_file_issue for {testcase.key.id()}') filed = False @@ -375,24 +388,25 @@ def _file_issue(testcase, issue_tracker, throttler): return filed -def _set_testcase_stuck_state(testcase: data_types.Testcase, state: bool): +def _set_testcase_stuck_state(testcase: data_types.Testcase, + state: bool) -> None: if testcase.stuck_in_triage == state: return testcase.stuck_in_triage = state testcase.put() -untriaged_testcases = {} +untriaged_testcases: dict[tuple[str, str], int] = {} -def _increment_untriaged_testcase_count(job, status): +def _increment_untriaged_testcase_count(job: str, status: str) -> None: identifier = (job, status) if identifier not in untriaged_testcases: untriaged_testcases[identifier] = 0 untriaged_testcases[identifier] += 1 -def _emit_untriaged_testcase_count_metric(): +def _emit_untriaged_testcase_count_metric() -> None: for (job, status) in untriaged_testcases: monitoring_metrics.UNTRIAGED_TESTCASE_COUNT.set( untriaged_testcases[(job, status)], @@ -402,23 +416,24 @@ def _emit_untriaged_testcase_count_metric(): }) -PENDING_ANALYZE = 'pending_analyze' -PENDING_CRITICAL_TASKS = 'pending_critical_tasks' -PENDING_PROGRESSION = 'pending_progression' -PENDING_GROUPING = 'pending_grouping' -PENDING_FILING = 'pending_filing' +PENDING_ANALYZE: str = 'pending_analyze' +PENDING_CRITICAL_TASKS: str = 'pending_critical_tasks' +PENDING_PROGRESSION: str = 'pending_progression' +PENDING_GROUPING: str = 'pending_grouping' +PENDING_FILING: str = 'pending_filing' def _emit_untriaged_testcase_age_metric(testcase: data_types.Testcase, - step: str): + step: str) -> None: """Emmits a metric to track age of untriaged testcases.""" - if not testcase.get_age_in_seconds(): + testcase_age = testcase.get_age_in_seconds() + if not testcase_age: return logs.info(f'Emiting UNTRIAGED_TESTCASE_AGE for testcase {testcase.key.id()} ' - f'(age = {testcase.get_age_in_seconds()}), step = {step}') + f'(age = {testcase_age}), step = {step}') monitoring_metrics.UNTRIAGED_TESTCASE_AGE.add( - testcase.get_age_in_seconds() / 3600, + testcase_age / 3600, labels={ 'job': testcase.job_type, 'platform': testcase.platform, @@ -426,7 +441,8 @@ def _emit_untriaged_testcase_age_metric(testcase: data_types.Testcase, }) -def _triage_testcase(testcase, excluded_jobs, all_jobs, throttler): +def _triage_testcase(testcase: data_types.Testcase, excluded_jobs: list[str], + all_jobs: list[str], throttler: 'Throttler') -> None: """Files a bug for a given testcase.""" testcase_id = testcase.key.id() critical_tasks_completed = data_handler.critical_tasks_completed(testcase) @@ -564,13 +580,13 @@ def _triage_testcase(testcase, excluded_jobs, all_jobs, throttler): @logs.cron_log_context() -def main(): +def main() -> bool: """Files bugs.""" try: logs.info('Grouping testcases.') grouper.group_testcases() logs.info('Grouping done.') - except: + except: # pylint: disable=bare-except logs.error('Error occurred while grouping test cases.') return False @@ -607,21 +623,23 @@ def main(): class Throttler: """Bug throttler""" - def __init__(self): - self._bug_filed_per_job_per_24hrs = collections.defaultdict(int) - self._bug_filed_per_project_per_24hrs = collections.defaultdict(int) - self._bug_throttling_cutoff = datetime.datetime.now() - datetime.timedelta( - hours=24) + def __init__(self) -> None: + self._bug_filed_per_job_per_24hrs: dict[str | None, + int] = collections.defaultdict(int) + self._bug_filed_per_project_per_24hrs: dict[ + str | None, int] = collections.defaultdict(int) + self._bug_throttling_cutoff: datetime.datetime = datetime.datetime.now( + ) - datetime.timedelta(hours=24) for bug in ndb_utils.get_all_from_query( data_types.FiledBug.query( data_types.FiledBug.timestamp >= self._bug_throttling_cutoff)): self._bug_filed_per_job_per_24hrs[bug.job_type] += 1 self._bug_filed_per_project_per_24hrs[bug.project_name] += 1 - self._max_bugs_per_job_per_24hrs = {} - self._max_bugs_per_project_per_24hrs = {} + self._max_bugs_per_job_per_24hrs: dict[str | None, int | None] = {} + self._max_bugs_per_project_per_24hrs: dict[str | None, int] = {} - def _get_job_bugs_filing_max(self, job_type): + def _get_job_bugs_filing_max(self, job_type: str | None) -> int | None: """Gets the maximum number of bugs that can be filed for a given job.""" if job_type in self._max_bugs_per_job_per_24hrs: return self._max_bugs_per_job_per_24hrs[job_type] @@ -631,14 +649,14 @@ def _get_job_bugs_filing_max(self, job_type): if job and 'MAX_BUGS_PER_24HRS' in job.get_environment(): try: max_bugs = int(job.get_environment()['MAX_BUGS_PER_24HRS']) - except Exception: + except: # pylint: disable=bare-except logs.error('Invalid environment value of \'MAX_BUGS_PER_24HRS\' ' f'for job type {job_type}.') self._max_bugs_per_job_per_24hrs[job_type] = max_bugs return max_bugs - def _get_project_bugs_filing_max(self, job_type): + def _get_project_bugs_filing_max(self, job_type: str | None) -> int: """Gets the maximum number of bugs that can be filed per project.""" project = data_handler.get_project_name(job_type) if project in self._max_bugs_per_project_per_24hrs: @@ -650,14 +668,14 @@ def _get_project_bugs_filing_max(self, job_type): max_bugs = MAX_BUGS_PER_PROJECT_PER_24HRS_DEFAULT try: max_bugs = int(config.get('max_bugs_per_project_per_24hrs', max_bugs)) - except: + except: # pylint: disable=bare-except logs.error('Invalid config value of \'max_bugs_per_project_per_24hrs\' ' f'for issue tracker {issue_tracker_name}') self._max_bugs_per_project_per_24hrs[project] = max_bugs return max_bugs - def should_throttle(self, testcase): + def should_throttle(self, testcase: data_types.Testcase) -> bool: """Returns whether the current bug needs to be throttled.""" job_bugs_filing_max = self._get_job_bugs_filing_max(testcase.job_type) diff --git a/src/clusterfuzz/_internal/datastore/data_handler.py b/src/clusterfuzz/_internal/datastore/data_handler.py index ce48a0c9f95..fb69a92cf58 100644 --- a/src/clusterfuzz/_internal/datastore/data_handler.py +++ b/src/clusterfuzz/_internal/datastore/data_handler.py @@ -13,17 +13,19 @@ # limitations under the License. """Data handler functions.""" -import collections +from collections.abc import Generator +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import Sequence import datetime import os import re import shlex import time -from typing import Generator -from typing import Mapping -from typing import Sequence -from typing import Type +from typing import Any +from typing import NamedTuple from typing import TypeAlias +from typing import TypeVar from google.cloud import ndb @@ -48,35 +50,36 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import shell -DATA_BUNDLE_DEFAULT_BUCKET_IAM_ROLE = 'roles/storage.objectAdmin' -DEFAULT_FAIL_RETRIES = 3 -DEFAULT_FAIL_WAIT = 1.5 -GOMA_DIR_LINE_REGEX = re.compile(r'^\s*goma_dir\s*=') -HEARTBEAT_LAST_UPDATE_KEY = 'heartbeat_update' -INPUT_DIR = 'inputs' -MEMCACHE_TTL_IN_SECONDS = 30 * 60 +DATA_BUNDLE_DEFAULT_BUCKET_IAM_ROLE: str = 'roles/storage.objectAdmin' +DEFAULT_FAIL_RETRIES: int = 3 +DEFAULT_FAIL_WAIT: float = 1.5 +GOMA_DIR_LINE_REGEX: re.Pattern[str] = re.compile(r'^\s*goma_dir\s*=') +HEARTBEAT_LAST_UPDATE_KEY: str = 'heartbeat_update' +INPUT_DIR: str = 'inputs' +MEMCACHE_TTL_IN_SECONDS: int = 30 * 60 -NUM_TESTCASE_QUALITY_BITS = 3 -MAX_TESTCASE_QUALITY = 2**NUM_TESTCASE_QUALITY_BITS - 1 +NUM_TESTCASE_QUALITY_BITS: int = 3 +MAX_TESTCASE_QUALITY: int = 2**NUM_TESTCASE_QUALITY_BITS - 1 # Value and dimension map for some crash types (timeout, ooms). -CRASH_TYPE_VALUE_REGEX_MAP = { +CRASH_TYPE_VALUE_REGEX_MAP: dict[str, str] = { 'Timeout': r'.*-timeout=(\d+)', 'Out-of-memory': r'.*-rss_limit_mb=(\d+)', } -CRASH_TYPE_DIMENSION_MAP = { +CRASH_TYPE_DIMENSION_MAP: dict[str, str] = { 'Timeout': 'secs', 'Out-of-memory': 'MB', } -TESTCASE_REPORT_URL = 'https://{domain}/testcase?key={testcase_id}' -TESTCASE_DOWNLOAD_URL = 'https://{domain}/download?testcase_id={testcase_id}' -TESTCASE_REVISION_RANGE_URL = ( +TESTCASE_REPORT_URL: str = 'https://{domain}/testcase?key={testcase_id}' +TESTCASE_DOWNLOAD_URL: str = ( + 'https://{domain}/download?testcase_id={testcase_id}') +TESTCASE_REVISION_RANGE_URL: str = ( 'https://{domain}/revisions?job={job_type}&range={revision_range}') -TESTCASE_REVISION_URL = ( +TESTCASE_REVISION_URL: str = ( 'https://{domain}/revisions?job={job_type}&revision={revision}') -FILE_UNREPRODUCIBLE_TESTCASE_TEXT = ( +FILE_UNREPRODUCIBLE_TESTCASE_TEXT: str = ( '************************* UNREPRODUCIBLE *************************\n' 'Note: This crash might not be reproducible with the provided testcase. ' 'That said, for the past %d days, we\'ve been seeing this crash ' @@ -93,9 +96,15 @@ data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)) FilterValue: TypeAlias = str | int | float | bool | datetime.datetime +_ModelT = TypeVar('_ModelT', bound=data_types.Model) + + +class FuzzerDisplay(NamedTuple): + engine: str | None + target: str | None + name: str | None + fully_qualified_name: str | None -FuzzerDisplay = collections.namedtuple( - 'FuzzerDisplay', ['engine', 'target', 'name', 'fully_qualified_name']) # ------------------------------------------------------------------------------ # Testcase, TestcaseUploadMetadata database related functions @@ -103,21 +112,21 @@ @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def get_all_project_names(): +def get_all_project_names() -> list[str]: """Return all project names.""" query = data_types.Job.query( projection=[data_types.Job.project], distinct=True) - return sorted([job.project for job in query]) + return sorted([job.project for job in query if job.project]) -def get_domain(): +def get_domain() -> str: """Get current domain.""" default_domain = '{app_id}.appspot.com'.format( app_id=utils.get_application_id()) return local_config.GAEConfig().get('domains.main', default=default_domain) -def get_testcase_by_id(testcase_id): +def get_testcase_by_id(testcase_id: int | str) -> data_types.Testcase: """Return the testcase with the given id. Raises InvalidTestcaseError if no such testcase exists. """ @@ -136,15 +145,15 @@ def get_testcase_by_id(testcase_id): return testcase -def find_testcase(project_name, - crash_type, - crash_state, - security_flag, - testcase_to_exclude=None, - fuzz_target=None): +def find_testcase(project_name: str | None, + crash_type: str | None, + crash_state: str | None, + security_flag: bool | None, + testcase_to_exclude: data_types.Testcase | None = None, + fuzz_target: str | None = None) -> data_types.Testcase | None: """Find an open test case matching certain parameters.""" # Prepare the query. - query_args = [ + query_args: list[Any] = [ data_types.Testcase.project_name == project_name, data_types.Testcase.crash_type == crash_type, data_types.Testcase.crash_state == crash_state, @@ -214,9 +223,9 @@ def find_testcase(project_name, return testcase -def get_crash_type_string(testcase): +def get_crash_type_string(testcase: data_types.Testcase) -> str: """Return a crash type string for a testcase.""" - crash_type = ' '.join(testcase.crash_type.splitlines()) + crash_type = ' '.join((testcase.crash_type or '').splitlines()) if crash_type not in CRASH_TYPE_VALUE_REGEX_MAP: return crash_type @@ -230,10 +239,12 @@ def get_crash_type_string(testcase): CRASH_TYPE_DIMENSION_MAP[crash_type]) -def filter_stacktrace(stacktrace, blob_name=None, signed_upload_url=None): +def filter_stacktrace(stacktrace: str | bytes | None, + blob_name: str | None = None, + signed_upload_url: str | None = None) -> str: """Filters stacktrace and returns content appropriate for storage as an appengine entity.""" - unicode_stacktrace = utils.decode_to_unicode(stacktrace) + unicode_stacktrace = utils.decode_to_unicode(stacktrace or '') if len(unicode_stacktrace) <= data_types.STACKTRACE_LENGTH_LIMIT: return unicode_stacktrace # TODO(alhijazi): Once the migration is done, callers are expected to @@ -251,6 +262,7 @@ def filter_stacktrace(stacktrace, blob_name=None, signed_upload_url=None): return '%s%s' % (data_types.BLOBSTORE_STACK_PREFIX, blob_name) tmpdir = environment.get_value('BOT_TMPDIR') + assert tmpdir is not None tmp_stacktrace_file = os.path.join(tmpdir, 'stacktrace.tmp') try: @@ -267,7 +279,7 @@ def filter_stacktrace(stacktrace, blob_name=None, signed_upload_url=None): return '%s%s' % (data_types.BLOBSTORE_STACK_PREFIX, key) -def get_issue_summary(testcase): +def get_issue_summary(testcase: data_types.Testcase) -> str: """Gets an issue description string for a testcase.""" # Get summary prefix. Note that values for fuzzers take priority over those # from job definitions. @@ -286,20 +298,23 @@ def get_issue_summary(testcase): if issue_summary: issue_summary += ': ' + testcase_crash_type = testcase.crash_type or '' + testcase_crash_state = testcase.crash_state or '' + # For ASSERTs and CHECK failures, we should just use the crash type and the # first line of the crash state as titles. Note that ASSERT_NOT_REACHED should # be handled by the general case. - if testcase.crash_type in [ + if testcase_crash_type in [ 'ASSERT', 'CHECK failure', 'Security CHECK failure', 'Security DCHECK failure' ]: issue_summary += ( - testcase.crash_type + ': ' + testcase.crash_state.splitlines()[0]) + testcase_crash_type + ': ' + testcase_crash_state.splitlines()[0]) return issue_summary # Special case for bad-cast style testcases. - if testcase.crash_type == 'Bad-cast': - filtered_crash_state_lines = testcase.crash_state.splitlines() + if testcase_crash_type == 'Bad-cast': + filtered_crash_state_lines = testcase_crash_state.splitlines() # Add the to/from line (this should always exist). issue_summary += filtered_crash_state_lines[0] @@ -311,34 +326,37 @@ def get_issue_summary(testcase): return issue_summary # Add first lines from crash type and crash_state. - if testcase.crash_type: + if testcase_crash_type: filtered_crash_type = re.sub(r'UNKNOWN( READ| WRITE)?', 'Crash', - testcase.crash_type.splitlines()[0]) + testcase_crash_type.splitlines()[0]) issue_summary += filtered_crash_type else: issue_summary += 'Unknown error' - if testcase.crash_state == 'NULL' or not testcase.crash_state: + if testcase_crash_state == 'NULL' or not testcase_crash_state: # Special case for empty stacktrace. issue_summary += ' with empty stacktrace' else: - issue_summary += ' in ' + testcase.crash_state.splitlines()[0] + issue_summary += ' in ' + testcase_crash_state.splitlines()[0] return issue_summary -def get_reproduction_help_url(testcase, config): +def get_reproduction_help_url(testcase: data_types.Testcase, + config: Any) -> str | None: """Return url to reproduce the bug.""" return get_value_from_job_definition_or_environment( testcase.job_type, 'HELP_URL', default=config.reproduction_help_url) -def get_fuzzer_display(testcase): +def get_fuzzer_display(testcase: data_types.Testcase) -> FuzzerDisplay: fuzz_target = get_fuzz_target(testcase.overridden_fuzzer_name) return get_fuzzer_display_unprivileged(testcase, fuzz_target) -def get_fuzzer_display_unprivileged(testcase, fuzz_target): +def get_fuzzer_display_unprivileged( + testcase: data_types.Testcase, + fuzz_target: data_types.FuzzTarget | None) -> FuzzerDisplay: """Return FuzzerDisplay tuple.""" if (testcase.overridden_fuzzer_name == testcase.fuzzer_name or not testcase.overridden_fuzzer_name): @@ -362,7 +380,8 @@ def get_fuzzer_display_unprivileged(testcase, fuzz_target): fully_qualified_name=fuzz_target.fully_qualified_name()) -def filter_arguments(arguments, fuzz_target_name=None): +def filter_arguments(arguments: str, + fuzz_target_name: str | None = None) -> str: """Filter arguments, removing testcase argument and fuzz target binary names.""" # Filter out %TESTCASE*% argument. @@ -373,7 +392,7 @@ def filter_arguments(arguments, fuzz_target_name=None): return arguments.strip() -def get_arguments(testcase): +def get_arguments(testcase: data_types.Testcase) -> str: """Return minimized arguments, without testcase argument and fuzz target binary itself (for engine fuzzers).""" arguments = ( @@ -387,7 +406,7 @@ def get_arguments(testcase): return filter_arguments(arguments, fuzz_target) -def _get_memory_tool_options(testcase): +def _get_memory_tool_options(testcase: data_types.Testcase) -> list[str]: """Return memory tool options as a string to pass on command line.""" env = testcase.get_metadata('env') if not env: @@ -407,7 +426,8 @@ def _get_memory_tool_options(testcase): return result -def _get_bazel_test_args(arguments, sanitizer_options): +def _get_bazel_test_args(arguments: str, + sanitizer_options: Sequence[str]) -> str: """Return arguments to pass to a bazel test.""" result = [] for sanitizer_option in sanitizer_options: @@ -419,7 +439,8 @@ def _get_bazel_test_args(arguments, sanitizer_options): return ' '.join(result) -def format_issue_information(testcase, format_string): +def format_issue_information(testcase: data_types.Testcase, + format_string: str) -> str: """Format a string with information from the testcase.""" arguments = get_arguments(testcase) fuzzer_display = get_fuzzer_display(testcase) @@ -461,7 +482,8 @@ def format_issue_information(testcase, format_string): return result -def get_formatted_reproduction_help(testcase): +def get_formatted_reproduction_help( + testcase: data_types.Testcase) -> str | None: """Return url to reproduce the bug.""" help_format = get_value_from_job_definition_or_environment( testcase.job_type, 'HELP_FORMAT') @@ -475,7 +497,7 @@ def get_formatted_reproduction_help(testcase): return format_issue_information(testcase, help_format) -def get_plaintext_help_text(testcase, config): +def get_plaintext_help_text(testcase: data_types.Testcase, config: Any) -> str: """Get the help text for this testcase for display in issue descriptions.""" # Prioritize a HELP_FORMAT message if available. formatted_help = get_formatted_reproduction_help(testcase) @@ -490,7 +512,7 @@ def get_plaintext_help_text(testcase, config): return '' -def get_fixed_range_url(testcase): +def get_fixed_range_url(testcase: data_types.Testcase) -> str | None: """Return url to testcase fixed range.""" # Testcase is not fixed yet. if not testcase.fixed: @@ -506,10 +528,10 @@ def get_fixed_range_url(testcase): revision_range=testcase.fixed) -def get_issue_description(testcase, - reporter=None, - show_reporter=False, - hide_crash_state=False): +def get_issue_description(testcase: data_types.Testcase, + reporter: str | None = None, + show_reporter: bool = False, + hide_crash_state: bool = False) -> str: """Returns testcase as string.""" # Get issue tracker configuration parameters. config = db_config.get() @@ -564,6 +586,7 @@ def get_issue_description(testcase, crash_state = '...see report...' else: crash_state = testcase.crash_state + assert crash_state is not None content_string += 'Crash State:\n%s\n' % ( utils.indent_string(crash_state + '\n', 2)) @@ -624,7 +647,8 @@ def get_issue_description(testcase, return content_string -def get_stacktrace(testcase, stack_attribute='crash_stacktrace'): +def get_stacktrace(testcase: data_types.Testcase, + stack_attribute: str = 'crash_stacktrace') -> str: """Returns the stacktrace for a test case. This may require a blobstore read. @@ -636,10 +660,13 @@ def get_stacktrace(testcase, stack_attribute='crash_stacktrace'): # For App Engine, we can't write to local file, so use blobs.read_key instead. if environment.is_running_on_app_engine(): key = result[len(data_types.BLOBSTORE_STACK_PREFIX):] - return str(blobs.read_key(key), 'utf-8', errors='replace') + blob_data = blobs.read_key(key) + assert blob_data is not None + return str(blob_data, 'utf-8', errors='replace') key = result[len(data_types.BLOBSTORE_STACK_PREFIX):] tmpdir = environment.get_value('BOT_TMPDIR') + assert tmpdir is not None tmp_stacktrace_file = os.path.join(tmpdir, 'stacktrace.tmp') blobs.read_blob_to_disk(key, tmp_stacktrace_file) @@ -654,7 +681,7 @@ def get_stacktrace(testcase, stack_attribute='crash_stacktrace'): return result -def handle_duplicate_entry(testcase): +def handle_duplicate_entry(testcase: data_types.Testcase) -> None: """Handles duplicates and deletes unreproducible one.""" # Caller ensures that our testcase object is up-to-date. If someone else # already marked us as a duplicate, no more work to do. @@ -713,7 +740,9 @@ def handle_duplicate_entry(testcase): (existing_testcase_id, testcase_id)) -def is_first_attempt_for_task(task_name, testcase, reset_after_retry=False): +def is_first_attempt_for_task(task_name: str, + testcase: data_types.Testcase, + reset_after_retry: bool = False) -> bool: """Returns true if this task is tried atleast once. Only applicable for analyze and progression tasks.""" retry_key = f'{task_name}_retry' @@ -733,13 +762,13 @@ def is_first_attempt_for_task(task_name, testcase, reset_after_retry=False): @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def get_issue_tracker_name(job_type=None): +def get_issue_tracker_name(job_type: str | None = None) -> str | None: """Return issue tracker name for a job type.""" return get_value_from_job_definition_or_environment(job_type, 'ISSUE_TRACKER') @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def get_project_name(job_type): +def get_project_name(job_type: str | None) -> str: """Return project name for a job type.""" default_project_name = utils.default_project_name() return get_value_from_job_definition(job_type, 'PROJECT_NAME', @@ -747,12 +776,13 @@ def get_project_name(job_type): @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def get_main_repo(job_type): +def get_main_repo(job_type: str | None) -> str | None: """Return project name for a job type.""" return get_value_from_job_definition(job_type, 'MAIN_REPO') -def _get_security_severity(crash, job_type, gestures): +def _get_security_severity(crash: Any, job_type: str, + gestures: Any) -> int | None: """Get security severity.""" if crash.security_flag: return severity_analyzer.get_security_severity( @@ -761,12 +791,16 @@ def _get_security_severity(crash, job_type, gestures): return None -def store_testcase(crash, fuzzed_keys, minimized_keys, regression, fixed, - one_time_crasher_flag, crash_revision, comment, - absolute_path, fuzzer_name, fully_qualified_fuzzer_name, - job_type, archived, archive_filename, http_flag, gestures, - redzone, disable_ubsan, window_argument, timeout_multiplier, - minimized_arguments, trusted): +def store_testcase( + crash: Any, fuzzed_keys: str | None, minimized_keys: str | None, + regression: str | None, fixed: str | None, one_time_crasher_flag: bool, + crash_revision: int | str | None, comment: str | None, + absolute_path: str | None, fuzzer_name: str, + fully_qualified_fuzzer_name: str | None, job_type: str, archived: bool, + archive_filename: str | None, http_flag: bool, gestures: list[str] | None, + redzone: int | None, disable_ubsan: bool, window_argument: str | None, + timeout_multiplier: float | int | str, minimized_arguments: str | None, + trusted: bool) -> int: """Create a testcase and store it in the datastore using remote api.""" # Initialize variable to prevent invalid values. if archived: @@ -793,7 +827,8 @@ def store_testcase(crash, fuzzed_keys, minimized_keys, regression, fixed, testcase.security_severity = _get_security_severity(crash, job_type, gestures) testcase.one_time_crasher_flag = one_time_crasher_flag - testcase.crash_revision = crash_revision + testcase.crash_revision = int( + crash_revision) if crash_revision is not None else None testcase.absolute_path = absolute_path testcase.fuzzer_name = fuzzer_name testcase.overridden_fuzzer_name = fully_qualified_fuzzer_name or fuzzer_name @@ -862,7 +897,7 @@ def set_build_metadata_to_testcase(testcase: data_types.Testcase, build_key: str | None = None, build_url: str | None = None, gn_args: str | None = None, - update: bool = False): + update: bool = False) -> None: """Set testcase metadata fields related to the build metadata.""" dirty_flag = False build_key = ( @@ -893,7 +928,9 @@ def set_initial_testcase_metadata(testcase: data_types.Testcase) -> None: testcase.platform_id = environment.get_platform_id() -def update_testcase_comment(testcase, task_state, message=None): +def update_testcase_comment(testcase: data_types.Testcase, + task_state: str, + message: str | None = None) -> None: """Add task status and message to the test case's comment field.""" bot_name = environment.get_value('BOT_NAME', 'Unknown') task_name = environment.get_value('TASK_NAME', 'Unknown') @@ -904,18 +941,20 @@ def update_testcase_comment(testcase, task_state, message=None): # For some tasks like blame, progression and impact, we need to delete lines # from old task executions to avoid clutter. + testcase_comments = testcase.comments or '' if (task_name in ['blame', 'progression', 'impact'] and task_state == data_types.TaskState.STARTED): pattern = r'.*?: %s.*\n' % task_string - testcase.comments = re.sub(pattern, '', testcase.comments) + testcase_comments = re.sub(pattern, '', testcase_comments) - testcase.comments += '[%s] %s: %s %s' % (timestamp, bot_name, task_string, + testcase_comments += '[%s] %s: %s %s' % (timestamp, bot_name, task_string, task_state) if message: - testcase.comments += ': %s' % message.rstrip('.') + testcase_comments += ': %s' % message.rstrip('.') task_comments = environment.get_value('TASK_COMMENTS', '') environment.set_value('TASK_COMMENTS', task_comments + message + '\n') - testcase.comments += '.\n' + testcase_comments += '.\n' + testcase.comments = testcase_comments # Truncate if too long. if len(testcase.comments) > data_types.TESTCASE_COMMENTS_LENGTH_LIMIT: @@ -936,7 +975,7 @@ def update_testcase_comment(testcase, task_state, message=None): f'{message} (testcase {testcase.key.id()}, job {testcase.job_type}).') -def get_open_testcase_id_iterator(): +def get_open_testcase_id_iterator() -> Generator[int, None, None]: """Get an iterator for open testcase ids.""" keys = ndb_utils.get_all_from_query( data_types.Testcase.query( @@ -948,7 +987,7 @@ def get_open_testcase_id_iterator(): yield key.id() -def critical_tasks_completed(testcase): +def critical_tasks_completed(testcase: data_types.Testcase) -> bool: """Check to see if all critical tasks have finished running on a test case.""" if testcase.status == 'Unreproducible': # These tasks don't apply to unreproducible testcases. @@ -960,7 +999,7 @@ def critical_tasks_completed(testcase): # For non-chromium projects, impact and blame tasks are not applicable. if not utils.is_chromium(): - return testcase.minimized_keys and testcase.regression + return bool(testcase.minimized_keys and testcase.regression) # Only require impact if it is feasible to run it. Don't require it if not # (when regression fails). @@ -975,7 +1014,7 @@ def critical_tasks_completed(testcase): # ------------------------------------------------------------------------------ -def get_build_state(job_type, crash_revision): +def get_build_state(job_type: str, crash_revision: int) -> int: """Return whether a build is unmarked, good or bad.""" build = data_types.BuildMetadata.query( data_types.BuildMetadata.job_type == job_type, @@ -990,10 +1029,11 @@ def get_build_state(job_type, crash_revision): return data_types.BuildState.GOOD -def add_build_metadata(job_type, - crash_revision, - is_bad_build, - console_output=None): +def add_build_metadata( + job_type: str, + crash_revision: int, + is_bad_build: bool, + console_output: str | bytes | None = None) -> data_types.BuildMetadata: """Add build metadata.""" build = data_types.BuildMetadata() build.bad_build = is_bad_build @@ -1020,7 +1060,8 @@ def add_build_metadata(job_type, # ------------------------------------------------------------------------------ -def create_data_bundle_bucket_and_iams(data_bundle_name, emails): +def create_data_bundle_bucket_and_iams(data_bundle_name: str, + emails: Sequence[str]) -> bool: """Creates a data bundle bucket and adds iams for access.""" bucket_name = get_data_bundle_bucket_name(data_bundle_name) location = local_config.ProjectConfig().get('data_bundle_bucket_location') @@ -1061,7 +1102,7 @@ def create_data_bundle_bucket_and_iams(data_bundle_name, emails): return bool(storage.set_bucket_iam_policy(client, bucket_name, iam_policy)) -def bucket_domain_suffix(): +def bucket_domain_suffix() -> str: domain = local_config.ProjectConfig().get('bucket_domain_suffix') if not domain: domain = '%s.appspot.com' % utils.get_application_id() @@ -1069,16 +1110,19 @@ def bucket_domain_suffix(): return domain -def get_data_bundle_bucket_name(data_bundle_name): +def get_data_bundle_bucket_name(data_bundle_name: str) -> str: """Return data bundle bucket name on GCS.""" domain = bucket_domain_suffix() return '%s-corpus.%s' % (data_bundle_name, domain) -def get_value_from_fuzzer_environment_string(fuzzer_name, - variable_pattern, - default=None): +def get_value_from_fuzzer_environment_string(fuzzer_name: str | None, + variable_pattern: str, + default: Any = None) -> Any: """Get a specific environment variable's value for a fuzzer.""" + if not fuzzer_name: + return default + fuzzer = data_types.Fuzzer.query(data_types.Fuzzer.name == fuzzer_name).get() if not fuzzer or not fuzzer.additional_environment_string: return default @@ -1092,7 +1136,8 @@ def get_value_from_fuzzer_environment_string(fuzzer_name, # ------------------------------------------------------------------------------ -def get_task_status(name, create_if_needed=False): +def get_task_status( + name: str, create_if_needed: bool = False) -> data_types.TaskStatus | None: """Return the TaskStatus object with the given name.""" metadata = ndb.Key(data_types.TaskStatus, name).get() if not metadata and create_if_needed: @@ -1101,7 +1146,9 @@ def get_task_status(name, create_if_needed=False): return metadata -def update_task_status(task_name, status, expiry_interval=None): +def update_task_status(task_name: str, + status: str, + expiry_interval: int | float | None = None) -> bool: """Updates status for a task. Used to ensure that a single instance of a task is running at any given time.""" bot_name = environment.get_value('BOT_NAME') @@ -1113,15 +1160,16 @@ def update_task_status(task_name, status, expiry_interval=None): if expiry_interval is None: logs.error('expiry_interval is None and TASK_LEASE_SECONDS not set.') - def _try_update_status(): + def _try_update_status() -> bool: """Try update metadata.""" task_status = get_task_status(task_name, create_if_needed=True) + assert task_status is not None # If another bot is already working on this task, bail out with error. if (status == data_types.TaskState.STARTED and task_status.status == data_types.TaskState.STARTED and - not dates.time_has_expired( - task_status.time, seconds=expiry_interval - 1)): + expiry_interval is not None and not dates.time_has_expired( + task_status.time, seconds=int(expiry_interval) - 1)): return False task_status.bot_name = bot_name @@ -1141,7 +1189,8 @@ def _try_update_status(): # Failing to update 'completed' status causes another bot # that picked up this job to bail out. logs.error('Unable to update %s task metadata. Retrying.' % task_name) - time.sleep(utils.random_number(1, failure_wait_interval)) + assert failure_wait_interval is not None + time.sleep(utils.random_number(1, int(failure_wait_interval))) # ------------------------------------------------------------------------------ @@ -1149,7 +1198,7 @@ def _try_update_status(): # ------------------------------------------------------------------------------ -def update_heartbeat(force_update=False): +def update_heartbeat(force_update: bool = False) -> int: """Updates heartbeat with current timestamp and log data.""" if environment.is_uworker(): # Uworkers can't update heartbeats. @@ -1188,7 +1237,7 @@ def update_heartbeat(force_update=False): return 1 -def bot_run_timed_out(): +def bot_run_timed_out() -> bool: """Return true if our run timed out.""" run_timeout = environment.get_value('RUN_TIMEOUT') if not run_timeout: @@ -1198,13 +1247,13 @@ def bot_run_timed_out(): if not start_time: return False - start_time = datetime.datetime.utcfromtimestamp(start_time) + start_datetime = datetime.datetime.utcfromtimestamp(float(start_time)) # Actual run timeout takes off the duration for one task. average_task_duration = environment.get_value('AVERAGE_TASK_DURATION', 0) - actual_run_timeout = run_timeout - average_task_duration + actual_run_timeout = float(run_timeout) - float(average_task_duration) - return dates.time_has_expired(start_time, seconds=actual_run_timeout) + return dates.time_has_expired(start_datetime, seconds=int(actual_run_timeout)) # ------------------------------------------------------------------------------ @@ -1212,8 +1261,8 @@ def bot_run_timed_out(): # ------------------------------------------------------------------------------ -def check_job_supports_untrusted_workloads(job_name: str, - platform_id: str = None) -> None: +def check_job_supports_untrusted_workloads( + job_name: str, platform_id: str | None = None) -> None: """Checks that the named job can run untrusted testcases or fuzzers. Raises: @@ -1237,14 +1286,14 @@ def check_job_supports_untrusted_workloads(job_name: str, # are implemented as Linux containers. Thus, non-Linux jobs cannot support # untrusted workloads in this setup. - if (platform_id or job.platform).lower() != 'linux': + if (platform_id or job.platform or '').lower() != 'linux': raise ValueError( f'Job "{job_name}" does not support running untrusted workloads. ' 'Untrusted workloads on Chrome are only supported on Linux.') @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def get_component_name(job_type): +def get_component_name(job_type: str) -> str: """Gets component name for a job type.""" job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: @@ -1260,7 +1309,7 @@ def get_component_name(job_type): @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def get_repository_for_component(component): +def get_repository_for_component(component: str) -> str: """Get the repository based on component.""" default_repository = '' repository = '' @@ -1277,9 +1326,9 @@ def get_repository_for_component(component): return repository or default_repository -def get_value_from_environment_string(environment_string, - variable_pattern, - default=None): +def get_value_from_environment_string(environment_string: str, + variable_pattern: str, + default: Any = None) -> Any: """Return the first value matching the pattern from the environment string.""" pattern = r'%s\s*=\s*(.*)' % variable_pattern match = re.search(pattern, environment_string) @@ -1289,7 +1338,9 @@ def get_value_from_environment_string(environment_string, return match.group(1).strip() -def get_value_from_job_definition(job_type, variable_pattern, default=None): +def get_value_from_job_definition(job_type: str | None, + variable_pattern: str, + default: Any = None) -> Any: """Get a specific environment variable's value from a job definition.""" if not job_type: return default @@ -1306,9 +1357,9 @@ def get_value_from_job_definition(job_type, variable_pattern, default=None): return job.get_environment().get(variable_pattern, default) -def get_value_from_job_definition_or_environment(job_type, - variable_pattern, - default=None): +def get_value_from_job_definition_or_environment(job_type: str | None, + variable_pattern: str, + default: Any = None) -> Any: """Gets a specific environment variable's value from a job definition. If not found, it returns the value from current environment.""" return get_value_from_job_definition( @@ -1317,7 +1368,8 @@ def get_value_from_job_definition_or_environment(job_type, default=environment.get_value(variable_pattern, default)) -def get_additional_values_for_variable(variable_name, job_type, fuzzer_name): +def get_additional_values_for_variable(variable_name: str, job_type: str | None, + fuzzer_name: str | None) -> list[str]: """Helper function to read a list of additional items from a job definition and fuzzer's additional environment string.""" value_list_strings = [ @@ -1325,7 +1377,7 @@ def get_additional_values_for_variable(variable_name, job_type, fuzzer_name): get_value_from_fuzzer_environment_string(fuzzer_name, variable_name), ] - additional_values = [] + additional_values: list[str] = [] for value_list_string in value_list_strings: if value_list_string: # Ignore whitespace between commas. @@ -1339,7 +1391,7 @@ def get_additional_values_for_variable(variable_name, job_type, fuzzer_name): # ------------------------------------------------------------------------------ -def is_notification_sent(testcase_id, user_email): +def is_notification_sent(testcase_id: int | str, user_email: str) -> bool: """Return true if this notification has already been sent.""" notification = data_types.Notification.query( data_types.Notification.testcase_id == testcase_id, @@ -1347,10 +1399,10 @@ def is_notification_sent(testcase_id, user_email): return bool(notification) -def create_notification_entry(testcase_id, user_email): +def create_notification_entry(testcase_id: int | str, user_email: str) -> None: """Create a entry log for sent notification.""" notification = data_types.Notification() - notification.testcase_id = testcase_id + notification.testcase_id = int(testcase_id) notification.user_email = user_email notification.put() @@ -1360,31 +1412,32 @@ def create_notification_entry(testcase_id, user_email): # ------------------------------------------------------------------------------ -def create_user_uploaded_testcase(key, - original_key, - archive_state, - filename, - file_path_input, - timeout, - job, - queue, - http_flag, - gestures, - additional_arguments, - bug_information, - crash_revision, - uploader_email, - platform_id, - app_launch_command, - fuzzer_name, - fully_qualified_fuzzer_name, - fuzzer_binary_name, - bundled, - retries, - bug_summary_update_flag, - quiet_flag, - additional_metadata=None, - crash_data=None): +def create_user_uploaded_testcase( + key: str | None, + original_key: str | None, + archive_state: int, + filename: str, + file_path_input: str | None, + timeout: int | float | None, + job: data_types.Job, + queue: str | None, + http_flag: bool | None, + gestures: list[str] | None, + additional_arguments: str | None, + bug_information: str | None, + crash_revision: int | str | None, + uploader_email: str, + platform_id: str | None, + app_launch_command: str | None, + fuzzer_name: str | None, + fully_qualified_fuzzer_name: str | None, + fuzzer_binary_name: str | None, + bundled: bool, + retries: int | None, + bug_summary_update_flag: bool | None, + quiet_flag: bool | None, + additional_metadata: Mapping[str, Any] | None = None, + crash_data: Any = None) -> int: """Create a testcase object, metadata, and task for a user uploaded test.""" testcase = data_types.Testcase() if crash_data: @@ -1392,9 +1445,11 @@ def create_user_uploaded_testcase(key, testcase.crash_type = crash_data.crash_type testcase.crash_state = crash_data.crash_state testcase.crash_address = crash_data.crash_address - testcase.crash_stacktrace = crash_data.crash_stacktrace - + testcase.crash_stacktrace = filter_stacktrace(crash_data.crash_stacktrace) testcase.status = 'Processed' + testcase.security_severity = ( + crash_data.user_security_severity + if hasattr(crash_data, 'user_security_severity') else None) testcase.security_flag = crash_analyzer.is_security_issue( testcase.crash_stacktrace, testcase.crash_type, testcase.crash_address) testcase.regression = 'NA' @@ -1404,7 +1459,7 @@ def create_user_uploaded_testcase(key, testcase.minimized_keys = 'NA' # analyze_task sets this for non-external reproductions. - testcase.platform = job.platform.lower() + testcase.platform = (job.platform or '').lower() testcase.platform_id = testcase.platform else: testcase.crash_type = '' @@ -1422,7 +1477,8 @@ def create_user_uploaded_testcase(key, testcase.bug_information = '' testcase.fixed = '' testcase.one_time_crasher_flag = False - testcase.crash_revision = crash_revision + testcase.crash_revision = int( + crash_revision) if crash_revision is not None else None testcase.fuzzer_name = fuzzer_name testcase.overridden_fuzzer_name = fully_qualified_fuzzer_name or fuzzer_name testcase.job_type = job.name @@ -1436,7 +1492,7 @@ def create_user_uploaded_testcase(key, testcase.archive_filename = filename else: testcase.absolute_path = filename - testcase.gestures = gestures + testcase.gestures = gestures or [] if bug_information and bug_information.isdigit() and int(bug_information): testcase.bug_information = bug_information if platform_id: @@ -1475,7 +1531,7 @@ def create_user_uploaded_testcase(key, metadata.testcase_id = testcase_id metadata.blobstore_key = key metadata.original_blobstore_key = original_key - metadata.timeout = timeout + metadata.timeout = int(timeout) if timeout is not None else None metadata.bundled = bundled metadata.retries = retries if bundled: @@ -1502,7 +1558,9 @@ def create_user_uploaded_testcase(key, return testcase.key.id() -def check_uploaded_testcase_duplicate(testcase, metadata): +def check_uploaded_testcase_duplicate( + testcase: data_types.Testcase, + metadata: data_types.TestcaseUploadMetadata) -> bool: """Check if the uploaded testcase is a duplicate.""" existing_testcase = find_testcase(testcase.project_name, testcase.crash_type, testcase.crash_state, @@ -1530,12 +1588,16 @@ def check_uploaded_testcase_duplicate(testcase, metadata): return duplicate_testcase.key.id() == testcase.key.id() -def close_invalid_uploaded_testcase(testcase, metadata, status): +def close_invalid_uploaded_testcase(testcase: data_types.Testcase, + metadata: data_types.TestcaseUploadMetadata, + status: str) -> None: testcase.open = False mark_invalid_uploaded_testcase(testcase, metadata, status) -def mark_invalid_uploaded_testcase(testcase, metadata, status): +def mark_invalid_uploaded_testcase(testcase: data_types.Testcase, + metadata: data_types.TestcaseUploadMetadata, + status: str) -> None: """Closes an invalid testcase and updates metadata.""" testcase.status = status testcase.minimized_keys = 'NA' @@ -1554,7 +1616,7 @@ def mark_invalid_uploaded_testcase(testcase, metadata, status): # ------------------------------------------------------------------------------ -def delete_group(group_id, update_testcases=True): +def delete_group(group_id: int | str, update_testcases: bool = True) -> None: """Delete the testcase group with the specified id if it exists.""" # Remove all testcases from the group. if update_testcases: @@ -1568,7 +1630,7 @@ def delete_group(group_id, update_testcases=True): group.key.delete() -def get_testcase_ids_in_group(group_id): +def get_testcase_ids_in_group(group_id: int | str) -> list[int]: """Return the all testcase ids in the specified group.""" if not group_id or not str(group_id).isdigit(): return [] @@ -1579,7 +1641,7 @@ def get_testcase_ids_in_group(group_id): return [key.id() for key in query] -def get_testcases_in_group(group_id): +def get_testcases_in_group(group_id: int | str) -> list[data_types.Testcase]: """Return the all testcases in the specified group.""" # Fetch by keys (strongly consistent) to avoid stale results from query # (eventually consistent). @@ -1594,7 +1656,7 @@ def get_testcases_in_group(group_id): return testcases -def remove_testcase_from_group(testcase): +def remove_testcase_from_group(testcase: data_types.Testcase | None) -> None: """Removes a testcase from group.""" if not testcase: return @@ -1604,7 +1666,7 @@ def remove_testcase_from_group(testcase): testcase.put() -def update_group_bug(group_id): +def update_group_bug(group_id: int | str | None) -> None: """Update group bug information for a group.""" if not group_id: # No associated group, no work to do. Bail out. @@ -1640,7 +1702,8 @@ def update_group_bug(group_id): retries=DEFAULT_FAIL_RETRIES, delay=DEFAULT_FAIL_WAIT, function='datastore.data_handler.get_entity_by_type_and_id') -def get_entity_by_type_and_id(entity_type, entity_id): +def get_entity_by_type_and_id(entity_type: type[_ModelT], + entity_id: int | str | None) -> _ModelT | None: """Return the datastore object with the given type and id if it exists.""" if not entity_id or not str(entity_id).isdigit() or int(entity_id) == 0: return None @@ -1648,8 +1711,9 @@ def get_entity_by_type_and_id(entity_type, entity_id): return entity_type.get_by_id(int(entity_id)) -def _apply_filters(query: ndb.Query, entity_kind: Type[data_types.Model], - equality_filters: Mapping[str, FilterValue] | None): +def _apply_filters( + query: ndb.Query, entity_kind: type[data_types.Model], + equality_filters: Mapping[str, FilterValue] | None) -> ndb.Query: """Applies equality filters to a query.""" if not equality_filters: return query @@ -1664,8 +1728,8 @@ def _apply_filters(query: ndb.Query, entity_kind: Type[data_types.Model], return query -def _apply_order(query: ndb.Query, entity_kind: Type[data_types.Model], - order_by: Sequence[str] | None): +def _apply_order(query: ndb.Query, entity_kind: type[data_types.Model], + order_by: Sequence[str] | None) -> ndb.Query: """Applies ordering to a query.""" if not order_by: return query @@ -1683,7 +1747,7 @@ def _apply_order(query: ndb.Query, entity_kind: Type[data_types.Model], def get_entities_query( - entity_kind: Type[data_types.Model], + entity_kind: type[data_types.Model], equality_filters: Mapping[str, FilterValue] | None = None, order_by: Sequence[str] | None = None) -> ndb.Query: """Returns a query for the entity kind with equality filters and ordering.""" @@ -1693,18 +1757,20 @@ def get_entities_query( return query -def get_entities_ids(entity_kind: Type[data_types.Model], - equality_filters: Mapping[str, FilterValue] | None = None, - order_by: Sequence[str] | None = None) -> Generator: +def get_entities_ids( + entity_kind: type[data_types.Model], + equality_filters: Mapping[str, FilterValue] | None = None, + order_by: Sequence[str] | None = None) -> Generator[int | str, None, None]: """Yields IDs of entities matching optional filters and ordering.""" query = get_entities_query(entity_kind, equality_filters, order_by) yield from ( key.id() for key in ndb_utils.get_all_from_query(query, keys_only=True)) -def get_entities(entity_kind: Type[data_types.Model], - equality_filters: Mapping[str, FilterValue] | None = None, - order_by: Sequence[str] | None = None) -> Generator: +def get_entities( + entity_kind: type[_ModelT], + equality_filters: Mapping[str, FilterValue] | None = None, + order_by: Sequence[str] | None = None) -> Generator[_ModelT, None, None]: """Yields entities matching optional filters and ordering.""" query = get_entities_query(entity_kind, equality_filters, order_by) yield from ndb_utils.get_all_from_query(query) @@ -1715,7 +1781,8 @@ def get_entities(entity_kind: Type[data_types.Model], # ------------------------------------------------------------------------------ -def get_or_create_testcase_variant(testcase_id, job_type): +def get_or_create_testcase_variant(testcase_id: int | str, + job_type: str) -> data_types.TestcaseVariant: """Get a testcase variant entity, and create if needed.""" testcase_id = int(testcase_id) variant = data_types.TestcaseVariant.query( @@ -1727,7 +1794,8 @@ def get_or_create_testcase_variant(testcase_id, job_type): return variant -def get_testcase_variant(testcase_id, job_type): +def get_testcase_variant(testcase_id: int | str, + job_type: str) -> data_types.TestcaseVariant | None: """Get a testcase variant entity""" testcase_id = int(testcase_id) return data_types.TestcaseVariant.query( @@ -1743,9 +1811,16 @@ def get_testcase_variant(testcase_id, job_type): FUZZ_TARGET_UPDATE_FAIL_DELAY = 2 -def record_fuzz_target(engine_name, binary_name, job_type): +def record_fuzz_target(engine_name: str | None, binary_name: str | None, + job_type: str | None) -> data_types.FuzzTarget | None: """Records exsistence of fuzz target to the DB.""" - result = record_fuzz_targets(engine_name, [binary_name], job_type)[0] + if not engine_name or not binary_name or not job_type: + return None + + targets = record_fuzz_targets(engine_name, [binary_name], job_type) + if not targets: + return None + result = targets[0] project = get_project_name(job_type) key_name = data_types.fuzz_target_fully_qualified_name( @@ -1760,18 +1835,19 @@ def record_fuzz_target(engine_name, binary_name, job_type): return result -def get_or_create_multi_entities_from_keys(mapping): +def get_or_create_multi_entities_from_keys( + mapping: Mapping[str, _ModelT]) -> list[_ModelT]: """Gets or creates multiple db entities.""" keys = list(mapping.keys()) entities = ndb_utils.get_multi( [ndb.Key(value.__class__, key) for key, value in mapping.items()]) - entities = dict(zip(keys, entities)) + entities_dict = dict(zip(keys, entities)) new_entities = [ - mapping[key] for key, entity in entities.items() if not entity + mapping[key] for key, entity in entities_dict.items() if not entity ] new_entities = ndb_utils.get_multi(ndb_utils.put_multi(new_entities)) - all_entities = [entity for entity in entities.values() if entity] + ( - new_entities) + all_entities: list[_ModelT] = [entity for entity in entities if entity + ] + new_entities return all_entities @@ -1779,15 +1855,17 @@ def get_or_create_multi_entities_from_keys(mapping): retries=FUZZ_TARGET_UPDATE_FAIL_RETRIES, delay=FUZZ_TARGET_UPDATE_FAIL_DELAY, function='datastore.data_handler.record_fuzz_targets') -def record_fuzz_targets(engine_name, binaries, job_type): +def record_fuzz_targets( + engine_name: str | None, binaries: Sequence[str], + job_type: str | None) -> list[data_types.FuzzTarget] | None: """Record existence of fuzz targets to the DB.""" # TODO(metzman): All of this code assumes that fuzzing jobs are behaving # reasonably and won't try to DoS us by putting bogus fuzzers in the db. # This should be changed by limiting the number of fuzz targets saved and # putting an expiration on them. binaries = [binary for binary in binaries if binary] - if not binaries: - logs.error('Expected binaries.') + if not binaries or not engine_name or not job_type: + logs.error('Expected binaries, engine_name, and job_type.') return None project = get_project_name(job_type) @@ -1820,7 +1898,7 @@ def record_fuzz_targets(engine_name, binaries, job_type): return fuzz_targets -def get_fuzz_target(name): +def get_fuzz_target(name: str | None) -> data_types.FuzzTarget | None: """Get FuzzTarget by fully qualified name.""" if not name: return None @@ -1828,15 +1906,19 @@ def get_fuzz_target(name): return ndb.Key(data_types.FuzzTarget, name).get() -def get_fuzz_target_job(fuzz_target_name, job): +def get_fuzz_target_job(fuzz_target_name: str, + job: str) -> data_types.FuzzTargetJob | None: """Get FuzzTargetJob by fully qualified name and job.""" return ndb.Key(data_types.FuzzTargetJob, data_types.fuzz_target_job_key(fuzz_target_name, job)).get() -def get_fuzz_targets(engine=None, project=None, binary=None): +def get_fuzz_targets( + engine: str | None = None, + project: str | None = None, + binary: str | None = None) -> Iterator[data_types.FuzzTarget]: """Return a Datastore query for fuzz targets.""" - query = data_types.FuzzTarget().query() + query = data_types.FuzzTarget.query() if engine: query = query.filter(data_types.FuzzTarget.engine == engine) @@ -1850,22 +1932,22 @@ def get_fuzz_targets(engine=None, project=None, binary=None): return ndb_utils.get_all_from_query(query) -def get_fuzzing_engines(): +def get_fuzzing_engines() -> list[str]: """Return the fuzzing engines currently running.""" query = data_types.FuzzTarget.query( projection=[data_types.FuzzTarget.engine], distinct=True) - return [f.engine for f in ndb_utils.get_all_from_query(query)] + return [f.engine for f in ndb_utils.get_all_from_query(query) if f.engine] -def is_fuzzing_engine(name): +def is_fuzzing_engine(name: str) -> bool: """Return whether or not |name| is a fuzzing engine.""" query = data_types.FuzzTarget.query(data_types.FuzzTarget.engine == name) return bool(query.count(limit=1)) @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def get_all_fuzzer_names_including_children(include_parents=False, - project=None): +def get_all_fuzzer_names_including_children( + include_parents: bool = False, project: str | None = None) -> list[str]: """Returns all fuzzer names, including expanded child fuzzers.""" all_fuzzers = set() engine_fuzzers = get_fuzzing_engines() @@ -1884,15 +1966,18 @@ def get_all_fuzzer_names_including_children(include_parents=False, @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) -def get_all_job_type_names(project=None): +def get_all_job_type_names(project: str | None = None) -> list[str]: """Return all job type names.""" query = data_types.Job.query(projection=['name']) if project: query = query.filter(data_types.Job.project == project) - return sorted([job.name for job in query]) + return sorted([job.name for job in query if job.name]) -def get_coverage_information(fuzzer_name, date, create_if_needed=False): +def get_coverage_information( + fuzzer_name: str, + date: datetime.date | datetime.datetime, + create_if_needed: bool = False) -> data_types.CoverageInformation | None: """Get coverage information, or create if it doesn't exist.""" coverage_info = ndb.Key( data_types.CoverageInformation, @@ -1905,7 +1990,8 @@ def get_coverage_information(fuzzer_name, date, create_if_needed=False): return coverage_info -def close_testcase_with_error(testcase, error_message): +def close_testcase_with_error(testcase: data_types.Testcase, + error_message: str) -> None: """Close testcase (fixed=NA) with an error message.""" update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) testcase.fixed = 'NA' @@ -1913,7 +1999,7 @@ def close_testcase_with_error(testcase, error_message): testcase.put() -def clear_progression_pending(testcase): +def clear_progression_pending(testcase: data_types.Testcase) -> None: """If we marked progression as pending for this testcase, clear that state.""" if not testcase.get_metadata('progression_pending'): return @@ -1921,10 +2007,10 @@ def clear_progression_pending(testcase): testcase.delete_metadata('progression_pending', update_testcase=False) -def update_progression_completion_metadata(testcase, - revision, - is_crash=False, - message=None): +def update_progression_completion_metadata(testcase: data_types.Testcase, + revision: int | str, + is_crash: bool = False, + message: str | None = None) -> None: """Update metadata the progression task completes.""" clear_progression_pending(testcase) testcase.set_metadata('last_tested_revision', revision, update_testcase=False) diff --git a/src/clusterfuzz/_internal/datastore/data_types.py b/src/clusterfuzz/_internal/datastore/data_types.py index ab2846c46fc..ce86da6169a 100644 --- a/src/clusterfuzz/_internal/datastore/data_types.py +++ b/src/clusterfuzz/_internal/datastore/data_types.py @@ -15,6 +15,9 @@ import datetime import re +from typing import Any +from typing import cast +from typing import TypeVar from google.cloud import ndb @@ -27,20 +30,20 @@ # pylint: disable=no-member,arguments-differ # Prefix used when a large testcase is stored in the blobstore. -BLOBSTORE_STACK_PREFIX = 'BLOB_KEY=' +BLOBSTORE_STACK_PREFIX: str = 'BLOB_KEY=' # List of builtin fuzzers. -BUILTIN_FUZZERS = ['afl', 'libFuzzer'] +BUILTIN_FUZZERS: list[str] = ['afl', 'libFuzzer'] # Time to look back to find a corpus backup that is marked public. -CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS = 90 +CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS: int = 90 # Marker to indicate end of crash stacktrace. Anything after that is excluded # from being stored as part of crash stacktrace (e.g. merge content, etc). -CRASH_STACKTRACE_END_MARKER = 'CRASH OUTPUT ENDS HERE' +CRASH_STACKTRACE_END_MARKER: str = 'CRASH OUTPUT ENDS HERE' # Skips using crash state similarity for these types. -CRASH_TYPES_WITH_UNIQUE_STATE = [ +CRASH_TYPES_WITH_UNIQUE_STATE: list[str] = [ 'Missing-library', 'Out-of-memory', 'Overwrites-const-input', @@ -53,35 +56,35 @@ # Maximum size allowed for an appengine entity type. # Explicily kept slightly lower than 1 MB. -ENTITY_SIZE_LIMIT = 900000 +ENTITY_SIZE_LIMIT: int = 900000 # Minimum number of unreproducible crashes to see before filing it. -FILE_UNREPRODUCIBLE_TESTCASE_MIN_CRASH_THRESHOLD = 100 +FILE_UNREPRODUCIBLE_TESTCASE_MIN_CRASH_THRESHOLD: int = 100 # Minimum number of unreproducible crashes to see before filing it for android. -FILE_UNREPRODUCIBLE_TESTCASE_MIN_STARTUP_CRASH_THRESHOLD = 14 +FILE_UNREPRODUCIBLE_TESTCASE_MIN_STARTUP_CRASH_THRESHOLD: int = 14 # Heartbeat wait interval. -HEARTBEAT_WAIT_INTERVAL = 10 * 60 +HEARTBEAT_WAIT_INTERVAL: int = 10 * 60 # Android device heartbeat wait interval. -ANDROID_HEARTBEAT_WAIT_INTERVAL = 60 +ANDROID_HEARTBEAT_WAIT_INTERVAL: int = 60 # Time to wait after a report is marked fixed and before filing another similar # one (hours). -MIN_ELAPSED_TIME_SINCE_FIXED = 2 * 24 +MIN_ELAPSED_TIME_SINCE_FIXED: int = 2 * 24 # Time to wait for grouping task to finish, before filing the report (hours). -MIN_ELAPSED_TIME_SINCE_REPORT = 4 +MIN_ELAPSED_TIME_SINCE_REPORT: int = 4 # Valid name check for fuzzer, job, etc. -NAME_CHECK_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$') +NAME_CHECK_REGEX: re.Pattern = re.compile(r'^[a-zA-Z0-9_-]+$') # Regex to match special chars in project name. -SPECIAL_CHARS_REGEX = re.compile('[^a-zA-Z0-9_-]') +SPECIAL_CHARS_REGEX: re.Pattern = re.compile('[^a-zA-Z0-9_-]') # List of supported platforms. -PLATFORMS = [ +PLATFORMS: list[str] = [ 'LINUX', 'ANDROID', 'CHROMEOS', @@ -93,48 +96,52 @@ # Maximum size allowed for an appengine pubsub request. # Explicily kept slightly lower than 1 MB. -PUBSUB_REQUEST_LIMIT = 900000 +PUBSUB_REQUEST_LIMIT: int = 900000 # We store at most 3 stacktraces per Testcase entity (original, second, latest). -STACKTRACE_LENGTH_LIMIT = ENTITY_SIZE_LIMIT // 3 +STACKTRACE_LENGTH_LIMIT: int = ENTITY_SIZE_LIMIT // 3 # Maximum size allowed for testcase comments. # 1MiB (maximum Datastore entity size) - ENTITY_SIZE_LIMIT (our limited entity # size with breathing room), divided by 2 to leave room for other things in the # entity. This is around 74KB. -TESTCASE_COMMENTS_LENGTH_LIMIT = (1024 * 1024 - ENTITY_SIZE_LIMIT) // 2 +TESTCASE_COMMENTS_LENGTH_LIMIT: int = (1024 * 1024 - ENTITY_SIZE_LIMIT) // 2 # Maximum number of testcase entities to query in one batch. -TESTCASE_ENTITY_QUERY_LIMIT = 256 +TESTCASE_ENTITY_QUERY_LIMIT: int = 256 # Deadlines for testcase filing, closures and deletions (in days). -DUPLICATE_TESTCASE_NO_BUG_DEADLINE = 3 -CLOSE_TESTCASE_WITH_CLOSED_BUG_DEADLINE = 14 -FILE_CONSISTENT_UNREPRODUCIBLE_TESTCASE_DEADLINE = 14 -NOTIFY_CLOSED_BUG_WITH_OPEN_TESTCASE_DEADLINE = 7 -UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE = 7 -UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE = 14 +DUPLICATE_TESTCASE_NO_BUG_DEADLINE: int = 3 +CLOSE_TESTCASE_WITH_CLOSED_BUG_DEADLINE: int = 14 +FILE_CONSISTENT_UNREPRODUCIBLE_TESTCASE_DEADLINE: int = 14 +NOTIFY_CLOSED_BUG_WITH_OPEN_TESTCASE_DEADLINE: int = 7 +UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE: int = 7 +UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE: int = 14 # TODO(ochang): Find some way to remove these. -CHROMIUM_ISSUE_PREDATOR_AUTO_CC_LABEL = 'Test-Predator-Auto-CC' -CHROMIUM_ISSUE_PREDATOR_AUTO_COMPONENTS_LABEL = 'Test-Predator-Auto-Components' -CHROMIUM_ISSUE_PREDATOR_AUTO_OWNER_LABEL = 'Test-Predator-Auto-Owner' -CHROMIUM_ISSUE_PREDATOR_WRONG_COMPONENTS_LABEL = ( +CHROMIUM_ISSUE_PREDATOR_AUTO_CC_LABEL: str = 'Test-Predator-Auto-CC' +CHROMIUM_ISSUE_PREDATOR_AUTO_COMPONENTS_LABEL: str = ( + 'Test-Predator-Auto-Components') +CHROMIUM_ISSUE_PREDATOR_AUTO_OWNER_LABEL: str = 'Test-Predator-Auto-Owner' +CHROMIUM_ISSUE_PREDATOR_WRONG_COMPONENTS_LABEL: str = ( 'Test-Predator-Wrong-Components') -CHROMIUM_ISSUE_PREDATOR_WRONG_CL_LABEL = 'Test-Predator-Wrong-CLs' +CHROMIUM_ISSUE_PREDATOR_WRONG_CL_LABEL: str = 'Test-Predator-Wrong-CLs' -MISSING_VALUE_STRING = '---' +MISSING_VALUE_STRING: str = '---' -COVERAGE_INFORMATION_DATE_FORMAT = '%Y%m%d' +COVERAGE_INFORMATION_DATE_FORMAT: str = '%Y%m%d' +_ModelT = TypeVar('_ModelT', bound=ndb.Model) -def clone_entity(e, **extra_args): + +def clone_entity(e: _ModelT, **extra_args: Any) -> _ModelT: """Clones a DataStore entity and returns the clone.""" ent_class = e.__class__ # pylint: disable=protected-access,unnecessary-dunder-call + properties = cast(dict[str, Any], ent_class._properties) props = { v._code_name: v.__get__(e, ent_class) - for v in ent_class._properties.values() + for v in properties.values() if not isinstance(v, ndb.ComputedProperty) } props.update(extra_args) @@ -150,12 +157,12 @@ class SecuritySeverity: MISSING = 4 @classmethod - def is_valid(cls, security_severity): + def is_valid(cls, security_severity: Any) -> bool: """Return bool on whether a severity is valid.""" return (security_severity in [cls.CRITICAL, cls.HIGH, cls.MEDIUM, cls.LOW]) @classmethod - def list(cls): + def list(cls) -> list[dict[str, Any]]: """Return the list of severities for a dropdown menu.""" return [ { @@ -287,7 +294,7 @@ class Fuzzer(Model): created_at = ndb.DateTimeProperty() # Additionally allows '.' and '@' over NAME_CHECK_REGEX. - VALID_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_@.-]+$') + VALID_NAME_REGEX: re.Pattern = re.compile(r'^[a-zA-Z0-9_@.-]+$') # Last update time. timestamp = ndb.DateTimeProperty() @@ -385,7 +392,7 @@ class Fuzzer(Model): # on disk |FUZZ_INPUTS_DISK|, rather than smaller tmpfs one (FUZZ_INPUTS). has_large_testcases = ndb.BooleanProperty(default=False) - def get_config_dict(self): + def get_config_dict(self) -> dict[str, Any]: """Returns a dict containing the required config to upload a fuzzer.""" return self.to_dict(exclude=self._EXCLUDED_FIELDS_FOR_CONFIG) @@ -399,6 +406,7 @@ class BuildCrashStatsJobHistory(Model): class Testcase(Model): """Represents a single testcase.""" + # Crash on an invalid read/write. crash_type = ndb.StringProperty() @@ -631,26 +639,26 @@ class Testcase(Model): # Defaults to false, since most testcases are fuzzer produced. analyze_pending = ndb.BooleanProperty(default=False) - def is_chromium(self): + def is_chromium(self) -> bool: return self.project_name in ('chromium', 'chromium-testing') - def has_blame(self): + def has_blame(self) -> bool: return self.is_chromium() - def has_impacts(self): - return self.is_chromium() and not self.one_time_crasher_flag + def has_impacts(self) -> bool: + return self.is_chromium() and not bool(self.one_time_crasher_flag) - def impacts_production(self): + def impacts_production(self) -> bool: return (bool(self.impact_extended_stable_version) or bool(self.impact_stable_version) or bool(self.impact_beta_version)) - def is_status_unreproducible(self): - return self.status and self.status.startswith('Unreproducible') + def is_status_unreproducible(self) -> bool: + return bool(self.status and self.status.startswith('Unreproducible')) - def is_crash(self): + def is_crash(self) -> bool: return bool(self.crash_state) - def populate_indices(self): + def populate_indices(self) -> None: """Populate keywords for fast test case list searching.""" self.keywords = list( search_tokenizer.tokenize(self.crash_state) @@ -698,10 +706,10 @@ def populate_indices(self): self.impact_stable_version_indices = [] self.impact_beta_version_indices = [] - def _pre_put_hook(self): + def _pre_put_hook(self) -> None: self.populate_indices() - def _post_put_hook(self, _): + def _post_put_hook(self, future: Any) -> None: if not self.key: # Failed put. An exception will be thrown automatically afterwards. return @@ -710,39 +718,41 @@ def _post_put_hook(self, _): f'Updated testcase {self.key.id()} (bug {self.bug_information or "-"}).' ) - def set_impacts_as_na(self): + def set_impacts_as_na(self) -> None: self.impact_stable_version = self.impact_beta_version = None self.impact_extended_stable_version = None self.impact_stable_version_likely = self.impact_beta_version_likely = False self.impact_extended_stable_version_likely = False self.is_impact_set_flag = False - def _ensure_metadata_is_cached(self): + def _ensure_metadata_is_cached(self) -> None: """Ensure that the metadata for this has been cached.""" if hasattr(self, 'metadata_cache'): return try: - cache = json_utils.loads(self.additional_metadata) + cache = json_utils.loads(self.additional_metadata or '') except (TypeError, ValueError): cache = {} - setattr(self, 'metadata_cache', cache) + # pylint: disable=attribute-defined-outside-init + self.metadata_cache = cache # Returns testcase.created in case it is present, as it is # the source of truth for creation time. If missing, returns # testcase.timestamp as a proxy for creation time. - def get_created_time(self) -> ndb.DateTimeProperty: + def get_created_time(self) -> datetime.datetime | None: return self.created if self.created else self.timestamp - def get_age_in_seconds(self): + def get_age_in_seconds(self) -> float | None: current_time = datetime.datetime.utcnow() - if not self.get_created_time(): + created_time = self.get_created_time() + if not created_time: return None - testcase_age = current_time - self.get_created_time() + testcase_age = current_time - created_time return testcase_age.total_seconds() - def get_metadata(self, key=None, default=None): + def get_metadata(self, key: str | None = None, default: Any = None) -> Any: """Get metadata for a test case. Slow on first access.""" self._ensure_metadata_is_cached() @@ -755,7 +765,8 @@ def get_metadata(self, key=None, default=None): except KeyError: return default - def set_metadata(self, key, value, update_testcase=True): + def set_metadata(self, key: str, value: Any, + update_testcase: bool = True) -> None: """Set metadata for a test case.""" self._ensure_metadata_is_cached() self.metadata_cache[key] = value @@ -764,7 +775,7 @@ def set_metadata(self, key, value, update_testcase=True): if update_testcase: self.put() - def delete_metadata(self, key, update_testcase=True): + def delete_metadata(self, key: str, update_testcase: bool = True) -> None: """Remove metadata key for a test case.""" self._ensure_metadata_is_cached() @@ -777,11 +788,11 @@ def delete_metadata(self, key, update_testcase=True): if update_testcase: self.put() - def actual_fuzzer_name(self): + def actual_fuzzer_name(self) -> str: """Actual fuzzer name, uses one from overridden attribute if available.""" - return self.overridden_fuzzer_name or self.fuzzer_name + return self.overridden_fuzzer_name or self.fuzzer_name or '' - def get_fuzz_target(self): + def get_fuzz_target(self) -> 'FuzzTarget | None': """Get the associated FuzzTarget entity for this test case.""" name = self.actual_fuzzer_name() if not name: @@ -811,7 +822,7 @@ class TestcaseGroup(Model): class DataBundle(Model): """Represents a data bundle associated with a fuzzer.""" - VALID_NAME_REGEX = NAME_CHECK_REGEX + VALID_NAME_REGEX: re.Pattern = NAME_CHECK_REGEX # The data bundle's name (important for identifying shared bundles). name = ndb.StringProperty() @@ -987,7 +998,7 @@ class JobTemplate(Model): class Job(Model): """Definition of a job type used by the bots.""" - VALID_NAME_REGEX = NAME_CHECK_REGEX + VALID_NAME_REGEX: re.Pattern = NAME_CHECK_REGEX # Job type name. name = ndb.StringProperty() @@ -1030,18 +1041,18 @@ class Job(Model): # value here is the subscription used for receiving reproduction updates. external_updates_subscription = ndb.StringProperty() - def is_external(self): + def is_external(self) -> bool: """Whether this job is external.""" return (bool(self.external_reproduction_topic) or bool(self.external_updates_subscription)) - def get_environment(self): + def get_environment(self) -> dict[str, Any]: """Get the environment as a dict for this job, including any environment variables in its template.""" if not self.templates: return environment.parse_environment_definition(self.environment_string) - job_environment = {} + job_environment: dict[str, Any] = {} for template_name in self.templates: template = JobTemplate.query(JobTemplate.name == template_name).get() if not template: @@ -1058,7 +1069,7 @@ def get_environment(self): job_environment.update(environment_overrides) return job_environment - def get_environment_string(self): + def get_environment_string(self) -> str: """Get the environment string for this job, including any environment variables in its template. Avoid using this if possible.""" environment_string = '' @@ -1068,13 +1079,13 @@ def get_environment_string(self): return environment_string - def populate_indices(self): + def populate_indices(self) -> None: """Populate keywords for fast job searching.""" self.keywords = list( search_tokenizer.tokenize(self.name) | search_tokenizer.tokenize(self.project)) - def _pre_put_hook(self): + def _pre_put_hook(self) -> None: """Pre-put hook.""" self.project = self.get_environment().get('PROJECT_NAME', utils.default_project_name()) @@ -1116,13 +1127,13 @@ class Heartbeat(Model): # Keywords is used for searching. keywords = ndb.StringProperty(repeated=True) - def populate_indices(self): + def populate_indices(self) -> None: """Populate keywords for fast job searching.""" self.keywords = list( search_tokenizer.tokenize(self.bot_name) | search_tokenizer.tokenize(self.task_payload)) - def _pre_put_hook(self): + def _pre_put_hook(self) -> None: """Pre-put hook.""" self.populate_indices() @@ -1205,7 +1216,7 @@ class WindowRateLimitTask(Model): it will have a different lifecycle (it's not needed after the window completes). This should have a TTL as TASK_RATE_LIMIT_WINDOW in task_rate_limiting.py (6 hours).""" - TASK_RATE_LIMIT_WINDOW = datetime.timedelta(hours=6) + TASK_RATE_LIMIT_WINDOW: datetime.timedelta = datetime.timedelta(hours=6) timestamp = ndb.DateTimeProperty(auto_now_add=True, indexed=True) # Only use this for TTL. It should only be saved to by ClusterFuzz, not read. @@ -1216,7 +1227,7 @@ class WindowRateLimitTask(Model): job_name = ndb.StringProperty(indexed=True) status = ndb.StringProperty(choices=[TaskState.ERROR, TaskState.FINISHED]) - def _pre_put_hook(self): + def _pre_put_hook(self) -> None: self.ttl_expiry_timestamp = ( datetime.datetime.now() + self.TASK_RATE_LIMIT_WINDOW) @@ -1266,7 +1277,7 @@ class ReportMetadata(Model): minidump_key = ndb.TextProperty(default='') # Processed crash bytes. - serialized_crash_stack_frames = ndb.BlobProperty(default='', indexed=False) + serialized_crash_stack_frames = ndb.BlobProperty(default=b'', indexed=False) # Id of the associated testcase. testcase_id = ndb.StringProperty(default='') @@ -1301,33 +1312,35 @@ class FuzzTarget(Model): # Binary name. binary = ndb.StringProperty() - def _pre_put_hook(self): + def _pre_put_hook(self) -> None: """Pre-put hook.""" self.key = ndb.Key(FuzzTarget, self.fully_qualified_name()) - def fully_qualified_name(self): + def fully_qualified_name(self) -> str: """Get the fully qualified name for this fuzz target.""" - return fuzz_target_fully_qualified_name(self.engine, self.project, - self.binary) + return fuzz_target_fully_qualified_name( + cast(str, self.engine), self.project, cast(str, self.binary)) - def project_qualified_name(self): + def project_qualified_name(self) -> str: """Get the name qualified by project.""" - return fuzz_target_project_qualified_name(self.project, self.binary) + return fuzz_target_project_qualified_name(self.project, + cast(str, self.binary)) -def fuzz_target_fully_qualified_name(engine, project, binary): +def fuzz_target_fully_qualified_name(engine: str, project: str | None, + binary: str) -> str: """Get a fuzz target's fully qualified name.""" return engine + '_' + fuzz_target_project_qualified_name(project, binary) -def normalized_name(name): +def normalized_name(name: str) -> str: """Return normalized name with special chars like slash, colon, etc normalized to hyphen(-). This is important as otherwise these chars break local and cloud storage paths.""" return SPECIAL_CHARS_REGEX.sub('-', name).strip('-') -def fuzz_target_project_qualified_name(project, binary): +def fuzz_target_project_qualified_name(project: str | None, binary: str) -> str: """Get a fuzz target's project qualified name.""" binary = normalized_name(binary) if not project: @@ -1367,10 +1380,12 @@ class FuzzTargetJob(Model): # Approximate last time this target was run. last_run = ndb.DateTimeProperty() - def _pre_put_hook(self): + def _pre_put_hook(self) -> None: """Pre-put hook.""" - self.key = ndb.Key(FuzzTargetJob, - fuzz_target_job_key(self.fuzz_target_name, self.job)) + self.key = ndb.Key( + FuzzTargetJob, + fuzz_target_job_key( + cast(str, self.fuzz_target_name), cast(str, self.job))) class FuzzStrategyProbability(Model): @@ -1382,7 +1397,7 @@ class FuzzStrategyProbability(Model): engine = ndb.StringProperty() -def fuzz_target_job_key(fuzz_target_name, job): +def fuzz_target_job_key(fuzz_target_name: str, job: str) -> str: """Return the key for FuzzTargetJob.""" return f'{fuzz_target_name}/{job}' @@ -1467,18 +1482,23 @@ class CoverageInformation(Model): # Link to the HTML report. html_report_url = ndb.StringProperty() - def _pre_put_hook(self): + def _pre_put_hook(self) -> None: """Pre-put hook.""" - self.key = ndb.Key(CoverageInformation, - coverage_information_key(self.fuzzer, self.date)) + self.key = ndb.Key( + CoverageInformation, + coverage_information_key( + cast(str, self.fuzzer), + cast(datetime.date | datetime.datetime, self.date))) -def coverage_information_date_to_string(date): +def coverage_information_date_to_string( + date: datetime.date | datetime.datetime) -> str: """Returns string representation of the date in a format used for coverage.""" return date.strftime(COVERAGE_INFORMATION_DATE_FORMAT) -def coverage_information_key(project_qualified_fuzzer_name, date): +def coverage_information_key(project_qualified_fuzzer_name: str, + date: datetime.date | datetime.datetime) -> str: """Constructs an ndb key for CoverageInformation entity.""" date_string = coverage_information_date_to_string(date) return project_qualified_fuzzer_name + '-' + date_string @@ -1545,7 +1565,7 @@ class ClusterInfo(Model): # Information about CPUs in each cluster. clusters = ndb.StructuredProperty(ClusterInfo, repeated=True) - def get_cluster_info(self, name): + def get_cluster_info(self, name: str) -> 'ClusterInfo | None': return next((info for info in self.clusters if info.cluster == name), None) @@ -1585,9 +1605,9 @@ class FuzzerJob(Model): multiplier = ndb.FloatProperty(default=1.0) @property - def actual_weight(self): + def actual_weight(self) -> float: """Get the actual weight for this job.""" - return self.weight * self.multiplier + return cast(float, self.weight) * cast(float, self.multiplier) class FuzzerJobs(Model): @@ -1658,7 +1678,7 @@ class TestcaseVariant(Model): class TestcaseLifecycleEvent(Model): """Represents an event from a testcase lifecycle.""" # Events' TTL, currently set to ~5Y. - TESTCASE_EVENT_TTL = datetime.timedelta(days=1826) + TESTCASE_EVENT_TTL: datetime.timedelta = datetime.timedelta(days=1826) ### Event definition. # Event type (testcase_creation, issue_filed, etc). @@ -1775,7 +1795,7 @@ class TestcaseLifecycleEvent(Model): # Task-related fuzzer name. task_fuzzer = ndb.StringProperty() - def _pre_put_hook(self): + def _pre_put_hook(self) -> None: self.ttl_expiry_timestamp = ( datetime.datetime.now() + self.TESTCASE_EVENT_TTL) @@ -1790,7 +1810,7 @@ class FuzzerTaskEvent(Model): will be used to assist with tracing clusterfuzz execution. """ # Fuzzer task events' TTL, currently set to 2 months. - FUZZER_EVENT_TTL = datetime.timedelta(days=60) + FUZZER_EVENT_TTL: datetime.timedelta = datetime.timedelta(days=60) ### Event definition. # Event type (mostly task_execution). @@ -1847,7 +1867,7 @@ class FuzzerTaskEvent(Model): # Task-related fuzzer name. task_fuzzer = ndb.TextProperty() - def _pre_put_hook(self): + def _pre_put_hook(self) -> None: self.ttl_expiry_timestamp = ( datetime.datetime.now() + self.FUZZER_EVENT_TTL) diff --git a/src/clusterfuzz/_internal/datastore/fuzz_target_utils.py b/src/clusterfuzz/_internal/datastore/fuzz_target_utils.py index 2f182eeec8d..df7aca7ae3d 100644 --- a/src/clusterfuzz/_internal/datastore/fuzz_target_utils.py +++ b/src/clusterfuzz/_internal/datastore/fuzz_target_utils.py @@ -12,13 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. """Helper functions related to fuzz target entities.""" +from collections.abc import Iterable +from collections.abc import Iterator + from google.cloud import ndb from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.datastore import ndb_utils -def get_fuzz_targets_for_target_jobs(target_jobs): +def get_fuzz_targets_for_target_jobs( + target_jobs: Iterable[data_types.FuzzTargetJob] +) -> list[data_types.FuzzTarget | None]: """Return corresponding FuzzTargets for the given FuzzTargetJobs.""" target_keys = [ ndb.Key(data_types.FuzzTarget, t.fuzz_target_name) for t in target_jobs @@ -26,10 +31,11 @@ def get_fuzz_targets_for_target_jobs(target_jobs): return ndb_utils.get_multi(target_keys) -def get_fuzz_target_jobs(fuzz_target_name=None, - engine=None, - job=None, - limit=None): +def get_fuzz_target_jobs( + fuzz_target_name: str | None = None, + engine: str | None = None, + job: str | None = None, + limit: int | None = None) -> Iterator[data_types.FuzzTargetJob]: """Return a Datastore query for fuzz target to job mappings.""" query = data_types.FuzzTargetJob.query() diff --git a/src/clusterfuzz/_internal/datastore/locks.py b/src/clusterfuzz/_internal/datastore/locks.py index 6c3c34a864c..17628dc8c89 100644 --- a/src/clusterfuzz/_internal/datastore/locks.py +++ b/src/clusterfuzz/_internal/datastore/locks.py @@ -17,6 +17,7 @@ import datetime import random import time +from typing import cast from google.cloud import ndb from google.cloud.ndb import exceptions @@ -26,25 +27,25 @@ from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -DEFAULT_MAX_HOLD_SECONDS = 60 * 10 +DEFAULT_MAX_HOLD_SECONDS: int = 60 * 10 -LOCK_CHECK_SLEEP_MULTIPLIER = 10 -LOCK_CHECK_TIMEOUT = 60 * 60 +LOCK_CHECK_SLEEP_MULTIPLIER: int = 10 +LOCK_CHECK_TIMEOUT: int = 60 * 60 -MAX_WAIT_EXPONENT = 6 -NUM_STATISTICS_SHARDS = 50 +MAX_WAIT_EXPONENT: int = 6 +NUM_STATISTICS_SHARDS: int = 50 -TRANSACTION_RETRIES = 0 +TRANSACTION_RETRIES: int = 0 -def _get_current_lock_zone(): +def _get_current_lock_zone() -> str | None: """Get the current zone for locking purposes.""" if environment.get_value('LOCAL_DEVELOPMENT', False): return 'local' platform = environment.get_platform_group().lower() platform_group_mappings = db_config.get_value('platform_group_mappings') - for mapping in platform_group_mappings.splitlines(): + for mapping in cast(str, platform_group_mappings).splitlines(): if ';' not in mapping: continue @@ -58,7 +59,7 @@ def _get_current_lock_zone(): return platform -def _get_key_name_with_lock_zone(key_name): +def _get_key_name_with_lock_zone(key_name: str) -> str | None: """Return the lock key name with the current lock zone.""" current_zone = _get_current_lock_zone() if not current_zone: @@ -68,7 +69,11 @@ def _get_key_name_with_lock_zone(key_name): return current_zone + ';' + key_name -def _try_acquire_lock(key_name, expiration_time, holder): +def _try_acquire_lock( + key_name: str, + expiration_time: datetime.datetime, + holder: str | None, +) -> data_types.Lock: """Actual lock acquire that runs in a transaction.""" lock_entity = ndb.Key(data_types.Lock, key_name).get() @@ -79,6 +84,7 @@ def _try_acquire_lock(key_name, expiration_time, holder): lock_entity.put() return lock_entity + assert lock_entity.expiration_time is not None if lock_entity.expiration_time <= datetime.datetime.utcnow(): # Lock was expired, try to take over the lock. lock_entity.expiration_time = expiration_time @@ -88,16 +94,18 @@ def _try_acquire_lock(key_name, expiration_time, holder): return lock_entity -def acquire_lock(key_name, - max_hold_seconds=DEFAULT_MAX_HOLD_SECONDS, - retries=None, - by_zone=True): +def acquire_lock( + key_name: str, + max_hold_seconds: int = DEFAULT_MAX_HOLD_SECONDS, + retries: int | None = None, + by_zone: bool = True, +) -> datetime.datetime | None: """Acquire a lock for the given key name. Returns the expiration time if succeeded, otherwise None. The lock holder is responsible for making sure it doesn't assume the lock is still held after the expiration time.""" logs.info('Acquiring lock for %s.' % key_name) failed_acquires = 0 - total_wait = 0 + total_wait: float = 0 wait_exponent = 1 if by_zone: @@ -113,9 +121,10 @@ def acquire_lock(key_name, while total_wait < LOCK_CHECK_TIMEOUT: try: lock_entity = ndb.transaction( - lambda: _try_acquire_lock(key_name, - expiration_time=datetime.datetime.utcnow() + - expiration_delta, holder=bot_name), + lambda: _try_acquire_lock( + key_name, + expiration_time=datetime.datetime.utcnow() + expiration_delta, + holder=bot_name), retries=TRANSACTION_RETRIES) if lock_entity.holder == bot_name: @@ -143,7 +152,11 @@ def acquire_lock(key_name, return None -def release_lock(key_name, force_release=False, by_zone=True): +def release_lock( + key_name: str, + force_release: bool = False, + by_zone: bool = True, +) -> None: """Release a lock for the given key name.""" logs.info('Releasing lock for %s.' % key_name) bot_name = environment.get_value('BOT_NAME') diff --git a/src/clusterfuzz/_internal/datastore/ndb_init.py b/src/clusterfuzz/_internal/datastore/ndb_init.py index c91e5c45bad..10dc84caa5c 100644 --- a/src/clusterfuzz/_internal/datastore/ndb_init.py +++ b/src/clusterfuzz/_internal/datastore/ndb_init.py @@ -13,22 +13,29 @@ # limitations under the License. """NDB initialization.""" +from collections.abc import Generator import contextlib import functools import os import threading +from typing import Callable +from typing import ParamSpec +from typing import TypeVar from google.cloud import ndb from google.cloud.ndb import context as context_module from clusterfuzz._internal.base import utils -_ndb_client = None +_ndb_client: ndb.Client | None = None _ndb_client_lock = threading.Lock() -_initial_pid = None +_initial_pid: int | None = None +_P = ParamSpec('_P') +_R = TypeVar('_R') -def _client(): + +def _client() -> ndb.Client: """Get or initialize the NDB client.""" global _ndb_client global _initial_pid @@ -43,7 +50,7 @@ def _client(): @contextlib.contextmanager -def context(): +def context() -> Generator[context_module.Context, None, None]: """Get the NDB context.""" if _initial_pid != os.getpid(): # Forked, clear the existing context to avoid issues. @@ -68,12 +75,12 @@ def context(): yield ndb_context -def thread_wrapper(func): +def thread_wrapper(func: Callable[_P, _R]) -> Callable[_P, _R]: """Wrapper for thread targets to initialize an NDB context, since contexts are thread local.""" @functools.wraps(func) - def _wrapper(*args, **kwargs): + def _wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: with context(): return func(*args, **kwargs) diff --git a/src/clusterfuzz/_internal/datastore/ndb_utils.py b/src/clusterfuzz/_internal/datastore/ndb_utils.py index 15a94301078..39be78aaec3 100644 --- a/src/clusterfuzz/_internal/datastore/ndb_utils.py +++ b/src/clusterfuzz/_internal/datastore/ndb_utils.py @@ -13,28 +13,38 @@ # limitations under the License. """NDB utilities. Provides utility functions for NDB.""" +from __future__ import annotations + +from collections.abc import Iterable +from collections.abc import Iterator +from typing import Any +from typing import TypeVar + from google.cloud import ndb -_GET_BATCH_SIZE = 1000 -_MODIFY_BATCH_SIZE = 500 +_ModelT = TypeVar('_ModelT', bound=ndb.Model) +_T = TypeVar('_T') + +_GET_BATCH_SIZE: int = 1000 +_MODIFY_BATCH_SIZE: int = 500 -def is_true(boolean_prop): +def is_true(boolean_prop: Any) -> Any: """Helper for boolean property filters to avoid lint errors.""" return boolean_prop == True # pylint: disable=g-explicit-bool-comparison,singleton-comparison -def is_false(boolean_prop): +def is_false(boolean_prop: Any) -> Any: """Helper for boolean property filters to avoid lint errors.""" return boolean_prop == False # pylint: disable=g-explicit-bool-comparison,singleton-comparison -def get_all_from_model(model): +def get_all_from_model(model: type[_ModelT]) -> Iterator[_ModelT]: """Get all results from a ndb.Model.""" return get_all_from_query(model.query()) -def get_all_from_query(query, **kwargs): +def get_all_from_query(query: ndb.Query, **kwargs: Any) -> Iterator[Any]: """Return all entities based on the query by paging, to avoid query expirations on App Engine.""" # TODO(ochang): Queries no longer expire with new NDB. Remove this and all @@ -43,14 +53,14 @@ def get_all_from_query(query, **kwargs): yield from query.iter(**kwargs) -def _gen_chunks(values, size): +def _gen_chunks(values: Iterable[_T], size: int) -> Iterator[list[_T]]: """Generate chunks of iterable.""" - values = list(values) - for i in range(0, len(values), size): - yield values[i:i + size] + values_list = list(values) + for i in range(0, len(values_list), size): + yield values_list[i:i + size] -def get_multi(keys): +def get_multi(keys: Iterable[ndb.Key]) -> list[Any]: """Get multiple entities, working around a limitation in the NDB library with the maximum number of keys allowed.""" result = [] @@ -60,7 +70,7 @@ def get_multi(keys): return result -def put_multi(entities): +def put_multi(entities: Iterable[ndb.Model]) -> list[ndb.Key]: """Put multiple entities, working around a limitation in the NDB library with the maximum number of keys allowed.""" result = [] @@ -70,7 +80,7 @@ def put_multi(entities): return result -def delete_multi(keys): +def delete_multi(keys: Iterable[ndb.Key]) -> None: """Delete multiple entities, working around a limitation in the NDB library with the maximum number of keys allowed.""" for chunk in _gen_chunks(keys, _MODIFY_BATCH_SIZE): diff --git a/src/clusterfuzz/_internal/datastore/search_tokenizer.py b/src/clusterfuzz/_internal/datastore/search_tokenizer.py index fa0a546323b..2d2eee614e7 100644 --- a/src/clusterfuzz/_internal/datastore/search_tokenizer.py +++ b/src/clusterfuzz/_internal/datastore/search_tokenizer.py @@ -14,16 +14,21 @@ """search_tokenizer tokenizes a string into tokens according to our keyword searching use cases.""" +from collections.abc import Iterator import re +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from clusterfuzz._internal.datastore import data_types -def tokenize(s): + +def tokenize(s: object) -> set[str]: """Tokenize a string, by line, into atomic tokens and complex tokens.""" if not s: s = '' s = '%s' % s - tokens = set() + tokens: set[str] = set() lines = s.splitlines() for line in lines: @@ -35,9 +40,9 @@ def tokenize(s): return tokens -def tokenize_bug_information(testcase): +def tokenize_bug_information(testcase: 'data_types.Testcase') -> list[str]: """Tokenize bug information for searching.""" - bug_indices = [] + bug_indices: list[str] = [] if testcase.bug_information: bug_indices.append(testcase.bug_information.lower().strip()) @@ -47,12 +52,12 @@ def tokenize_bug_information(testcase): return bug_indices -def tokenize_impact_version(version): +def tokenize_impact_version(version: str | None) -> list[str]: """Tokenize impact.""" if not version: return [] - tokens = set() + tokens: set[str] = set() splitted = version.split('.') for index in range(len(splitted)): tokens.add('.'.join(splitted[0:(index + 1)])) @@ -60,26 +65,26 @@ def tokenize_impact_version(version): return [t for t in tokens if t.strip()] -def prepare_search_keyword(s): +def prepare_search_keyword(s: str) -> str: """Prepare the search keywords into the form that is appropriate for searching according to our tokenization algorithm.""" return s.lower().strip() -def _is_camel_case_ab(s, index): +def _is_camel_case_ab(s: str, index: int) -> bool: """Determine if the index is at 'aB', which is the start of a camel token. For example, with 'workAt', this function detects 'kA'.""" return index >= 1 and s[index - 1].islower() and s[index].isupper() -def _is_camel_case_abb(s, index): +def _is_camel_case_abb(s: str, index: int) -> bool: """Determine if the index ends at 'ABb', which is the start of a camel token. For example, with 'HTMLParser', this function detects 'LPa'.""" return (index >= 2 and s[index - 2].isupper() and s[index - 1].isupper() and s[index].islower()) -def _token_indices(s): +def _token_indices(s: str) -> Iterator[tuple[int, int]]: """Iterate through (end_current_token_index, start_next_token_index) of `s`, which is tokenized based on non-alphanumeric characters and camel casing. For example, 'aa:bbCC' have 3 tokens: 'aa', 'bb', 'CC'. @@ -105,7 +110,7 @@ def _token_indices(s): yield (length - 1), length -def _complex_tokenize(s, limit): +def _complex_tokenize(s: str, limit: int) -> set[str]: """Tokenize a string into complex tokens. For example, a:b:c is tokenized into ['a', 'b', 'c', 'a:b', 'a:b:c', 'b:c']. This method works recursively. It generates all possible complex tokens starting from the first token. Then, @@ -116,7 +121,7 @@ def _complex_tokenize(s, limit): if not s: return set() - tokens = [] + tokens: list[str] = [] second_token_index = len(s) count = 0 for end_index, next_start_index in _token_indices(s): @@ -127,6 +132,6 @@ def _complex_tokenize(s, limit): if count >= limit: break - tokens = {t.lower() for t in tokens if t.strip()} - tokens |= _complex_tokenize(s[second_token_index:], limit=limit) - return tokens + token_set: set[str] = {t.lower() for t in tokens if t.strip()} + token_set |= _complex_tokenize(s[second_token_index:], limit=limit) + return token_set diff --git a/src/clusterfuzz/_internal/fuzzer_utils/tests.py b/src/clusterfuzz/_internal/fuzzer_utils/tests.py index 894c11e6550..23f22e1349a 100644 --- a/src/clusterfuzz/_internal/fuzzer_utils/tests.py +++ b/src/clusterfuzz/_internal/fuzzer_utils/tests.py @@ -15,6 +15,7 @@ import os import random +from typing import Sequence from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot import testcase_manager @@ -23,10 +24,11 @@ # FIXME: Importing data_handler module is heavyweight and # runs into exception with |oauth2client|. Skip and initialize # constant directly. -LOCK_FILENAME = '.lock' +LOCK_FILENAME: str = '.lock' -def create_testcase_list_file(testcase_file_paths, input_directory): +def create_testcase_list_file(testcase_file_paths: Sequence[str], + input_directory: str) -> None: """Store list of fuzzed testcases from fuzzer in a bot specific testcase list file.""" if not testcase_file_paths: @@ -34,42 +36,43 @@ def create_testcase_list_file(testcase_file_paths, input_directory): return bot_testcases_file_path = utils.get_bot_testcases_file_path(input_directory) - with open(bot_testcases_file_path, 'wb') as bot_testcases_file_handle: + with open(bot_testcases_file_path, 'w') as bot_testcases_file_handle: bot_testcases_file_handle.write('\n'.join(testcase_file_paths)) -def is_valid_testcase_file(file_path, - check_if_exists=True, - size_limit=None, - allowed_extensions=None): +def is_valid_testcase_file( + file_path: str, + check_if_exists: bool = True, + size_limit: int | None = None, + allowed_extensions: Sequence[str] | set[str] | None = None) -> bool: """Return true if the file looks like a testcase file.""" return utils.is_valid_testcase_file(file_path, check_if_exists, size_limit, allowed_extensions) -def is_locked(input_directory): +def is_locked(input_directory: str) -> bool: """Returns true if the data bundle is locked and unavailable for use.""" lock_file_path = os.path.join(input_directory, LOCK_FILENAME) return os.path.exists(lock_file_path) -def get_random_testcases(input_directory, max_testcases): +def get_random_testcases(input_directory: str, max_testcases: int) -> list[str]: """Returns list of |max_testcases| testcases.""" testcases_list = get_testcases(input_directory) return random.SystemRandom().sample(testcases_list, max_testcases) -def get_testcases(input_directory): +def get_testcases(input_directory: str) -> list[str]: """Returns list of testcase files.""" testcase_list_file_path = os.path.join( input_directory, testcase_manager.TESTCASE_LIST_FILENAME) if not os.path.exists(testcase_list_file_path): return [] - with open(testcase_list_file_path, 'rb') as testcase_list_file_handle: + with open(testcase_list_file_path, 'r') as testcase_list_file_handle: testcase_relative_file_paths = testcase_list_file_handle.read().splitlines() - testcase_file_paths = [] + testcase_file_paths: list[str] = [] for testcase_relative_file_path in testcase_relative_file_paths: # Discard junk paths. if not testcase_relative_file_path.strip(): diff --git a/src/clusterfuzz/_internal/fuzzing/corpus_manager.py b/src/clusterfuzz/_internal/fuzzing/corpus_manager.py index fa8115a6293..5fafb2ce9d4 100644 --- a/src/clusterfuzz/_internal/fuzzing/corpus_manager.py +++ b/src/clusterfuzz/_internal/fuzzing/corpus_manager.py @@ -13,13 +13,16 @@ # limitations under the License. """Functions for corpus synchronization with GCS.""" +from collections.abc import Callable +from collections.abc import Sequence import datetime import itertools import os import re import shutil import tempfile -from typing import Optional +from typing import Any +from typing import cast from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot.tasks import task_types @@ -29,6 +32,7 @@ from clusterfuzz._internal.protos import uworker_msg_pb2 from clusterfuzz._internal.system import archive from clusterfuzz._internal.system import environment +from clusterfuzz._internal.system import new_process from clusterfuzz._internal.system import shell try: @@ -36,32 +40,33 @@ # Disable "invalid-name" because fixing the issue will cause pylint to # complain the None assignment is incorrectly named. - DEFAULT_GSUTIL_RUNNER = gsutil.GSUtilRunner # pylint: disable=invalid-name -except: + DEFAULT_GSUTIL_RUNNER: Any = gsutil.GSUtilRunner # pylint: disable=invalid-name +except: # pylint: disable=bare-except # This is expected to fail on App Engine. - gsutil = None + gsutil = None # type: ignore DEFAULT_GSUTIL_RUNNER = None -BACKUP_ARCHIVE_FORMAT = 'zip' -CORPUS_FILES_SYNC_TIMEOUT = 60 * 60 -LATEST_BACKUP_TIMESTAMP = 'latest' -PUBLIC_BACKUP_TIMESTAMP = 'public' -REGRESSIONS_GCS_PATH_SUFFIX = '_regressions' +BACKUP_ARCHIVE_FORMAT: str = 'zip' +CORPUS_FILES_SYNC_TIMEOUT: int = 60 * 60 +LATEST_BACKUP_TIMESTAMP: str = 'latest' +PUBLIC_BACKUP_TIMESTAMP: str = 'public' +REGRESSIONS_GCS_PATH_SUFFIX: str = '_regressions' -ZIPPED_PATH_PREFIX = 'zipped' -PARTIAL_ZIPCORPUS_PREFIX = 'partial' -BASE_ZIPCORPUS_PREFIX = 'base' +ZIPPED_PATH_PREFIX: str = 'zipped' +PARTIAL_ZIPCORPUS_PREFIX: str = 'partial' +BASE_ZIPCORPUS_PREFIX: str = 'base' -RSYNC_ERROR_REGEX = (br'CommandException:\s*(\d+)\s*files?/objects? ' - br'could not be copied/removed') +RSYNC_ERROR_REGEX: bytes = (br'CommandException:\s*(\d+)\s*files?/objects? ' + br'could not be copied/removed') -MAX_SYNC_ERRORS = 10 +MAX_SYNC_ERRORS: int = 10 # Default used by shutil. -COPY_BUFFER_SIZE = 16 * 1024 +COPY_BUFFER_SIZE: int = 16 * 1024 -def _rsync_errors_below_threshold(gsutil_result, max_errors): +def _rsync_errors_below_threshold(gsutil_result: new_process.ProcessResult, + max_errors: int) -> bool: """Check if the number of errors during rsync is lower than our threshold.""" match = re.search(RSYNC_ERROR_REGEX, gsutil_result.output, re.MULTILINE) if not match: @@ -77,7 +82,8 @@ def _rsync_errors_below_threshold(gsutil_result, max_errors): return error_count <= max_errors -def _handle_rsync_result(gsutil_result, max_errors): +def _handle_rsync_result(gsutil_result: new_process.ProcessResult, + max_errors: int) -> bool: """Handle rsync result.""" if gsutil_result.return_code == 0: sync_succeeded = True @@ -90,12 +96,12 @@ def _handle_rsync_result(gsutil_result, max_errors): return sync_succeeded and not gsutil_result.timed_out -def _count_corpus_files(directory): +def _count_corpus_files(directory: str) -> int: """Count the number of corpus files.""" return shell.get_directory_file_count(directory) -def rename_file_to_sha(filepath, directory=None): +def rename_file_to_sha(filepath: str, directory: str | None = None) -> str: if directory is None: directory = os.path.dirname(filepath) sha1sum = utils.file_hash(filepath) @@ -104,15 +110,15 @@ def rename_file_to_sha(filepath, directory=None): return new_filepath -def legalize_filenames(filepaths): +def legalize_filenames(filepaths: Sequence[str]) -> list[str]: """Convert the name of every file in |filepaths| a name that is legal on Windows. Returns list of legally named files.""" if environment.is_trusted_host(): - return filepaths + return cast(list[str], filepaths) illegal_chars = {'<', '>', ':', '\\', '|', '?', '*'} - legally_named = [] - failed_to_move_filepaths = [] + legally_named: list[str] = [] + failed_to_move_filepaths: list[str] = [] for filepath in filepaths: _, basename = os.path.split(filepath) if not any(char in illegal_chars for char in basename): @@ -136,7 +142,7 @@ def legalize_filenames(filepaths): return legally_named -def legalize_corpus_files(directory): +def legalize_corpus_files(directory: str) -> None: """Convert the name of every corpus file in |directory| to a name that is allowed on Windows.""" # Iterate through return value of legalize_filenames to convert every @@ -149,11 +155,13 @@ class GcsCorpus: """Google Cloud Storage corpus.""" # TODO(metzman): Consider merging this with FuzzTargetCorpus. - def __init__(self, - bucket_name, - bucket_path='/', - log_results=True, - gsutil_runner_func=DEFAULT_GSUTIL_RUNNER): + def __init__( + self, + bucket_name: str, + bucket_path: str = '/', + log_results: bool = True, + gsutil_runner_func: Callable[[], Any] | None = DEFAULT_GSUTIL_RUNNER, + ) -> None: """Inits the GcsCorpus. Args: @@ -163,17 +171,17 @@ def __init__(self, self._bucket_name = bucket_name self._bucket_path = bucket_path self._log_results = log_results - self._gsutil_runner = gsutil_runner_func() + self._gsutil_runner: Any = cast(Callable[[], Any], gsutil_runner_func)() @property - def bucket_name(self): + def bucket_name(self) -> str: return self._bucket_name @property - def bucket_path(self): + def bucket_path(self) -> str: return self._bucket_path - def get_gcs_url(self): + def get_gcs_url(self) -> str: """Build corpus GCS URL for gsutil. Returns: A string giving the GCS URL. @@ -188,9 +196,9 @@ def get_gcs_url(self): return url def rsync_from_disk(self, - directory, - timeout=CORPUS_FILES_SYNC_TIMEOUT, - delete=True): + directory: str, + timeout: int = CORPUS_FILES_SYNC_TIMEOUT, + delete: bool = True) -> bool: """Upload local files to GCS and remove files which do not exist locally. Args: @@ -210,9 +218,9 @@ def rsync_from_disk(self, return _handle_rsync_result(result, max_errors=MAX_SYNC_ERRORS) def rsync_to_disk(self, - directory, - timeout=CORPUS_FILES_SYNC_TIMEOUT, - delete=True): + directory: str, + timeout: int = CORPUS_FILES_SYNC_TIMEOUT, + delete: bool = True) -> bool: """Run gsutil to download corpus files from GCS. Args: @@ -232,7 +240,9 @@ def rsync_to_disk(self, # Allow a small number of files to fail to be synced. return _handle_rsync_result(result, max_errors=MAX_SYNC_ERRORS) - def upload_files(self, file_paths, timeout=CORPUS_FILES_SYNC_TIMEOUT): + def upload_files(self, + file_paths: Sequence[str], + timeout: int = CORPUS_FILES_SYNC_TIMEOUT) -> Any: """Upload files to the GCS. Args: @@ -255,13 +265,15 @@ def upload_files(self, file_paths, timeout=CORPUS_FILES_SYNC_TIMEOUT): class FuzzTargetCorpus(GcsCorpus): """Engine fuzzer (libFuzzer, AFL) specific corpus.""" - def __init__(self, - engine, - project_qualified_target_name, - quarantine=False, - log_results=True, - include_regressions=False, - gsutil_runner_func=DEFAULT_GSUTIL_RUNNER): + def __init__( + self, + engine: str, + project_qualified_target_name: str, + quarantine: bool = False, + log_results: bool = True, + include_regressions: bool = False, + gsutil_runner_func: Callable[[], Any] | None = DEFAULT_GSUTIL_RUNNER, + ) -> None: """Inits the FuzzTargetCorpus. Args: @@ -295,7 +307,7 @@ def __init__(self, gsutil_runner_func=gsutil_runner_func, ) - self._regressions_corpus = GcsCorpus( + self._regressions_corpus: GcsCorpus | None = GcsCorpus( sync_corpus_bucket_name, f'/{self._engine}/{self._project_qualified_target_name}' f'{REGRESSIONS_GCS_PATH_SUFFIX}', @@ -303,17 +315,17 @@ def __init__(self, gsutil_runner_func=gsutil_runner_func) if include_regressions else None @property - def engine(self): + def engine(self) -> str: return self._engine @property - def project_qualified_target_name(self): + def project_qualified_target_name(self) -> str: return self._project_qualified_target_name def rsync_from_disk(self, - directory, - timeout=CORPUS_FILES_SYNC_TIMEOUT, - delete=True): + directory: str, + timeout: int = CORPUS_FILES_SYNC_TIMEOUT, + delete: bool = True) -> bool: """Upload local files to GCS and remove files which do not exist locally. Overridden to have additional logging. @@ -336,9 +348,9 @@ def rsync_from_disk(self, return result def rsync_to_disk(self, - directory, - timeout=CORPUS_FILES_SYNC_TIMEOUT, - delete=True): + directory: str, + timeout: int = CORPUS_FILES_SYNC_TIMEOUT, + delete: bool = True) -> bool: """Run gsutil to download corpus files from GCS. Overridden to have additional logging. @@ -374,11 +386,12 @@ class ProtoFuzzTargetCorpus(FuzzTargetCorpus): """Implementation of GCS corpus that uses protos (uworker-compatible) for fuzz targets.""" - def __init__(self, - engine, - project_qualified_target_name, - proto_corpus, - allow_engine_override=True): + def __init__( + self, + engine: str, + project_qualified_target_name: str, + proto_corpus: 'uworker_msg_pb2.FuzzTargetCorpus', # pylint: disable=no-member + allow_engine_override: bool = True) -> None: # TODO(metzman): Do we need project_qualified_target_name? # This is used to let AFL share corpora with libFuzzer. @@ -390,13 +403,16 @@ def __init__(self, self.proto_corpus = proto_corpus proto_corpus.engine = self._engine proto_corpus.project_qualified_target_name = project_qualified_target_name - self._filenames_to_delete_urls_mapping = {} + self._filenames_to_delete_urls_mapping: dict[str, str | None] = {} - def serialize(self): + def serialize(self) -> 'uworker_msg_pb2.FuzzTargetCorpus': # pylint: disable=no-member return self.proto_corpus @classmethod - def deserialize(cls, proto_corpus): + def deserialize( + cls, + proto_corpus: 'uworker_msg_pb2.FuzzTargetCorpus' # pylint: disable=no-member + ) -> 'ProtoFuzzTargetCorpus': return ProtoFuzzTargetCorpus( proto_corpus.engine, proto_corpus.project_qualified_target_name, @@ -404,9 +420,9 @@ def deserialize(cls, proto_corpus): allow_engine_override=False) def rsync_from_disk(self, - directory, - timeout=CORPUS_FILES_SYNC_TIMEOUT, - delete=True): + directory: str, + timeout: int = CORPUS_FILES_SYNC_TIMEOUT, + delete: bool = True) -> bool: """Upload local files to GCS and remove files which do not exist locally. Overridden to have additional logging. @@ -420,7 +436,7 @@ def rsync_from_disk(self, A bool indicating whether or not the command succeeded. """ filenames_to_delete_dict = self._filenames_to_delete_urls_mapping.copy() - filepaths_to_upload = [] + filepaths_to_upload: list[str] = [] logs.info('Rsyncing corpus from disk.') for filepath in shell.get_files_list(directory): filepath = rename_file_to_sha(filepath) @@ -434,7 +450,7 @@ def rsync_from_disk(self, filepaths_to_upload.append(filepath) logs.info('Uploading corpus.') - results = self.upload_files(filepaths_to_upload) + results: list[bool] = self.upload_files(filepaths_to_upload) logs.info('Done uploading corpus.') filenames_to_delete = [ delete_url for delete_url in set(filenames_to_delete_dict.values()) @@ -464,9 +480,9 @@ def rsync_from_disk(self, return results.count(False) < MAX_SYNC_ERRORS def rsync_to_disk(self, - directory, - timeout=CORPUS_FILES_SYNC_TIMEOUT, - delete=True) -> bool: + directory: str, + timeout: int = CORPUS_FILES_SYNC_TIMEOUT, + delete: bool = True) -> bool: """Sync fuzz target corpus to disk.""" if not self._sync_corpus_to_disk(self.proto_corpus.corpus, directory): return False @@ -480,7 +496,10 @@ def rsync_to_disk(self, logs.info(f'{num_files} corpus files downloaded.') return True - def _sync_corpus_to_disk(self, corpus, directory): + def _sync_corpus_to_disk( + self, + corpus: 'uworker_msg_pb2.Corpus', # pylint: disable=no-member + directory: str) -> bool: """Syncs a corpus from GCS to disk.""" shell.create_directory(directory, create_intermediates=True) if corpus.backup_url: @@ -490,7 +509,8 @@ def _sync_corpus_to_disk(self, corpus, directory): temp_zipfile = tempfile.NamedTemporaryFile( dir=tmpdir, suffix='.zip', delete=False) try: - storage.download_signed_url_to_file(corpus.backup_url, temp_zipfile) + storage.download_signed_url_to_file(corpus.backup_url, + cast(Any, temp_zipfile)) temp_zipfile.close() with archive.open(temp_zipfile.name) as reader: reader.extract_all(directory) @@ -501,7 +521,8 @@ def _sync_corpus_to_disk(self, corpus, directory): finally: os.unlink(temp_zipfile.name) - results = storage.download_signed_urls(corpus.corpus_urls, directory) + results = storage.download_signed_urls( + cast(Any, corpus.corpus_urls), directory) fails = 0 # Convert this to a dict so proto's map doesn't return a default value for # missing keys (this hides errors). @@ -517,7 +538,11 @@ def _sync_corpus_to_disk(self, corpus, directory): # TODO(metzman): Add timeout and tolerance for missing URLs. return fails < MAX_SYNC_ERRORS - def upload_files(self, file_paths, timeout=CORPUS_FILES_SYNC_TIMEOUT) -> bool: + def upload_files( + self, + file_paths: Sequence[str], + timeout: int = CORPUS_FILES_SYNC_TIMEOUT, + ) -> list[bool]: del timeout num_upload_urls = len(self.proto_corpus.corpus.upload_urls) if len(file_paths) > num_upload_urls: @@ -526,8 +551,8 @@ def upload_files(self, file_paths, timeout=CORPUS_FILES_SYNC_TIMEOUT) -> bool: file_paths = file_paths[:num_upload_urls] logs.info(f'Uploading {len(file_paths)} corpus files.') - results = storage.upload_signed_urls(self.proto_corpus.corpus.upload_urls, - file_paths) + results: list[bool] = storage.upload_signed_urls( + cast(Any, self.proto_corpus.corpus.upload_urls), cast(Any, file_paths)) # Make sure we don't reuse upload_urls. urls_remaining = self.proto_corpus.corpus.upload_urls[len(results):] @@ -536,12 +561,13 @@ def upload_files(self, file_paths, timeout=CORPUS_FILES_SYNC_TIMEOUT) -> bool: return results - def get_gcs_url(self): + def get_gcs_url(self) -> str: return self.proto_corpus.corpus.gcs_url -def gcs_url_for_backup_file(backup_bucket_name, fuzzer_name, - project_qualified_target_name, date): +def gcs_url_for_backup_file( + backup_bucket_name: str | None, fuzzer_name: str | None, + project_qualified_target_name: str | None, date: Any) -> str | None: """Build GCS URL for corpus backup file for the given date. Returns: @@ -559,7 +585,8 @@ def gcs_url_for_backup_file(backup_bucket_name, fuzzer_name, return f'{backup_dir.rstrip("/")}/{backup_file}' -def backup_corpus(dated_backup_signed_url, corpus, directory): +def backup_corpus(dated_backup_signed_url: str | None, corpus: Any, + directory: str) -> bool: """Archive and store corpus as a backup. Args: @@ -603,9 +630,8 @@ def backup_corpus(dated_backup_signed_url, corpus, directory): return backup_succeeded -def gcs_url_for_backup_directory( - backup_bucket_name, fuzzer_name, - project_qualified_target_name) -> Optional[str]: +def gcs_url_for_backup_directory(backup_bucket_name: str, fuzzer_name: str, + project_qualified_target_name: str) -> str: """Build GCS URL for corpus backup directory. Returns: @@ -615,13 +641,13 @@ def gcs_url_for_backup_directory( f'{project_qualified_target_name}/') -def _get_regressions_corpus_gcs_url(bucket_name, bucket_path): +def _get_regressions_corpus_gcs_url(bucket_name: str, bucket_path: str) -> str: """Return gcs path to directory containing crash regressions.""" return _get_gcs_url( bucket_name, bucket_path, suffix=REGRESSIONS_GCS_PATH_SUFFIX) -def _get_gcs_url(bucket_name, bucket_path, suffix=''): +def _get_gcs_url(bucket_name: str, bucket_path: str, suffix: str = '') -> str: """Build corpus GCS URL for gsutil. Returns: A string giving the GCS URL. @@ -639,7 +665,7 @@ def _get_gcs_url(bucket_name, bucket_path, suffix=''): def get_proto_data_bundle_corpus( - data_bundle) -> uworker_msg_pb2.DataBundleCorpus: # pylint: disable=no-member + data_bundle: Any) -> uworker_msg_pb2.DataBundleCorpus: # pylint: disable=no-member """Returns a data bundle corpus that can be used by uworkers or trusted workers to download the data bundle files using the fastest means available to them.""" @@ -666,24 +692,27 @@ def get_proto_data_bundle_corpus( return data_bundle_corpus -def sync_data_bundle_corpus_to_disk(data_bundle_corpus, directory): +def sync_data_bundle_corpus_to_disk( + data_bundle_corpus: 'uworker_msg_pb2.DataBundleCorpus', # pylint: disable=no-member + directory: str) -> bool: + """Syncs a data bundle corpus to disk.""" if (not task_types.task_main_runs_on_uworker() and not environment.is_uworker()): # Fast path for when we don't need an untrusted worker to run a task. - return gsutil.GSUtilRunner().rsync( + return cast(Any, gsutil).GSUtilRunner().rsync( data_bundle_corpus.gcs_url, directory, delete=False).return_code == 0 - results = storage.download_signed_urls(data_bundle_corpus.corpus_urls, - directory) + results = storage.download_signed_urls( + cast(Any, data_bundle_corpus.corpus_urls), directory) fails = [result.url for result in results if not result.url] return len(fails) < MAX_SYNC_ERRORS -def get_proto_corpus(bucket_name, - bucket_path, - max_upload_urls, - include_delete_urls=False, - max_download_urls=None, - backup_url=None): +def get_proto_corpus(bucket_name: str, + bucket_path: str, + max_upload_urls: int, + include_delete_urls: bool = False, + max_download_urls: int | None = None, + backup_url: str | None = None) -> 'uworker_msg_pb2.Corpus': # pylint: disable=no-member """Returns a proto representation of a corpus.""" gcs_url = _get_gcs_url(bucket_name, bucket_path) corpus = uworker_msg_pb2.Corpus(gcs_url=gcs_url) # pylint: disable=no-member @@ -699,8 +728,8 @@ def get_proto_corpus(bucket_name, if backup_exists: # Corpus backup can take up to 24 hours, get any corpus element before the # backup was made. - corpus.backup_url = storage.get_signed_download_url(backup_url) - backup = list(storage.get_blobs(backup_url, single_file=True)) + corpus.backup_url = storage.get_signed_download_url(cast(str, backup_url)) + backup = list(storage.get_blobs(cast(str, backup_url), single_file=True)) start_time = backup[0]['updated'] - datetime.timedelta(days=1) blobs = storage.get_blobs(gcs_url) urls = (f'{storage.GS_PREFIX}/{bucket_name}/{blob["name"]}' @@ -726,9 +755,9 @@ def get_proto_corpus(bucket_name, return corpus -def get_target_bucket_and_path(engine, - project_qualified_target_name, - quarantine=False): +def get_target_bucket_and_path(engine: str, + project_qualified_target_name: str, + quarantine: bool = False) -> tuple[str, str]: """Gets target and bucket path for the corpus.""" engine = os.getenv('CORPUS_FUZZER_NAME_OVERRIDE', engine) if quarantine: @@ -740,14 +769,14 @@ def get_target_bucket_and_path(engine, return sync_corpus_bucket_name, f'/{engine}/{project_qualified_target_name}' -def get_fuzz_target_corpus(engine, - project_qualified_target_name, - quarantine=False, - include_regressions=False, - include_delete_urls=False, - max_upload_urls=10000, - max_download_urls=None, - use_backup=False): +def get_fuzz_target_corpus(engine: str, + project_qualified_target_name: str, + quarantine: bool = False, + include_regressions: bool = False, + include_delete_urls: bool = False, + max_upload_urls: int = 10000, + max_download_urls: int | None = None, + use_backup: bool = False) -> ProtoFuzzTargetCorpus: """Copies the corpus from gcs to disk. Can run on uworker.""" fuzz_target_corpus = uworker_msg_pb2.FuzzTargetCorpus() # pylint: disable=no-member bucket_name, bucket_path = get_target_bucket_and_path( @@ -787,14 +816,16 @@ def get_fuzz_target_corpus(engine, fuzz_target_corpus) -def get_regressions_signed_upload_url(engine, project_qualified_target_name): +def get_regressions_signed_upload_url( + engine: str, project_qualified_target_name: str) -> str: bucket, path = get_target_bucket_and_path(engine, project_qualified_target_name) regression_url = _get_regressions_corpus_gcs_url(bucket, path) return storage.get_arbitrary_signed_upload_url(regression_url) -def get_pruning_corpora_urls(engine, project_qualified_name): +def get_pruning_corpora_urls(engine: str, + project_qualified_name: str) -> tuple[str, str]: bucket_name, bucket_path = get_target_bucket_and_path( engine, project_qualified_name, False) gcs_url = _get_gcs_url(bucket_name, bucket_path) @@ -804,7 +835,9 @@ def get_pruning_corpora_urls(engine, project_qualified_name): return gcs_url, quarantine_gcs_url -def get_corpuses_for_pruning(engine, project_qualified_name): +def get_corpuses_for_pruning( + engine: str, project_qualified_name: str +) -> tuple[ProtoFuzzTargetCorpus, ProtoFuzzTargetCorpus]: """Returns a fuzz target corpus and quarantine corpus for pruning.""" # We need to include upload URLs because of corpus pollination. This is # unfortunate as it is probably rarely used. diff --git a/src/clusterfuzz/_internal/fuzzing/fuzzer_selection.py b/src/clusterfuzz/_internal/fuzzing/fuzzer_selection.py index 09be079e2d5..bfb190c1980 100644 --- a/src/clusterfuzz/_internal/fuzzing/fuzzer_selection.py +++ b/src/clusterfuzz/_internal/fuzzing/fuzzer_selection.py @@ -13,7 +13,8 @@ # limitations under the License. """Helper functions to update fuzzer-job mappings, and select fuzzers to run.""" -import collections +from typing import cast +from typing import NamedTuple from google.cloud import ndb @@ -25,11 +26,15 @@ from clusterfuzz._internal.platforms.android import constants from clusterfuzz._internal.system import environment + # Used to prepare targets to be passed to utils.random_weighted_choice. -WeightedTarget = collections.namedtuple('WeightedTarget', ['target', 'weight']) +class WeightedTarget(NamedTuple): + target: str + weight: float -def update_mappings_for_fuzzer(fuzzer, mappings=None): +def update_mappings_for_fuzzer(fuzzer: data_types.Fuzzer, + mappings: list[str] | None = None) -> None: """Clear existing mappings for a fuzzer, and replace them.""" if mappings is None: # Make a copy in case we need to modify it. @@ -38,15 +43,15 @@ def update_mappings_for_fuzzer(fuzzer, mappings=None): query = data_types.FuzzerJob.query() query = query.filter(data_types.FuzzerJob.fuzzer == fuzzer.name) fuzzer_job_entities = ndb_utils.get_all_from_query(query) - old_mappings = {} + old_mappings: dict[str, data_types.FuzzerJob] = {} for fuzzer_job in fuzzer_job_entities: old_mappings[fuzzer_job.job] = fuzzer_job - new_mappings = [] + new_mappings: list[data_types.FuzzerJob] = [] if mappings: - jobs = ndb_utils.get_all_from_query(data_types.Job.query().filter( + jobs_list = ndb_utils.get_all_from_query(data_types.Job.query().filter( data_types.Job.name.IN(mappings))) - jobs = {job.name: job for job in jobs} + jobs = {job.name: job for job in jobs_list} else: jobs = {} @@ -80,18 +85,19 @@ def update_mappings_for_fuzzer(fuzzer, mappings=None): fuzzer.put() -def update_mappings_for_job(job, mappings): +def update_mappings_for_job(job: data_types.Job, mappings: list[str]) -> None: """Clear existing mappings for a job, and replace them.""" - existing_fuzzers = { - fuzzer.name: fuzzer + existing_fuzzers: dict[str, data_types.Fuzzer] = { + cast(str, fuzzer.name): fuzzer for fuzzer in data_types.Fuzzer.query() if job.name in fuzzer.jobs } - modified_fuzzers = [] + modified_fuzzers: list[data_types.Fuzzer] = [] for fuzzer_name in mappings: fuzzer = existing_fuzzers.pop(fuzzer_name, None) if fuzzer: + # If we already have this mapping, we don't need to do anything. continue fuzzer = data_types.Fuzzer.query( @@ -101,37 +107,39 @@ def update_mappings_for_job(job, mappings): job.name)) continue - fuzzer.jobs.append(job.name) + cast(list[str], fuzzer.jobs).append(cast(str, job.name)) modified_fuzzers.append(fuzzer) update_mappings_for_fuzzer(fuzzer) # Removing the remaining values in exisiting_fuzzers as # they are no longer mapped. for fuzzer in existing_fuzzers.values(): - fuzzer.jobs.remove(job.name) + cast(list[str], fuzzer.jobs).remove(cast(str, job.name)) modified_fuzzers.append(fuzzer) update_mappings_for_fuzzer(fuzzer) ndb.put_multi(modified_fuzzers) -def update_platform_for_job(job_name, new_platform): +def update_platform_for_job(job_name: str, new_platform: str) -> None: """Update platform for all mappings for a particular job.""" query = data_types.FuzzerJob.query() query = query.filter(data_types.FuzzerJob.job == job_name) mappings = ndb_utils.get_all_from_query(query) - new_mappings = [] + new_mappings: list[data_types.FuzzerJob] = [] for mapping in mappings: mapping.platform = new_platform new_mappings.append(mapping) ndb_utils.put_multi(new_mappings) -def get_fuzz_task_payload(platform=None): +def get_fuzz_task_payload( + platform: str | None = None) -> tuple[str | None, str | None]: """Select a fuzzer that can run on this platform.""" if not platform: queue_override = environment.get_value('QUEUE_OVERRIDE') platform = queue_override if queue_override else environment.platform() + platform = cast(str, platform) platforms = [platform] base_platform = platform.split(':')[0] @@ -149,7 +157,7 @@ def get_fuzz_task_payload(platform=None): query = data_types.FuzzerJobs.query() query = query.filter(data_types.FuzzerJobs.platform.IN(platforms)) - mappings = [] + mappings: list[data_types.FuzzerJob] = [] for entity in query: mappings.extend(entity.fuzzer_jobs) else: @@ -191,11 +199,12 @@ def get_fuzz_task_payload(platform=None): return selection.fuzzer, selection.job -def select_fuzz_target(targets, target_weights): +def select_fuzz_target(targets: list[str], + target_weights: dict[str, float]) -> str: """Select a fuzz target from a list of potential targets.""" assert targets - weighted_targets = [] + weighted_targets: list[WeightedTarget] = [] for target in targets: weight = target_weights.get(target, 1.0) weighted_targets.append(WeightedTarget(target, weight)) @@ -203,26 +212,26 @@ def select_fuzz_target(targets, target_weights): return utils.random_weighted_choice(weighted_targets).target -def get_fuzz_target_weights(): +def get_fuzz_target_weights() -> dict[str, float]: """Get a list of fuzz target weights based on the current fuzzer.""" job_type = environment.get_value('JOB_NAME') target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job_type)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs) - weights = {} + weights: dict[str, float] = {} for fuzz_target, target_job in zip(fuzz_targets, target_jobs): if not fuzz_target: logs.error('Skipping weight assignment for fuzz target ' f'{target_job.fuzz_target_name}.') continue - weights[fuzz_target.binary] = target_job.weight + weights[cast(str, fuzz_target.binary)] = cast(float, target_job.weight) return weights -def get_job_list(jobs_str): +def get_job_list(jobs_str: str | None) -> list[str]: if jobs_str: return [job.strip() for job in jobs_str.split(',')] diff --git a/src/clusterfuzz/_internal/fuzzing/gesture_handler.py b/src/clusterfuzz/_internal/fuzzing/gesture_handler.py index 0d5d5d6ff0b..cf5c17e9f90 100644 --- a/src/clusterfuzz/_internal/fuzzing/gesture_handler.py +++ b/src/clusterfuzz/_internal/fuzzing/gesture_handler.py @@ -19,7 +19,7 @@ from clusterfuzz._internal.system import environment -def get_gestures(gesture_count): +def get_gestures(gesture_count: int) -> list[str]: """Return a list of random gestures.""" plt = environment.platform() diff --git a/src/clusterfuzz/_internal/fuzzing/leak_blacklist.py b/src/clusterfuzz/_internal/fuzzing/leak_blacklist.py index f95dd783b93..2a9f63efc57 100644 --- a/src/clusterfuzz/_internal/fuzzing/leak_blacklist.py +++ b/src/clusterfuzz/_internal/fuzzing/leak_blacklist.py @@ -13,8 +13,10 @@ # limitations under the License. """Highlights and generates suppressions for LSAN reports.""" +from collections.abc import Sequence import os import re +from typing import cast from clusterfuzz._internal.base import errors from clusterfuzz._internal.base import memoize @@ -25,20 +27,21 @@ from clusterfuzz._internal.system import environment # Constants for highlighting. -DIRECT_LEAK_LABEL = 'Direct-leak' -DIRECT_LEAK_REGEX = re.compile(r'^ *Direct leak of') -FIRST_LEAK_DIVIDER = ('%s\nThe following leaks are not necessarily related ' - 'to the first leak.\n\n' % ('=' * 80)) -STACK_REGEX = re.compile(r'^ *#[0-9]+\s0x[A-Za-z0-9]+') -STACK_START_REGEX = re.compile(r'^ *#0 ') -BLANK_LINE_REGEX = re.compile(r'^\s*$') +DIRECT_LEAK_LABEL: str = 'Direct-leak' +DIRECT_LEAK_REGEX: re.Pattern[str] = re.compile(r'^ *Direct leak of') +FIRST_LEAK_DIVIDER: str = ( + '%s\nThe following leaks are not necessarily related ' + 'to the first leak.\n\n' % ('=' * 80)) +STACK_REGEX: re.Pattern[str] = re.compile(r'^ *#[0-9]+\s0x[A-Za-z0-9]+') +STACK_START_REGEX: re.Pattern[str] = re.compile(r'^ *#0 ') +BLANK_LINE_REGEX: re.Pattern[str] = re.compile(r'^\s*$') -LSAN_TOOL_NAME = 'lsan' -LSAN_SUPPRESSION_LINE = 'leak:{function}\n' -LSAN_HEADER_COMMENT = '# This is a LSAN suppressions file.\n' +LSAN_TOOL_NAME: str = 'lsan' +LSAN_SUPPRESSION_LINE: str = 'leak:{function}\n' +LSAN_HEADER_COMMENT: str = '# This is a LSAN suppressions file.\n' -def create_empty_local_blacklist(): +def create_empty_local_blacklist() -> None: """Creates an empty local blacklist.""" lsan_suppressions_path = get_local_blacklist_file_path() with open(lsan_suppressions_path, 'w') as local_blacklist: @@ -46,7 +49,7 @@ def create_empty_local_blacklist(): local_blacklist.write(LSAN_HEADER_COMMENT) -def cleanup_global_blacklist(): +def cleanup_global_blacklist() -> None: """Cleans out closed and deleted testcases from the global blacklist.""" blacklists_to_delete = [] global_blacklists = data_types.Blacklist.query( @@ -55,7 +58,7 @@ def cleanup_global_blacklist(): testcase_id = blacklist.testcase_id try: - testcase = data_handler.get_testcase_by_id(testcase_id) + testcase = data_handler.get_testcase_by_id(cast(int, testcase_id)) except errors.InvalidTestcaseError: testcase = None @@ -67,7 +70,7 @@ def cleanup_global_blacklist(): @memoize.wrap(memoize.Memcache(60 * 10)) -def get_global_blacklisted_functions(): +def get_global_blacklisted_functions() -> list[str]: """Gets global blacklisted functions.""" # Copy global blacklist into local blacklist. global_blacklists = data_types.Blacklist.query( @@ -80,8 +83,9 @@ def get_global_blacklisted_functions(): return blacklisted_functions -def copy_global_to_local_blacklist(blacklisted_functions, - excluded_testcase=None): +def copy_global_to_local_blacklist( + blacklisted_functions: Sequence[str], + excluded_testcase: data_types.Testcase | None = None) -> None: """Copies contents of global blacklist into local blacklist file, excluding a particular testcase (if any).""" lsan_suppressions_path = get_local_blacklist_file_path() @@ -100,16 +104,17 @@ def copy_global_to_local_blacklist(blacklisted_functions, LSAN_SUPPRESSION_LINE.format(function=function_name)) -def get_leak_function_for_blacklist(testcase): +def get_leak_function_for_blacklist( + testcase: data_types.Testcase) -> str | None: """Return leak function to be used for blacklisting.""" - crash_functions = testcase.crash_state.splitlines() + crash_functions = cast(str, testcase.crash_state).splitlines() if not crash_functions: return None return crash_functions[0] -def get_local_blacklist_file_path(): +def get_local_blacklist_file_path() -> str: """Return the file path to the local blacklist text file.""" local_blacklist_path = os.path.join(environment.get_suppressions_directory(), 'lsan_suppressions.txt') @@ -122,13 +127,15 @@ def get_local_blacklist_file_path(): return local_blacklist_path -def should_be_blacklisted(testcase): +def should_be_blacklisted(testcase: data_types.Testcase) -> bool: """Returns True if testcase is reproducible and not deleted.""" - return (testcase.open and testcase.crash_type == DIRECT_LEAK_LABEL and - not testcase.one_time_crasher_flag) + return cast( + bool, testcase.open and testcase.crash_type == DIRECT_LEAK_LABEL and + not testcase.one_time_crasher_flag) -def add_crash_to_global_blacklist_if_needed(testcase): +def add_crash_to_global_blacklist_if_needed( + testcase: data_types.Testcase) -> data_types.Blacklist | bool: """Adds relevant function from testcase crash state to global blacklist.""" testcase_id = testcase.key.id() if not should_be_blacklisted(testcase): @@ -163,7 +170,7 @@ def add_crash_to_global_blacklist_if_needed(testcase): return blacklist_item -def highlight_first_direct_leak(crash_stacktrace): +def highlight_first_direct_leak(crash_stacktrace: str) -> str: """Highlights the first direct leak in a report. Args: diff --git a/src/clusterfuzz/_internal/fuzzing/strategy.py b/src/clusterfuzz/_internal/fuzzing/strategy.py index 8cec1d16c74..0595fa1dbdf 100755 --- a/src/clusterfuzz/_internal/fuzzing/strategy.py +++ b/src/clusterfuzz/_internal/fuzzing/strategy.py @@ -13,18 +13,25 @@ # limitations under the License. """Fuzzing strategies for fuzzing engines like libFuzzer, AFL, etc.""" -from collections import namedtuple +from typing import NamedTuple + # Named tuple for each strategy. The manually_enable field signifies whether a # given strategy should be considered separately from the multi-armed bandit # strategy seleciton process. Strategies with this field enabled are not # included in the multi-armed bandit cron job query. -Strategy = namedtuple('Strategy', 'name probability manually_enable') +class Strategy(NamedTuple): + name: str + probability: float + manually_enable: bool + # Number of testcases to use for the corpus subset strategy. # See https://crbug.com/682311 for more information. # Size 100 has a slightly higher chance as it seems to be the best one so far. -CORPUS_SUBSET_NUM_TESTCASES = [10, 20, 50, 75, 75, 100, 100, 100, 125, 125, 150] +CORPUS_SUBSET_NUM_TESTCASES: list[int] = [ + 10, 20, 50, 75, 75, 100, 100, 100, 125, 125, 150 +] # Supported fuzzing strategies. CORPUS_MUTATION_RADAMSA_STRATEGY = Strategy( @@ -41,7 +48,7 @@ # Keep this strategy order for strategy combination tracking as strategy # combinations are tracked as strings. -LIBFUZZER_STRATEGY_LIST = [ +LIBFUZZER_STRATEGY_LIST: list[Strategy] = [ CORPUS_MUTATION_RADAMSA_STRATEGY, RANDOM_MAX_LENGTH_STRATEGY, VALUE_PROFILE_STRATEGY, @@ -50,23 +57,23 @@ USE_EXTRA_SANITIZERS_STRATEGY, ] -AFL_STRATEGY_LIST = [ +AFL_STRATEGY_LIST: list[Strategy] = [ CORPUS_MUTATION_RADAMSA_STRATEGY, CORPUS_SUBSET_STRATEGY, ] # Lists of prefix and boolean strategies maintained for libFuzzer stats. -LIBFUZZER_STRATEGIES_WITH_PREFIX_VALUE = [ +LIBFUZZER_STRATEGIES_WITH_PREFIX_VALUE: list[Strategy] = [ CORPUS_SUBSET_STRATEGY, FORK_STRATEGY, ] -LIBFUZZER_STRATEGIES_WITH_PREFIX_VALUE_TYPE = { +LIBFUZZER_STRATEGIES_WITH_PREFIX_VALUE_TYPE: dict[str, type] = { 'corpus_subset': int, 'fork': int, } -LIBFUZZER_STRATEGIES_WITH_BOOLEAN_VALUE = [ +LIBFUZZER_STRATEGIES_WITH_BOOLEAN_VALUE: list[Strategy] = [ CORPUS_MUTATION_RADAMSA_STRATEGY, RANDOM_MAX_LENGTH_STRATEGY, VALUE_PROFILE_STRATEGY, diff --git a/src/clusterfuzz/_internal/google_cloud_utils/big_query.py b/src/clusterfuzz/_internal/google_cloud_utils/big_query.py index 76ad50b1469..997e47c397b 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/big_query.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/big_query.py @@ -15,10 +15,13 @@ because it requires oauth2client 4.0.0. But our appengine requires oauth2client 1.4.2. Therefore, we implement our own BigQuery client.""" -import collections +from collections.abc import Sequence import datetime import re import time +from typing import Any +from typing import NamedTuple +from typing import TYPE_CHECKING from googleapiclient import discovery @@ -29,26 +32,39 @@ from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment +if TYPE_CHECKING: + from clusterfuzz._internal.datastore import data_types + # pylint: disable=no-member -REQUEST_TIMEOUT = 60 -QUERY_TIMEOUT = 10 * 60 -QUERY_MAX_RESULTS = 10000 -QUERY_RETRY_COUNT = 3 -QUERY_RETRY_DELAY = 3 +REQUEST_TIMEOUT: int = 60 +QUERY_TIMEOUT: int = 10 * 60 +QUERY_MAX_RESULTS: int = 10000 +QUERY_RETRY_COUNT: int = 3 +QUERY_RETRY_DELAY: int = 3 # The actual limit is 10,000, but the documentation suggests using batches of # 500 rows at most (https://cloud.google.com/bigquery/quotas#streaminginserts). -INSERT_BATCH_SIZE = 500 +INSERT_BATCH_SIZE: int = 500 + +VALID_FIELD_NAME_REGEX: re.Pattern = re.compile(r'^[a-zA-Z0-9_]+$') + + +class Insert(NamedTuple): + row: dict[str, Any] + insert_id: str | None = None -VALID_FIELD_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_]+$') + +class QueryResult(NamedTuple): + rows: list[dict[str, Any]] + total_count: int @retry.wrap( retries=QUERY_RETRY_COUNT, delay=QUERY_RETRY_DELAY, function='google_cloud_utils.big_query.get_api_client') -def get_api_client(): +def get_api_client() -> Any: """Return an api client for bigquery.""" return discovery.build( 'bigquery', @@ -57,12 +73,12 @@ def get_api_client(): credentials=credentials.get_default()[0]) -def get_bucket(): +def get_bucket() -> str | None: """Return bucket for bigquery stats.""" return local_config.ProjectConfig().get('bigquery.bucket') -def cast(value, field): +def cast(value: Any, field: dict[str, Any]) -> Any: """Cast value to appropriate type.""" if value is None: return None @@ -82,9 +98,10 @@ def cast(value, field): raise RuntimeError(f'The type field[{"type"}] is unsupported.') -def convert_row(raw_row, fields): +def convert_row(raw_row: dict[str, Any], + fields: Sequence[dict[str, Any]]) -> dict[str, Any]: """Convert a single raw row (from BigQuery) to a dict.""" - row = {} + row: dict[str, Any] = {} for index, raw_value in enumerate(raw_row['f']): field = fields[index] @@ -98,11 +115,11 @@ def convert_row(raw_row, fields): return row -def convert(result): +def convert(result: dict[str, Any]) -> list[dict[str, Any]]: """Convert a query result into an array of dicts, each of which represents a row.""" fields = result['schema']['fields'] - rows = [] + rows: list[dict[str, Any]] = [] for raw_row in result.get('rows', []): rows.append(convert_row(raw_row, fields)) @@ -111,7 +128,8 @@ def convert(result): @environment.local_noop -def write_range(table_id, testcase, range_name, start, end): +def write_range(table_id: str, testcase: 'data_types.Testcase', range_name: str, + start: int | str, end: int | str) -> None: """Write a range to BigQuery. This is applicable for regression and fixed ranges.""" client = Client(dataset_id='main', table_id=table_id) @@ -132,14 +150,15 @@ def write_range(table_id, testcase, range_name, start, end): insert_id='%s:%s:%s' % (testcase.key.id(), start, end)) ]) - for error in result.get('insertErrors', []): + for error in (result.get('insertErrors', []) if result else []): logs.error( ("Ignoring error writing the testcase's %s range (%s) to " 'BigQuery.' % (range_name, testcase.key.id())), exception=ValueError(error)) -def _get_max_results(max_results, limit, count_so_far): +def _get_max_results(max_results: int, limit: int | None, + count_so_far: int) -> int: """Get an appropriate max_results.""" # limit is None means we get every record (no limit). if limit is None: @@ -148,25 +167,22 @@ def _get_max_results(max_results, limit, count_so_far): return min(max_results, limit - count_so_far) -Insert = collections.namedtuple('Insert', ['row', 'insert_id']) -QueryResult = collections.namedtuple('QueryResult', ['rows', 'total_count']) - - class Client: """BigQuery client.""" - def __init__(self, dataset_id=None, table_id=None): - self.project_id = utils.get_application_id() - self.dataset_id = dataset_id - self.table_id = table_id + def __init__(self, dataset_id: str | None = None, + table_id: str | None = None) -> None: + self.project_id: str | None = utils.get_application_id() + self.dataset_id: str | None = dataset_id + self.table_id: str | None = table_id - self.client = get_api_client() + self.client: Any = get_api_client() @retry.wrap( retries=QUERY_RETRY_COUNT, delay=QUERY_RETRY_DELAY, function='google_cloud_utils.big_query.Client.raw_query') - def raw_query(self, query, max_results): + def raw_query(self, query: str, max_results: int) -> dict[str, Any]: # pylint: disable=line-too-long """Perform a query and return result. @@ -195,7 +211,8 @@ def raw_query(self, query, max_results): retries=QUERY_RETRY_COUNT, delay=QUERY_RETRY_DELAY, function='google_cloud_utils.big_query.Client.get_query_results') - def get_query_results(self, job_id, page_token, start_index, max_results): + def get_query_results(self, job_id: str, page_token: str | None, + start_index: int, max_results: int) -> dict[str, Any]: # pylint: disable=line-too-long """Perform a query and return result. @@ -218,8 +235,8 @@ def get_query_results(self, job_id, page_token, start_index, max_results): startIndex=start_index, pageToken=page_token).execute() - def wait_for_completion(self, job_id, offset, max_results, start_time, - timeout): + def wait_for_completion(self, job_id: str, offset: int, max_results: int, + start_time: float, timeout: float) -> dict[str, Any]: """Wait for job completion and return the first page.""" while True: result = self.get_query_results( @@ -237,13 +254,13 @@ def wait_for_completion(self, job_id, offset, max_results, start_time, time.sleep(1) def query(self, - query, - timeout=QUERY_TIMEOUT, - max_results=QUERY_MAX_RESULTS, - offset=0, - limit=None): + query: str, + timeout: float = QUERY_TIMEOUT, + max_results: int = QUERY_MAX_RESULTS, + offset: int = 0, + limit: int | None = None) -> QueryResult: """Performs a query and returns an array of dicts.""" - rows = [] + rows: list[dict[str, Any]] = [] start_time = time.time() result = self.raw_query(query, max_results=0) @@ -277,7 +294,7 @@ def query(self, retries=QUERY_RETRY_COUNT, delay=QUERY_RETRY_DELAY, function='google_cloud_utils.big_query.Client.get_job') - def get_job(self, job_id): + def get_job(self, job_id: str) -> dict[str, Any]: # pylint: disable=line-too-long """Get the job. @@ -296,7 +313,8 @@ def get_job(self, job_id): retries=QUERY_RETRY_COUNT, delay=QUERY_RETRY_DELAY, function='google_cloud_utils.big_query.Client.insert_from_query') - def insert_from_query(self, dataset_id, table_id, job_id, query): + def insert_from_query(self, dataset_id: str, table_id: str, job_id: str, + query: str) -> dict[str, Any]: # pylint: disable=line-too-long """Insert rows to the table from a query. @@ -338,7 +356,7 @@ def insert_from_query(self, dataset_id, table_id, job_id, query): retries=QUERY_RETRY_COUNT, delay=QUERY_RETRY_DELAY, function='google_cloud_utils.big_query.Client.insert') - def _insert_batch(self, inserts): + def _insert_batch(self, inserts: Sequence[Insert]) -> dict[str, Any]: # pylint: disable=line-too-long """Insert a single batch of rows to BigQuery. @@ -361,7 +379,7 @@ def _insert_batch(self, inserts): body=body).execute() @environment.local_noop - def insert(self, inserts): + def insert(self, inserts: Sequence[Insert]) -> dict[str, Any]: # pylint: disable=line-too-long """Insert the given rows and batch them if needed. @@ -382,11 +400,11 @@ def insert(self, inserts): if not result: # Use result from the first batch, appending errors from the rest. - result = response + result = response or {} continue # If there are new errors from the current batch, append to the result. - new_errors = response.get('insertErrors') + new_errors = response.get('insertErrors') if response else None if not new_errors: continue diff --git a/src/clusterfuzz/_internal/google_cloud_utils/blobs.py b/src/clusterfuzz/_internal/google_cloud_utils/blobs.py index d6bf0140501..2b26be3c2c7 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/blobs.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/blobs.py @@ -15,6 +15,7 @@ import os import re +from typing import Any import uuid from google.cloud import ndb @@ -27,8 +28,8 @@ from . import storage -FAIL_NUM_RETRIES = 2 -FAIL_WAIT = 1.5 +FAIL_NUM_RETRIES: int = 2 +FAIL_WAIT: float = 1.5 # pylint: disable=invalid-name @@ -44,7 +45,7 @@ class BlobInfo(data_types.Model): upload_id = ndb.StringProperty() @classmethod - def _get_kind(cls): + def _get_kind(cls) -> str: if environment.get_value('DATASTORE_EMULATOR_HOST'): # Datastore emulator does not allow writing entities with names of the # format "__*__". @@ -67,7 +68,7 @@ class BlobsError(Exception): """Base exception for blobs module.""" -def _is_gcs_key(blob_key): +def _is_gcs_key(blob_key: str) -> bool: """Return whether if the key is a GCS key.""" gcs_key_pattern = re.compile( r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$') @@ -75,12 +76,12 @@ def _is_gcs_key(blob_key): return bool(gcs_key_pattern.match(blob_key)) -def _get_gcs_blob_path(blob_key): +def _get_gcs_blob_path(blob_key: str) -> str: """Return the full path to the blob on GCS.""" return '/%s/%s' % (storage.blobs_bucket(), blob_key) -def get_gcs_path(blob_key): +def get_gcs_path(blob_key: str) -> str | None: """Return GCS path for the blob.""" if _is_gcs_key(blob_key): return _get_gcs_blob_path(blob_key) @@ -98,7 +99,7 @@ def get_gcs_path(blob_key): retries=FAIL_NUM_RETRIES, delay=FAIL_WAIT, function='google_cloud_utils.blobs.get_blob_size') -def get_blob_size(blob_key): +def get_blob_size(blob_key: str | None) -> int | None: """Returns blob size for a given blob key.""" if not blob_key or blob_key == 'NA': return None @@ -110,7 +111,7 @@ def get_blob_size(blob_key): return blob_info.size -def get_blob_info(blob_key): +def get_blob_info(blob_key: str) -> storage.GcsBlobInfo | None: """Get the GcsBlobInfo for the given key. Always returns a storage.GcsBlobInfo, even for legacy blobs.""" if _is_gcs_key(blob_key): @@ -128,7 +129,7 @@ def get_blob_info(blob_key): delay=FAIL_WAIT, function='google_cloud_utils.blobs.delete_blob', retry_on_false=True) -def delete_blob(blob_key): +def delete_blob(blob_key: str) -> bool: """Delete a blob key.""" blob_info = get_blob_info(blob_key) if not blob_info: @@ -142,7 +143,7 @@ def delete_blob(blob_key): delay=FAIL_WAIT, function='google_cloud_utils.blobs.write_blob', retry_on_false=True) -def write_blob(file_handle_or_path): +def write_blob(file_handle_or_path: Any) -> str: """Write a single file testcase to GCS.""" blobs_bucket = storage.blobs_bucket() blob_name = generate_new_blob_name() @@ -171,7 +172,7 @@ def write_blob(file_handle_or_path): delay=FAIL_WAIT, function='google_cloud_utils.blobs.read_blob_to_disk', retry_on_false=True) -def read_blob_to_disk(blob_key, local_file): +def read_blob_to_disk(blob_key: str, local_file: str) -> bool: """Copy data stored in the blobstore to a local file.""" assert not environment.is_running_on_app_engine() @@ -180,16 +181,23 @@ def read_blob_to_disk(blob_key, local_file): os.makedirs(directory) gcs_path = get_gcs_path(blob_key) + assert gcs_path is not None return storage.copy_file_from(gcs_path, local_file) -def read_key(blob_key): +def read_key(blob_key: str | None) -> bytes | None: """Returns data associated with a blobstore key.""" + if not blob_key: + return None + gcs_path = get_gcs_path(blob_key) + if not gcs_path: + return None + return storage.read_data(gcs_path) -def get_legacy_blob_info(blob_key): +def get_legacy_blob_info(blob_key: str) -> BlobInfo | None: """Return legacy blob info information.""" legacy_blob_info = ndb.Key(BlobInfo, blob_key).get() if not legacy_blob_info: @@ -208,31 +216,33 @@ def get_legacy_blob_info(blob_key): return legacy_blob_info -def get_blob_mapping(blob_key): +def get_blob_mapping(blob_key: str) -> _blobmigrator_BlobKeyMapping | None: """Return blob mapping information.""" return ndb.Key(_blobmigrator_BlobKeyMapping, blob_key).get() -def generate_new_blob_name(): +def generate_new_blob_name() -> str: """Generate a new blob name.""" return str(uuid.uuid4()).lower() -def get_signed_download_url(blob_key): +def get_signed_download_url(blob_key: str) -> str: """Returns a signed download URL that can be used to upload the blob pointed to by |blob_key|.""" gcs_path = get_gcs_path(blob_key) + assert gcs_path is not None return storage.get_signed_download_url(gcs_path) -def get_signed_upload_url(blob_key): +def get_signed_upload_url(blob_key: str) -> str: """Returns a signed download URL that can be used to download the blob pointed to by |blob_key|.""" gcs_path = get_gcs_path(blob_key) + assert gcs_path is not None return storage.get_signed_upload_url(gcs_path) -def get_blob_signed_upload_url(): +def get_blob_signed_upload_url() -> tuple[str, str]: """Returns a pair of (blob_name,signed_upload_url) to be used from utask_main to upload blobs.""" bucket = storage.blobs_bucket() diff --git a/src/clusterfuzz/_internal/google_cloud_utils/compute_engine.py b/src/clusterfuzz/_internal/google_cloud_utils/compute_engine.py index 3646cf39403..7cd8d6bede7 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/compute_engine.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/compute_engine.py @@ -16,6 +16,7 @@ import datetime import random import time +from typing import Any from googleapiclient.discovery import build @@ -23,18 +24,19 @@ from clusterfuzz._internal.base import persistent_cache from clusterfuzz._internal.metrics import logs -GCE_INSTANCE_INFO_KEY = 'gce_instance_info' -NUM_RETRIES = 10 -OPERATION_TIMEOUT = 15 * 60 -POLL_INTERVAL = 30 -SLEEP_TIME = 10 +GCE_INSTANCE_INFO_KEY: str = 'gce_instance_info' +NUM_RETRIES: int = 10 +OPERATION_TIMEOUT: int = 15 * 60 +POLL_INTERVAL: int = 30 +SLEEP_TIME: int = 10 # pylint: disable=no-member # TODO(ochang): Allow batching. -def _add_metadata_key_value(items, key, value): +def _add_metadata_key_value(items: list[dict[str, Any]], key: str, + value: Any) -> None: """Adds a metadata key/value to the "items" list from an instance's metadata.""" replaced_existing = False @@ -48,7 +50,8 @@ def _add_metadata_key_value(items, key, value): items.append({'key': key, 'value': value}) -def _do_operation_with_retries(operation, project, zone, wait_for_completion): +def _do_operation_with_retries(operation: Any, project: str, zone: str, + wait_for_completion: bool) -> bool: """Execute an operation, retrying on any exceptions.""" response = _execute_api_call_with_retries(operation) if response is None: @@ -72,21 +75,23 @@ def _do_operation_with_retries(operation, project, zone, wait_for_completion): return False -def _do_instance_operation(operation, instance_name, project, zone, - wait_for_completion): +def _do_instance_operation(operation: str, instance_name: str, project: str, + zone: str, wait_for_completion: bool) -> bool: """Do a start/reset/stop on the compute engine instance in the given project and zone.""" api = _get_api() operation_func = getattr(api.instances(), operation) - operation = operation_func(instance=instance_name, project=project, zone=zone) + operation_obj = operation_func( + instance=instance_name, project=project, zone=zone) return _do_operation_with_retries( - operation, project, zone, wait_for_completion=wait_for_completion) + operation_obj, project, zone, wait_for_completion=wait_for_completion) -def _execute_api_call_with_retries(api_func): +def _execute_api_call_with_retries(api_func: Any) -> Any: """Execute the given API call, retrying if necessary. Returns the response if successful, or None.""" + response = None last_exception = None for i in range(NUM_RETRIES + 1): try: @@ -115,12 +120,13 @@ def _execute_api_call_with_retries(api_func): return response -def _get_api(): +def _get_api() -> Any: """Return the compute engine api object.""" return build('compute', 'v1', cache_discovery=False) -def _get_instance_info(instance_name, project, zone): +def _get_instance_info(instance_name: str, project: str, + zone: str) -> dict[str, Any] | None: """Return the instance information for a given instance.""" api = _get_api() instance_info_func = api.instances().get( @@ -128,7 +134,9 @@ def _get_instance_info(instance_name, project, zone): return _execute_api_call_with_retries(instance_info_func) -def _get_metadata_and_fingerprint(instance_name, project, zone): +def _get_metadata_and_fingerprint( + instance_name: str, project: str, + zone: str) -> tuple[list[dict[str, Any]] | None, str | None]: """Return the metadata values and fingerprint for the given instance.""" instance_info = _get_instance_info(instance_name, project, zone) if not instance_info: @@ -140,7 +148,8 @@ def _get_metadata_and_fingerprint(instance_name, project, zone): return metadata_items, fingerprint -def _wait_for_operation(response, project, zone): +def _wait_for_operation(response: dict[str, Any], project: str, + zone: str) -> None: """Wait for the given operation to complete.""" if 'status' in response and response['status'] == 'DONE': return @@ -170,7 +179,8 @@ def _wait_for_operation(response, project, zone): logs.error('Compute engine operation %s timed out.' % str(operation)) -def add_metadata(instance_name, project, zone, key, value, wait_for_completion): +def add_metadata(instance_name: str, project: str, zone: str, key: str, + value: Any, wait_for_completion: bool) -> bool: """Add metadata to an existing instance. Replaces existing metadata values with the same key.""" existing_metadata, fingerprint = _get_metadata_and_fingerprint( @@ -194,12 +204,12 @@ def add_metadata(instance_name, project, zone, key, value, wait_for_completion): wait_for_completion) -def create_disk(disk_name, - source_image, - size_gb, - project, - zone, - wait_for_completion=False): +def create_disk(disk_name: str, + source_image: str, + size_gb: int | str, + project: str, + zone: str, + wait_for_completion: bool = False) -> bool: """Create a disk.""" api = _get_api() operation = api.disks().insert( @@ -215,7 +225,10 @@ def create_disk(disk_name, operation, project, zone, wait_for_completion=wait_for_completion) -def delete_disk(disk_name, project, zone, wait_for_completion=False): +def delete_disk(disk_name: str, + project: str, + zone: str, + wait_for_completion: bool = False) -> bool: """Delete a disk.""" api = _get_api() operation = api.disks().delete(disk=disk_name, project=project, zone=zone) @@ -224,11 +237,12 @@ def delete_disk(disk_name, project, zone, wait_for_completion=False): operation, project, zone, wait_for_completion=wait_for_completion) -def recreate_instance_with_disks(instance_name, - project, - zone, - additional_metadata=None, - wait_for_completion=False): +def recreate_instance_with_disks( + instance_name: str, + project: str, + zone: str, + additional_metadata: dict[str, Any] | None = None, + wait_for_completion: bool = False) -> bool: """Recreate an instance and its disk.""" # Get existing instance information. # First, try to get instance info from cache. @@ -315,7 +329,8 @@ def recreate_instance_with_disks(instance_name, operation, project, zone, wait_for_completion=wait_for_completion) -def remove_metadata(instance_name, project, zone, key, wait_for_completion): +def remove_metadata(instance_name: str, project: str, zone: str, key: str, + wait_for_completion: bool) -> bool: """Remove a metadata key/value from an existing instance.""" existing_metadata, fingerprint = _get_metadata_and_fingerprint( instance_name, project, zone) @@ -345,7 +360,10 @@ def remove_metadata(instance_name, project, zone, key, wait_for_completion): wait_for_completion) -def reset_instance(instance_name, project, zone, wait_for_completion=False): +def reset_instance(instance_name: str, + project: str, + zone: str, + wait_for_completion: bool = False) -> bool: """Reset an instance.""" return _do_instance_operation('reset', instance_name, project, zone, wait_for_completion) diff --git a/src/clusterfuzz/_internal/google_cloud_utils/compute_metadata.py b/src/clusterfuzz/_internal/google_cloud_utils/compute_metadata.py index 362e25a6d80..4b76a448f45 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/compute_metadata.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/compute_metadata.py @@ -27,7 +27,7 @@ _DELAY = 1 -def _get_raw(path, timeout=None): +def _get_raw(path: str, timeout: float | int | None = None) -> str: """Internal helper to get metadata without retries.""" attribute_url = _METADATA_URL + path headers = {'Metadata-Flavor': 'Google'} @@ -43,12 +43,12 @@ def _get_raw(path, timeout=None): retries=_RETRIES, delay=_DELAY, function='python.google_cloud_utils.compute_metadata.get') -def get(path): +def get(path: str) -> str: """Get GCE metadata value.""" return _get_raw(path) -def is_gce(): +def is_gce() -> bool: """Return whether or not we're on GCE.""" try: sock = socket.create_connection((_METADATA_SERVER, 80)) @@ -59,14 +59,14 @@ def is_gce(): return True -def get_preempted_status(): +def get_preempted_status() -> str: """Gets the preemption status of the instance.""" # We use a short timeout and no retries because this is called frequently # in a background loop and should fail fast. return _get_raw('instance/preempted', timeout=5) -def is_preemptible(): +def is_preemptible() -> bool: """Returns True if the instance is preemptible (or Spot).""" # Skip metadata queries on App Engine or K8s. These environments emulate or # proxy the metadata server but lack standard GCE scheduling keys, returning diff --git a/src/clusterfuzz/_internal/google_cloud_utils/credentials.py b/src/clusterfuzz/_internal/google_cloud_utils/credentials.py index e47e831efa1..e5f708f59fd 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/credentials.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/credentials.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. """Cloud credential helpers.""" +from collections.abc import Sequence import json +from typing import Any +import google.auth from google.auth import compute_engine from google.auth import credentials from google.auth import impersonated_credentials @@ -28,12 +31,6 @@ from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -try: - import google.auth -except ImportError: - # Can't be imported on App Engine. - pass - # Retry params. FAIL_RETRIES = 5 FAIL_WAIT = 10 @@ -46,7 +43,7 @@ _SIGNING_KEY_SECRET_ID = 'gcs-signer-key' -def _use_anonymous_credentials(): +def _use_anonymous_credentials() -> bool: """Returns whether or not to use anonymous credentials.""" if (environment.get_value('INTEGRATION') or environment.get_value('UNTRUSTED_RUNNER_TESTS') or @@ -63,22 +60,27 @@ def _use_anonymous_credentials(): delay=FAIL_WAIT, function='google_cloud_utils.credentials.get_default') @memoize.wrap(memoize.FifoInMemory(1)) -def get_default(scopes=None): +def get_default(scopes: Sequence[str] | None = None + ) -> tuple[credentials.Credentials, str | bytes | None]: """Get default Google Cloud credentials.""" if _use_anonymous_credentials(): return credentials.AnonymousCredentials(), '' return google.auth.default(scopes=scopes) -def get_storage_signing_service_account(): +def get_storage_signing_service_account() -> dict[str, Any] | None: """Gets a dedicated signing account for signing storage objects.""" if _use_anonymous_credentials(): return None project_id = utils.get_application_id() + if not project_id: + return None return json.loads(secret_manager.get(_SIGNING_KEY_SECRET_ID, project_id)) -def get_signing_credentials(service_account_info): +def get_signing_credentials( + service_account_info: dict[str, Any] | None +) -> tuple[credentials.Credentials, str | None] | None: """Returns signing credentials for signing URLs.""" if _use_anonymous_credentials(): return None @@ -98,13 +100,15 @@ def get_signing_credentials(service_account_info): creds.refresh(request) signing_creds = compute_engine.IDTokenCredentials( - request, '', service_account_email=creds.service_account_email) + request, + '', + service_account_email=getattr(creds, 'service_account_email', None)) token = creds.token return signing_creds, token def get_scoped_service_account_credentials( - scopes: list[str]) -> impersonated_credentials.Credentials | None: + scopes: Sequence[str]) -> impersonated_credentials.Credentials | None: """Gets scoped credentials by self-impersonating the service account.""" creds, _ = get_default() service_account_email = getattr(creds, 'service_account_email', None) diff --git a/src/clusterfuzz/_internal/google_cloud_utils/google_groups.py b/src/clusterfuzz/_internal/google_cloud_utils/google_groups.py index 9105afac3e1..29747ee05ea 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/google_groups.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/google_groups.py @@ -14,6 +14,7 @@ """Helper for google groups management.""" import threading +from typing import Any from urllib import parse from googleapiclient import discovery @@ -33,7 +34,7 @@ _local = threading.local() -def get_identity_api() -> discovery.Resource | None: +def get_identity_api() -> Any: """Return cloud identity api client.""" if not hasattr(_local, 'identity_service'): creds, _ = credentials.get_default() @@ -43,7 +44,7 @@ def get_identity_api() -> discovery.Resource | None: return _local.identity_service -def get_group_settings_api() -> discovery.Resource | None: +def get_group_settings_api() -> Any: """Return the groups settings api client.""" if not hasattr(_local, 'groups_settings_service'): scopes = ['https://www.googleapis.com/auth/apps.groups.settings'] diff --git a/src/clusterfuzz/_internal/google_cloud_utils/gsutil.py b/src/clusterfuzz/_internal/google_cloud_utils/gsutil.py index 86c1ef2e138..125271135bc 100755 --- a/src/clusterfuzz/_internal/google_cloud_utils/gsutil.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/gsutil.py @@ -13,8 +13,11 @@ # limitations under the License. """Functions for running gsutil.""" +from collections.abc import Callable +from collections.abc import Sequence import os import shutil +from typing import Any from clusterfuzz._internal.base import utils from clusterfuzz._internal.google_cloud_utils import storage @@ -26,12 +29,12 @@ FILES_SYNC_TIMEOUT = 5 * 60 * 60 -def use_gcloud_for_command(command): +def use_gcloud_for_command(command: str) -> bool: """Returns whether to use gcloud storage for the given command.""" return bool(environment.get_value(f'USE_GCLOUD_STORAGE_{command.upper()}')) -def get_gcloud_path(): +def get_gcloud_path() -> str | None: """Get path to gcloud executable.""" gcloud_executable = 'gcloud' if environment.platform() == 'WINDOWS': @@ -55,7 +58,7 @@ def get_gcloud_path(): return None -def get_gsutil_path(): +def get_gsutil_path() -> str | None: """Get path to gsutil executable. Returns: @@ -81,7 +84,7 @@ def get_gsutil_path(): return gsutil_absolute_path -def _multiprocessing_args(): +def _multiprocessing_args() -> list[str]: """Get multiprocessing args for gsutil.""" if utils.cpu_count() == 1: # GSUtil's default thread count is 5 as it assumes the common configuration @@ -91,7 +94,7 @@ def _multiprocessing_args(): return [] -def _filter_path(path, write=False): +def _filter_path(path: str, write: bool = False) -> str: """Filters path if needed. In local development environment, this uses local paths from an emulated GCS instead of real GCS. `write` indicates whether if `path` is a GCS write destination and that intermediate paths should be @@ -117,10 +120,18 @@ def _filter_path(path, write=False): class GSUtilRunner: """GSUtil/gcloud storage runner.""" - def __init__(self, process_runner=new_process.ProcessRunner): + def __init__( + self, + process_runner: Callable[ + ..., new_process.ProcessRunner] = new_process.ProcessRunner, + ) -> None: self._process_runner = process_runner - def _get_runner_and_args(self, use_gcloud_storage, quiet=False): + def _get_runner_and_args( + self, + use_gcloud_storage: bool, + quiet: bool = False, + ) -> tuple[new_process.ProcessRunner, list[str]]: """Get the process runner and default arguments.""" if use_gcloud_storage: executable_path = get_gcloud_path() @@ -142,7 +153,13 @@ def _get_runner_and_args(self, use_gcloud_storage, quiet=False): return runner, additional_args - def run_gsutil(self, arguments, use_gcloud_storage, quiet=False, **kwargs): + def run_gsutil( + self, + arguments: list[str], + use_gcloud_storage: bool, + quiet: bool = False, + **kwargs: Any, + ) -> new_process.ProcessResult: """Run GSUtil or gcloud storage.""" runner, additional_args = self._get_runner_and_args(use_gcloud_storage, quiet) @@ -189,12 +206,14 @@ def run_gsutil(self, arguments, use_gcloud_storage, quiet=False, **kwargs): arguments=arg_str) raise - def rsync(self, - source, - destination, - timeout=FILES_SYNC_TIMEOUT, - delete=True, - exclusion_pattern=None): + def rsync( + self, + source: str, + destination: str, + timeout: float = FILES_SYNC_TIMEOUT, + delete: bool = True, + exclusion_pattern: str | None = None, + ) -> new_process.ProcessResult: """Rsync with gsutil or gcloud storage.""" use_gcloud = use_gcloud_for_command('rsync') if use_gcloud: @@ -217,7 +236,12 @@ def rsync(self, return self.run_gsutil(command, use_gcloud, timeout=timeout, quiet=True) - def download_file(self, gcs_url, file_path, timeout=None): + def download_file( + self, + gcs_url: str, + file_path: str, + timeout: float | None = None, + ) -> bool: """Download a file from GCS.""" use_gcloud = use_gcloud_for_command('cp') command = ['cp', _filter_path(gcs_url), file_path] @@ -229,12 +253,14 @@ def download_file(self, gcs_url, file_path, timeout=None): return result.return_code == 0 - def upload_file(self, - file_path, - gcs_url, - timeout=None, - gzip=False, - metadata=None): + def upload_file( + self, + file_path: str, + gcs_url: str, + timeout: float | None = None, + gzip: bool = False, + metadata: dict[str, str] | None = None, + ) -> bool: """Upload a single file to a given GCS url.""" if not file_path or not gcs_url: return False @@ -301,7 +327,12 @@ def upload_file(self, return True - def upload_files_to_url(self, file_paths, gcs_url, timeout=None): + def upload_files_to_url( + self, + file_paths: Sequence[str], + gcs_url: str, + timeout: float | None = None, + ) -> bool: """Upload files to the given GCS url.""" if not file_paths or not gcs_url: return False diff --git a/src/clusterfuzz/_internal/google_cloud_utils/pubsub.py b/src/clusterfuzz/_internal/google_cloud_utils/pubsub.py index 3a06082179f..8054bf79d6d 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/pubsub.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/pubsub.py @@ -14,11 +14,16 @@ """Pub/Sub helpers.""" import base64 -import collections +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import Sequence import json import threading +from typing import Any +from typing import NamedTuple -import googleapiclient +import googleapiclient.discovery +import googleapiclient.errors import httplib2 from clusterfuzz._internal.base import retry @@ -30,15 +35,25 @@ _PUBSUB_FAIL_RETRIES = 5 _PUBSUB_FAIL_WAIT = 2 -Topic = collections.namedtuple('Topic', ['name', 'labels']) -Subscription = collections.namedtuple( - 'Subsciption', ['name', 'topic', 'ack_deadline', 'labels']) + +class Topic(NamedTuple): + name: str + labels: dict[str, str] | None + + +class Subscription(NamedTuple): + name: str + topic: str + ack_deadline: int + labels: dict[str, str] | None class Message: """Pubsub message.""" - def __init__(self, data=None, attributes=None): + def __init__(self, + data: bytes | None = None, + attributes: Mapping[str, Any] | None = None): self.data = data self.attributes = attributes @@ -46,8 +61,9 @@ def __init__(self, data=None, attributes=None): class ReceivedMessage(Message): """Received pubsub message.""" - def __init__(self, client, subscription, data, attributes, message_id, - publish_time, ack_id): + def __init__(self, client: 'PubSubClient', subscription: str, + data: bytes | None, attributes: Mapping[str, Any] | None, + message_id: str, publish_time: str, ack_id: str): super().__init__(data, attributes) self._client = client self.ack_id = ack_id @@ -55,11 +71,11 @@ def __init__(self, client, subscription, data, attributes, message_id, self.subscription = subscription self.publish_time = publish_time - def ack(self): + def ack(self) -> None: """Acknowledge the message.""" self._client.ack(self.subscription, [self.ack_id]) - def modify_ack_deadline(self, seconds): + def modify_ack_deadline(self, seconds: int | float) -> None: """Modify the acknowledgement deadline.""" self._client.modify_ack_deadline(self.subscription, [self.ack_id], seconds) @@ -67,7 +83,7 @@ def modify_ack_deadline(self, seconds): class PubSubClient: """Helper class that provides wrappers around some Pub/Sub functionality.""" - def __init__(self, emulator_host=None): + def __init__(self, emulator_host: str | None = None): self._local = threading.local() self._emulator_host = emulator_host @@ -75,7 +91,7 @@ def __init__(self, emulator_host=None): retries=_PUBSUB_FAIL_RETRIES, delay=_PUBSUB_FAIL_WAIT, function='google_cloud_utils.pubsub._api_client') - def _api_client(self): + def _api_client(self) -> Any: """Get the client for the current thread.""" if hasattr(self._local, 'api_client'): return self._local.api_client @@ -102,7 +118,7 @@ def _api_client(self): retries=_PUBSUB_FAIL_RETRIES, delay=_PUBSUB_FAIL_WAIT, function='google_cloud_utils.pubsub._execute_with_retry') - def _execute_with_retry(self, request): + def _execute_with_retry(self, request: Any) -> Any: """Execute a request (with retries).""" try: return request.execute() @@ -112,14 +128,15 @@ def _execute_with_retry(self, request): raise - def ack(self, subscription, ack_ids): + def ack(self, subscription: str, ack_ids: Sequence[str]) -> None: """Acknowledge messages.""" ack_body = {'ackIds': ack_ids} request = self._api_client().projects().subscriptions().acknowledge( subscription=subscription, body=ack_body) self._execute_with_retry(request) - def modify_ack_deadline(self, subscription, ack_ids, seconds): + def modify_ack_deadline(self, subscription: str, ack_ids: Sequence[str], + seconds: int | float) -> None: """Modify acknowledgement deadline of messages.""" body = { 'ackIds': ack_ids, @@ -130,7 +147,7 @@ def modify_ack_deadline(self, subscription, ack_ids, seconds): subscription=subscription, body=body) self._execute_with_retry(request) - def publish(self, topic, messages): + def publish(self, topic: str, messages: Sequence[Message]) -> list[str]: """Publish a message to a topic.""" request_body = { 'messages': [_message_to_dict(message) for message in messages] @@ -144,10 +161,11 @@ def publish(self, topic, messages): return sorted(response.get('messageIds')) - def pull_from_subscription(self, - subscription, - max_messages=_DEFAULT_MAX_MESSAGES, - acknowledge=False): + def pull_from_subscription( + self, + subscription: str, + max_messages: int = _DEFAULT_MAX_MESSAGES, + acknowledge: bool = False) -> list[ReceivedMessage]: """Pull messages from a subscription to a topic.""" request_body = { 'returnImmediately': True, @@ -174,9 +192,10 @@ def pull_from_subscription(self, for message in received_messages ] - def create_topic(self, name, labels=None): + def create_topic(self, name: str, + labels: dict[str, str] | None = None) -> None: """Create a new topic.""" - body = { + body: dict[str, Any] = { 'labels': {}, } @@ -187,12 +206,12 @@ def create_topic(self, name, labels=None): name=name, body=body) self._execute_with_retry(request) - def delete_topic(self, name): + def delete_topic(self, name: str) -> None: """Deletes a topic.""" request = self._api_client().projects().topics().delete(topic=name) self._execute_with_retry(request) - def get_topic(self, name): + def get_topic(self, name: str) -> Topic | None: """Get a topic, or None if it does not exist.""" request = self._api_client().projects().topics().get(topic=name) response = self._execute_with_retry(request) @@ -202,12 +221,16 @@ def get_topic(self, name): return Topic(response['name'], response.get('labels')) def create_subscription(self, - name, - topic, - ack_deadline=MAX_ACK_DEADLINE, - labels=None): + name: str, + topic: str, + ack_deadline: int = MAX_ACK_DEADLINE, + labels: dict[str, str] | None = None) -> None: """Create a new subscription.""" - body = {'topic': topic, 'ackDeadlineSeconds': ack_deadline, 'labels': {}} + body: dict[str, Any] = { + 'topic': topic, + 'ackDeadlineSeconds': ack_deadline, + 'labels': {} + } if labels: body['labels'] = labels @@ -216,13 +239,13 @@ def create_subscription(self, name=name, body=body) self._execute_with_retry(request) - def delete_subscription(self, name): + def delete_subscription(self, name: str) -> None: """Delete a subscription.""" request = self._api_client().projects().subscriptions().delete( subscription=name) self._execute_with_retry(request) - def get_subscription(self, name): + def get_subscription(self, name: str) -> Subscription | None: """Get a subscription, or None if it does not exist.""" request = self._api_client().projects().subscriptions().get( subscription=name) @@ -233,7 +256,7 @@ def get_subscription(self, name): return Subscription(response['name'], response['topic'], response['ackDeadlineSeconds'], response.get('labels')) - def list_topics(self, project, page_size=1000): + def list_topics(self, project: str, page_size: int = 1000) -> Iterator[str]: """List topics.""" request = self._api_client().projects().topics().list( project=project, pageSize=page_size) @@ -251,7 +274,8 @@ def list_topics(self, project, page_size=1000): project=project, pageToken=next_page_token, pageSize=page_size) response = self._execute_with_retry(request) - def list_topic_subscriptions(self, topic, page_size=1000): + def list_topic_subscriptions(self, topic: str, + page_size: int = 1000) -> Iterator[str]: """List topic subscriptions.""" request = self._api_client().projects().topics().subscriptions().list( topic=topic, pageSize=page_size) @@ -269,22 +293,22 @@ def list_topic_subscriptions(self, topic, page_size=1000): response = self._execute_with_retry(request) -def project_name(project): +def project_name(project: str) -> str: return 'projects/' + project -def subscription_name(project, name): +def subscription_name(project: str | None, name: str) -> str: """Get subscription name.""" return 'projects/{project}/subscriptions/{name}'.format( project=project, name=name) -def topic_name(project, name): +def topic_name(project: str | None, name: str) -> str: """Get topic name.""" return 'projects/{project}/topics/{name}'.format(project=project, name=name) -def parse_name(name): +def parse_name(name: str) -> tuple[str, str]: """Parse the topic or subscription name.""" components = name.split('/') if len(components) != 4: @@ -295,20 +319,21 @@ def parse_name(name): return project, name -def _decode_data(raw_message): +def _decode_data(raw_message: dict[str, Any]) -> bytes | None: """Decode Pub/Sub data.""" return (base64.b64decode(raw_message['data']) if 'data' in raw_message else None) -def raw_message_to_message(raw_message_response): +def raw_message_to_message(raw_message_response: dict[str, Any]) -> Message: """Convert a raw message response to a Message.""" raw_message = raw_message_response['message'] return Message(_decode_data(raw_message), raw_message.get('attributes')) -def _raw_message_to_received_message(client, subscription, - raw_message_response): +def _raw_message_to_received_message( + client: PubSubClient, subscription: str, + raw_message_response: dict[str, Any]) -> ReceivedMessage: """Convert a raw message response to a Message.""" raw_message = raw_message_response['message'] return ReceivedMessage(client, subscription, _decode_data(raw_message), @@ -317,9 +342,9 @@ def _raw_message_to_received_message(client, subscription, raw_message_response['ackId']) -def _message_to_dict(message): +def _message_to_dict(message: Message) -> dict[str, Any]: """Convert the message to a dict.""" - result = {} + result: dict[str, Any] = {} if message.data: result['data'] = base64.b64encode(message.data).decode('utf-8') diff --git a/src/clusterfuzz/_internal/google_cloud_utils/secret_manager.py b/src/clusterfuzz/_internal/google_cloud_utils/secret_manager.py index 5047da202ed..35675fc2e94 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/secret_manager.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/secret_manager.py @@ -15,12 +15,12 @@ from google.cloud import secretmanager -def get_secret_manager_client(): +def get_secret_manager_client() -> secretmanager.SecretManagerServiceClient: """Returns the secretmanager client.""" return secretmanager.SecretManagerServiceClient() -def get(secret_id, project): +def get(secret_id: str, project: str) -> bytes: """Returns the value of the secret identified by |secret_id| in |project|.""" client = get_secret_manager_client() diff --git a/src/clusterfuzz/_internal/google_cloud_utils/storage.py b/src/clusterfuzz/_internal/google_cloud_utils/storage.py index aad16899c4b..ac4b84048ac 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/storage.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/storage.py @@ -24,8 +24,13 @@ import shutil import threading import time +from typing import Any +from typing import Callable +from typing import Iterator from typing import List +from typing import Optional from typing import Tuple +from typing import Union import uuid from xml.etree import ElementTree as ET @@ -53,16 +58,19 @@ try: import google.cloud from google.cloud import storage as gcs - import google.cloud.storage.fileio + import google.cloud.exceptions + from google.cloud.storage import fileio except ImportError: # This is expected to fail on AppEngine. - pass + gcs = None # type: ignore + fileio = None # type: ignore try: import OpenSSL.SSL + _OPENSSL_ERROR: Any = OpenSSL.SSL.Error except ImportError: # This is expected to fail on non-linux platforms. - OpenSSL = None # pylint: disable=invalid-name + _OPENSSL_ERROR = None # Usually, authentication time have expiry of ~30 minutes, but keeping this # values lower to avoid failures and any future changes. @@ -101,7 +109,7 @@ # memory usage? _POOL_SIZE = 16 -_TRANSIENT_ERRORS = [ +_TRANSIENT_ERRORS: List[Any] = [ google.cloud.exceptions.GoogleCloudError, ConnectionError, requests.exceptions.ConnectionError, @@ -111,9 +119,9 @@ google.auth.exceptions.TransportError, ] -if OpenSSL: +if _OPENSSL_ERROR: # We haven't imported this on non-linux platforms. - _TRANSIENT_ERRORS.append(OpenSSL.SSL.Error) + _TRANSIENT_ERRORS.append(_OPENSSL_ERROR) SignedUrlDownloadResult = collections.namedtuple('SignedUrlDownloadResult', ['url', 'filepath']) @@ -122,77 +130,93 @@ class StorageProvider: """Core storage provider interface.""" - def create_bucket(self, name, object_lifecycle, cors, location): + def create_bucket(self, name: str, object_lifecycle: Optional[dict], + cors: Optional[list], location: Optional[str]) -> bool: """Create a new bucket.""" raise NotImplementedError - def get_bucket(self, name): + def get_bucket(self, name: str) -> Optional[dict]: """Get a bucket.""" raise NotImplementedError def list_blobs(self, - remote_path, - recursive=True, - names_only=False, - single_file=False): + remote_path: str, + recursive: bool = True, + names_only: bool = False, + single_file: bool = False) -> Iterator[dict]: """List the blobs under the remote path.""" raise NotImplementedError - def copy_file_from(self, remote_path, local_path): + def copy_file_from(self, remote_path: str, local_path: str) -> bool: """Copy file from a remote path to a local path.""" raise NotImplementedError - def copy_file_to(self, local_path_or_handle, remote_path, metadata=None): + def copy_file_to(self, + local_path_or_handle: Union[str, io.IOBase, Any], + remote_path: str, + metadata: Optional[dict] = None) -> bool: """Copy file from a local path to a remote path.""" raise NotImplementedError - def copy_blob(self, remote_source, remote_target): + def copy_blob(self, remote_source: str, remote_target: str) -> bool: """Copy a remote file to another remote location.""" raise NotImplementedError - def read_data(self, remote_path): + def read_data(self, remote_path: str) -> Optional[bytes]: """Read the data of a remote file.""" raise NotImplementedError - def write_data(self, data_or_fileobj, remote_path, metadata=None): + def write_data(self, + data_or_fileobj: Union[str, bytes, io.IOBase, Any], + remote_path: str, + metadata: Optional[dict] = None) -> bool: """Write the data of a remote file.""" raise NotImplementedError - def write_stream(self, stream, remote_path, metadata=None): + def write_stream(self, + stream: Any, + remote_path: str, + metadata: Optional[dict] = None) -> bool: """Write the data of a generator.""" raise NotImplementedError - def get(self, remote_path): + def get(self, remote_path: str) -> Optional[dict]: """Get information about a remote file.""" raise NotImplementedError - def delete(self, remote_path): + def delete(self, remote_path: str) -> bool: """Delete a remote file.""" raise NotImplementedError def sign_download_url(self, - remote_path, - minutes=SIGNED_URL_EXPIRATION_MINUTES): + remote_path: str, + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Signs a download URL for a remote file.""" raise NotImplementedError - def sign_upload_url(self, remote_path, minutes=SIGNED_URL_EXPIRATION_MINUTES): + def sign_upload_url(self, + remote_path: str, + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Signs an upload URL for a remote file.""" raise NotImplementedError - def download_signed_url(self, signed_url): + def download_signed_url(self, signed_url: str) -> Optional[bytes]: """Downloads |signed_url|.""" raise NotImplementedError - def upload_signed_url(self, data_or_fileobj, signed_url: str) -> bool: + def upload_signed_url(self, + data_or_fileobj: Union[str, bytes, io.IOBase, Any], + signed_url: str) -> bool: """Uploads |data| to |signed_url|.""" raise NotImplementedError - def sign_delete_url(self, remote_path, minutes=SIGNED_URL_EXPIRATION_MINUTES): + def sign_delete_url(self, + remote_path: str, + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Signs a DELETE URL for a remote file.""" raise NotImplementedError - def delete_signed_url(self, signed_url): + def delete_signed_url(self, signed_url: str) -> None: """Makes a DELETE HTTP request to |signed_url|.""" raise NotImplementedError @@ -200,17 +224,18 @@ def delete_signed_url(self, signed_url): class GcsProvider(StorageProvider): """GCS storage provider.""" - def _chunk_size(self): + def _chunk_size(self) -> Optional[int]: if environment.is_running_on_app_engine(): # To match App Engine URLFetch's request size limit. return 10 * 1024 * 1024 # 10 MiB. return None - def create_bucket(self, name, object_lifecycle, cors, location): + def create_bucket(self, name: str, object_lifecycle: Optional[dict], + cors: Optional[list], location: Optional[str]) -> bool: """Create a new bucket.""" project_id = utils.get_application_id() - request_body = {'name': name} + request_body: dict[str, Any] = {'name': name} if object_lifecycle: request_body['lifecycle'] = object_lifecycle @@ -221,6 +246,8 @@ def create_bucket(self, name, object_lifecycle, cors, location): request_body['location'] = location client = create_discovery_storage_client() + if not client: + return False try: client.buckets().insert(project=project_id, body=request_body).execute() except HttpError as e: @@ -229,9 +256,11 @@ def create_bucket(self, name, object_lifecycle, cors, location): return True - def get_bucket(self, name): + def get_bucket(self, name: str) -> Optional[dict]: """Get a bucket.""" client = create_discovery_storage_client() + if not client: + return None try: return client.buckets().get(bucket=name).execute() except HttpError as e: @@ -241,10 +270,10 @@ def get_bucket(self, name): raise def list_blobs(self, - remote_path, - recursive=True, - names_only=False, - single_file=False): + remote_path: str, + recursive: bool = True, + names_only: bool = False, + single_file: bool = False) -> Iterator[dict]: """List the blobs under the remote path.""" bucket_name, path = get_bucket_name_and_path(remote_path) @@ -298,7 +327,7 @@ def list_blobs(self, if iterations and iterations % 50 == 0: logs.error('Might be infinite looping.') - def copy_file_from(self, remote_path, local_path): + def copy_file_from(self, remote_path: str, local_path: str) -> bool: """Copy file from a remote path to a local path.""" client = _storage_client() bucket_name, path = get_bucket_name_and_path(remote_path) @@ -314,7 +343,10 @@ def copy_file_from(self, remote_path, local_path): return True - def copy_file_to(self, local_path_or_handle, remote_path, metadata=None): + def copy_file_to(self, + local_path_or_handle: Union[str, io.IOBase, Any], + remote_path: str, + metadata: Optional[dict] = None) -> bool: """Copy file from a local path to a remote path.""" client = _storage_client() bucket_name, path = get_bucket_name_and_path(remote_path) @@ -336,7 +368,7 @@ def copy_file_to(self, local_path_or_handle, remote_path, metadata=None): return True - def copy_blob(self, remote_source, remote_target): + def copy_blob(self, remote_source: str, remote_target: str) -> bool: """Copy a remote file to another remote location.""" source_bucket_name, source_path = get_bucket_name_and_path(remote_source) target_bucket_name, target_path = get_bucket_name_and_path(remote_target) @@ -354,7 +386,7 @@ def copy_blob(self, remote_source, remote_target): return True - def read_data(self, remote_path): + def read_data(self, remote_path: str) -> Optional[bytes]: """Read the data of a remote file.""" bucket_name, path = get_bucket_name_and_path(remote_path) @@ -370,7 +402,10 @@ def read_data(self, remote_path): logs.warning('Failed to read cloud storage file %s.' % remote_path) raise - def write_data(self, data_or_fileobj, remote_path, metadata=None): + def write_data(self, + data_or_fileobj: Union[str, bytes, io.IOBase, Any], + remote_path: str, + metadata: Optional[dict] = None) -> bool: """Write the data of a remote file.""" client = _storage_client() bucket_name, path = get_bucket_name_and_path(remote_path) @@ -391,7 +426,10 @@ def write_data(self, data_or_fileobj, remote_path, metadata=None): return True - def write_stream(self, stream, remote_path, metadata=None): + def write_stream(self, + stream: Any, + remote_path: str, + metadata: Optional[dict] = None) -> bool: """Writes the data from an iterator in chunks to a remote file.""" client = _storage_client() bucket_name, path = get_bucket_name_and_path(remote_path) @@ -401,7 +439,8 @@ def write_stream(self, stream, remote_path, metadata=None): blob = bucket.blob(path, chunk_size=self._chunk_size()) if metadata: blob.metadata = metadata - with gcs.fileio.BlobWriter(blob) as blob_writer: + assert fileio is not None + with fileio.BlobWriter(blob) as blob_writer: for data in stream: if isinstance(data, str): data = data.encode() @@ -412,9 +451,11 @@ def write_stream(self, stream, remote_path, metadata=None): return True - def get(self, remote_path): + def get(self, remote_path: str) -> Optional[dict]: """Get information about a remote file.""" client = create_discovery_storage_client() + if not client: + return None bucket, path = get_bucket_name_and_path(remote_path) try: @@ -425,7 +466,7 @@ def get(self, remote_path): raise - def delete(self, remote_path): + def delete(self, remote_path: str) -> bool: """Delete a remote file.""" client = _storage_client() bucket_name, path = get_bucket_name_and_path(remote_path) @@ -441,44 +482,48 @@ def delete(self, remote_path): def sign_download_url(self, remote_path: str, - minutes=SIGNED_URL_EXPIRATION_MINUTES) -> str: + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Signs a download URL for a remote file.""" return _sign_url(remote_path, method='GET', minutes=minutes) def sign_upload_url(self, remote_path: str, - minutes=SIGNED_URL_EXPIRATION_MINUTES) -> str: + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Signs an upload URL for a remote file.""" return _sign_url(remote_path, method='PUT', minutes=minutes) - def download_signed_url(self, signed_url): + def download_signed_url(self, signed_url: str) -> Optional[bytes]: """Downloads |signed_url|.""" return _download_url(signed_url) - def upload_signed_url(self, data_or_fileobj, signed_url: str) -> bool: + def upload_signed_url(self, + data_or_fileobj: Union[str, bytes, io.IOBase, Any], + signed_url: str) -> bool: """Uploads |data| to |signed_url|.""" requests.put( signed_url, data=data_or_fileobj, timeout=HTTP_TIMEOUT_SECONDS).raise_for_status() return True - def sign_delete_url(self, remote_path, minutes=SIGNED_URL_EXPIRATION_MINUTES): + def sign_delete_url(self, + remote_path: str, + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Signs a DELETE URL for a remote file.""" return _sign_url( remote_path, method='DELETE', minutes=SIGNED_URL_EXPIRATION_MINUTES) - def delete_signed_url(self, signed_url): + def delete_signed_url(self, signed_url: str) -> None: """Makes a DELETE HTTP request to |signed_url|.""" requests.delete(signed_url, timeout=HTTP_TIMEOUT_SECONDS).raise_for_status() def _sign_url(remote_path: str, - minutes=SIGNED_URL_EXPIRATION_MINUTES, - method='GET') -> str: + minutes: int = SIGNED_URL_EXPIRATION_MINUTES, + method: str = 'GET') -> str: """Returns a signed URL for |remote_path| with |method|.""" if _integration_test_env_doesnt_support_signed_urls(): return remote_path - minutes = datetime.timedelta(minutes=minutes) + expiration = datetime.timedelta(minutes=minutes) bucket_name, object_path = get_bucket_name_and_path(remote_path) signing_creds, access_token = _signing_creds() client = _storage_client() @@ -486,7 +531,7 @@ def _sign_url(remote_path: str, blob = bucket.blob(object_path) return blob.generate_signed_url( version='v4', - expiration=minutes, + expiration=expiration, method=method, credentials=signing_creds, access_token=access_token, @@ -499,15 +544,15 @@ class FileSystemProvider(StorageProvider): OBJECTS_DIR = 'objects' METADATA_DIR = 'metadata' - def __init__(self, filesystem_dir): + def __init__(self, filesystem_dir: str) -> None: self.filesystem_dir = os.path.abspath(filesystem_dir) - def _get_object_properties(self, remote_path): + def _get_object_properties(self, remote_path: str) -> dict: """Set local object properties.""" bucket, path = get_bucket_name_and_path(remote_path) fs_path = self.convert_path(remote_path) - data = { + data: dict[str, Any] = { 'bucket': bucket, 'name': path, } @@ -525,7 +570,7 @@ def _get_object_properties(self, remote_path): return data - def _get_metadata(self, bucket, path): + def _get_metadata(self, bucket: str, path: str) -> dict: """Get the metadata for a given object.""" fs_metadata_path = self._fs_path(bucket, path, self.METADATA_DIR) if os.path.exists(fs_metadata_path): @@ -534,19 +579,19 @@ def _get_metadata(self, bucket, path): return {} - def _fs_bucket_path(self, bucket): + def _fs_bucket_path(self, bucket: str) -> str: """Get the local FS path for the bucket.""" return os.path.join(self.filesystem_dir, bucket) - def _fs_objects_dir(self, bucket): + def _fs_objects_dir(self, bucket: str) -> str: """Get the local FS path for objects in the bucket.""" return os.path.join(self._fs_bucket_path(bucket), self.OBJECTS_DIR) - def _fs_path(self, bucket, path, directory): + def _fs_path(self, bucket: str, path: str, directory: str) -> str: """Get the local object/metadata FS path.""" return os.path.join(self._fs_bucket_path(bucket), directory, path) - def _write_metadata(self, remote_path, metadata): + def _write_metadata(self, remote_path: str, metadata: Optional[dict]) -> None: """Write metadata.""" if not metadata: return @@ -556,12 +601,14 @@ def _write_metadata(self, remote_path, metadata): with open(fs_metadata_path, 'w') as f: json.dump(metadata, f) - def convert_path(self, remote_path, directory=OBJECTS_DIR): + def convert_path(self, remote_path: str, directory: str = OBJECTS_DIR) -> str: """Get the local FS path for the remote path.""" bucket, path = get_bucket_name_and_path(remote_path) return self._fs_path(bucket, path, directory) - def convert_path_for_write(self, remote_path, directory=OBJECTS_DIR): + def convert_path_for_write(self, + remote_path: str, + directory: str = OBJECTS_DIR) -> str: """Get the local FS path for writing to the remote path. Creates any intermediate directories if necessary (except for the parent bucket directory).""" @@ -575,7 +622,8 @@ def convert_path_for_write(self, remote_path, directory=OBJECTS_DIR): return fs_path - def create_bucket(self, name, object_lifecycle, cors, location): + def create_bucket(self, name: str, object_lifecycle: Optional[dict], + cors: Optional[list], location: Optional[str]) -> bool: """Create a new bucket.""" bucket_path = self._fs_bucket_path(name) if os.path.exists(bucket_path): @@ -584,7 +632,7 @@ def create_bucket(self, name, object_lifecycle, cors, location): os.makedirs(bucket_path) return True - def get_bucket(self, name): + def get_bucket(self, name: str) -> Optional[dict]: """Get a bucket.""" bucket_path = self._fs_bucket_path(name) if not os.path.exists(bucket_path): @@ -594,22 +642,22 @@ def get_bucket(self, name): 'name': name, } - def _list_files_recursive(self, fs_path): + def _list_files_recursive(self, fs_path: str) -> Iterator[str]: """List files recursively.""" for root, _, filenames in shell.walk(fs_path): for filename in filenames: yield os.path.join(root, filename) - def _list_files_nonrecursive(self, fs_path): + def _list_files_nonrecursive(self, fs_path: str) -> Iterator[str]: """List files non-recursively.""" for filename in os.listdir(fs_path): yield os.path.join(fs_path, filename) def list_blobs(self, - remote_path, - recursive=True, - names_only=False, - single_file=False): + remote_path: str, + recursive: bool = True, + names_only: bool = False, + single_file: bool = False) -> Iterator[dict]: """List the blobs under the remote path.""" del names_only bucket, _ = get_bucket_name_and_path(remote_path) @@ -620,18 +668,21 @@ def list_blobs(self, else: file_paths = self._list_files_nonrecursive(fs_path) - for fs_path in file_paths: - path = os.path.relpath(fs_path, self._fs_objects_dir(bucket)) + for fs_file_path in file_paths: + path = os.path.relpath(fs_file_path, self._fs_objects_dir(bucket)) yield self._get_object_properties( get_cloud_storage_file_path(bucket, path)) - def copy_file_from(self, remote_path, local_path): + def copy_file_from(self, remote_path: str, local_path: str) -> bool: """Copy file from a remote path to a local path.""" fs_path = self.convert_path(remote_path) return shell.copy_file(fs_path, local_path) - def copy_file_to(self, local_path_or_handle, remote_path, metadata=None): + def copy_file_to(self, + local_path_or_handle: Union[str, io.IOBase, Any], + remote_path: str, + metadata: Optional[dict] = None) -> bool: """Copy file from a local path to a remote path.""" fs_path = self.convert_path_for_write(remote_path) @@ -645,13 +696,13 @@ def copy_file_to(self, local_path_or_handle, remote_path, metadata=None): self._write_metadata(remote_path, metadata) return True - def copy_blob(self, remote_source, remote_target): + def copy_blob(self, remote_source: str, remote_target: str) -> bool: """Copy a remote file to another remote location.""" fs_source_path = self.convert_path(remote_source) fs_target_path = self.convert_path_for_write(remote_target) return shell.copy_file(fs_source_path, fs_target_path) - def read_data(self, remote_path): + def read_data(self, remote_path: str) -> Optional[bytes]: """Read the data of a remote file.""" fs_path = self.convert_path(remote_path) if not os.path.exists(fs_path): @@ -660,7 +711,10 @@ def read_data(self, remote_path): with open(fs_path, 'rb') as f: return f.read() - def write_data(self, data_or_fileobj, remote_path, metadata=None): + def write_data(self, + data_or_fileobj: Union[str, bytes, io.IOBase, Any], + remote_path: str, + metadata: Optional[dict] = None) -> bool: """Write the data of a remote file.""" fs_path = self.convert_path_for_write(remote_path) if isinstance(data_or_fileobj, str): @@ -675,7 +729,10 @@ def write_data(self, data_or_fileobj, remote_path, metadata=None): self._write_metadata(remote_path, metadata) return True - def write_stream(self, stream, remote_path, metadata=None): + def write_stream(self, + stream: Any, + remote_path: str, + metadata: Optional[dict] = None) -> bool: """Write the data of a remote file.""" fs_path = self.convert_path_for_write(remote_path) @@ -688,7 +745,7 @@ def write_stream(self, stream, remote_path, metadata=None): self._write_metadata(remote_path, metadata) return True - def get(self, remote_path): + def get(self, remote_path: str) -> Optional[dict]: """Get information about a remote file.""" fs_path = self.convert_path(remote_path) if not os.path.exists(fs_path): @@ -696,7 +753,7 @@ def get(self, remote_path): return self._get_object_properties(remote_path) - def delete(self, remote_path): + def delete(self, remote_path: str) -> bool: """Delete a remote file.""" fs_path = self.convert_path(remote_path) shell.remove_file(fs_path) @@ -706,33 +763,39 @@ def delete(self, remote_path): return True def sign_download_url(self, - remote_path, - minutes=SIGNED_URL_EXPIRATION_MINUTES): + remote_path: str, + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Returns remote_path since we are pretending to sign a URL for download.""" del minutes return remote_path - def sign_upload_url(self, remote_path, minutes=SIGNED_URL_EXPIRATION_MINUTES): + def sign_upload_url(self, + remote_path: str, + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Returns remote_path since we are pretending to sign a URL for upload.""" del minutes return remote_path - def download_signed_url(self, signed_url): + def download_signed_url(self, signed_url: str) -> Optional[bytes]: """Downloads |signed_url|.""" return self.read_data(signed_url) - def upload_signed_url(self, data_or_fileobj, signed_url: str) -> bool: + def upload_signed_url(self, + data_or_fileobj: Union[str, bytes, io.IOBase, Any], + signed_url: str) -> bool: """Uploads |data| to |signed_url|.""" return self.write_data(data_or_fileobj, signed_url) - def sign_delete_url(self, remote_path, minutes=SIGNED_URL_EXPIRATION_MINUTES): + def sign_delete_url(self, + remote_path: str, + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Signs a DELETE URL for a remote file.""" del minutes return remote_path - def delete_signed_url(self, signed_url): + def delete_signed_url(self, signed_url: str) -> None: """Makes a DELETE HTTP request to |signed_url|.""" self.delete(signed_url) @@ -741,11 +804,11 @@ class GcsBlobInfo: """GCS blob info.""" def __init__(self, - bucket, - object_path, - filename=None, - size=None, - legacy_key=None): + bucket: str, + object_path: str, + filename: Optional[str] = None, + size: Optional[int] = None, + legacy_key: Optional[str] = None) -> None: self.bucket = bucket self.object_path = object_path @@ -763,18 +826,18 @@ def __init__(self, self.legacy_key = legacy_key - def key(self): + def key(self) -> str: if self.legacy_key: return self.legacy_key return self.object_path @property - def gcs_path(self): + def gcs_path(self) -> str: return '/%s/%s' % (self.bucket, self.object_path) @staticmethod - def from_key(key): + def from_key(key: str) -> Optional['GcsBlobInfo']: try: return GcsBlobInfo(blobs_bucket(), key) except Exception: @@ -782,13 +845,13 @@ def from_key(key): return None @staticmethod - def from_legacy_blob_info(blob_info): + def from_legacy_blob_info(blob_info: Any) -> 'GcsBlobInfo': bucket, path = get_bucket_name_and_path(blob_info.gs_object_name) return GcsBlobInfo(bucket, path, blob_info.filename, blob_info.size, blob_info.key.id()) -def _provider(): +def _provider() -> StorageProvider: """Get the current storage provider.""" local_buckets_path = environment.get_value('LOCAL_GCS_BUCKETS_PATH') if local_buckets_path: @@ -797,35 +860,36 @@ def _provider(): return GcsProvider() -def _create_storage_client_new(): +def _create_storage_client_new() -> Any: """Create a storage client.""" creds, project = credentials.get_default() if not project: project = utils.get_application_id() + assert gcs is not None return gcs.Client(project=project, credentials=creds) -def _storage_client(): +def _storage_client() -> Any: """Get the storage client, creating it if it does not exist.""" if not hasattr(_local, 'client'): _local.client = _create_storage_client_new() return _local.client -def _new_signing_creds(): +def _new_signing_creds() -> None: service_account = credentials.get_storage_signing_service_account() _local.signing_creds = credentials.get_signing_credentials(service_account) -def _signing_creds(): +def _signing_creds() -> Any: if not hasattr(_local, 'signing_creds'): _new_signing_creds() return _local.signing_creds -def get_bucket_name_and_path(cloud_storage_file_path): +def get_bucket_name_and_path(cloud_storage_file_path: str) -> Tuple[str, str]: """Return bucket name and path given a full cloud storage path.""" filtered_path = utils.strip_from_left(cloud_storage_file_path, GS_PREFIX) _, bucket_name_and_path = filtered_path.split('/', 1) @@ -839,12 +903,12 @@ def get_bucket_name_and_path(cloud_storage_file_path): return bucket_name, path -def get_cloud_storage_file_path(bucket, path): +def get_cloud_storage_file_path(bucket: str, path: str) -> str: """Get the full GCS file path.""" return GS_PREFIX + '/' + bucket + '/' + path -def _get_error_reason(http_error): +def _get_error_reason(http_error: HttpError) -> Optional[str]: """Get error reason from googleapiclient.errors.HttpError.""" try: data = json.loads(http_error.content.decode('utf-8')) @@ -856,10 +920,13 @@ def _get_error_reason(http_error): @environment.local_noop -def add_single_bucket_iam(storage, iam_policy, role, bucket_name, member): +def add_single_bucket_iam(storage: Any, iam_policy: dict, role: str, + bucket_name: str, member: str) -> Any: """Attempt to add a single bucket IAM. Returns the modified iam policy, or None on failure.""" - binding = get_bucket_iam_binding(iam_policy, role) + binding = get_or_create_bucket_iam_binding(iam_policy, role) + if not binding: + return None binding['members'].append(member) result = set_bucket_iam_policy(storage, bucket_name, iam_policy) @@ -869,7 +936,7 @@ def add_single_bucket_iam(storage, iam_policy, role, bucket_name, member): @environment.local_noop -def get_bucket_iam_binding(iam_policy, role): +def get_bucket_iam_binding(iam_policy: dict, role: str) -> Optional[dict]: """Get the binding matching a role, or None.""" return next(( binding for binding in iam_policy['bindings'] if binding['role'] == role), @@ -877,7 +944,8 @@ def get_bucket_iam_binding(iam_policy, role): @environment.local_noop -def get_or_create_bucket_iam_binding(iam_policy, role): +def get_or_create_bucket_iam_binding(iam_policy: dict, + role: str) -> Optional[dict]: """Get or create the binding matching a role.""" binding = get_bucket_iam_binding(iam_policy, role) if not binding: @@ -888,7 +956,7 @@ def get_or_create_bucket_iam_binding(iam_policy, role): @environment.local_noop -def remove_bucket_iam_binding(iam_policy, role): +def remove_bucket_iam_binding(iam_policy: dict, role: str) -> None: """Remove existing binding matching the role.""" iam_policy['bindings'] = [ binding for binding in iam_policy['bindings'] if binding['role'] != role @@ -896,7 +964,7 @@ def remove_bucket_iam_binding(iam_policy, role): @environment.local_noop -def get_bucket_iam_policy(storage, bucket_name): +def get_bucket_iam_policy(storage: Any, bucket_name: str) -> Optional[dict]: """Get bucket IAM policy.""" try: iam_policy = storage.buckets().getIamPolicy(bucket=bucket_name).execute() @@ -908,7 +976,8 @@ def get_bucket_iam_policy(storage, bucket_name): @environment.local_noop -def set_bucket_iam_policy(client, bucket_name, iam_policy): +def set_bucket_iam_policy(client: Any, bucket_name: str, + iam_policy: dict) -> Optional[dict]: """Set bucket IAM policy.""" filtered_iam_policy = copy.deepcopy(iam_policy) @@ -944,15 +1013,15 @@ def set_bucket_iam_policy(client, bucket_name, iam_policy): return None -def get_bucket(bucket_name: str): +def get_bucket(bucket_name: str) -> Optional[dict]: provider = _provider() return provider.get_bucket(bucket_name) -def create_bucket_if_needed(bucket_name, - object_lifecycle=None, - cors=None, - location=None): +def create_bucket_if_needed(bucket_name: str, + object_lifecycle: Optional[dict] = None, + cors: Optional[list] = None, + location: Optional[str] = None) -> bool: """Creates a GCS bucket.""" provider = _provider() if provider.get_bucket(bucket_name): @@ -966,18 +1035,21 @@ def create_bucket_if_needed(bucket_name, @environment.local_noop -def create_discovery_storage_client(): +def create_discovery_storage_client() -> Any: """Create a storage client using discovery APIs.""" return build('storage', 'v1', cache_discovery=False) -def generate_life_cycle_config(action, age=None, num_newer_versions=None): +def generate_life_cycle_config( + action: str, + age: Optional[int] = None, + num_newer_versions: Optional[int] = None) -> dict: """Generate GCS lifecycle management config. For the reference, see https://cloud.google.com/storage/docs/lifecycle and https://cloud.google.com/storage/docs/managing-lifecycles. """ - rule = {} + rule: dict[str, Any] = {} rule['action'] = {'type': action} rule['condition'] = {} if age is not None: @@ -994,7 +1066,7 @@ def generate_life_cycle_config(action, age=None, num_newer_versions=None): delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.copy_file_from', exception_types=_TRANSIENT_ERRORS) -def copy_file_from(cloud_storage_file_path, local_file_path): +def copy_file_from(cloud_storage_file_path: str, local_file_path: str) -> bool: """Saves a cloud storage file locally.""" if not _provider().copy_file_from(cloud_storage_file_path, local_file_path): return False @@ -1007,9 +1079,9 @@ def copy_file_from(cloud_storage_file_path, local_file_path): delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.copy_file_to', exception_types=_TRANSIENT_ERRORS) -def copy_file_to(local_file_path_or_handle, - cloud_storage_file_path, - metadata=None): +def copy_file_to(local_file_path_or_handle: Union[str, io.IOBase, Any], + cloud_storage_file_path: str, + metadata: Optional[dict] = None) -> bool: """Copy local file to a cloud storage path.""" if (isinstance(local_file_path_or_handle, str) and not os.path.exists(local_file_path_or_handle)): @@ -1025,7 +1097,8 @@ def copy_file_to(local_file_path_or_handle, delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.copy_blob', exception_types=_TRANSIENT_ERRORS) -def copy_blob(cloud_storage_source_path, cloud_storage_target_path): +def copy_blob(cloud_storage_source_path: str, + cloud_storage_target_path: str) -> bool: """Copy two blobs on GCS 'in the cloud' without touching local disk.""" return _provider().copy_blob(cloud_storage_source_path, cloud_storage_target_path) @@ -1036,7 +1109,7 @@ def copy_blob(cloud_storage_source_path, cloud_storage_target_path): delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.delete', exception_types=_TRANSIENT_ERRORS) -def delete(cloud_storage_file_path): +def delete(cloud_storage_file_path: str) -> bool: """Delete a cloud storage file given its path.""" return _provider().delete(cloud_storage_file_path) @@ -1045,7 +1118,7 @@ def delete(cloud_storage_file_path): retries=DEFAULT_FAIL_RETRIES, delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.exists') -def exists(cloud_storage_file_path, ignore_errors=False): +def exists(cloud_storage_file_path: str, ignore_errors: bool = False) -> bool: """Return whether if a cloud storage file exists.""" try: return bool(_provider().get(cloud_storage_file_path)) @@ -1062,7 +1135,7 @@ def exists(cloud_storage_file_path, ignore_errors=False): delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.last_updated', exception_types=_TRANSIENT_ERRORS) -def last_updated(cloud_storage_file_path): +def last_updated(cloud_storage_file_path: str) -> Optional[datetime.datetime]: """Return last updated value by parsing stats for all blobs under a cloud storage path.""" last_update = None @@ -1080,7 +1153,7 @@ def last_updated(cloud_storage_file_path): delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.read_data', exception_types=_TRANSIENT_ERRORS) -def read_data(cloud_storage_file_path): +def read_data(cloud_storage_file_path: str) -> Optional[bytes]: """Return content of a cloud storage file.""" return _provider().read_data(cloud_storage_file_path) @@ -1090,7 +1163,9 @@ def read_data(cloud_storage_file_path): delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.write_data', exception_types=_TRANSIENT_ERRORS) -def write_data(data_or_fileobj, cloud_storage_file_path, metadata=None): +def write_data(data_or_fileobj: Union[str, bytes, io.IOBase, Any], + cloud_storage_file_path: str, + metadata: Optional[dict] = None) -> bool: """Return content of a cloud storage file.""" return _provider().write_data( data_or_fileobj, cloud_storage_file_path, metadata=metadata) @@ -1101,7 +1176,9 @@ def write_data(data_or_fileobj, cloud_storage_file_path, metadata=None): delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.write_data', exception_types=_TRANSIENT_ERRORS) -def write_stream(stream, cloud_storage_file_path, metadata=None): +def write_stream(stream: Any, + cloud_storage_file_path: str, + metadata: Optional[dict] = None) -> bool: """Return content of a cloud storage file.""" return _provider().write_stream( stream, cloud_storage_file_path, metadata=metadata) @@ -1112,11 +1189,12 @@ def write_stream(stream, cloud_storage_file_path, metadata=None): delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.get_blobs', exception_types=_TRANSIENT_ERRORS) -def get_blobs(*args, **kwargs): +def get_blobs(*args: Any, **kwargs: Any) -> Iterator[dict]: yield from _provider().list_blobs(*args, **kwargs) -def get_blobs_no_retry(cloud_storage_path, recursive=True): +def get_blobs_no_retry(cloud_storage_path: str, + recursive: bool = True) -> Iterator[dict]: """Return blobs under the given cloud storage path.""" yield from _provider().list_blobs(cloud_storage_path, recursive=recursive) @@ -1126,7 +1204,8 @@ def get_blobs_no_retry(cloud_storage_path, recursive=True): delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.list_blobs', exception_types=_TRANSIENT_ERRORS) -def list_blobs(cloud_storage_path, recursive=True): +def list_blobs(cloud_storage_path: str, + recursive: bool = True) -> Iterator[str]: """Return blob names under the given cloud storage path.""" for blob in _provider().list_blobs( cloud_storage_path, recursive=recursive, names_only=True): @@ -1137,7 +1216,7 @@ def list_blobs(cloud_storage_path, recursive=True): retries=DEFAULT_FAIL_RETRIES, delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.get') -def get(cloud_storage_file_path): +def get(cloud_storage_file_path: str) -> Optional[dict]: """Get GCS object data.""" return _provider().get(cloud_storage_file_path) @@ -1147,9 +1226,11 @@ def get(cloud_storage_file_path): retries=DEFAULT_FAIL_RETRIES, delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.get_acl') -def get_acl(cloud_storage_file_path, entity): +def get_acl(cloud_storage_file_path: str, entity: str) -> Optional[dict]: """Get the access control for a file.""" client = create_discovery_storage_client() + if not client: + return None bucket, path = get_bucket_name_and_path(cloud_storage_file_path) try: @@ -1167,9 +1248,12 @@ def get_acl(cloud_storage_file_path, entity): retries=DEFAULT_FAIL_RETRIES, delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.set_acl') -def set_acl(cloud_storage_file_path, entity, role='READER'): +def set_acl(cloud_storage_file_path: str, entity: str, + role: str = 'READER') -> Optional[dict]: """Set the access control for a file.""" client = create_discovery_storage_client() + if not client: + return None bucket, path = get_bucket_name_and_path(cloud_storage_file_path) try: @@ -1185,7 +1269,7 @@ def set_acl(cloud_storage_file_path, entity, role='READER'): raise -def get_object_size(cloud_storage_file_path): +def get_object_size(cloud_storage_file_path: str) -> Optional[int]: """Get the metadata for a file.""" gcs_object = get(cloud_storage_file_path) if not gcs_object: @@ -1195,7 +1279,7 @@ def get_object_size(cloud_storage_file_path): @memoize.wrap(memoize.FifoInMemory(1)) -def blobs_bucket(): +def blobs_bucket() -> str: """Get the blobs bucket name.""" # Allow tests to override blobs bucket name safely. test_blobs_bucket = environment.get_value('TEST_BLOBS_BUCKET') @@ -1207,13 +1291,13 @@ def blobs_bucket(): @memoize.wrap(memoize.FifoInMemory(1)) -def use_async_http(): - enabled = local_config.ProjectConfig().get('async_http.enabled', False) +def use_async_http() -> bool: + enabled: bool = local_config.ProjectConfig().get('async_http.enabled', False) logs.info(f'Using async HTTP: {enabled}.') return enabled -def uworker_input_bucket(): +def uworker_input_bucket() -> Optional[str]: """Returns the bucket where uworker input is done.""" test_uworker_input_bucket = environment.get_value('TEST_UWORKER_INPUT_BUCKET') if test_uworker_input_bucket: @@ -1221,13 +1305,13 @@ def uworker_input_bucket(): assert not environment.get_value('PY_UNITTESTS') # TODO(metzman): Use local config. - bucket = environment.get_value('UWORKER_INPUT_BUCKET') + bucket: Optional[str] = environment.get_value('UWORKER_INPUT_BUCKET') if not bucket: logs.error('UWORKER_INPUT_BUCKET is not defined.') return bucket -def uworker_output_bucket(): +def uworker_output_bucket() -> Optional[str]: """Returns the bucket where uworker I/O is done.""" test_uworker_output_bucket = environment.get_value( 'TEST_UWORKER_OUTPUT_BUCKET') @@ -1236,21 +1320,24 @@ def uworker_output_bucket(): assert not environment.get_value('PY_UNITTESTS') # TODO(metzman): Use local config. - bucket = environment.get_value('UWORKER_OUTPUT_BUCKET') + bucket: Optional[str] = environment.get_value('UWORKER_OUTPUT_BUCKET') if not bucket: logs.error('UWORKER_OUTPUT_BUCKET is not defined.') return bucket -def _integration_test_env_doesnt_support_signed_urls(): - return environment.get_value('UTASK_TESTS') or environment.get_value( - 'UNTRUSTED_RUNNER_TESTS') +def _integration_test_env_doesnt_support_signed_urls() -> bool: + return (environment.get_value('UTASK_TESTS') or + environment.get_value('UNTRUSTED_RUNNER_TESTS')) class ExpiredSignedUrlError(errors.Error): """Expired Signed URL.""" - def __init__(self, message, url=None, response_text=None): + def __init__(self, + message: str, + url: Optional[str] = None, + response_text: Optional[str] = None) -> None: super().__init__(message) self.url = url self.response_text = response_text @@ -1262,7 +1349,7 @@ def __init__(self, message, url=None, response_text=None): delay=1, function='google_cloud_utils.storage._download_url', exception_types=_TRANSIENT_ERRORS) -def _download_url(url): +def _download_url(url: str) -> Optional[bytes]: """Downloads |url| and returns the contents.""" if _integration_test_env_doesnt_support_signed_urls(): return read_data(url) @@ -1270,7 +1357,7 @@ def _download_url(url): if not response.ok: try: element_tree = ET.fromstring(response.text) - error = element_tree.find('Code').text + error = element_tree.find('Code').text # type: ignore if error == 'ExpiredToken': raise ExpiredSignedUrlError('Expired token for signed URL.', url, response.text) @@ -1287,49 +1374,53 @@ def _download_url(url): retries=DEFAULT_FAIL_RETRIES, delay=DEFAULT_FAIL_WAIT, function='google_cloud_utils.storage.upload_signed_url') -def upload_signed_url(data_or_fileobj, url: str) -> bool: +def upload_signed_url(data_or_fileobj: Union[str, bytes, io.IOBase, Any], + url: str) -> bool: """Uploads data to the |signed_url|.""" return _provider().upload_signed_url(str_to_bytes(data_or_fileobj), url) -def download_signed_url(url): +def download_signed_url(url: str) -> Optional[bytes]: """Returns contents of |url|.""" return _provider().download_signed_url(url) -def str_to_bytes(data): +def str_to_bytes(data: Union[str, bytes, Any]) -> Any: if not isinstance(data, str): return data return bytes(data, 'utf-8') -def download_signed_url_to_filepath(url, filepath: str): +def download_signed_url_to_filepath(url: str, filepath: str) -> str: os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, 'wb') as fp: download_signed_url_to_file(url, fp) return filepath -def download_signed_url_to_file(url, file: io.IOBase): +def download_signed_url_to_file(url: str, file: io.IOBase) -> None: contents = download_signed_url(url) file.write(contents) -def get_signed_upload_url(remote_path, minutes=SIGNED_URL_EXPIRATION_MINUTES): +def get_signed_upload_url(remote_path: str, + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Returns a signed upload URL for |remote_path|. Does not download the contents.""" provider = _provider() return provider.sign_upload_url(remote_path, minutes=minutes) -def get_signed_download_url(remote_path, minutes=SIGNED_URL_EXPIRATION_MINUTES): +def get_signed_download_url( + remote_path: str, minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: """Returns a signed download URL for |remote_path|. Does not download the contents.""" provider = _provider() return provider.sign_download_url(remote_path, minutes=minutes) -def _error_tolerant_download_signed_url_to_file(url_and_path) -> bool: +def _error_tolerant_download_signed_url_to_file( + url_and_path: Tuple[str, str]) -> bool: url, path = url_and_path try: download_signed_url_to_filepath(url, path) @@ -1338,7 +1429,7 @@ def _error_tolerant_download_signed_url_to_file(url_and_path) -> bool: return False -def _error_tolerant_upload_signed_url(url_and_path) -> bool: +def _error_tolerant_upload_signed_url(url_and_path: Tuple[str, str]) -> bool: url, path = url_and_path try: with open(path, 'rb') as fp: @@ -1349,19 +1440,19 @@ def _error_tolerant_upload_signed_url(url_and_path) -> bool: return False -def delete_signed_url(url: str): +def delete_signed_url(url: str) -> None: """Makes a DELETE HTTP request to |url|.""" _provider().delete_signed_url(url) -def _error_tolerant_delete_signed_url(url: str): +def _error_tolerant_delete_signed_url(url: str) -> None: try: delete_signed_url(url) except Exception: logs.warning(f'Failed to delete: {url}') -def _should_use_threads(urls): +def _should_use_threads(urls: List[str]) -> bool: """Returns True if threads should be used for operations on |urls|.""" if environment.get_value('USE_THREADED_STORAGE_OPS'): if urls: @@ -1388,7 +1479,8 @@ def upload_signed_urls(signed_urls: List[str], files: List[str]) -> List[bool]: return result -def sign_delete_url(remote_path, minutes=SIGNED_URL_EXPIRATION_MINUTES): +def sign_delete_url(remote_path: str, + minutes: int = SIGNED_URL_EXPIRATION_MINUTES) -> str: return _provider().sign_delete_url(remote_path, minutes) @@ -1409,7 +1501,8 @@ def download_signed_urls(signed_urls: List[str], urls_and_filepaths = list(zip(signed_urls, filepaths)) - def synchronous_download_urls(urls_and_filepaths): + def synchronous_download_urls( + urls_and_filepaths: List[Tuple[str, str]]) -> List[bool]: use_threads = _should_use_threads(signed_urls) if use_threads: # TODO(paulovlb): Remove this once fixed. @@ -1432,7 +1525,7 @@ def synchronous_download_urls(urls_and_filepaths): return download_results -def delete_signed_urls(urls): +def delete_signed_urls(urls: List[str]) -> None: """Deletes signed URLs.""" if not urls: return @@ -1458,24 +1551,26 @@ def _sign_urls_for_existing_file( return (download_url, delete_url) -def _mappable_sign_urls_for_existing_file(url_and_include_delete_urls): +def _mappable_sign_urls_for_existing_file( + url_and_include_delete_urls: Tuple[str, bool]) -> Tuple[str, str]: url, include_delete_urls = url_and_include_delete_urls - return _sign_urls_for_existing_file(url, include_delete_urls) + return _sign_urls_for_existing_file((url, include_delete_urls)) -def sign_urls_for_existing_files(urls, include_delete_urls): +def sign_urls_for_existing_files( + urls: Any, include_delete_urls: bool) -> Iterator[Tuple[str, str]]: logs.info('Signing URLs for existing files.') args = ((url, include_delete_urls) for url in urls) yield from parallel_map(_sign_urls_for_existing_file, args) logs.info('Done signing URLs for existing files.') -def get_arbitrary_signed_upload_url(remote_directory): +def get_arbitrary_signed_upload_url(remote_directory: str) -> str: return list( get_arbitrary_signed_upload_urls(remote_directory, num_uploads=1))[0] -def parallel_map(func, argument_list): +def parallel_map(func: Callable[..., Any], argument_list: Any) -> Iterator[Any]: """Wrapper around pool.map so we don't do it on OSS-Fuzz hosts which will OOM.""" max_pool_size = 2 @@ -1495,7 +1590,8 @@ def parallel_map(func, argument_list): yield finished_call.result(timeout=timeout) -def get_arbitrary_signed_upload_urls(remote_directory: str, num_uploads: int): +def get_arbitrary_signed_upload_urls(remote_directory: str, + num_uploads: int) -> Iterator[str]: """Returns |num_uploads| number of signed upload URLs to upload files with unique arbitrary names to remote_directory.""" # We don't verify there are no collisions for uuid4s because it's extremely diff --git a/src/clusterfuzz/_internal/issue_management/google_issue_tracker/__init__.py b/src/clusterfuzz/_internal/issue_management/google_issue_tracker/__init__.py index c752b15e4c2..9bfa4671372 100644 --- a/src/clusterfuzz/_internal/issue_management/google_issue_tracker/__init__.py +++ b/src/clusterfuzz/_internal/issue_management/google_issue_tracker/__init__.py @@ -13,11 +13,17 @@ # limitations under the License. """Google issue tracker.""" +from typing import Any +from typing import Optional + # pylint: disable=line-too-long from clusterfuzz._internal.issue_management.google_issue_tracker.issue_tracker import \ IssueTracker -def get_issue_tracker(project, config, issue_tracker_client=None): +def get_issue_tracker(project: str, + config: Any, + issue_tracker_client: Optional[Any] = None + ) -> IssueTracker: """Gets an IssueTracker for the project.""" return IssueTracker(project, issue_tracker_client, config) diff --git a/src/clusterfuzz/_internal/issue_management/google_issue_tracker/client.py b/src/clusterfuzz/_internal/issue_management/google_issue_tracker/client.py index da8353573bd..ca6e9264599 100644 --- a/src/clusterfuzz/_internal/issue_management/google_issue_tracker/client.py +++ b/src/clusterfuzz/_internal/issue_management/google_issue_tracker/client.py @@ -13,6 +13,9 @@ # limitations under the License. """Gets a Google Issue Tracker HTTP client.""" +from typing import Any +from typing import cast + import google.auth import google_auth_httplib2 from googleapiclient import discovery @@ -21,26 +24,27 @@ from clusterfuzz._internal.base import retry -_ROLE_ACCOUNT = "cluster-fuzz@appspot.gserviceaccount.com" +_ROLE_ACCOUNT: str = 'cluster-fuzz@appspot.gserviceaccount.com' -_DISCOVERY_URL = ('https://issuetracker.googleapis.com/$discovery/rest?' - 'version=v1&labels=GOOGLE_PUBLIC') -_SCOPE = 'https://www.googleapis.com/auth/buganizer' -_REQUEST_TIMEOUT = 60 +_DISCOVERY_URL: str = ('https://issuetracker.googleapis.com/$discovery/rest?' + 'version=v1&labels=GOOGLE_PUBLIC') +_SCOPE: str = 'https://www.googleapis.com/auth/buganizer' +_REQUEST_TIMEOUT: int = 60 HttpError = errors.HttpError UnknownApiNameOrVersion = errors.UnknownApiNameOrVersion -def user(): +def user() -> str: return _ROLE_ACCOUNT -def build_http(): +def build_http() -> google_auth_httplib2.AuthorizedHttp: """Builds a httplib2.Http.""" creds, _ = google.auth.default() - if creds.requires_scopes: - creds = creds.with_scopes([_SCOPE]) + creds_any = cast(Any, creds) + if creds_any.requires_scopes: + creds = creds_any.with_scopes([_SCOPE]) return google_auth_httplib2.AuthorizedHttp( creds, http=httplib2.Http(timeout=_REQUEST_TIMEOUT)) @@ -51,7 +55,10 @@ def build_http(): exception_types=[UnknownApiNameOrVersion], function='issue_issue_management.google_issue_tracker,client.' '_call_discovery') -def _call_discovery(api, http): +def _call_discovery( + api: str, + http: httplib2.Http | google_auth_httplib2.AuthorizedHttp, +) -> discovery.Resource: """Calls the discovery service. Retries upto twice if there are any UnknownApiNameOrVersion errors. @@ -64,7 +71,10 @@ def _call_discovery(api, http): static_discovery=False) -def build(api='issuetracker', http=None): +def build( + api: str = 'issuetracker', + http: httplib2.Http | google_auth_httplib2.AuthorizedHttp | None = None, +) -> discovery.Resource: """Builds a google api client for buganizer.""" if not http: http = build_http() diff --git a/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py b/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py index 9827c84cf2d..00e298d5ae4 100644 --- a/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py +++ b/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py @@ -19,33 +19,47 @@ import enum import functools import re +from typing import Any +from typing import Callable +from typing import Dict +from typing import Iterable +from typing import Iterator from typing import List from typing import Optional from typing import Sequence +from typing import Set from typing import Tuple +from typing import Union import urllib.parse from google.auth import exceptions import googleapiclient.http -from clusterfuzz._internal.issue_management import issue_tracker from clusterfuzz._internal.issue_management.google_issue_tracker import client +from clusterfuzz._internal.issue_management.issue_tracker import \ + Action as BaseAction +from clusterfuzz._internal.issue_management.issue_tracker import \ + Issue as BaseIssue +from clusterfuzz._internal.issue_management.issue_tracker import \ + IssueTracker as BaseIssueTracker +from clusterfuzz._internal.issue_management.issue_tracker import ChangeList +from clusterfuzz._internal.issue_management.issue_tracker import LabelStore from clusterfuzz._internal.metrics import logs -_NUM_RETRIES = 3 +_NUM_RETRIES: int = 3 # TODO: Make these configuration settings instead of hardcoded values in code. # These custom fields use repeated enums. -_CHROMIUM_OS_CUSTOM_FIELD_ID = '1223084' -_CHROMIUM_COMPONENT_TAGS_CUSTOM_FIELD_ID = '1222907' +_CHROMIUM_OS_CUSTOM_FIELD_ID: str = '1223084' +_CHROMIUM_COMPONENT_TAGS_CUSTOM_FIELD_ID: str = '1222907' -_OSS_FUZZ_REPORTED_CUSTOM_FIELD_ID = '1349561' -_OSS_FUZZ_PROJECT_CUSTOM_FIELD_ID = '1349507' +_OSS_FUZZ_REPORTED_CUSTOM_FIELD_ID: str = '1349561' +_OSS_FUZZ_PROJECT_CUSTOM_FIELD_ID: str = '1349507' -_SEVERITY_LABEL_PREFIX = 'Security_Severity-' +_SEVERITY_LABEL_PREFIX: str = 'Security_Severity-' -_DEFAULT_SEVERITY = 'S4' -_DEFAULT_PRIORITY = 'P4' +_DEFAULT_SEVERITY: str = 'S4' +_DEFAULT_PRIORITY: str = 'P4' class IssueAccessLevel(str, enum.Enum): @@ -55,7 +69,8 @@ class IssueAccessLevel(str, enum.Enum): LIMIT_VIEW_TRUSTED = 'LIMIT_VIEW_TRUSTED' -def _get_access_limit_from_labels(labels: issue_tracker.LabelStore): +def _get_access_limit_from_labels(labels: LabelStore | List[str], + ) -> Optional[IssueAccessLevel]: """ Extracts the access limit label from labels and returns the corresponding IssueAccessLevel or None.""" limit_view_label = _extract_label(labels, 'LIMIT_') @@ -69,12 +84,13 @@ def _get_access_limit_from_labels(labels: issue_tracker.LabelStore): return None -def retry_on_invalid_gaia_accounts(func): +def retry_on_invalid_gaia_accounts(func: Callable[..., Any], + ) -> Callable[..., Any]: """Decorator to retry saving an issue, skipping CCs without a GAIA account.""" @functools.wraps(func) - def wrapper(self, *args, **kwargs): + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: try: return func(self, *args, **kwargs) except Exception as e: @@ -99,8 +115,7 @@ class IssueTrackerPermissionError(IssueTrackerError): """Permission error.""" -def _extract_all_labels(labels: issue_tracker.LabelStore, - prefix: str) -> List[str]: +def _extract_all_labels(labels: LabelStore, prefix: str) -> List[str]: """Extract all label values.""" results = [] labels_to_remove = [] @@ -132,7 +147,8 @@ def _sanitize_oses(oses: Sequence[str]) -> List[str]: return result -def _extract_label(labels: Sequence[str], prefix: str) -> Optional[str]: +def _extract_label(labels: LabelStore | List[str], + prefix: str) -> Optional[str]: """Extract a label value.""" for label in labels: if not label.startswith(prefix): @@ -143,7 +159,7 @@ def _extract_label(labels: Sequence[str], prefix: str) -> Optional[str]: return None -def _get_labels(labels: Sequence[str], prefix: str) -> List[str]: +def _get_labels(labels: Iterable[str], prefix: str) -> List[str]: """Return the values of all labels with the given prefix.""" results = [] for label in labels: @@ -153,7 +169,8 @@ def _get_labels(labels: Sequence[str], prefix: str) -> List[str]: return results -def _get_oss_fuzz_reported_value(labels: Sequence[str]) -> Optional[dict]: +def _get_oss_fuzz_reported_value(labels: LabelStore | List[str], + ) -> Optional[Dict[str, int]]: """Return OSS-Fuzz Reported custom field value.""" added_reported = _extract_label(labels, 'Reported-') if not added_reported: @@ -172,7 +189,7 @@ def _get_oss_fuzz_reported_value(labels: Sequence[str]) -> Optional[dict]: } -def _get_severity_from_labels(labels: Sequence[str]) -> Optional[str]: +def _get_severity_from_labels(labels: Iterable[str]) -> Optional[str]: """Return the value of the first severity label, if any.""" # Ignore case to match `issue_tracker.LabelStore.remove_by_prefix()`. values = _get_labels((l.lower() for l in labels), @@ -192,7 +209,7 @@ def _get_severity_from_labels(labels: Sequence[str]) -> Optional[str]: return severity -def _get_severity_from_label_value(value): +def _get_severity_from_label_value(value: str) -> str: """Convert a severity label value into a Google issue tracker severity.""" value = value.lower() if value == 'critical': @@ -213,35 +230,36 @@ def _parse_date_label(date_value: str) -> Tuple[int, int, int]: return int(year), int(month), int(day) -class Issue(issue_tracker.Issue): +class Issue(BaseIssue): """Issue tracker issue.""" - def __init__(self, data, is_new, tracker): - self._data = data - self._is_new = is_new - self._issue_tracker = tracker + def __init__(self, data: Dict[str, Any], is_new: bool, + tracker: 'IssueTracker') -> None: + self._data: Dict[str, Any] = data + self._is_new: bool = is_new + self._issue_tracker: 'IssueTracker' = tracker ccs = data['issueState'].get('ccs', []) - self._ccs = issue_tracker.LabelStore( + self._ccs: LabelStore = LabelStore( [user['emailAddress'] for user in ccs if 'emailAddress' in user]) collaborators = data['issueState'].get('collaborators', []) - self._collaborators = issue_tracker.LabelStore([ + self._collaborators: LabelStore = LabelStore([ user['emailAddress'] for user in collaborators if 'emailAddress' in user ]) labels = [ str(hotlist_id) for hotlist_id in data['issueState'].get('hotlistIds', []) ] - self._labels = issue_tracker.LabelStore(labels) + self._labels: LabelStore = LabelStore(labels) component_tags = self._get_component_tags() - self._component_tags = issue_tracker.LabelStore(component_tags) + self._component_tags: LabelStore = LabelStore(component_tags) - self._body = None - self._changed = set() - self._issue_access_limit = IssueAccessLevel.LIMIT_NONE + self._body: Optional[str] = None + self._changed: Set[str] = set() + self._issue_access_limit: IssueAccessLevel = IssueAccessLevel.LIMIT_NONE @property - def is_unrestricted(self): + def is_unrestricted(self) -> bool: """Whether the issue has no view restrictions (i.e. is public). Checks the accessLimit field from the Buganizer API response. An issue @@ -251,7 +269,7 @@ def is_unrestricted(self): {}).get('accessLevel') return access_limit == IssueAccessLevel.LIMIT_NONE - def _get_component_tags(self): + def _get_component_tags(self) -> List[str]: """Returns the value of the Component Tags custom field.""" custom_fields = self._data['issueState'].get('customFields', []) for cf in custom_fields: @@ -261,7 +279,7 @@ def _get_component_tags(self): return enum_values.get('values') or [] return [] - def _get_component_paths(self, component_tags): + def _get_component_paths(self, component_tags: Iterable[str]) -> List[str]: """Converts component IDs from component tags into component paths. Eg: component_id=1456567 will be translated into @@ -281,7 +299,8 @@ def _get_component_paths(self, component_tags): component_paths.add(ct) return sorted(component_paths) - def _filter_custom_field_enum_values(self, custom_field_id, values): + def _filter_custom_field_enum_values(self, custom_field_id: str, + values: Sequence[str]) -> List[str]: """Filters out invalid enum values from the provided values.""" filtered_values = [] for cf in self._data.get('customFields', []): @@ -296,7 +315,7 @@ def _filter_custom_field_enum_values(self, custom_field_id, values): break return filtered_values - def _filter_labels(self): + def _filter_labels(self) -> None: """Filters out and logs labels that are not hotlist IDs.""" logs.info( 'google_issue_tracker: Labels before filtering: %s' % list(self.labels)) @@ -311,7 +330,7 @@ def _filter_labels(self): logs.info( 'google_issue_tracker: Labels after filtering: %s' % list(self.labels)) - def _reset_tracking(self): + def _reset_tracking(self) -> None: """Resets diff tracking.""" self._changed.clear() self._ccs.reset_tracking() @@ -319,7 +338,7 @@ def _reset_tracking(self): self._labels.reset_tracking() self._component_tags.reset_tracking() - def apply_extension_fields(self, extension_fields): + def apply_extension_fields(self, extension_fields: Dict[str, Any]) -> None: """Applies _ext_ prefixed extension fields.""" if extension_fields.get('_ext_collaborators'): logs.info('google_issue_tracker: In apply_extension_fields for ' @@ -335,27 +354,27 @@ def apply_extension_fields(self, extension_fields): self._issue_access_limit = extension_fields['_ext_issue_access_limit'] @property - def issue_tracker(self): + def issue_tracker(self) -> 'IssueTracker': """The issue tracker for this issue.""" return self._issue_tracker @property - def id(self): + def id(self) -> int: """The issue identifier.""" return int(self._data['issueId']) @property - def title(self): + def title(self) -> Optional[str]: """The issue title.""" return self._data['issueState'].get('title') @title.setter - def title(self, new_title): + def title(self, new_title: Optional[str]) -> None: self._changed.add('title') self._data['issueState']['title'] = new_title @property - def reporter(self): + def reporter(self) -> Optional[str]: """The issue reporter.""" reporter = self._data['issueState'].get('reporter') if not reporter: @@ -363,17 +382,17 @@ def reporter(self): return reporter['emailAddress'] @reporter.setter - def reporter(self, new_reporter): + def reporter(self, new_reporter: Optional[str]) -> None: self._changed.add('reporter') self._data['issueState']['reporter'] = _make_user(new_reporter) @property - def merged_into(self): + def merged_into(self) -> Optional[Union[int, str]]: """The issue that this is merged into.""" return self._data['issueState'].get('canonicalIssueId') @property - def closed_time(self): + def closed_time(self) -> Optional[datetime.datetime]: """When the issue was closed.""" resolved_time = self._data.get('resolvedTime') if not resolved_time: @@ -381,22 +400,22 @@ def closed_time(self): return _parse_datetime(resolved_time) @property - def is_open(self): + def is_open(self) -> bool: """Whether the issue is open.""" return self.status in ['NEW', 'ASSIGNED', 'ACCEPTED'] @property - def status(self): + def status(self) -> Optional[str]: """The issue status.""" return self._data['issueState']['status'] @status.setter - def status(self, new_status): + def status(self, new_status: Optional[str]) -> None: self._changed.add('status') self._data['issueState']['status'] = new_status @property - def body(self): + def body(self) -> Optional[str]: """The issue body.""" if self._body is not None: return self._body @@ -419,11 +438,11 @@ def body(self): return self._body @body.setter - def body(self, new_body): + def body(self, new_body: Optional[str]) -> None: self._body = new_body @property - def assignee(self): + def assignee(self) -> Optional[str]: """The issue assignee.""" assignee = self._data['issueState'].get('assignee') if not assignee: @@ -431,43 +450,43 @@ def assignee(self): return assignee['emailAddress'] @assignee.setter - def assignee(self, new_assignee): + def assignee(self, new_assignee: Optional[str]) -> None: self._changed.add('assignee') self._data['issueState']['assignee'] = _make_user(new_assignee) @property - def created_time(self): + def created_time(self) -> Optional[str]: """The time at which this issue was created.""" return self._data['createdTime'] @property - def ccs(self): + def ccs(self) -> LabelStore: """The issue CC list.""" return self._ccs @property - def labels(self): + def labels(self) -> LabelStore: """The issue labels list.""" return self._labels @property - def component_id(self): + def component_id(self) -> Optional[Union[int, str]]: """The issue's component ID.""" return self._data['issueState']['componentId'] @component_id.setter - def component_id(self, component_id): + def component_id(self, component_id: Union[int, str]) -> None: """Setter for component_id.""" self._changed.add('component_id') self._data['issueState']['componentId'] = component_id @property - def components(self): + def components(self) -> LabelStore: """The issue's component tags.""" return self._component_tags @property - def actions(self): + def actions(self) -> Iterator['Action']: """Gets the issue actions.""" page_token = None while True: @@ -494,7 +513,7 @@ def _os_custom_field_values(self) -> List[str]: return [] @property - def _foundin_versions(self): + def _foundin_versions(self) -> List[str]: """FoundIn versions.""" foundin_versions = self._data['issueState'].get('foundInVersions') if not foundin_versions: @@ -502,7 +521,7 @@ def _foundin_versions(self): return foundin_versions @property - def _verifier(self): + def _verifier(self) -> Optional[str]: """The issue verifier.""" verifier = self._data['issueState'].get('verifier') if not verifier: @@ -510,17 +529,19 @@ def _verifier(self): return verifier['emailAddress'] @_verifier.setter - def _verifier(self, new_verifier): + def _verifier(self, new_verifier: Optional[str]) -> None: self._changed.add('_verifier') self._data['issueState']['verifier'] = _make_user(new_verifier) - def _add_update_single(self, - update_body, - added, - removed, - field_name, - api_field_name, - modifier=None): + def _add_update_single( + self, + update_body: Dict[str, Any], + added: List[str], + removed: List[str], + field_name: str, + api_field_name: str, + modifier: Optional[Callable[[Any], Any]] = None, + ) -> None: """Prepares a single field update.""" if field_name not in self._changed: return @@ -533,16 +554,18 @@ def _add_update_single(self, else: removed.append(api_field_name) - def _set_severity(self, severity: str): + def _set_severity(self, severity: str) -> None: self._data['issueState']['severity'] = severity - def _add_update_collection(self, - update_body, - added, - removed, - field_name, - api_field_name, - modifier=None): + def _add_update_collection( + self, + update_body: Dict[str, Any], + added: List[str], + removed: List[str], + field_name: str, + api_field_name: str, + modifier: Optional[Callable[[Any], Any]] = None, + ) -> None: """Prepares a collection field update.""" collection = getattr(self, field_name) collection_added = list(collection.added) @@ -558,9 +581,11 @@ def _add_update_collection(self, collection_removed = modifier(collection_removed) update_body['remove'][api_field_name] = collection_removed - def _update_issue(self, new_comment=None, notify=True): + def _update_issue(self, + new_comment: Optional[str] = None, + notify: bool = True) -> Dict[str, Any]: """Updates an existing issue.""" - update_body = { + update_body: Dict[str, Any] = { 'add': {}, 'addMask': '', 'remove': {}, @@ -574,8 +599,8 @@ def _update_issue(self, new_comment=None, notify=True): if not self.assignee: self.assignee = client.user() # Add updates. - added = [] - removed = [] + added: List[str] = [] + removed: List[str] = [] self._add_update_single(update_body, added, removed, 'status', 'status') self._add_update_single(update_body, added, removed, 'assignee', 'assignee', _make_user) @@ -593,7 +618,7 @@ def _update_issue(self, new_comment=None, notify=True): # Custom fields are modified by providing the complete value of the # customFieldId. - custom_field_entries = [] + custom_field_entries: List[Dict[str, Any]] = [] # Special case OS custom field. added_oses = _get_labels(self.labels.added, 'OS-') @@ -728,8 +753,14 @@ def _update_issue(self, new_comment=None, notify=True): issueId=str(self.id))) return result + # pylint: disable=dangerous-default-value @retry_on_invalid_gaia_accounts - def save(self, new_comment=None, notify=True, skip_emails=[]): # pylint: disable=dangerous-default-value + def save( + self, + new_comment: Optional[str] = None, + notify: bool = True, + skip_emails: Sequence[str] = [], + ) -> None: """Saves the issue.""" logs.info(f'Skipping supposed non gaia emails emails: {skip_emails}.') if self._is_new: @@ -739,7 +770,7 @@ def save(self, new_comment=None, notify=True, skip_emails=[]): # pylint: disabl self._data['issueState']['type'] = issue_type self._data['issueState']['priority'] = priority - custom_field_entries = [] + custom_field_entries: List[Dict[str, Any]] = [] oses = _sanitize_oses(_extract_all_labels(self.labels, 'OS-')) if oses: oses.sort() @@ -838,12 +869,12 @@ def save(self, new_comment=None, notify=True, skip_emails=[]): # pylint: disabl logs.info('google_issue_tracker: self._data: %s' % self._data) -class Action(issue_tracker.Action): +class Action(BaseAction): """Issue tracker action.""" # FieldUpdates give us the integer value for statuses, so we need to keep this # mapping. - INT_TO_STATUS = { + INT_TO_STATUS: Dict[Any, str] = { 1: 'NEW', 2: 'ASSIGNED', 3: 'ACCEPTED', @@ -856,10 +887,10 @@ class Action(issue_tracker.Action): 10: 'DUPLICATE', } - def __init__(self, data): - self._data = data + def __init__(self, data: Dict[str, Any]) -> None: + self._data: Dict[str, Any] = data - def _get_actual_value(self, value): + def _get_actual_value(self, value: Optional[Dict[str, Any]]) -> Optional[Any]: """Gets the actual value of a field update google.protobuf.Any value.""" if value is None: return None @@ -869,13 +900,14 @@ def _get_actual_value(self, value): return value['value'] raise IssueTrackerError('Unknown value type: ' + value['type']) - def _get_actual_values(self, values): + def _get_actual_values( + self, values: Optional[Sequence[Dict[str, Any]]]) -> Optional[List[Any]]: """Gets the actual values of field update values.""" if values is None: return None return [self._get_actual_value(value) for value in values] - def _get_field_update(self, field_name): + def _get_field_update(self, field_name: str) -> Optional[Dict[str, Any]]: """Gets the FieldUpdate for a field name.""" if 'fieldUpdates' not in self._data: return None @@ -885,7 +917,8 @@ def _get_field_update(self, field_name): None, ) - def _get_field_update_single(self, field_name): + def _get_field_update_single( + self, field_name: str) -> Tuple[Optional[Any], Optional[Any]]: """Gets a single field update.""" update = self._get_field_update(field_name) if not update: @@ -898,7 +931,8 @@ def _get_field_update_single(self, field_name): self._get_actual_value(single_value_update.get('newValue')), ) - def _get_field_update_changes(self, field_name): + def _get_field_update_changes( + self, field_name: str) -> Tuple[Optional[List[Any]], Optional[List[Any]]]: """Gets a collection field update.""" update = self._get_field_update(field_name) if not update: @@ -912,29 +946,29 @@ def _get_field_update_changes(self, field_name): ) @property - def author(self): + def author(self) -> Optional[str]: """The author of the action.""" return self._data['author']['emailAddress'] @property - def comment(self): + def comment(self) -> Optional[str]: """Represents a comment.""" if 'issueComment' not in self._data: return None return self._data['issueComment']['comment'] @property - def title(self): + def title(self) -> Optional[str]: """The new issue title.""" return self._get_field_update_single('title')[1] @property - def status(self): + def status(self) -> Optional[str]: """The new issue status.""" return self.INT_TO_STATUS.get(self._get_field_update_single('status')[1]) @property - def assignee(self): + def assignee(self) -> Optional[str]: """The new issue assignee.""" assignee = self._get_field_update_single('assignee')[1] if not assignee: @@ -942,10 +976,10 @@ def assignee(self): return assignee @property - def ccs(self): + def ccs(self) -> ChangeList: """The issue CC change list.""" removed, added = self._get_field_update_changes('ccs') - change_list = issue_tracker.ChangeList() + change_list = ChangeList() if added: change_list.added.extend(added) if removed: @@ -953,12 +987,12 @@ def ccs(self): return change_list @property - def labels(self): + def labels(self) -> ChangeList: """The issue labels change list.""" # We need to use the snake_case version of the field here, as that's the # string value the backend actually uses. removed, added = self._get_field_update_changes('hotlist_ids') - change_list = issue_tracker.ChangeList() + change_list = ChangeList() if added: change_list.added.extend(added) if removed: @@ -966,29 +1000,31 @@ def labels(self): return change_list @property - def components(self): + def components(self) -> ChangeList: # Component tags will be handled by _update_issue and save. - return issue_tracker.ChangeList() + return ChangeList() -class IssueTracker(issue_tracker.IssueTracker): +class IssueTracker(BaseIssueTracker): """Google issue tracker implementation.""" - def __init__(self, project, http_client, config): - self._project = project - self._client = http_client - self._default_component_id = config['default_component_id'] - self._type = config['type'] if hasattr(config, 'type') else None - self._url = config['url'] + def __init__(self, project: str, http_client: Optional[Any], + config: Any) -> None: + self._project: str = project + self._client: Optional[Any] = http_client + self._default_component_id: Union[int, str] = config['default_component_id'] + self._type: Optional[str] = config['type'] if hasattr(config, + 'type') else None + self._url: str = config['url'] @property - def client(self): + def client(self) -> Any: """HTTP Client.""" if self._client is None: self._client = client.build() return self._client - def _execute(self, request): + def _execute(self, request: Any) -> Any: """Executes a request.""" http = None for _ in range(2): @@ -1007,16 +1043,16 @@ def _execute(self, request): raise IssueTrackerError(str(e)) @property - def project(self): + def project(self) -> str: """Gets the project name of this issue tracker.""" return self._project @property - def type(self): + def type(self) -> Optional[str]: """The type of the tracker - e.g. monorail, google-issue-tracker, etc.""" return self._type - def new_issue(self): + def new_issue(self) -> Issue: """Creates an unsaved new issue.""" data = { 'issueState': { @@ -1031,7 +1067,8 @@ def new_issue(self): } return Issue(data, True, self) - def _get_relative_component_path(self, component_id): + def _get_relative_component_path( + self, component_id: Union[int, str]) -> Optional[str]: """Gets the component path relative to the default component path. For component_id=1456567 (Sparkplug) and @@ -1058,11 +1095,13 @@ def _get_relative_component_path(self, component_id): component_name = component.get('name', '') if parent_component_id == str(self._default_component_id): return component_name - return self._get_relative_component_path( - parent_component_id) + ">" + component_name + parent_path = self._get_relative_component_path(parent_component_id) + if not parent_path: + return None + return parent_path + ">" + component_name return None - def get_issue(self, issue_id): + def get_issue(self, issue_id: Union[int, str]) -> Optional[Issue]: """Gets the issue with the given ID.""" try: issue = self._execute(self.client.issues().get(issueId=str(issue_id))) @@ -1074,7 +1113,7 @@ def get_issue(self, issue_id): logs.error('Failed to retrieve issue.', issue_id=issue_id) return None - def get_description(self, issue_id): + def get_description(self, issue_id: Union[int, str]) -> Optional[str]: """Gets the content of the description for the issue with the given ID.""" try: comments = self._execute( @@ -1090,7 +1129,8 @@ def get_description(self, issue_id): logs.error('Failed to retrieve issue description.', issue_id=issue_id) return None - def get_attachment_metadata(self, issue_id): + def get_attachment_metadata( + self, issue_id: Union[int, str]) -> Optional[List[Dict[str, Any]]]: """Gets the attachment metadata of an issue with the given ID.""" try: attachment_metadata = self._execute( @@ -1105,7 +1145,7 @@ def get_attachment_metadata(self, issue_id): 'Failed to retrieve issue attachment metadata.', issue_id=issue_id) return None - def get_attachment(self, resource_name): + def get_attachment(self, resource_name: str) -> Optional[Any]: """Gets the attachment for the given resource name.""" try: http_request = self.client.media().download(resourceName=resource_name) @@ -1121,14 +1161,20 @@ def get_attachment(self, resource_name): 'Failed to retrieve attachment for resource name: %s' % resource_name) return None - def find_issues(self, keywords=None, only_open=None): + def find_issues( + self, + keywords: Optional[Union[str, List[str]]] = None, + only_open: Optional[bool] = None, + ) -> Iterator[Issue]: """Finds issues.""" return self.find_issues_with_filters(keywords=keywords, only_open=only_open) - def find_issues_with_filters(self, - keywords=None, - query_filters=None, - only_open=None): + def find_issues_with_filters( + self, + keywords: Optional[Union[str, List[str]]] = None, + query_filters: Optional[Any] = None, + only_open: Optional[bool] = None, + ) -> Iterator[Issue]: """Finds issues with additional query filters.""" page_token = None while True: @@ -1147,15 +1193,21 @@ def find_issues_with_filters(self, if not page_token: break - def find_issues_url(self, keywords=None, only_open=None): + def find_issues_url( + self, + keywords: Optional[Union[str, List[str]]] = None, + only_open: Optional[bool] = None, + ) -> str: """Finds issues (web URL).""" return self.find_issues_url_with_filters( keywords=keywords, only_open=only_open) - def find_issues_url_with_filters(self, - keywords=None, - query_filters=None, - only_open=None): + def find_issues_url_with_filters( + self, + keywords: Optional[Union[str, List[str]]] = None, + query_filters: Optional[Any] = None, + only_open: Optional[bool] = None, + ) -> str: """Finds issues (web URL) with additional query filters.""" return (self._url + '?' + urllib.parse.urlencode({ 'q': @@ -1165,33 +1217,33 @@ def find_issues_url_with_filters(self, only_open=only_open), })) - def issue_url(self, issue_id): + def issue_url(self, issue_id: Union[int, str]) -> str: """Returns the issue URL with the given ID.""" return self._url + '/' + str(issue_id) @property - def label_type(self): + def label_type(self) -> str: """Label type.""" return 'hotlist' - def label_text(self, label): + def label_text(self, label: Union[int, str]) -> str: """Text for a label (with label type).""" return 'hotlistid:' + str(label) -def _make_user(email): +def _make_user(email: Optional[str]) -> Dict[str, Optional[str]]: """Makes a User.""" return { 'emailAddress': email, } -def _make_users(emails): +def _make_users(emails: Iterable[str]) -> List[Dict[str, Optional[str]]]: """Makes Users.""" return [_make_user(email) for email in emails] -def _parse_datetime(date_string): +def _parse_datetime(date_string: str) -> datetime.datetime: """Parses a datetime.""" datetime_obj, _, microseconds_string = date_string.rstrip('Z').partition('.') datetime_obj = datetime.datetime.strptime(datetime_obj, '%Y-%m-%dT%H:%M:%S') @@ -1201,14 +1253,28 @@ def _parse_datetime(date_string): return datetime_obj -def _get_query(string_keywords, only_open, query_filters): +def _get_query( + string_keywords: Optional[Union[str, Iterable[str]]], + only_open: Optional[bool], + query_filters: Optional[Iterable[str]], +) -> str: """Gets a search query.""" - query = ' '.join('"{}"'.format(keyword) for keyword in string_keywords) + if not string_keywords: + query = '' + elif isinstance(string_keywords, str): + query = f'"{string_keywords}"' + else: + query = ' '.join('"{}"'.format(keyword) for keyword in string_keywords) + if query_filters: - query += ' ' + if query: + query += ' ' query += ' '.join('{}'.format(keyword) for keyword in query_filters) if only_open: - query += ' status:open' + if query: + query += ' status:open' + else: + query = 'status:open' return query @@ -1216,7 +1282,7 @@ def _get_query(string_keywords, only_open, query_filters): # steps to work. List of steps taken (for posterity)- # 1. gcloud iam service-accounts keys create --iam-account=${service_account} \ # --key-file-type=json /tmp/sa-key -# 2. pipenv shell +# 2. source .venv/bin/activate # 3. GOOGLE_APPLICATION_CREDENTIALS=/tmp/sa-key PYTHONPATH=$PYTHONPATH:src/ \ # python src/clusterfuzz/_internal/issue_management/google_issue_tracker/\ # issue_tracker.py diff --git a/src/clusterfuzz/_internal/issue_management/issue_filer.py b/src/clusterfuzz/_internal/issue_management/issue_filer.py index 1c093d34075..eef2d951495 100644 --- a/src/clusterfuzz/_internal/issue_management/issue_filer.py +++ b/src/clusterfuzz/_internal/issue_management/issue_filer.py @@ -15,6 +15,10 @@ import itertools import re +from typing import Any +from typing import Callable +from typing import Iterable +from typing import Sequence from clusterfuzz._internal.base import external_users from clusterfuzz._internal.base import utils @@ -26,12 +30,14 @@ from clusterfuzz._internal.google_cloud_utils import pubsub from clusterfuzz._internal.issue_management import issue_tracker_policy from clusterfuzz._internal.issue_management import oss_fuzz_github +from clusterfuzz._internal.issue_management.issue_tracker import Issue +from clusterfuzz._internal.issue_management.issue_tracker import IssueTracker from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -_MAX_STACKTRACE_LEN_FOR_PARSING = 300000 +_MAX_STACKTRACE_LEN_FOR_PARSING: int = 300000 -NON_CRASH_TYPES = [ +NON_CRASH_TYPES: list[str] = [ 'Data race', 'Direct-leak', 'Float-cast-overflow', @@ -43,7 +49,7 @@ 'Unsigned-integer-overflow', ] -MEMORY_TOOLS_LABELS = [ +MEMORY_TOOLS_LABELS: list[dict[str, str]] = [ { 'token': 'AddressSanitizer', 'label': 'Memory-AddressSanitizer' @@ -74,10 +80,12 @@ }, ] -STACKFRAME_LINE_REGEX = re.compile(r'\s*#\d+\s+0x[0-9A-Fa-f]+\s*') +STACKFRAME_LINE_REGEX: re.Pattern[str] = re.compile( + r'\s*#\d+\s+0x[0-9A-Fa-f]+\s*') -def platform_substitution(label, testcase, _): +def platform_substitution(label: str, testcase: data_types.Testcase, + _: Any) -> list[str]: """Platform substitution.""" platform = None if environment.is_chromeos_job(testcase.job_type): @@ -97,17 +105,18 @@ def platform_substitution(label, testcase, _): return [label.replace('%PLATFORM%', platform)] -def current_date(): +def current_date() -> str: """Date format.""" return utils.utcnow().date().isoformat() -def date_substitution(label, *_): +def date_substitution(label: str, *_: Any) -> list[str]: """Date substitution.""" return [label.replace('%YYYY-MM-DD%', current_date())] -def sanitizer_substitution(label, testcase, _): +def sanitizer_substitution(label: str, testcase: data_types.Testcase, + _: Any) -> list[str]: """Sanitizer substitution.""" stacktrace = data_handler.get_stacktrace(testcase) memory_tool_labels = get_memory_tool_labels(stacktrace) @@ -118,22 +127,25 @@ def sanitizer_substitution(label, testcase, _): ] -def severity_substitution(label, testcase, security_severity): +def severity_substitution(label: str, testcase: data_types.Testcase, + security_severity: int | str | None) -> list[str]: """Severity substitution.""" # Use severity from testcase if one is not available. if security_severity is None: security_severity = testcase.security_severity # Set to default high severity if we can't determine it automatically. - if not data_types.SecuritySeverity.is_valid(security_severity): - security_severity = data_types.SecuritySeverity.HIGH + if security_severity is None or not data_types.SecuritySeverity.is_valid( + security_severity): + severity_int: int | None = data_types.SecuritySeverity.HIGH + else: + severity_int = int(security_severity) - security_severity_string = severity_analyzer.severity_to_string( - security_severity) + security_severity_string = severity_analyzer.severity_to_string(severity_int) return [label.replace('%SEVERITY%', security_severity_string)] -def impact_to_string(impact): +def impact_to_string(impact: int) -> str: """Convert an impact value to a human-readable string.""" impact_map = { data_types.SecurityImpact.EXTENDED_STABLE: 'Extended', @@ -147,30 +159,34 @@ def impact_to_string(impact): return impact_map[impact] -def _get_impact_from_labels(labels): +def _get_impact_from_labels(labels: Iterable[str]) -> int: """Get the impact from the label list.""" - labels = [label.lower() for label in labels] - if 'security_impact-extended' in labels: + labels_list = [label.lower() for label in labels] + if 'security_impact-extended' in labels_list: return data_types.SecurityImpact.EXTENDED_STABLE - if 'security_impact-stable' in labels: + if 'security_impact-stable' in labels_list: return data_types.SecurityImpact.STABLE - if 'security_impact-beta' in labels: + if 'security_impact-beta' in labels_list: return data_types.SecurityImpact.BETA - if 'security_impact-head' in labels: + if 'security_impact-head' in labels_list: return data_types.SecurityImpact.HEAD - if 'security_impact-none' in labels: + if 'security_impact-none' in labels_list: return data_types.SecurityImpact.NONE return data_types.SecurityImpact.MISSING -def update_issue_impact_labels(testcase, issue, policy): +def update_issue_impact_labels( + testcase: data_types.Testcase, + issue: Issue, + policy: issue_tracker_policy.IssueTrackerPolicy, +) -> None: """Update impact labels on issue.""" if testcase.one_time_crasher_flag: return existing_impact = _get_impact_from_labels(issue.labels) - if testcase.regression.startswith('0:'): + if (testcase.regression or '').startswith('0:'): # If the regression range starts from the start of time, # then we assume that the bug impacts stable. new_impact = data_types.SecurityImpact.EXTENDED_STABLE @@ -208,7 +224,8 @@ def update_issue_impact_labels(testcase, issue, policy): impact_to_string(new_impact))) -def update_issue_foundin_labels(testcase, issue): +def update_issue_foundin_labels(testcase: data_types.Testcase, + issue: Issue) -> None: """Updates FoundIn- labels on issue.""" if not testcase.is_impact_set_flag: return @@ -226,13 +243,18 @@ def update_issue_foundin_labels(testcase, issue): issue.labels.add('FoundIn-' + found_milestone) -def apply_substitutions(policy, label, testcase, security_severity=None): +def apply_substitutions( + policy: issue_tracker_policy.IssueTrackerPolicy, + label: str | None, + testcase: data_types.Testcase, + security_severity: int | str | None = None, +) -> list[str]: """Apply label substitutions.""" if label is None: # If the label is not configured, then nothing to subsitute. return [] - label_substitutions = ( + label_substitutions: tuple[tuple[str, Callable[..., list[str]]], ...] = ( ('%PLATFORM%', platform_substitution), ('%YYYY-MM-DD%', date_substitution), ('%SANITIZER%', sanitizer_substitution), @@ -242,8 +264,8 @@ def apply_substitutions(policy, label, testcase, security_severity=None): for marker, handler in label_substitutions: if marker in label: return [ - policy.substitution_mapping(label) - for label in handler(label, testcase, security_severity) + policy.substitution_mapping(sub_label) + for sub_label in handler(label, testcase, security_severity) ] # No match found. Return mapped value if it exists else the original label @@ -251,12 +273,12 @@ def apply_substitutions(policy, label, testcase, security_severity=None): return [policy.substitution_mapping(label)] -def get_label_pattern(label): +def get_label_pattern(label: str) -> re.Pattern[str]: """Get the label pattern regex.""" return re.compile('^' + re.sub(r'%.*?%', r'(.*)', label) + '$', re.IGNORECASE) -def get_memory_tool_labels(stacktrace): +def get_memory_tool_labels(stacktrace: str) -> list[str]: """Distinguish memory tools used and return corresponding labels.""" if len(stacktrace) > _MAX_STACKTRACE_LEN_FOR_PARSING: # Some stacktraces to too huge to do this on. @@ -274,7 +296,7 @@ def get_memory_tool_labels(stacktrace): return labels -def _get_from_metadata(testcase, name): +def _get_from_metadata(testcase: data_types.Testcase, name: str) -> list[str]: """Get values from testcase metadata.""" return utils.parse_delimited( testcase.get_metadata(name, ''), @@ -283,7 +305,7 @@ def _get_from_metadata(testcase, name): remove_empty=True) -def notify_issue_update(testcase, status): +def notify_issue_update(testcase: data_types.Testcase, status: str) -> None: """Notify that an issue update occurred (i.e. issue was filed or closed).""" topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic') if not topic: @@ -311,11 +333,13 @@ def notify_issue_update(testcase, status): oss_fuzz_github.close_issue(testcase) -def file_issue(testcase, - issue_tracker, - security_severity=None, - user_email=None, - additional_ccs=None): +def file_issue( + testcase: data_types.Testcase, + issue_tracker: IssueTracker, + security_severity: int | str | None = None, + user_email: str | None = None, + additional_ccs: Sequence[str] | None = None, +) -> tuple[int | str | None, Exception | None]: """File an issue for the given test case.""" logs.info(f'Filing new issue for testcase: {testcase.key.id()}.') @@ -324,9 +348,9 @@ def file_issue(testcase, data_types.Fuzzer.name == testcase.fuzzer_name).get() is_crash = not utils.sub_string_exists_in(NON_CRASH_TYPES, - testcase.crash_type) + testcase.crash_type or '') properties = policy.get_new_issue_properties( - is_security=testcase.security_flag, is_crash=is_crash) + is_security=bool(testcase.security_flag), is_crash=is_crash) issue = issue_tracker.new_issue() if issue_tracker.project == 'google-buganizer': @@ -451,11 +475,12 @@ def file_issue(testcase, security_severity): issue.labels.add(result) - issue.body += data_handler.format_issue_information( - testcase, properties.issue_body_footer) + info = data_handler.format_issue_information(testcase, + properties.issue_body_footer) + issue.body = (issue.body or '') + info if (should_restrict_issue and has_accountable_people and policy.deadline_policy_message): - issue.body += '\n\n' + policy.deadline_policy_message + issue.body = (issue.body or '') + '\n\n' + policy.deadline_policy_message for cc in ccs: issue.ccs.add(cc) @@ -517,7 +542,7 @@ def file_issue(testcase, if policy.fallback_policy_message: message = policy.fallback_policy_message.replace( '%COMPONENTS%', ' '.join(metadata_components)) - issue.body += '\n\n' + message + issue.body = (issue.body or '') + '\n\n' + message logs.info('Secondary attempt to the save the issue.') issue.save() else: diff --git a/src/clusterfuzz/_internal/issue_management/issue_tracker.py b/src/clusterfuzz/_internal/issue_management/issue_tracker.py index b9f1bb852bb..c1492aa89fd 100644 --- a/src/clusterfuzz/_internal/issue_management/issue_tracker.py +++ b/src/clusterfuzz/_internal/issue_management/issue_tracker.py @@ -5,6 +5,7 @@ # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,34 +13,45 @@ # limitations under the License. """Issue tracker interface.""" +import datetime +from typing import Any +from typing import Dict +from typing import Iterable +from typing import Iterator +from typing import List +from typing import Optional +from typing import Pattern +from typing import Union +from typing import ValuesView + class LabelStore: """Label storage which tracks changes. Case insensitive, but preserves case.""" - def __init__(self, seq=()): - self._backing = {} - self._added = {} - self._removed = {} + def __init__(self, seq: Iterable[str] = ()): + self._backing: Dict[str, str] = {} + self._added: Dict[str, str] = {} + self._removed: Dict[str, str] = {} for item in seq: self._backing[item.lower()] = item - def __iter__(self): + def __iter__(self) -> Iterator[str]: yield from self._backing.values() - def __contains__(self, item): + def __contains__(self, item: str) -> bool: return item.lower() in self._backing @property - def added(self): + def added(self) -> ValuesView[str]: return self._added.values() @property - def removed(self): + def removed(self) -> ValuesView[str]: return self._removed.values() - def add(self, label): + def add(self, label: Optional[object]) -> None: """Add a new label.""" if not label: return @@ -53,7 +65,7 @@ def add(self, label): self._backing[key] = label - def remove(self, label): + def remove(self, label: object) -> None: """Remove a label.""" label = str(label) key = label.lower() @@ -67,40 +79,40 @@ def remove(self, label): del self._backing[key] - def reset_tracking(self): + def reset_tracking(self) -> None: """Reset tracking.""" self._added.clear() self._removed.clear() - def clear(self): + def clear(self) -> None: """Clear the store, and reset tracking.""" self._added.clear() self._removed.update(self._backing) self._backing.clear() - def get_by_prefix(self, prefix): + def get_by_prefix(self, prefix: str) -> Iterator[str]: """Get labels with the given prefix.""" for item in self: if item.lower().startswith(prefix.lower()): yield item - def has_with_prefix(self, prefix): + def has_with_prefix(self, prefix: str) -> bool: """Return whether if there is an item with the given prefix.""" return bool(next(self.get_by_prefix(prefix), None)) - def remove_by_prefix(self, prefix): + def remove_by_prefix(self, prefix: str) -> None: """Remove labels with a given prefix.""" for item in list(self): if item.lower().startswith(prefix.lower()): self.remove(item) - def get_by_pattern(self, re_pattern): + def get_by_pattern(self, re_pattern: Pattern[str]) -> Iterator[str]: """Get labels with the given pattern.""" for item in self: if re_pattern.match(item): yield item - def has_with_pattern(self, re_pattern): + def has_with_pattern(self, re_pattern: Pattern[str]) -> bool: """Return whether if there is an item with the given pattern.""" return bool(next(self.get_by_pattern(re_pattern), None)) @@ -109,106 +121,107 @@ class Issue: """Represents an issue.""" @property - def issue_tracker(self): + def issue_tracker(self) -> 'IssueTracker': """The issue tracker for this issue.""" raise NotImplementedError @property - def id(self): + def id(self) -> Optional[int]: """The issue identifier.""" raise NotImplementedError @property - def title(self): + def title(self) -> Optional[str]: """The issue title.""" raise NotImplementedError @title.setter - def title(self, new_title): + def title(self, new_title: Optional[str]) -> None: raise NotImplementedError @property - def reporter(self): + def reporter(self) -> Optional[str]: """The issue reporter.""" raise NotImplementedError @reporter.setter - def reporter(self, new_reporter): + def reporter(self, new_reporter: Optional[str]) -> None: raise NotImplementedError @property - def merged_into(self): + def merged_into(self) -> Optional[Union[int, str]]: """The issue that this is merged into.""" raise NotImplementedError @property - def closed_time(self): + def closed_time(self) -> Optional[datetime.datetime]: """When the issue was closed.""" raise NotImplementedError @property - def is_open(self): + def is_open(self) -> bool: """Whether the issue is open.""" raise NotImplementedError @property - def status(self): + def status(self) -> Optional[str]: """The issue status.""" raise NotImplementedError @status.setter - def status(self, new_status): + def status(self, new_status: Optional[str]) -> None: raise NotImplementedError @property - def body(self): + def body(self) -> Optional[str]: """The issue body.""" raise NotImplementedError @body.setter - def body(self, new_body): + def body(self, new_body: Optional[str]) -> None: raise NotImplementedError @property - def assignee(self): + def assignee(self) -> Optional[str]: """The issue assignee.""" raise NotImplementedError @assignee.setter - def assignee(self, new_assignee): + def assignee(self, new_assignee: Optional[str]) -> None: raise NotImplementedError @property - def ccs(self): + def ccs(self) -> LabelStore: """The issue CC list.""" raise NotImplementedError @property - def labels(self): + def labels(self) -> LabelStore: """The issue labels list.""" raise NotImplementedError @property - def components(self): + def components(self) -> LabelStore: """The issue component list.""" raise NotImplementedError @property - def actions(self): + def actions(self) -> Iterable['Action']: """Get the issue actions.""" raise NotImplementedError - def save(self, new_comment=None, notify=True): + def save(self, new_comment: Optional[str] = None, + notify: bool = True) -> None: """Save the issue.""" raise NotImplementedError @property - def is_unrestricted(self): + def is_unrestricted(self) -> bool: """Whether the issue has no view restrictions (i.e. is public).""" return False # pylint: disable=unused-argument - def apply_extension_fields(self, extension_fields): + def apply_extension_fields(self, extension_fields: Dict[str, Any]) -> None: """Applies _ext_ prefixed extension fields to the issue.""" return @@ -216,51 +229,51 @@ def apply_extension_fields(self, extension_fields): class ChangeList: """Records a change in a list.""" - def __init__(self): - self.added = [] - self.removed = [] + def __init__(self) -> None: + self.added: List[str] = [] + self.removed: List[str] = [] class Action: """Represents an action on an issue (e.g. a comment).""" @property - def author(self): + def author(self) -> Optional[str]: """The author of the action.""" raise NotImplementedError @property - def comment(self): + def comment(self) -> Optional[str]: """Represents a comment.""" raise NotImplementedError @property - def title(self): + def title(self) -> Optional[str]: """The new issue title.""" raise NotImplementedError @property - def status(self): + def status(self) -> Optional[str]: """The new issue status.""" raise NotImplementedError @property - def assignee(self): + def assignee(self) -> Optional[str]: """The new issue assignee.""" raise NotImplementedError @property - def ccs(self): + def ccs(self) -> ChangeList: """The issue CC change list.""" raise NotImplementedError @property - def labels(self): + def labels(self) -> ChangeList: """The issue labels change list.""" raise NotImplementedError @property - def components(self): + def components(self) -> ChangeList: """The issue component change list.""" raise NotImplementedError @@ -269,33 +282,33 @@ class IssueTracker: """Issue tracker interface.""" @property - def project(self): + def project(self) -> str: """Get the project name of this issue tracker.""" raise NotImplementedError @property - def label_type(self): + def label_type(self) -> str: """Label type.""" return 'label' # default - def label_text(self, label): + def label_text(self, label: str) -> str: """Text for a label (with label type).""" return label + ' ' + self.label_type - def new_issue(self): + def new_issue(self) -> Issue: """Create an unsaved new issue.""" raise NotImplementedError - def get_issue(self, issue_id): + def get_issue(self, issue_id: Union[int, str]) -> Optional[Issue]: """Get the issue with the given ID.""" raise NotImplementedError - def get_original_issue(self, issue_id): + def get_original_issue(self, issue_id: Union[int, str]) -> Optional[Issue]: """Retrieve the original issue object traversing the list of duplicates.""" # Caller might pass |issue_id| as int, so change it to str so that # circular chain checks in loop actually work. original_issue_id = str(issue_id) - seen_issue_ids = [] + seen_issue_ids: List[str] = [] while True: original_issue = self.get_issue(original_issue_id) if not original_issue: @@ -314,28 +327,35 @@ def get_original_issue(self, issue_id): return original_issue - def find_issues(self, keywords=None, only_open=None): + def find_issues( + self, + keywords: Optional[Union[str, List[str]]] = None, + only_open: Optional[bool] = None) -> Optional[Iterable[Issue]]: """Find issues.""" raise NotImplementedError - def find_issues_with_filters(self, - keywords=None, - query_filters=None, - only_open=None): + def find_issues_with_filters( + self, + keywords: Optional[Union[str, List[str]]] = None, + query_filters: Optional[Any] = None, + only_open: Optional[bool] = None) -> Optional[Iterable[Issue]]: """Find issues.""" raise NotImplementedError - def find_issues_url(self, keywords=None, only_open=None): + def find_issues_url(self, + keywords: Optional[Union[str, List[str]]] = None, + only_open: Optional[bool] = None) -> Optional[str]: """Find issues (web URL).""" raise NotImplementedError - def find_issues_url_with_filters(self, - keywords=None, - query_filters=None, - only_open=None): + def find_issues_url_with_filters( + self, + keywords: Optional[Union[str, List[str]]] = None, + query_filters: Optional[Any] = None, + only_open: Optional[bool] = None) -> Optional[str]: """Find issues.""" raise NotImplementedError - def issue_url(self, issue_id): + def issue_url(self, issue_id: Union[int, str]) -> Optional[str]: """Return the issue URL with the given ID.""" raise NotImplementedError diff --git a/src/clusterfuzz/_internal/issue_management/issue_tracker_policy.py b/src/clusterfuzz/_internal/issue_management/issue_tracker_policy.py index 0210b4c929a..28f9f9bdc53 100644 --- a/src/clusterfuzz/_internal/issue_management/issue_tracker_policy.py +++ b/src/clusterfuzz/_internal/issue_management/issue_tracker_policy.py @@ -5,6 +5,7 @@ # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,14 +13,26 @@ # limitations under the License. """Issue tracker policy.""" -from collections import namedtuple +from typing import Any +from typing import Dict +from typing import List +from typing import Mapping +from typing import NamedTuple +from typing import Optional +from typing import Sequence from clusterfuzz._internal.config import local_config -Status = namedtuple('Status', - ['assigned', 'duplicate', 'wontfix', 'fixed', 'verified']) -EXPECTED_STATUSES = [ +class Status(NamedTuple): + assigned: str + duplicate: str + wontfix: str + fixed: str + verified: str + + +EXPECTED_STATUSES: List[str] = [ 'assigned', 'duplicate', 'wontfix', @@ -28,7 +41,7 @@ 'new', ] -EXTENSION_PREFIX = '_ext_' +EXTENSION_PREFIX: str = '_ext_' class ConfigurationError(Exception): @@ -38,17 +51,17 @@ class ConfigurationError(Exception): class NewIssuePolicy: """New issue policy.""" - def __init__(self): - self.status = '' - self.ccs = [] - self.labels = [] - self.issue_body_footer = '' + def __init__(self) -> None: + self.status: str = '' + self.ccs: List[str] = [] + self.labels: List[str] = [] + self.issue_body_footer: str = '' # Contains _ext_ prefixed extension fields. Eg: _ext_collaborators. - self.extension_fields = {} + self.extension_fields: Dict[str, Any] = {} -def _to_str_list(values): +def _to_str_list(values: Sequence[Any]) -> List[str]: """Convert a list to a list of strs.""" return [str(value) for value in values] @@ -56,8 +69,8 @@ def _to_str_list(values): class IssueTrackerPolicy: """Represents an issue tracker policy.""" - def __init__(self, data): - self._data = data + def __init__(self, data: Mapping[str, Any]) -> None: + self._data: Mapping[str, Any] = data if 'status' not in self._data: raise ConfigurationError('Status not set in policies.') @@ -69,13 +82,14 @@ def __init__(self, data): raise ConfigurationError( 'Expected status {} is not set.'.format(status)) - def status(self, status_type): + def status(self, status_type: str) -> str: """Get the actual status string for the given type.""" return self._data['status'][status_type] - def get_extension_fields(self, issue_type): + def get_extension_fields(self, + issue_type: Mapping[str, Any]) -> Dict[str, Any]: """Returns all _ext_ prefixed items from issue_type.""" - extension_fields = {} + extension_fields: Dict[str, Any] = {} # Extension fields are dynamically added to the policy # depending on which (if any) have been set in the config for k, v in issue_type.items(): @@ -83,7 +97,7 @@ def get_extension_fields(self, issue_type): extension_fields[k] = v return extension_fields - def label(self, label_type): + def label(self, label_type: str) -> Optional[str]: """Get the actual label string for the given type.""" label = self._data['labels'].get(label_type) if label is None: @@ -91,9 +105,9 @@ def label(self, label_type): return str(label) - def label_for_crash_type(self, crash_type): + def label_for_crash_type(self, crash_type: str | None) -> Optional[str]: """Get the actual label string for the given crash type.""" - if 'crash_types' not in self._data['labels']: + if not crash_type or 'crash_types' not in self._data['labels']: return None crash_type = crash_type.splitlines()[0].lower() @@ -103,7 +117,7 @@ def label_for_crash_type(self, crash_type): return str(label) - def label_for_crash_category(self, crash_category): + def label_for_crash_category(self, crash_category: str) -> Optional[str]: """Get the actual label string for the given crash subtype.""" if 'crash_categories' not in self._data['labels']: return None @@ -115,7 +129,7 @@ def label_for_crash_category(self, crash_category): return str(label) - def substitution_mapping(self, label): + def substitution_mapping(self, label: str) -> str: """Get an explicit substitution mapping.""" if 'substitutions' not in self._data: return label @@ -127,26 +141,27 @@ def substitution_mapping(self, label): return str(mapped) @property - def deadline_policy_message(self): + def deadline_policy_message(self) -> Optional[str]: """Get the deadline policy message, if it exists.""" return self._data.get('deadline_policy_message') @property - def unreproducible_component(self): + def unreproducible_component(self) -> Optional[str]: """Get the component for unreproducible bugs, if it exists.""" return self._data.get('unreproducible_component') @property - def fallback_component(self): + def fallback_component(self) -> Optional[str]: """Get the component for fallback (when save fails) issues, if it exists.""" return self._data.get('fallback_component') @property - def fallback_policy_message(self): + def fallback_policy_message(self) -> Optional[str]: """Get the fallback policy message, if it exists.""" return self._data.get('fallback_policy_message') - def get_new_issue_properties(self, is_security, is_crash): + def get_new_issue_properties(self, is_security: bool, + is_crash: bool) -> NewIssuePolicy: """Get the properties to apply to a new issue.""" policy = NewIssuePolicy() @@ -164,7 +179,9 @@ def get_new_issue_properties(self, is_security, is_crash): return policy - def _apply_new_issue_properties(self, policy, issue_type, is_crash): + def _apply_new_issue_properties(self, policy: NewIssuePolicy, + issue_type: Optional[Mapping[str, Any]], + is_crash: bool) -> None: """Apply issue policies.""" if not issue_type: return @@ -195,7 +212,7 @@ def _apply_new_issue_properties(self, policy, issue_type, is_crash): for k, v in self.get_extension_fields(issue_type).items(): policy.extension_fields[k] = v - def get_existing_issue_properties(self): + def get_existing_issue_properties(self) -> NewIssuePolicy: """Get the properties to apply to a new issue.""" policy = NewIssuePolicy() @@ -205,7 +222,7 @@ def get_existing_issue_properties(self): return policy -def get(project_name): +def get(project_name: str) -> IssueTrackerPolicy: """Get policy.""" issue_tracker_config = local_config.IssueTrackerConfig() project_config = issue_tracker_config.get(project_name) @@ -220,7 +237,7 @@ def get(project_name): return IssueTrackerPolicy(project_config['policies']) -def get_empty(): +def get_empty() -> IssueTrackerPolicy: """Get an empty policy.""" return IssueTrackerPolicy({ 'status': { diff --git a/src/clusterfuzz/_internal/issue_management/issue_tracker_utils.py b/src/clusterfuzz/_internal/issue_management/issue_tracker_utils.py index a305bae6800..57888e5eb30 100644 --- a/src/clusterfuzz/_internal/issue_management/issue_tracker_utils.py +++ b/src/clusterfuzz/_internal/issue_management/issue_tracker_utils.py @@ -13,6 +13,15 @@ # limitations under the License. """Utilities for managing issue tracker instance.""" +from typing import Any +from typing import Callable +from typing import cast +from typing import Dict +from typing import List +from typing import Optional +from typing import ParamSpec +from typing import TypeVar + from clusterfuzz._internal.base import memoize from clusterfuzz._internal.config import local_config from clusterfuzz._internal.datastore import data_types @@ -21,18 +30,26 @@ from clusterfuzz._internal.issue_management import issue_tracker_policy from clusterfuzz._internal.issue_management import jira from clusterfuzz._internal.issue_management import monorail +from clusterfuzz._internal.issue_management.issue_tracker import Issue +from clusterfuzz._internal.issue_management.issue_tracker import IssueTracker from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -_ISSUE_TRACKER_CACHE_CAPACITY = 8 -_ISSUE_TRACKER_CONSTRUCTORS = { - 'monorail': monorail.get_issue_tracker, - 'jira': jira.get_issue_tracker, - 'google_issue_tracker': google_issue_tracker.get_issue_tracker, -} +_P = ParamSpec('_P') +_R = TypeVar('_R') + +_ISSUE_TRACKER_CACHE_CAPACITY: int = 8 +_ISSUE_TRACKER_CONSTRUCTORS: Dict[str, Callable[..., Optional[ + IssueTracker]]] = { + 'monorail': monorail.get_issue_tracker, + 'jira': jira.get_issue_tracker, + 'google_issue_tracker': google_issue_tracker.get_issue_tracker, + } -def register_issue_tracker(tracker_type, constructor): +def register_issue_tracker( + tracker_type: str, + constructor: Callable[..., Optional[IssueTracker]]) -> None: """Register an issue tracker implementation.""" if tracker_type in _ISSUE_TRACKER_CONSTRUCTORS: raise ValueError( @@ -40,17 +57,18 @@ def register_issue_tracker(tracker_type, constructor): _ISSUE_TRACKER_CONSTRUCTORS[tracker_type] = constructor -def _get_issue_tracker_project_name(testcase=None): +def _get_issue_tracker_project_name( + testcase: Optional[data_types.Testcase] = None) -> Optional[str]: """Return issue tracker project name given a testcase or default.""" from clusterfuzz._internal.datastore import data_handler job_type = testcase.job_type if testcase else None issue_tracker_name = data_handler.get_issue_tracker_name(job_type) - logs.info( - f'For testcase {testcase.key}, using issue tracker {issue_tracker_name}') + logs.info(f'For testcase {cast(Any, testcase).key}, using issue tracker ' + f'{issue_tracker_name}') return issue_tracker_name -def request_or_task_cache(func): +def request_or_task_cache(func: Callable[_P, _R]) -> Callable[_P, _R]: """Cache that lasts for a bot task's lifetime, or an App Engine request lifetime.""" if environment.is_running_on_app_engine(): @@ -60,7 +78,8 @@ def request_or_task_cache(func): @request_or_task_cache -def get_issue_tracker(project_name=None): +def get_issue_tracker( + project_name: Optional[str] = None) -> Optional[IssueTracker]: """Get the issue tracker with the given type and name.""" issue_tracker_config = local_config.IssueTrackerConfig() if not project_name: @@ -83,7 +102,8 @@ def get_issue_tracker(project_name=None): return constructor(project_name, issue_project_config) -def get_issue_tracker_for_testcase(testcase): +def get_issue_tracker_for_testcase( + testcase: data_types.Testcase) -> Optional[IssueTracker]: """Get the issue tracker with the given type and name.""" issue_tracker_project_name = _get_issue_tracker_project_name(testcase) if not issue_tracker_project_name or issue_tracker_project_name == 'disabled': @@ -92,7 +112,9 @@ def get_issue_tracker_for_testcase(testcase): return get_issue_tracker(issue_tracker_project_name) -def get_issue_tracker_policy_for_testcase(testcase): +def get_issue_tracker_policy_for_testcase( + testcase: data_types.Testcase +) -> Optional[issue_tracker_policy.IssueTrackerPolicy]: """Get the issue tracker with the given type and name.""" issue_tracker_project_name = _get_issue_tracker_project_name(testcase) if not issue_tracker_project_name or issue_tracker_project_name == 'disabled': @@ -101,40 +123,42 @@ def get_issue_tracker_policy_for_testcase(testcase): return issue_tracker_policy.get(issue_tracker_project_name) -def get_issue_for_testcase(testcase): +def get_issue_for_testcase(testcase: data_types.Testcase) -> Optional[Issue]: """Return issue object associated with testcase.""" if not testcase.bug_information: # Do not check |testcase.group_bug_information| as we look for an issue # associated with the testcase directly, not through a group of testcases. return None - issue_tracker = get_issue_tracker_for_testcase(testcase) - if not issue_tracker: + issue_tracker_instance = get_issue_tracker_for_testcase(testcase) + if not issue_tracker_instance: return None issue_id = testcase.bug_information - return issue_tracker.get_original_issue(issue_id) + return issue_tracker_instance.get_original_issue(issue_id) -def get_search_keywords(testcase): +def get_search_keywords(testcase: data_types.Testcase) -> List[str]: """Get search keywords for a testcase.""" - crash_state_lines = testcase.crash_state.splitlines() + crash_state_lines = cast(str, testcase.crash_state).splitlines() # Use top 2 frames for searching. return crash_state_lines[:2] -def get_similar_issues(issue_tracker, testcase, only_open=True): +def get_similar_issues(issue_tracker: IssueTracker, + testcase: data_types.Testcase, + only_open: bool = True) -> List[Issue]: """Get issue objects that seem to be related to a particular test case.""" # Get list of issues using the search query. keywords = get_search_keywords(testcase) issues = issue_tracker.find_issues(keywords=keywords, only_open=only_open) if issues: - issues = list(issues) + issue_list: List[Issue] = list(issues) else: - issues = [] + issue_list = [] - issue_ids = [issue.id for issue in issues] + issue_ids = [issue.id for issue in issue_list] # Add issues from similar testcases sharing the same group id. if testcase.group_id: @@ -160,23 +184,25 @@ def get_similar_issues(issue_tracker, testcase, only_open=True): if (only_open and (not issue.is_open or not testcase.open)): continue - issues.append(issue) + issue_list.append(issue) issue_ids.append(issue_id) - return issues + return issue_list -def get_similar_issues_url(issue_tracker, testcase, only_open=True): +def get_similar_issues_url(issue_tracker: IssueTracker, + testcase: data_types.Testcase, + only_open: bool = True) -> Optional[str]: """Get similar issues web URL.""" keywords = get_search_keywords(testcase) return issue_tracker.find_issues_url(keywords=keywords, only_open=only_open) -def get_issue_url(testcase): +def get_issue_url(testcase: data_types.Testcase) -> Optional[str]: """Return issue url for a testcase. This is used when rendering a testcase, details page, therefore it accounts for |group_bug_information| as well.""" - issue_tracker = get_issue_tracker_for_testcase(testcase) - if not issue_tracker: + issue_tracker_instance = get_issue_tracker_for_testcase(testcase) + if not issue_tracker_instance: return None issue_id = ( @@ -186,10 +212,10 @@ def get_issue_url(testcase): return None # Use str(issue_id) as |group_bug_information| might be an integer. - return issue_tracker.issue_url(str(issue_id)) + return issue_tracker_instance.issue_url(str(issue_id)) -def was_label_added(issue, label): +def was_label_added(issue: Issue, label: Optional[str]) -> bool: """Check if a label was ever added to an issue.""" if not label: return False diff --git a/src/clusterfuzz/_internal/issue_management/jira/__init__.py b/src/clusterfuzz/_internal/issue_management/jira/__init__.py index 32d0cddbdc2..9cd886ee6a1 100644 --- a/src/clusterfuzz/_internal/issue_management/jira/__init__.py +++ b/src/clusterfuzz/_internal/issue_management/jira/__init__.py @@ -14,12 +14,20 @@ """Jira issue tracker.""" import datetime +from typing import Any +from typing import cast +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union from urllib.parse import urljoin from dateutil import parser from clusterfuzz._internal.config import db_config from clusterfuzz._internal.issue_management import issue_tracker +from clusterfuzz._internal.issue_management.issue_tracker import Action +from clusterfuzz._internal.issue_management.issue_tracker import LabelStore from clusterfuzz._internal.issue_management.jira.issue_tracker_manager import \ IssueTrackerManager @@ -27,102 +35,103 @@ class Issue(issue_tracker.Issue): """Represents an issue.""" - def __init__(self, itm, jira_issue): + def __init__(self, itm: IssueTrackerManager, jira_issue: Any) -> None: self.itm = itm self.jira_issue = jira_issue - self._ccs = issue_tracker.LabelStore(self.itm.get_watchers(self.jira_issue)) - self._components = issue_tracker.LabelStore( - self.jira_issue.fields.components) - self._labels = issue_tracker.LabelStore(self.jira_issue.fields.labels) + self._ccs = LabelStore(self.itm.get_watchers(self.jira_issue)) + self._components = LabelStore(self.jira_issue.fields.components) + self._labels = LabelStore(self.jira_issue.fields.labels) @property - def issue_tracker(self): + def issue_tracker(self) -> 'IssueTracker': """The IssueTracker for this issue.""" return IssueTracker(self.itm) @property - def id(self): + def id(self) -> int: """The issue identifier.""" return int(self.jira_issue.id) @property - def key(self): + def key(self) -> str: """The issue key (e.g. FUZZ-123).""" - return self.jira_issue.key + return str(self.jira_issue.key) @property - def title(self): + def title(self) -> Optional[str]: """The issue title.""" return self.jira_issue.fields.summary @title.setter - def title(self, new_title): + def title(self, new_title: Optional[str]) -> None: self.jira_issue.fields.summary = new_title @property - def reporter(self): + def reporter(self) -> Optional[str]: """The issue reporter.""" return self.jira_issue.fields.reporter @reporter.setter - def reporter(self, new_reporter): + def reporter(self, new_reporter: Optional[str]) -> None: self.jira_issue.fields.reporter = new_reporter @property - def is_open(self): + def is_open(self) -> bool: """Whether the issue is open.""" return self.jira_issue.fields.resolution is None @property - def closed_time(self): + def closed_time(self) -> Optional[datetime.datetime]: return datetime.datetime.fromtimestamp( - parser.parse(self.jira_issue.fields.resolutiondate).timestamp()) + cast(datetime.datetime, + parser.parse(self.jira_issue.fields.resolutiondate)).timestamp()) @property - def status(self): + def status(self) -> Optional[str]: """The issue status.""" return self.jira_issue.fields.status @status.setter - def status(self, new_status): + def status(self, new_status: Optional[str]) -> None: self.jira_issue.fields.status = new_status @property - def body(self): + def body(self) -> Optional[str]: """The issue body.""" return self.jira_issue.fields.description @body.setter - def body(self, new_body): + def body(self, new_body: Optional[str]) -> None: self.jira_issue.fields.description = new_body @property - def assignee(self): + def assignee(self) -> Optional[str]: """The issue assignee.""" return self.jira_issue.fields.assignee @assignee.setter - def assignee(self, new_assignee): + def assignee(self, new_assignee: Optional[str]) -> None: self.jira_issue.fields.assignee = new_assignee @property - def ccs(self): + def ccs(self) -> LabelStore: """The issue CC list.""" return self._ccs @property - def labels(self): + def labels(self) -> LabelStore: """The issue labels list.""" return self._labels @property - def components(self): + def components(self) -> LabelStore: """The issue component list.""" return self._components # FIXME: Add support for notify arguments - def save(self, new_comment=None, notify=True): # pylint: disable=unused-argument + def save(self, new_comment: Optional[str] = None, + notify: bool = True) -> None: # pylint: disable=unused-argument """Save the issue.""" # add new comment to issue if new_comment: @@ -149,36 +158,38 @@ def save(self, new_comment=None, notify=True): # pylint: disable=unused-argumen self.itm.save(self) @property - def actions(self): + def actions(self) -> Tuple[Action, ...]: return () @property - def merged_into(self): + def merged_into(self) -> Optional[Union[int, str]]: pass class IssueTracker(issue_tracker.IssueTracker): """Issue tracker interface.""" - def __init__(self, itm): + def __init__(self, itm: IssueTrackerManager) -> None: self._itm = itm @property - def project(self): + def project(self) -> str: return self._itm.project_name - def new_issue(self): + def new_issue(self) -> Issue: jira_issue = self._itm.create() return Issue(self._itm, jira_issue) - def get_issue(self, issue_id): + def get_issue(self, issue_id: Union[int, str]) -> Optional[Issue]: jira_issue = self._itm.get_issue(issue_id) if not jira_issue: return None return Issue(self._itm, jira_issue) - def find_issues(self, keywords=None, only_open=False): + def find_issues(self, + keywords: Optional[Union[str, List[str]]] = None, + only_open: Optional[bool] = False) -> List[Issue]: """Find issues.""" search_text = 'project = {project_name}' + _get_search_text(keywords) search_text = search_text.format(project_name=self._itm.project_name) @@ -188,41 +199,46 @@ def find_issues(self, keywords=None, only_open=False): return [Issue(self._itm, issue) for issue in issues] def find_issues_with_filters(self, - keywords=None, - query_filters=None, - only_open=None): + keywords: Optional[Any] = None, + query_filters: Optional[Any] = None, + only_open: Optional[bool] = None) -> List[Issue]: """Find issues given query filters.""" # Jira treats keywords and query filters the same for queries. - return self.find_issues(keywords + query_filters, only_open) + return self.find_issues(keywords + query_filters, only_open) # type: ignore - def issue_url(self, issue_id): + def issue_url(self, issue_id: Union[int, str]) -> Optional[str]: """Return the issue URL with the given ID.""" issue = self.get_issue(issue_id) if not issue: return None config = db_config.get() - url = urljoin(config.jira_url, f'/browse/{str(issue.key)}') + url = urljoin(cast(Any, config).jira_url, f'/browse/{str(issue.key)}') return url - def find_issues_url(self, keywords=None, only_open=None): + def find_issues_url(self, + keywords: Optional[Union[str, List[str]]] = None, + only_open: Optional[bool] = None) -> Optional[str]: search_text = 'project = {project_name}' + _get_search_text(keywords) search_text = search_text.format(project_name=self._itm.project_name) if only_open: search_text += ' AND resolution = Unresolved' config = db_config.get() - return urljoin(config.jira_url, f'/issues/?jql={search_text}') + return urljoin(cast(Any, config).jira_url, f'/issues/?jql={search_text}') - def find_issues_url_with_filters(self, - keywords=None, - query_filters=None, - only_open=None): + def find_issues_url_with_filters( + self, + keywords: Optional[Any] = None, + query_filters: Optional[Any] = None, + only_open: Optional[bool] = None) -> Optional[str]: """Return the issue URL with the given ID and additional query filters.""" # Jira treats keywords and query filters the same for queries. - return self.find_issues_url(keywords + query_filters, only_open) + return self.find_issues_url( + cast(Any, keywords) + cast(Any, query_filters), only_open) -def _get_issue_tracker_manager_for_project(project_name): +def _get_issue_tracker_manager_for_project( + project_name: Optional[str]) -> Optional[IssueTrackerManager]: """Return jira issue tracker manager for the given project.""" # If there is no issue tracker set, bail out. if not project_name or project_name == 'disabled': @@ -231,7 +247,8 @@ def _get_issue_tracker_manager_for_project(project_name): return IssueTrackerManager(project_name=project_name) -def get_issue_tracker(project_name, config): # pylint: disable=unused-argument +def get_issue_tracker(project_name: Optional[str], + config: Any) -> Optional[IssueTracker]: # pylint: disable=unused-argument """Get the issue tracker for the project name.""" itm = _get_issue_tracker_manager_for_project(project_name) if itm is None: @@ -240,7 +257,7 @@ def get_issue_tracker(project_name, config): # pylint: disable=unused-argument return IssueTracker(itm) -def _get_search_text(keywords): +def _get_search_text(keywords: Any) -> str: """Get search text.""" jira_special_characters = '+-&|!(){}[]^~*?\\:' search_text = '' diff --git a/src/clusterfuzz/_internal/issue_management/jira/issue_tracker_manager.py b/src/clusterfuzz/_internal/issue_management/jira/issue_tracker_manager.py index df035ccd7b0..57e54e0c8b0 100644 --- a/src/clusterfuzz/_internal/issue_management/jira/issue_tracker_manager.py +++ b/src/clusterfuzz/_internal/issue_management/jira/issue_tracker_manager.py @@ -14,47 +14,58 @@ """Issue tracker manager functions.""" import json +from typing import Any +from typing import cast +from typing import Dict +from typing import List +from typing import Optional +from typing import Union import jira +import jira.resilientsession +import jira.resources from clusterfuzz._internal.config import db_config from clusterfuzz._internal.metrics import logs -UNCREATED_JIRA_ISSUE_ID = -1 +UNCREATED_JIRA_ISSUE_ID: int = -1 class IssueTrackerManager: """Issue tracker manager.""" - def __init__(self, project_name): + def __init__(self, project_name: str) -> None: """Construct an issue tracker manager instance based on parameters.""" - self._client = None - self.project_name = project_name + self._client: Optional[jira.JIRA] = None + self.project_name: str = project_name @property - def client(self): + def client(self) -> jira.JIRA: """HTTP Client.""" if self._client is None: self._client = self._create_client() return self._client - def _create_client(self): + def _create_client(self) -> jira.JIRA: """Return a client object for querying the issue tracker.""" config = db_config.get() + assert config is not None + assert config.jira_credentials is not None + assert config.jira_url is not None credentials = json.loads(config.jira_credentials) jira_url = config.jira_url jira_client = jira.JIRA( jira_url, auth=(credentials['username'], credentials['password'])) return jira_client - def save(self, issue): + def save(self, issue: Any) -> None: """Save an issue.""" if issue.id == UNCREATED_JIRA_ISSUE_ID: return self._create(issue) return self._update(issue) - def create(self): + def create(self) -> jira.resources.Issue: """Create an issue object locally.""" raw_fields = { 'id': f'{UNCREATED_JIRA_ISSUE_ID}', @@ -69,7 +80,7 @@ def create(self): raw_fields) return jira_issue - def _transition_issue_status_if_updated(self, issue): + def _transition_issue_status_if_updated(self, issue: Any) -> None: """Transitions the status of the issue if updated. Jira has a separate endpoint to transition status.""" # Brittle - we should be pulling the equivalent of 'new' from the policy. @@ -84,7 +95,8 @@ def _transition_issue_status_if_updated(self, issue): if not isinstance(issue.status, jira.resources.Resource): self.client.transition_issue(issue.jira_issue, transition=issue.status) - def _add_watchers(self, issue, update_if_different=False): + def _add_watchers(self, issue: Any, + update_if_different: bool = False) -> None: """Add watchers to the ticket. Jira has a separate endpoint to add watchers.""" @@ -106,7 +118,7 @@ def _add_watchers(self, issue, update_if_different=False): except Exception: logs.error(f'Error adding watcher {watcher} to issue {issue.id}') - def _get_issue_fields(self, issue): + def _get_issue_fields(self, issue: Any) -> Dict[str, Any]: """Get issue fields to populate the ticket""" # Get labels from LabelStore. labels = list(issue.labels) @@ -114,7 +126,7 @@ def _get_issue_fields(self, issue): # Get components from LabelStore. components = list(issue.components) - fields = { + fields: Dict[str, Any] = { 'summary': issue.title, 'description': issue.body, 'labels': labels, @@ -135,7 +147,7 @@ def _get_issue_fields(self, issue): fields['priority'] = {'name': 'Major - P2'} return fields - def _create(self, issue): + def _create(self, issue: Any) -> None: """Create an issue.""" fields = self._get_issue_fields(issue) @@ -143,7 +155,7 @@ def _create(self, issue): self._add_watchers(jira_issue) issue.jira_issue = jira_issue - def _update(self, issue): + def _update(self, issue: Any) -> None: """Update an issue.""" update_fields = self._get_issue_fields(issue) @@ -151,7 +163,7 @@ def _update(self, issue): self._add_watchers(issue, update_if_different=True) issue.jira_issue.update(fields=update_fields) - def get_watchers(self, issue): + def get_watchers(self, issue: Any) -> List[str]: """Retrieve list of watchers.""" if issue.id == UNCREATED_JIRA_ISSUE_ID: return [] @@ -161,17 +173,19 @@ def get_watchers(self, issue): watchers.append(watcher.name) return watchers - def get_issue(self, issue_id): + def get_issue(self, + issue_id: Union[str, int]) -> Optional[jira.resources.Issue]: """Retrieve an issue object with a specific id.""" issue = self.client.issue(str(issue_id)) return issue - def get_issue_count(self, query_string): + def get_issue_count(self, query_string: str) -> int: """Return number of issues for a given query.""" issues = self.client.search_issues(query_string) return len(issues) - def get_issues(self, query_string, max_results=10000): + def get_issues(self, query_string: str, + max_results: int = 10000) -> List[jira.resources.Issue]: """Return all issues for a given query.""" issues = self.client.search_issues(query_string, maxResults=max_results) - return issues + return cast(List[jira.resources.Issue], issues) diff --git a/src/clusterfuzz/_internal/issue_management/monorail/__init__.py b/src/clusterfuzz/_internal/issue_management/monorail/__init__.py index 693e7c9113e..641ff2fc812 100644 --- a/src/clusterfuzz/_internal/issue_management/monorail/__init__.py +++ b/src/clusterfuzz/_internal/issue_management/monorail/__init__.py @@ -13,66 +13,77 @@ # limitations under the License. """Monorail issue tracker.""" +import datetime +from typing import Any +from typing import cast +from typing import Iterable +from typing import Iterator +from typing import List +from typing import Optional +from typing import Union import urllib.parse from clusterfuzz._internal.issue_management import issue_tracker +from clusterfuzz._internal.issue_management.issue_tracker import ChangeList +from clusterfuzz._internal.issue_management.issue_tracker import LabelStore +from clusterfuzz._internal.issue_management.monorail.comment import Comment from clusterfuzz._internal.issue_management.monorail.issue import \ Issue as MonorailIssue -from clusterfuzz._internal.issue_management.monorail.issue import ChangeList # pylint: disable=line-too-long from clusterfuzz._internal.issue_management.monorail.issue_tracker_manager import \ IssueTrackerManager -ISSUE_TRACKER_URL = ( +ISSUE_TRACKER_URL: str = ( 'https://bugs.chromium.org/p/{project}/issues/detail?id={id}') -ISSUE_TRACKER_SEARCH_URL = ( +ISSUE_TRACKER_SEARCH_URL: str = ( 'https://bugs.chromium.org/p/{project}/issues/list?{params}') class Issue(issue_tracker.Issue): """Represents an issue.""" - def __init__(self, monorail_issue): + def __init__(self, monorail_issue: MonorailIssue) -> None: self._monorail_issue = monorail_issue # These mirror the underlying MonorailIssue data structures, to make it more # opaque to the client about how issue updates are done. For instance, when # a `label` is removed, what actually happens is `-label` is added. This # should not be visible to the client. - self._ccs = issue_tracker.LabelStore(self._monorail_issue.cc) - self._components = issue_tracker.LabelStore(self._monorail_issue.components) - self._labels = issue_tracker.LabelStore(self._monorail_issue.labels) + self._ccs = LabelStore(self._monorail_issue.cc) + self._components = LabelStore(self._monorail_issue.components) + self._labels = LabelStore(self._monorail_issue.labels) @property - def issue_tracker(self): + def issue_tracker(self) -> 'IssueTracker': """The IssueTracker for this issue.""" + assert self._monorail_issue.itm is not None return IssueTracker(self._monorail_issue.itm) @property - def id(self): + def id(self) -> Optional[int]: """The issue identifier.""" return self._monorail_issue.id @property - def title(self): + def title(self) -> Optional[str]: """The issue title.""" return self._monorail_issue.summary @title.setter - def title(self, new_title): + def title(self, new_title: Optional[str]) -> None: self._monorail_issue.summary = new_title @property - def reporter(self): + def reporter(self) -> Optional[str]: """The issue reporter.""" return self._monorail_issue.reporter @reporter.setter - def reporter(self, new_reporter): + def reporter(self, new_reporter: Optional[str]) -> None: self._monorail_issue.reporter = new_reporter @property - def merged_into(self): + def merged_into(self) -> Optional[Union[int, str]]: """The issue that this is merged into.""" if self._monorail_issue.merged_into_project != self.issue_tracker.project: # Don't consider duplicates in a different issue project. @@ -81,62 +92,65 @@ def merged_into(self): return self._monorail_issue.merged_into @property - def is_open(self): + def is_open(self) -> bool: """Whether the issue is open.""" return self._monorail_issue.open @property - def closed_time(self): + def closed_time(self) -> Optional[datetime.datetime]: return self._monorail_issue.closed @property - def status(self): + def status(self) -> Optional[str]: """The issue status.""" return self._monorail_issue.status @status.setter - def status(self, new_status): + def status(self, new_status: Optional[str]) -> None: self._monorail_issue.status = new_status @property - def body(self): + def body(self) -> Optional[str]: """The issue body.""" return self._monorail_issue.body @body.setter - def body(self, new_body): + def body(self, new_body: Optional[str]) -> None: self._monorail_issue.body = new_body @property - def assignee(self): + def assignee(self) -> Optional[str]: """The issue assignee.""" return self._monorail_issue.owner @assignee.setter - def assignee(self, new_assignee): + def assignee(self, new_assignee: Optional[str]) -> None: self._monorail_issue.owner = new_assignee @property - def ccs(self): + def ccs(self) -> LabelStore: """The issue CC list.""" return self._ccs @property - def labels(self): + def labels(self) -> LabelStore: """The issue labels list.""" return self._labels @property - def components(self): + def components(self) -> LabelStore: """The issue component list.""" return self._components @property - def actions(self): + def actions(self) -> Iterator['Action']: """Get the issue actions.""" - return (Action(comment) for comment in self._monorail_issue.get_comments()) + return ( + Action(comment) + for comment in cast(Iterable[Any], self._monorail_issue.get_comments())) - def save(self, new_comment=None, notify=True): + def save(self, new_comment: Optional[str] = None, + notify: bool = True) -> None: """Save the issue.""" # Apply actual label changes to the underlying MonorailIssue. @@ -167,46 +181,46 @@ def save(self, new_comment=None, notify=True): class Action(issue_tracker.Action): """Monorail Action.""" - def __init__(self, monorail_comment): + def __init__(self, monorail_comment: Comment) -> None: self._monorail_comment = monorail_comment @property - def author(self): + def author(self) -> Optional[str]: """The author of the action.""" return self._monorail_comment.author @property - def comment(self): + def comment(self) -> Optional[str]: """Represents a comment.""" return self._monorail_comment.comment @property - def title(self): + def title(self) -> Optional[str]: """The new issue title.""" return self._monorail_comment.summary @property - def status(self): + def status(self) -> Optional[str]: """The new issue status.""" return self._monorail_comment.status @property - def assignee(self): + def assignee(self) -> Optional[str]: """The new issue assignee.""" return self._monorail_comment.owner @property - def ccs(self): + def ccs(self) -> ChangeList: """The issue CC change list.""" return _to_change_list(self._monorail_comment.cc) @property - def labels(self): + def labels(self) -> ChangeList: """The issue labels change list.""" return _to_change_list(self._monorail_comment.labels) @property - def components(self): + def components(self) -> ChangeList: """The issue component change list.""" return _to_change_list(self._monorail_comment.components) @@ -214,26 +228,28 @@ def components(self): class IssueTracker(issue_tracker.IssueTracker): """Issue tracker interface.""" - def __init__(self, itm): + def __init__(self, itm: IssueTrackerManager) -> None: self._itm = itm @property - def project(self): + def project(self) -> str: return self._itm.project_name - def new_issue(self): + def new_issue(self) -> Issue: monorail_issue = MonorailIssue() monorail_issue.itm = self._itm return Issue(monorail_issue) - def get_issue(self, issue_id): + def get_issue(self, issue_id: Union[int, str]) -> Optional[Issue]: monorail_issue = self._itm.get_issue(int(issue_id)) if not monorail_issue: return None return Issue(monorail_issue) - def find_issues(self, keywords=None, only_open=False): + def find_issues(self, + keywords: Optional[Union[str, List[str]]] = None, + only_open: Optional[bool] = False) -> Optional[List[Issue]]: """Find issues.""" if not keywords: return None @@ -247,15 +263,19 @@ def find_issues(self, keywords=None, only_open=False): issues = self._itm.get_issues(search_text, can=can) return [Issue(issue) for issue in issues] - def find_issues_with_filters(self, - keywords=None, - query_filters=None, - only_open=None): + def find_issues_with_filters( + self, + keywords: Optional[Any] = None, + query_filters: Optional[Any] = None, + only_open: Optional[bool] = None) -> Optional[List[Issue]]: """Find issues given additional filters.""" # Monorail treats keywords and query filters the same for queries. - return self.find_issues(keywords + query_filters, only_open) + return self.find_issues( + cast(Any, keywords) + cast(Any, query_filters), only_open) - def find_issues_url(self, keywords=None, only_open=False): + def find_issues_url(self, + keywords: Optional[Union[str, List[str]]] = None, + only_open: Optional[bool] = False) -> Optional[str]: """Find issues (web URL).""" if not keywords: return None @@ -274,21 +294,23 @@ def find_issues_url(self, keywords=None, only_open=False): 'q': search_text, })) - def find_issues_url_with_filters(self, - keywords=None, - query_filters=None, - only_open=None): + def find_issues_url_with_filters( + self, + keywords: Optional[Any] = None, + query_filters: Optional[Any] = None, + only_open: Optional[bool] = None) -> Optional[str]: # Monorail treats keywords and query filters the same for queries. - return self.find_issues_url(keywords + query_filters, only_open) + return self.find_issues_url( + cast(Any, keywords) + cast(Any, query_filters), only_open) - def issue_url(self, issue_id): + def issue_url(self, issue_id: Union[int, str]) -> str: """Return the issue URL with the given ID.""" return ISSUE_TRACKER_URL.format(project=self.project, id=issue_id) -def _to_change_list(monorail_list): +def _to_change_list(monorail_list: Optional[Iterable[str]]) -> ChangeList: """Convert a list of changed items to a issue_tracker.ChangeList.""" - change_list = issue_tracker.ChangeList() + change_list = ChangeList() if not monorail_list: return change_list @@ -301,7 +323,8 @@ def _to_change_list(monorail_list): return change_list -def _get_issue_tracker_manager_for_project(project_name): +def _get_issue_tracker_manager_for_project( + project_name: Optional[str]) -> Optional[IssueTrackerManager]: """Return monorail issue tracker manager for the given project.""" # If there is no issue tracker set, bail out. if not project_name or project_name == 'disabled': @@ -310,7 +333,7 @@ def _get_issue_tracker_manager_for_project(project_name): return IssueTrackerManager(project_name=project_name) -def _get_search_text(keywords): +def _get_search_text(keywords: Any) -> str: """Get search text.""" search_text = ' '.join(['"{}"'.format(keyword) for keyword in keywords]) search_text = search_text.replace(':', ' ') @@ -319,7 +342,8 @@ def _get_search_text(keywords): return search_text -def get_issue_tracker(project_name, config): # pylint: disable=unused-argument +def get_issue_tracker(project_name: Optional[str], + config: Any) -> Optional[IssueTracker]: # pylint: disable=unused-argument """Get the issue tracker for the project name.""" # TODO(ochang): Make this lazy. itm = _get_issue_tracker_manager_for_project(project_name) diff --git a/src/clusterfuzz/_internal/issue_management/monorail/comment.py b/src/clusterfuzz/_internal/issue_management/monorail/comment.py index 275b30a503c..1cf0562cd41 100644 --- a/src/clusterfuzz/_internal/issue_management/monorail/comment.py +++ b/src/clusterfuzz/_internal/issue_management/monorail/comment.py @@ -13,42 +13,45 @@ # limitations under the License. """Issue tracker comments.""" +import datetime import re +from typing import List +from typing import Optional class Comment: """Class representing a single comment update.""" - def __init__(self): - self.author = None - self.cc = None - self.comment = None - self.components = [] - self.created = None - self.labels = [] - self.summary = None - self.status = None - self.owner = None - self.id = 0 - - def has_label_containing(self, expression): + def __init__(self) -> None: + self.author: Optional[str] = None + self.cc: Optional[List[str]] = None + self.comment: Optional[str] = None + self.components: List[str] = [] + self.created: Optional[datetime.datetime] = None + self.labels: List[str] = [] + self.summary: Optional[str] = None + self.status: Optional[str] = None + self.owner: Optional[str] = None + self.id: int = 0 + + def has_label_containing(self, expression: str) -> bool: return any(self.get_labels_containing(expression)) - def get_labels_containing(self, expression): + def get_labels_containing(self, expression: str) -> List[str]: regex = re.compile(expression, re.DOTALL | re.IGNORECASE) return [label for label in self.labels if regex.search(label)] - def has_label_matching(self, expression): + def has_label_matching(self, expression: str) -> bool: return any(self.get_labels_matching(expression)) - def get_labels_matching(self, expression): + def get_labels_matching(self, expression: str) -> List[str]: regex = re.compile(expression + r'\Z', re.DOTALL | re.IGNORECASE) return [label for label in self.labels if regex.match(label)] - def get_labels_by_prefix(self, prefix): + def get_labels_by_prefix(self, prefix: str) -> List[str]: return self.get_labels_matching('%s.*' % prefix) - def has_label(self, value): + def has_label(self, value: str) -> bool: for label in self.labels: if label.lower() == value.lower(): return True diff --git a/src/clusterfuzz/_internal/issue_management/monorail/credential_storage.py b/src/clusterfuzz/_internal/issue_management/monorail/credential_storage.py index d2869e3da70..aec4ad4b4e5 100644 --- a/src/clusterfuzz/_internal/issue_management/monorail/credential_storage.py +++ b/src/clusterfuzz/_internal/issue_management/monorail/credential_storage.py @@ -22,7 +22,7 @@ class CredentialStorage(Storage): """Instead of reading a file, just parse a config entry.""" - def locked_get(self): + def locked_get(self) -> Credentials | None: """Return Credentials.""" content = db_config.get_value('client_credentials') if not content: @@ -36,8 +36,8 @@ def locked_get(self): return credentials - def locked_put(self, credentials): # pylint: disable=unused-argument + def locked_put(self, credentials: Credentials) -> None: # pylint: disable=unused-argument pass - def locked_delete(self): + def locked_delete(self) -> None: pass diff --git a/src/clusterfuzz/_internal/issue_management/monorail/issue.py b/src/clusterfuzz/_internal/issue_management/monorail/issue.py index f7693b64f20..e1995e48ec8 100644 --- a/src/clusterfuzz/_internal/issue_management/monorail/issue.py +++ b/src/clusterfuzz/_internal/issue_management/monorail/issue.py @@ -13,28 +13,40 @@ # limitations under the License. """Helper classes for managing issues.""" +import datetime import re +from typing import Any +from typing import Dict +from typing import Iterable +from typing import List +from typing import Optional +from typing import Set +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from .comment import Comment + from .issue_tracker_manager import IssueTrackerManager -def get_values_containing(target, expression): + +def get_values_containing(target: Iterable[str], expression: str) -> List[str]: regex = re.compile(expression, re.DOTALL | re.IGNORECASE) return [value for value in target if regex.search(value)] -def get_values_matching(target, expression): +def get_values_matching(target: Iterable[str], expression: str) -> List[str]: regex = re.compile(expression + r'\Z', re.DOTALL | re.IGNORECASE) return [value for value in target if regex.match(value)] -def has_values_containing(target, expression): +def has_values_containing(target: Iterable[str], expression: str) -> bool: return any(get_values_containing(target, expression)) -def has_values_matching(target, expression): +def has_values_matching(target: Iterable[str], expression: str) -> bool: return any(get_values_matching(target, expression)) -def has_value(target, value): +def has_value(target: Iterable[str], value: str) -> bool: for target_value in target: if target_value.lower() == value.lower(): return True @@ -45,12 +57,12 @@ def has_value(target, value): class ChangeList(list): """List that tracks changes for incremental updates.""" - def __init__(self, seq=()): + def __init__(self, seq: Iterable[Any] = ()) -> None: super().__init__(seq) - self.added = set() - self.removed = set() + self.added: Set[Any] = set() + self.removed: Set[Any] = set() - def append(self, p_object): + def append(self, p_object: Any) -> None: list.append(self, p_object) if p_object in self.removed: @@ -58,7 +70,7 @@ def append(self, p_object): else: self.added.add(p_object) - def remove(self, value): + def remove(self, value: Any) -> None: list.remove(self, value) if value in self.added: @@ -66,10 +78,10 @@ def remove(self, value): else: self.removed.add(value) - def is_changed(self): + def is_changed(self) -> bool: return (len(self.added) + len(self.removed)) > 0 - def reset(self): + def reset(self) -> None: self.added.clear() self.removed.clear() @@ -77,47 +89,48 @@ def reset(self): class Issue: """Class representing a single issue.""" - def __init__(self): - self.blocking = None - self.blocked_on = None - self.body = None - self.depends_on = None - self.cc = ChangeList() - self.closed = None - self.comment = '' - self.components = ChangeList() - self.created = None - self.id = 0 - self.labels = ChangeList() - self.merged_into = None - self.merged_into_project = None - self.open = False - self.owner = None - self.reporter = None - self.status = None - self.stars = 0 - self.summary = None - self.updated = None - - self.dirty = False - self.send_email = True - self.new = True - self.itm = None - self.project_name = None - self.comments = None - self.comment_count = 0 - self.first_comment = None - self.last_comment = None - self.changed = set() - - def __getattribute__(self, item): + def __init__(self) -> None: + self.blocking: Optional[List[int]] = None + self.blocked_on: Optional[List[int]] = None + self.body: Optional[str] = None + self.depends_on: Optional[List[int]] = None + self.cc: ChangeList = ChangeList() + self.closed: Optional[datetime.datetime] = None + self.comment: Optional[str] = '' + self.components: ChangeList = ChangeList() + self.created: Optional[datetime.datetime] = None + self.id: int = 0 + self.labels: ChangeList = ChangeList() + self.merged_into: Optional[int] = None + self.merged_into_project: Optional[str] = None + self.open: bool = False + self.owner: Optional[str] = None + self.reporter: Optional[str] = None + self.status: Optional[str] = None + self.stars: int = 0 + self.summary: Optional[str] = None + self.updated: Optional[datetime.datetime] = None + + self.dirty: bool = False + self.send_email: bool = True + self.new: bool = True + self.itm: Optional['IssueTrackerManager'] = None + self.project_name: Optional[str] = None + self.comments: Optional[List['Comment']] = None + self.comment_count: int = 0 + self.first_comment: Optional['Comment'] = None + self.last_comment: Optional['Comment'] = None + self.changed: Set[str] = set() + + def __getattribute__(self, item: str) -> Any: if item in ['body'] and not object.__getattribute__(self, item): comment = self.get_first_comment() - self.__setattr__(item, comment.comment) + if comment: + self.__setattr__(item, comment.comment) return object.__getattribute__(self, item) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: self.__dict__[name] = value if 'changed' in self.__dict__: self.__dict__['changed'].add(name) @@ -138,7 +151,7 @@ def __setattr__(self, name, value): if 'changed' in self.__dict__: self.changed.clear() - def __getstate__(self): + def __getstate__(self) -> Dict[str, Any]: """Ensure that we don't pickle the itm. This would raise an exception due to the way the apiary folks did their @@ -148,137 +161,143 @@ def __getstate__(self): del result_dict['itm'] return result_dict - def __setstate__(self, new_dict): + def __setstate__(self, new_dict: Dict[str, Any]) -> None: self.__dict__.update(new_dict) self.itm = None - def _remove_tracked_value(self, target, value): + def _remove_tracked_value(self, target: ChangeList, value: str) -> None: for existing_value in target: if existing_value.lower() == value.lower(): target.remove(existing_value) self.dirty = True return - def add_component(self, component): + def add_component(self, component: str) -> None: if not self.has_component(component): self.components.append(component) self.dirty = True - def remove_component(self, component): + def remove_component(self, component: str) -> None: if self.has_component(component): self._remove_tracked_value(self.components, component) self.add_component('-%s' % component) - def remove_components_by_prefix(self, prefix): + def remove_components_by_prefix(self, prefix: str) -> None: components = self.get_components_by_prefix(prefix) for component in components: - self.remove_label(component) + self.remove_component(component) - def add_label(self, label): + def add_label(self, label: str) -> None: if not self.has_label(label): self.labels.append(label) self.dirty = True - def remove_label(self, label): + def remove_label(self, label: str) -> None: if self.has_label(label): self._remove_tracked_value(self.labels, label) self.add_label('-%s' % label) - def remove_label_by_prefix(self, prefix): + def remove_label_by_prefix(self, prefix: str) -> None: labels = self.get_labels_by_prefix(prefix) for label in labels: self.remove_label(label) - def add_cc(self, cc): + def add_cc(self, cc: str) -> None: if not self.has_cc(cc): self.cc.append(cc) self.dirty = True - def remove_cc(self, cc): + def remove_cc(self, cc: str) -> None: if self.has_cc(cc): self.cc.remove(cc) self.dirty = True - def get_components_by_prefix(self, prefix): + def get_components_by_prefix(self, prefix: str) -> List[str]: return get_values_matching(self.components, '%s.*' % prefix) - def get_components_containing(self, expression): + def get_components_containing(self, expression: str) -> List[str]: return get_values_containing(self.components, expression) - def get_components_matching(self, expression): + def get_components_matching(self, expression: str) -> List[str]: return get_values_matching(self.components, expression) - def has_components_containing(self, expression): + def has_components_containing(self, expression: str) -> bool: return has_values_containing(self.components, expression) - def has_components_matching(self, expression): + def has_components_matching(self, expression: str) -> bool: return has_values_matching(self.components, expression) - def has_component(self, value): + def has_component(self, value: str) -> bool: return has_value(self.components, value) - def get_labels_by_prefix(self, prefix): + def get_labels_by_prefix(self, prefix: str) -> List[str]: return get_values_matching(self.labels, '%s.*' % prefix) - def get_labels_containing(self, expression): + def get_labels_containing(self, expression: str) -> List[str]: return get_values_containing(self.labels, expression) - def get_labels_matching(self, expression): + def get_labels_matching(self, expression: str) -> List[str]: return get_values_matching(self.labels, expression) - def has_label_by_prefix(self, prefix): + def has_label_by_prefix(self, prefix: str) -> bool: return has_values_containing(self.labels, '%s.*' % prefix) - def has_label_containing(self, expression): + def has_label_containing(self, expression: str) -> bool: return has_values_containing(self.labels, expression) - def has_label_matching(self, expression): + def has_label_matching(self, expression: str) -> bool: return has_values_matching(self.labels, expression) - def has_label(self, value): + def has_label(self, value: str) -> bool: return has_value(self.labels, value) - def has_cc(self, value): + def has_cc(self, value: str) -> bool: return has_value(self.cc, value) - def has_comment_with_label(self, label): - for comment in self.get_comments(): + def has_comment_with_label(self, label: str) -> bool: + comments = self.get_comments() + if not comments: + return False + for comment in comments: if comment.has_label(label): return True return False - def has_comment_with_label_by_prefix(self, prefix): - for comment in self.get_comments(): + def has_comment_with_label_by_prefix(self, prefix: str) -> bool: + comments = self.get_comments() + if not comments: + return False + for comment in comments: if comment.get_labels_by_prefix(prefix): return True return False - def get_comments(self): + def get_comments(self) -> Optional[List['Comment']]: if not self.comments and self.itm: self.comments = self.itm.get_comments(self.id) self.comment_count = len(self.comments) return self.comments - def get_first_comment(self): + def get_first_comment(self) -> Optional['Comment']: if not self.first_comment and self.itm: self.first_comment = self.itm.get_first_comment(self.id) return self.first_comment - def get_last_comment(self): + def get_last_comment(self) -> Optional['Comment']: if not self.last_comment and self.itm: self.last_comment = self.itm.get_last_comment(self.id) return self.last_comment - def get_comment_count(self): + def get_comment_count(self) -> int: if not self.comment_count and self.itm: self.comment_count = self.itm.get_comment_count(self.id) return self.comment_count - def save(self, send_email=None): + def save(self, send_email: Optional[bool] = None) -> None: if self.itm: self.itm.save(self, send_email) - def refresh(self): + def refresh(self) -> 'Issue': if self.itm: self.comments = None self.last_comment = None diff --git a/src/clusterfuzz/_internal/issue_management/monorail/issue_tracker_manager.py b/src/clusterfuzz/_internal/issue_management/monorail/issue_tracker_manager.py index a880b29f3c9..6b6e7952bdc 100644 --- a/src/clusterfuzz/_internal/issue_management/monorail/issue_tracker_manager.py +++ b/src/clusterfuzz/_internal/issue_management/monorail/issue_tracker_manager.py @@ -14,8 +14,10 @@ """Issue tracker manager functions.""" import datetime +from typing import Any from googleapiclient import discovery +from googleapiclient import http as googleapiclient_http from clusterfuzz._internal.base import retry @@ -25,13 +27,13 @@ from .issue import Issue # Default value for issue tracker connection failures. -FAIL_RETRIES = 7 -FAIL_WAIT = 1 +FAIL_RETRIES: int = 7 +FAIL_WAIT: int = 1 # pylint: disable=no-member -def convert_entry_to_comment(entry): +def convert_entry_to_comment(entry: dict[str, Any]) -> Comment: """Convert an issue entry object into a comment object.""" comment = Comment() comment.author = entry['author']['name'] if 'author' in entry else None @@ -50,7 +52,11 @@ def convert_entry_to_comment(entry): return comment -def convert_entry_to_issue(entry, itm, old_issue=None): +def convert_entry_to_issue( + entry: dict[str, Any], + itm: 'IssueTrackerManager', + old_issue: Issue | None = None, +) -> Issue: """Convert an issue entry object into a issue object.""" if old_issue: issue = old_issue @@ -90,7 +96,7 @@ def convert_entry_to_issue(entry, itm, old_issue=None): return issue -def parse_datetime(date_string): +def parse_datetime(date_string: str) -> datetime.datetime: """Parse a date time string into a datetime object.""" datetime_obj, _, microseconds_string = date_string.partition('.') datetime_obj = datetime.datetime.strptime(datetime_obj, '%Y-%m-%dT%H:%M:%S') @@ -104,15 +110,15 @@ def parse_datetime(date_string): class IssueTrackerManager: """Issue tracker manager.""" - CAN_ALL = 'all' - CAN_OPEN = 'open' - CAN_MY_OPEN_BUGS = 'owned' - CAN_REPORTED_BY_ME = 'reported' - CAN_STARRED_BY_ME = 'starred' - CAN_NEW = 'new' - CAN_VERIFY = 'to-verify' + CAN_ALL: str = 'all' + CAN_OPEN: str = 'open' + CAN_MY_OPEN_BUGS: str = 'owned' + CAN_REPORTED_BY_ME: str = 'reported' + CAN_STARRED_BY_ME: str = 'starred' + CAN_NEW: str = 'new' + CAN_VERIFY: str = 'to-verify' - CAN_VALUE_TO_ID_MAP = { + CAN_VALUE_TO_ID_MAP: dict[str, int] = { CAN_ALL: 1, CAN_OPEN: 2, CAN_MY_OPEN_BUGS: 3, @@ -121,18 +127,19 @@ class IssueTrackerManager: CAN_NEW: 6 } - API_DISCOVERY_URL = ('https://monorail-prod.appspot.com/_ah/api/discovery/' - 'v1/apis/{api}/{apiVersion}/rest') - API_NAME = 'monorail' - API_VERSION = 'v1' + API_DISCOVERY_URL: str = ( + 'https://monorail-prod.appspot.com/_ah/api/discovery/' + 'v1/apis/{api}/{apiVersion}/rest') + API_NAME: str = 'monorail' + API_VERSION: str = 'v1' - def __init__(self, project_name): + def __init__(self, project_name: str) -> None: """"Construct an issue tracker manager instance based on parameters.""" - self._client = None - self.project_name = project_name + self._client: Any = None + self.project_name: str = project_name @property - def client(self): + def client(self) -> Any: """HTTP Client.""" if self._client is None: self._client = self._create_client() @@ -146,7 +153,7 @@ def client(self): function= 'clusterfuzz._internal.issue_management.issue_tracker_manager._execute_with_retry' ) - def _execute_with_retry(self, query): + def _execute_with_retry(self, query: googleapiclient_http.HttpRequest) -> Any: """Execute a query (with retries).""" return query.execute() @@ -157,7 +164,7 @@ def _execute_with_retry(self, query): function= 'clusterfuzz._internal.issue_management.issue_tracker_manager._create_client' ) - def _create_client(self): + def _create_client(self) -> Any: """Return a client object for querying the issue tracker. Includes retry logic to handle occasional 503 backend errors. @@ -172,7 +179,7 @@ def _create_client(self): discoveryServiceUrl=self.API_DISCOVERY_URL, cache_discovery=False) - def save(self, issue, send_email=None): + def save(self, issue: Issue, send_email: bool | None = None) -> None: """Save an issue and optionally send update notification over email.""" if send_email is None: send_email = issue.send_email @@ -182,7 +189,7 @@ def save(self, issue, send_email=None): else: self._update(issue, send_email) - def _create(self, issue, send_email=True): + def _create(self, issue: Issue, send_email: bool = True) -> Issue: """Create an issue and optionally send update notification over email.""" cc = [{'name': user} for user in issue.cc] body = { @@ -204,7 +211,7 @@ def _create(self, issue, send_email=True): issue.new = False return issue - def _update(self, issue, send_email=True): + def _update(self, issue: Issue, send_email: bool = True) -> Issue: """Update an issue and optionally send update notification over email.""" if not issue.dirty: return issue @@ -240,13 +247,18 @@ def _update(self, issue, send_email=True): issue.dirty = False return issue - def add_comment(self, issue_id, comment, send_email=True): + def add_comment( + self, + issue_id: int, + comment: str, + send_email: bool = True, + ) -> None: """Add comment to an issue and potentially send an email update.""" issue = self.get_issue(issue_id) issue.comment = comment self.save(issue, send_email) - def get_comment_count(self, issue_id): + def get_comment_count(self, issue_id: int) -> int: """Get number of comments for an issue.""" feed = self._execute_with_retry(self.client.issues().comments().list( projectId=self.project_name, @@ -255,7 +267,7 @@ def get_comment_count(self, issue_id): maxResults=0)) return feed.get('totalResults', '0') - def get_comments(self, issue_id): + def get_comments(self, issue_id: int) -> list[Comment]: """Get all comments for an issue.""" comments = [] comments_feed = self._execute_with_retry( @@ -280,7 +292,7 @@ def get_comments(self, issue_id): return comments - def get_first_comment(self, issue_id): + def get_first_comment(self, issue_id: int) -> Comment | None: """Get first comment for an issue.""" feed = self._execute_with_retry(self.client.issues().comments().list( projectId=self.project_name, @@ -292,7 +304,7 @@ def get_first_comment(self, issue_id): return None - def get_last_comment(self, issue_id): + def get_last_comment(self, issue_id: int) -> Comment | None: """Get last comment for an issue.""" total_results = self.get_comment_count(issue_id) feed = self._execute_with_retry(self.client.issues().comments().list( @@ -305,13 +317,13 @@ def get_last_comment(self, issue_id): return None - def get_issue(self, issue_id): + def get_issue(self, issue_id: int) -> Issue: """Retrieve an issue object with a specific id.""" entry = self._execute_with_retry(self.client.issues().get( projectId=self.project_name, issueId=issue_id)) return convert_entry_to_issue(entry, self) - def refresh(self, issue): + def refresh(self, issue: Issue | None) -> Issue | None: """Refresh an issue object with latest updates.""" if issue and not issue.new: entry = self._execute_with_retry(self.client.issues().get( @@ -320,13 +332,13 @@ def refresh(self, issue): return issue - def get_all_issues(self): + def get_all_issues(self) -> list[Issue]: """Get all issues for the project.""" feed = self._execute_with_retry( self.client.issues().list(projectId=self.project_name)) return [convert_entry_to_issue(entry, self) for entry in feed['items']] - def get_issue_count(self, query_string, can=CAN_ALL): + def get_issue_count(self, query_string: str, can: str = CAN_ALL) -> int: """Return number of issues for a given query.""" feed = self._execute_with_retry(self.client.issues().list( can=can, @@ -340,7 +352,12 @@ def get_issue_count(self, query_string, can=CAN_ALL): return 0 - def get_issues(self, query_string, can=CAN_ALL, max_results=1000): + def get_issues( + self, + query_string: str, + can: str = CAN_ALL, + max_results: int = 1000, + ) -> list[Issue]: """Return all issues for a given query.""" block_count = 0 count = 0 @@ -358,11 +375,13 @@ def get_issues(self, query_string, can=CAN_ALL, max_results=1000): return issues - def get_issues_from_index(self, - query_string, - can=CAN_ALL, - max_results=1000, - start_index=0): + def get_issues_from_index( + self, + query_string: str, + can: str = CAN_ALL, + max_results: int = 1000, + start_index: int = 0, + ) -> tuple[list[Issue], int]: """Retrieve a set of issues for a query from a given start index.""" feed = self._execute_with_retry(self.client.issues().list( projectId=self.project_name, diff --git a/src/clusterfuzz/_internal/issue_management/oss_fuzz_github.py b/src/clusterfuzz/_internal/issue_management/oss_fuzz_github.py index f62ead8f52b..7651d23dca6 100644 --- a/src/clusterfuzz/_internal/issue_management/oss_fuzz_github.py +++ b/src/clusterfuzz/_internal/issue_management/oss_fuzz_github.py @@ -13,54 +13,61 @@ # limitations under the License. """Helper functions to file GitHub issues.""" +from typing import cast + import github +from github.Issue import Issue +from github.PaginatedList import PaginatedList +from github.Repository import Repository from clusterfuzz._internal.base import utils from clusterfuzz._internal.config import db_config from clusterfuzz._internal.datastore import data_handler +from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.metrics import logs -GITHUB_HANDLE = 'oss-fuzz-robot' +GITHUB_HANDLE: str = 'oss-fuzz-robot' -GITHUB_PREFIX = 'https://github.com/' -GIT_SUFFIX = '.git' +GITHUB_PREFIX: str = 'https://github.com/' +GIT_SUFFIX: str = '.git' -TESTCASE_REPORT_URL = 'https://{domain}/testcase?key={testcase_id}' +TESTCASE_REPORT_URL: str = 'https://{domain}/testcase?key={testcase_id}' -MONORAIL_URL = ( +MONORAIL_URL: str = ( 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id={bug_information}') -OSS_FUZZ_ISSUE_URL = 'https://github.com/google/oss-fuzz/issues/new' +OSS_FUZZ_ISSUE_URL: str = 'https://github.com/google/oss-fuzz/issues/new' -ISSUE_TITTLE_TEXT_PREFIX = 'OSS-Fuzz issue' +ISSUE_TITTLE_TEXT_PREFIX: str = 'OSS-Fuzz issue' -ISSUE_TITTLE_TEXT = '{issue_title_prefix} {bug_information}' +ISSUE_TITTLE_TEXT: str = '{issue_title_prefix} {bug_information}' -ISSUE_CONTENT_TEXT = ('OSS-Fuzz has found a bug in this project. Please see ' - f'{TESTCASE_REPORT_URL} ' - 'for details and reproducers.' - '\n\n' - 'This issue is mirrored from ' - f'{MONORAIL_URL} ' - 'and will auto-close if the status changes there.' - '\n\n' - 'If you have trouble accessing this report, ' - f'please file an issue at {OSS_FUZZ_ISSUE_URL}.' - '\n') +ISSUE_CONTENT_TEXT: str = ( + 'OSS-Fuzz has found a bug in this project. Please see ' + f'{TESTCASE_REPORT_URL} ' + 'for details and reproducers.' + '\n\n' + 'This issue is mirrored from ' + f'{MONORAIL_URL} ' + 'and will auto-close if the status changes there.' + '\n\n' + 'If you have trouble accessing this report, ' + f'please file an issue at {OSS_FUZZ_ISSUE_URL}.' + '\n') -ISSUE_ClOSE_COMMENT_TEXT = ('OSS-Fuzz has closed this bug. Please see ' - f'{MONORAIL_URL} ' - 'for details.') +ISSUE_ClOSE_COMMENT_TEXT: str = ('OSS-Fuzz has closed this bug. Please see ' + f'{MONORAIL_URL} ' + 'for details.') -def get_issue_title(testcase): +def get_issue_title(testcase: data_types.Testcase) -> str: """Generate the title of the issue""" return ISSUE_TITTLE_TEXT.format( issue_title_prefix=ISSUE_TITTLE_TEXT_PREFIX, bug_information=testcase.bug_information) -def get_issue_body(testcase): +def get_issue_body(testcase: data_types.Testcase) -> str: """Generate the body of the issue""" return ISSUE_CONTENT_TEXT.format( domain=data_handler.get_domain(), @@ -68,13 +75,13 @@ def get_issue_body(testcase): bug_information=testcase.bug_information) -def get_issue_close_comment(testcase): +def get_issue_close_comment(testcase: data_types.Testcase) -> str: """Generate the closing comment of the issue""" return ISSUE_ClOSE_COMMENT_TEXT.format( bug_information=testcase.bug_information) -def _get_access_token(): +def _get_access_token() -> github.Github: """Get access to GitHub with the oss-fuzz personal access token""" token = db_config.get_value('oss_fuzz_robot_github_personal_access_token') if not token: @@ -83,14 +90,15 @@ def _get_access_token(): return github.Github(token) -def _filing_enabled(testcase): +def _filing_enabled(testcase: data_types.Testcase) -> bool: """Check if the project YAML file requires to file a GitHub issue.""" require_issue = data_handler.get_value_from_job_definition( testcase.job_type, 'FILE_GITHUB_ISSUE', default='False') return utils.string_is_true(require_issue) -def _get_repo(testcase, access): +def _get_repo(testcase: data_types.Testcase, + access: github.Github) -> Repository | None: """Get the GitHub repository to file the issue""" repo_url = data_handler.get_value_from_job_definition(testcase.job_type, 'MAIN_REPO', '') @@ -113,7 +121,7 @@ def _get_repo(testcase, access): return target_repo -def _find_existing_issue(repo, issue_title): +def _find_existing_issue(repo: Repository, issue_title: str) -> Issue | None: """Checking if there is an existing open issue under the same name""" for issue in repo.get_issues(): if issue.title == issue_title: @@ -122,7 +130,7 @@ def _find_existing_issue(repo, issue_title): return None -def _post_issue(repo, testcase): +def _post_issue(repo: Repository, testcase: data_types.Testcase) -> Issue: """Post the issue to the Github repo of the project.""" issue_title = get_issue_title(testcase) issue_body = get_issue_body(testcase) @@ -130,13 +138,14 @@ def _post_issue(repo, testcase): repo.create_issue(title=issue_title, body=issue_body)) -def update_testcase_properties(testcase, repo, issue): +def update_testcase_properties(testcase: data_types.Testcase, repo: Repository, + issue: Issue) -> None: """Update the GitHub-related properties in the FiledBug entity.""" testcase.github_repo_id = repo.id testcase.github_issue_num = issue.number -def file_issue(testcase): +def file_issue(testcase: data_types.Testcase) -> None: """File an issue to the GitHub repo of the project""" if not _filing_enabled(testcase): return @@ -170,17 +179,18 @@ def file_issue(testcase): update_testcase_properties(testcase, repo, issue) -def _issue_recorded(testcase): +def _issue_recorded(testcase: data_types.Testcase) -> bool: """Verify the issue has been filed.""" - return testcase.github_repo_id and testcase.github_issue_num + return cast(bool, testcase.github_repo_id and testcase.github_issue_num) -def _get_issue(testcase, access): +def _get_issue(testcase: data_types.Testcase, + access: github.Github) -> Issue | None: """Locate the issue of the testcase.""" repo_id = testcase.github_repo_id issue_num = testcase.github_issue_num try: - repo = access.get_repo(repo_id) + repo = access.get_repo(cast(int, repo_id)) except github.UnknownObjectException: logs.error(f'Unable to locate the GitHub repository id {repo_id}.') return None @@ -191,21 +201,22 @@ def _get_issue(testcase, access): return None try: - target_issue = repo.get_issue(issue_num) + target_issue = repo.get_issue(cast(int, issue_num)) except github.UnknownObjectException: logs.error(f'Unable to locate the GitHub issue number {issue_num}.') target_issue = None return target_issue -def _close_issue_with_comment(testcase, issue): +def _close_issue_with_comment(testcase: data_types.Testcase, + issue: Issue) -> None: """Generate closing comment, comment, and close the GitHub issue.""" issue_close_comment = get_issue_close_comment(testcase) issue.create_comment(issue_close_comment) issue.edit(state='closed') -def close_issue(testcase): +def close_issue(testcase: data_types.Testcase) -> None: """Close the issue on GitHub, when the same issue is closed on Monorail.""" if not _issue_recorded(testcase): return @@ -227,7 +238,7 @@ def close_issue(testcase): f'in GitHub repository {testcase.github_repo_id}.') -def get_my_issues(): +def get_my_issues() -> PaginatedList: """Get all issues filed by oss-fuzz-robot.""" access_token = _get_access_token() return access_token.search_issues(f'author:{GITHUB_HANDLE}') diff --git a/src/clusterfuzz/_internal/k8s/service.py b/src/clusterfuzz/_internal/k8s/service.py index 3d3005271c5..fc25a93595f 100644 --- a/src/clusterfuzz/_internal/k8s/service.py +++ b/src/clusterfuzz/_internal/k8s/service.py @@ -16,6 +16,9 @@ import collections import ipaddress import os +from typing import Any +from typing import cast +from typing import NamedTuple import uuid import google.auth @@ -23,6 +26,7 @@ from googleapiclient import discovery import jinja2 from kubernetes import client as k8s_client +from kubernetes.client import rest import yaml from clusterfuzz._internal.base import feature_flags @@ -36,23 +40,24 @@ from clusterfuzz._internal.remote_task import remote_task_types from clusterfuzz._internal.system import environment -CLUSTER_NAME = project_config = local_config.ProjectConfig().get( +CLUSTER_NAME: str = local_config.ProjectConfig().get( 'cluster_name', 'clusterfuzz-cronjobs-gke') -K8S_JOBS_PENDING_LIMIT_DEFAULT = 1000 +K8S_JOBS_PENDING_LIMIT_DEFAULT: int = 1000 -KubernetesJobConfig = collections.namedtuple('KubernetesJobConfig', [ - 'job_type', - 'docker_image', - 'command', - 'disk_size_gb', - 'service_account_email', - 'clusterfuzz_release', - 'is_kata', -]) +class KubernetesJobConfig(NamedTuple): + job_type: str + docker_image: str + command: str + disk_size_gb: Any + service_account_email: str + clusterfuzz_release: str + is_kata: bool -def _get_config_names(remote_tasks: list[remote_task_types.RemoteTask]): + +def _get_config_names(remote_tasks: list[remote_task_types.RemoteTask] + ) -> dict[tuple[str, str], tuple[str, Any, str | None]]: """Gets the name of the configs for each batch_task. Returns a dict that is indexed by command and job_type for efficient lookup.""" @@ -134,7 +139,7 @@ def _get_k8s_job_configs(remote_tasks: list[remote_task_types.RemoteTask] def _create_job_body(config: KubernetesJobConfig, input_url: str, - service_account_name: str) -> dict: + service_account_name: str) -> dict[str, Any]: """Creates the body of a Kubernetes job.""" job_name = f'cf-job-{str(uuid.uuid4())}' @@ -161,13 +166,13 @@ def _create_job_body(config: KubernetesJobConfig, input_url: str, # Render the template and load as YAML. rendered_spec = template.render(context) - return yaml.safe_load(rendered_spec) + return cast(dict[str, Any], yaml.safe_load(rendered_spec)) class KubernetesService(remote_task_types.RemoteTaskInterface): """A remote task execution client for Kubernetes.""" - def __init__(self, k8s_config_loaded: bool = False): + def __init__(self, k8s_config_loaded: bool = False) -> None: # In e2e tests, the kubeconfig is already loaded by the test setup. if not k8s_config_loaded: self._load_gke_credentials() @@ -175,7 +180,7 @@ def __init__(self, k8s_config_loaded: bool = False): self._core_api = k8s_client.CoreV1Api() self._batch_api = k8s_client.BatchV1Api() - def _load_gke_credentials(self): + def _load_gke_credentials(self) -> None: """Loads GKE credentials and configures the Kubernetes client.""" credentials, _ = google.auth.default() project = environment.get_value('K8S_PROJECT') @@ -211,7 +216,7 @@ def _load_gke_credentials(self): try: ipaddress.ip_address(endpoint) - configuration.ssl_ca_cert = ca_cert_path + cast(Any, configuration).ssl_ca_cert = ca_cert_path except ValueError: # If the endpoint is a hostname, we assume it's using a public CA or # the system trust store should be used. @@ -220,13 +225,14 @@ def _load_gke_credentials(self): configuration.verify_ssl = True - def get_token(creds): + def get_token(creds: Any) -> dict[str, str]: request = google_requests.Request() if not creds.valid or creds.expired: creds.refresh(request) - return {'authorization': 'Bearer ' + creds.token} + return {'authorization': 'Bearer ' + cast(str, creds.token)} - configuration.refresh_api_key_hook = lambda _: get_token(credentials) + cast(Any, + configuration).refresh_api_key_hook = lambda _: get_token(credentials) configuration.api_key = get_token(credentials) k8s_client.Configuration.set_default(configuration) @@ -241,7 +247,7 @@ def _create_service_account_if_needed(self, self._core_api.read_namespaced_service_account(service_account_name, namespace) return service_account_name - except k8s_client.rest.ApiException as e: + except rest.ApiException as e: if e.status != 404: raise @@ -267,7 +273,7 @@ def create_job(self, config: KubernetesJobConfig, input_url: str) -> str: config.service_account_email) job_body = _create_job_body(config, input_url, service_account_name) self._batch_api.create_namespaced_job(body=job_body, namespace='default') - return job_body['metadata']['name'] + return cast(str, job_body['metadata']['name']) def _get_pending_jobs_count(self) -> int: """Returns the number of pending jobs.""" @@ -282,8 +288,9 @@ def _get_pending_jobs_count(self) -> int: logs.error(f'Failed to list pods: {e}') return 0 - def create_utask_main_job(self, module: str, job_type: str, - input_download_url: str): + def create_utask_main_job( + self, module: str, job_type: str, + input_download_url: str) -> list[remote_task_types.RemoteTask]: """Creates a single Kubernetes job for a uworker main task.""" command = task_utils.get_command_from_module(module) @@ -294,7 +301,8 @@ def create_utask_main_job(self, module: str, job_type: str, return uncreated_tasks def create_utask_main_jobs(self, - remote_tasks: list[remote_task_types.RemoteTask]): + remote_tasks: list[remote_task_types.RemoteTask] + ) -> list[remote_task_types.RemoteTask]: """Creates Kubernetes jobs for a list of uworker main tasks. This method groups the tasks by their workload specification to efficiently @@ -315,12 +323,14 @@ def create_utask_main_jobs(self, f'for k8s.') return remote_tasks - job_specs = collections.defaultdict(list) + job_specs: dict[KubernetesJobConfig, list[str]] = collections.defaultdict( + list) configs = _get_k8s_job_configs(remote_tasks) for remote_task in remote_tasks: logs.info( f'Scheduling {remote_task.command}, {remote_task.job_type} in K8s.') config = configs[(remote_task.command, remote_task.job_type)] + assert remote_task.input_download_url is not None job_specs[config].append(remote_task.input_download_url) logs.info('Creating Kubernetes jobs.') for config, input_urls in job_specs.items(): diff --git a/src/clusterfuzz/_internal/metrics/crash_stats.py b/src/clusterfuzz/_internal/metrics/crash_stats.py index edd5d49580a..de77291cb3b 100644 --- a/src/clusterfuzz/_internal/metrics/crash_stats.py +++ b/src/clusterfuzz/_internal/metrics/crash_stats.py @@ -16,12 +16,13 @@ import datetime import json import math +from typing import Any from clusterfuzz._internal.datastore import data_types from clusterfuzz._internal.google_cloud_utils import big_query from clusterfuzz._internal.system import environment -SQL = """ +SQL: str = """ WITH # Deduplicate rows in case build_crash_stats runs twice on the same hour. uniqueRows AS ( @@ -111,7 +112,7 @@ """ -def get_remainder_for_index(true_end, time_span): +def get_remainder_for_index(true_end: int, time_span: int) -> int: """Get remainder. This should be tested together with convert_index_to_hour.""" # The remainder needs +1 because the cut-off is at the end of true_end. @@ -120,7 +121,7 @@ def get_remainder_for_index(true_end, time_span): return (true_end % time_span) + 1 -def convert_index_to_hour(index, time_span, remainder): +def convert_index_to_hour(index: int, time_span: int, remainder: int) -> int: """Convert index to hour.""" # This needs -1 because the end hour is inclusive. For example, if the period # represents [2, 26), the end hour is 25. @@ -131,8 +132,17 @@ def convert_index_to_hour(index, time_span, remainder): return ((index + 1) * time_span) + remainder - 1 -def get(end, days, block, group_by, where_clause, group_having_clause, sort_by, - offset, limit): +def get( + end: int, + days: int, + block: str, + group_by: str, + where_clause: str, + group_having_clause: str, + sort_by: str, + offset: int, + limit: int | None = None, +) -> tuple[int, list[dict[str, Any]]]: """Query from BigQuery given the params.""" if where_clause: where_clause = '(%s) AND ' % where_clause @@ -203,12 +213,12 @@ def get(end, days, block, group_by, where_clause, group_having_clause, sort_by, return result.total_count, items -def get_datetime(hours): +def get_datetime(hours: int) -> datetime.datetime: """Get datetime obj from hours from epoch.""" return datetime.datetime.utcfromtimestamp(hours * 60 * 60) -def _get_first_or_last_successful_hour(is_last): +def _get_first_or_last_successful_hour(is_last: bool) -> int | None: """Get the first successful hour.""" order = data_types.BuildCrashStatsJobHistory.end_time_in_hours if is_last: @@ -221,12 +231,12 @@ def _get_first_or_last_successful_hour(is_last): return item.end_time_in_hours -def get_last_successful_hour(): +def get_last_successful_hour() -> int | None: """Get the last hour that ran successfully. We want to run the next hour.""" return _get_first_or_last_successful_hour(is_last=True) -def get_min_hour(): +def get_min_hour() -> int: """Get the first hour that ran successfully (for the date-time picker).""" hour = _get_first_or_last_successful_hour(is_last=False) @@ -239,7 +249,7 @@ def get_min_hour(): return (hour or 0) + 1 -def get_max_hour(): +def get_max_hour() -> int: """Get the last hour that can be selected by the date-time picker.""" hour = get_last_successful_hour() @@ -253,7 +263,8 @@ def get_max_hour(): @environment.local_noop -def get_last_crash_time(testcase): +def get_last_crash_time(testcase: data_types.Testcase, + ) -> datetime.datetime | None: """Return timestamp for last crash with same crash params as testcase.""" client = big_query.Client() diff --git a/src/clusterfuzz/_internal/metrics/events.py b/src/clusterfuzz/_internal/metrics/events.py index 7cd684ffe94..0d07bac429f 100644 --- a/src/clusterfuzz/_internal/metrics/events.py +++ b/src/clusterfuzz/_internal/metrics/events.py @@ -15,16 +15,16 @@ from abc import ABC from abc import abstractmethod +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Mapping +from collections.abc import Sequence from dataclasses import asdict from dataclasses import dataclass from dataclasses import field from dataclasses import InitVar import datetime from typing import Any -from typing import Generator -from typing import Mapping -from typing import Sequence -from typing import Type from clusterfuzz._internal.base import errors from clusterfuzz._internal.base.tasks import task_utils @@ -36,7 +36,7 @@ from clusterfuzz._internal.system import environment -def _get_datetime_now(tz_aware=False): +def _get_datetime_now(tz_aware: bool = False) -> datetime.datetime: """Returns the current UTC datetime (useful for testing).""" utc_time = datetime.datetime.now(datetime.timezone.utc) if not tz_aware: @@ -47,76 +47,76 @@ def _get_datetime_now(tz_aware=False): class EventTypes: """Specific event types.""" - FUZZER_TASK_EXECUTION = 'fuzzer_task_execution' - ISSUE_CLOSING = 'issue_closing' - ISSUE_FILING = 'issue_filing' - TASK_EXECUTION = 'task_execution' - TESTCASE_CREATION = 'testcase_creation' - TESTCASE_FIXED = 'testcase_fixed' - TESTCASE_GROUPING = 'testcase_grouping' - TESTCASE_REJECTION = 'testcase_rejection' + FUZZER_TASK_EXECUTION: str = 'fuzzer_task_execution' + ISSUE_CLOSING: str = 'issue_closing' + ISSUE_FILING: str = 'issue_filing' + TASK_EXECUTION: str = 'task_execution' + TESTCASE_CREATION: str = 'testcase_creation' + TESTCASE_FIXED: str = 'testcase_fixed' + TESTCASE_GROUPING: str = 'testcase_grouping' + TESTCASE_REJECTION: str = 'testcase_rejection' class TestcaseOrigin: """Testcase creation origins.""" - MANUAL_UPLOAD = 'manual_upload' - FUZZ_TASK = 'fuzz_task' - CORPUS_PRUNING = 'corpus_pruning' + MANUAL_UPLOAD: str = 'manual_upload' + FUZZ_TASK: str = 'fuzz_task' + CORPUS_PRUNING: str = 'corpus_pruning' class RejectionReason: """Explanation for the testcase rejection values.""" - ANALYZE_NO_REPRO = 'analyze_no_repro' - ANALYZE_FLAKE_ON_FIRST_ATTEMPT = 'analyze_flake_on_first_attempt' - CLEANUP_UNREPRODUCIBLE_NO_ISSUE = 'cleanup_unreproducible_no_issue' - CLEANUP_DUPLICATE_NO_ISSUE = 'cleanup_duplicate_no_issue' - CLEANUP_UNREPRODUCIBLE_WITH_ISSUE = 'cleanup_unreproducible_with_issue' - CLEANUP_ISSUE_CLOSED = 'cleanup_issue_closed' - CLEANUP_INVALID_JOB = 'cleanup_invalid_job' - GROUPER_DUPLICATE = 'grouper_duplicate' - GROUPER_OVERFLOW = 'grouper_overflow' - PROGRESSION_BUILD_NOT_FOUND = 'progression_build_not_found' - PROGRESSION_BAD_STATE_MIN_MAX = 'progression_bad_state_min_max' + ANALYZE_NO_REPRO: str = 'analyze_no_repro' + ANALYZE_FLAKE_ON_FIRST_ATTEMPT: str = 'analyze_flake_on_first_attempt' + CLEANUP_UNREPRODUCIBLE_NO_ISSUE: str = 'cleanup_unreproducible_no_issue' + CLEANUP_DUPLICATE_NO_ISSUE: str = 'cleanup_duplicate_no_issue' + CLEANUP_UNREPRODUCIBLE_WITH_ISSUE: str = 'cleanup_unreproducible_with_issue' + CLEANUP_ISSUE_CLOSED: str = 'cleanup_issue_closed' + CLEANUP_INVALID_JOB: str = 'cleanup_invalid_job' + GROUPER_DUPLICATE: str = 'grouper_duplicate' + GROUPER_OVERFLOW: str = 'grouper_overflow' + PROGRESSION_BUILD_NOT_FOUND: str = 'progression_build_not_found' + PROGRESSION_BAD_STATE_MIN_MAX: str = 'progression_bad_state_min_max' class TaskStage: """Task stage, usually applicable for untrusted tasks.""" - PREPROCESS = 'preprocess' - MAIN = 'main' - POSTPROCESS = 'postprocess' - NA = 'n/a' + PREPROCESS: str = 'preprocess' + MAIN: str = 'main' + POSTPROCESS: str = 'postprocess' + NA: str = 'n/a' class TaskStatus: """Task status.""" - STARTED = 'started' - FINISHED = 'finished' - POST_STARTED = 'postprocess_started' - POST_COMPLETED = 'postprocess_completed' - EXCEPTION = 'exception' + STARTED: str = 'started' + FINISHED: str = 'finished' + POST_STARTED: str = 'postprocess_started' + POST_COMPLETED: str = 'postprocess_completed' + EXCEPTION: str = 'exception' class TaskOutcome: """Task outcomes/exceptions to complement the uworker error types.""" # All caps to maintain style from error types proto. - PREPROCESS_NO_RETURN = 'PREPROCESS_NO_RETURN' - UNHANDLED_EXCEPTION = 'UNHANDLED_EXCEPTION' + PREPROCESS_NO_RETURN: str = 'PREPROCESS_NO_RETURN' + UNHANDLED_EXCEPTION: str = 'UNHANDLED_EXCEPTION' class ClosingReason: """Reason for closing an issue during cleanup.""" - TESTCASE_FIXED = 'testcase_fixed' - TESTCASE_UNREPRO = 'testcase_unreproducible' - TESTCASE_INVALID = 'testcase_invalid' + TESTCASE_FIXED: str = 'testcase_fixed' + TESTCASE_UNREPRO: str = 'testcase_unreproducible' + TESTCASE_INVALID: str = 'testcase_invalid' class GroupingReason: """Reason for grouping testcases.""" - SIMILAR_CRASH = 'similar_crash' - SAME_ISSUE = 'same_issue' - IDENTICAL_VARIANT = 'identical_variant' - GROUP_MERGE = 'group_merge' - UNGROUPED = 'ungrouped' + SIMILAR_CRASH: str = 'similar_crash' + SAME_ISSUE: str = 'same_issue' + IDENTICAL_VARIANT: str = 'identical_variant' + GROUP_MERGE: str = 'group_merge' + UNGROUPED: str = 'ungrouped' @dataclass(kw_only=True) @@ -136,7 +136,7 @@ class Event: operating_system: str | None = field(init=False, default=None) os_version: str | None = field(init=False, default=None) - def __post_init__(self, **kwargs): + def __post_init__(self, **kwargs: Any) -> None: del kwargs self.timestamp = _get_datetime_now() common_ctx = logs.get_common_log_context() @@ -162,14 +162,16 @@ class BaseTestcaseEvent(Event): testcase: InitVar[data_types.Testcase | None] = None # Testcase ID (either retrieved from testcase entity or directly set). - testcase_id: int | None = None + testcase_id: int | str | None = None # Testcase metadata (retrieved from the testcase entity, if available). fuzzer: str | None = field(init=False, default=None) job: str | None = field(init=False, default=None) crash_revision: int | None = field(init=False, default=None) - def __post_init__(self, testcase=None, **kwargs): + def __post_init__(self, + testcase: data_types.Testcase | None = None, + **kwargs: Any) -> None: if testcase is not None: if self.testcase_id is None: self.testcase_id = testcase.key.id() @@ -187,7 +189,7 @@ class BaseTaskEvent(Event): # Task name retrieved from environment var (if not directly set). task_name: str | None = None - def __post_init__(self, **kwargs): + def __post_init__(self, **kwargs: Any) -> None: if self.task_id is None: self.task_id = environment.get_value('CF_TASK_ID', None) if self.task_name is None: @@ -302,7 +304,7 @@ class FuzzerTaskExecutionEvent(BaseTaskEvent): # Mapping of specific event types to their data classes. -_EVENT_TYPE_CLASSES = { +_EVENT_TYPE_CLASSES: dict[str, Callable[[], Event]] = { EventTypes.FUZZER_TASK_EXECUTION: FuzzerTaskExecutionEvent, EventTypes.ISSUE_CLOSING: IssueClosingEvent, EventTypes.ISSUE_FILING: IssueFilingEvent, @@ -363,10 +365,10 @@ class NDBEventRepository(IEventRepository, EventHandler): the correct entity. """ # Maps `event_type` to a Datastore model. - _event_to_entity_map: dict[str, Type[data_types.Model]] = { + _event_to_entity_map: dict[str, type[data_types.Model]] = { EventTypes.FUZZER_TASK_EXECUTION: data_types.FuzzerTaskEvent, } - _default_entity = data_types.TestcaseLifecycleEvent + _default_entity: type[data_types.Model] = data_types.TestcaseLifecycleEvent def _serialize_event(self, event: Event) -> data_types.Model | None: """Converts an event object into the Datastore entity.""" @@ -388,7 +390,7 @@ def _deserialize_event(self, entity: data_types.Model) -> Event | None: raise TypeError( f'Datastore entity should contain an event_type: {entity.key}.') - event_type = entity.event_type # type: ignore + event_type: str = getattr(entity, 'event_type') event_class = _EVENT_TYPE_CLASSES.get(event_type, None) if event_class is None: event = Event(event_type=event_type) @@ -402,7 +404,7 @@ def _deserialize_event(self, entity: data_types.Model) -> Event | None: logs.error(f'Error deserializing Datastore entity to event: {entity}.') return None - def store_event(self, event: Event) -> int | None: + def store_event(self, event: Event) -> int | str | None: """Stores a Datastore entity and returns its ID.""" entity = self._serialize_event(event) if entity is None: @@ -462,8 +464,9 @@ class EventIssueNotification(EventHandler): `{ : True | list[]}` If set to True, all occurrences of the event type are disabled. """ + disabled_events: dict[str, Any] - def __init__(self, disabled_events: dict | None = None): + def __init__(self, disabled_events: dict[str, Any] | None = None) -> None: if disabled_events is None: disabled_events = {} self.disabled_events = disabled_events @@ -615,7 +618,7 @@ def emit(event: Event) -> None: def get_events( equality_filters: Mapping[str, data_handler.FilterValue] | None = None, - order_by: Sequence[str] | None = None) -> Generator: + order_by: Sequence[str] | None = None) -> Generator[Event, None, None]: """Yields events matching the equality filters and ordering.""" repository = get_repository() if repository: @@ -623,10 +626,11 @@ def get_events( equality_filters=equality_filters, order_by=order_by) -def get_events_from_testcase(testcase_id: int, - event_type: str | None = None, - task_name: str | None = None, - latest_first: bool = True) -> Generator: +def get_events_from_testcase( + testcase_id: int | str, + event_type: str | None = None, + task_name: str | None = None, + latest_first: bool = True) -> Generator[Event, None, None]: """Yields events from a testcase, with optional filters. If latest_first is True, events are yielded in reverse chronological order. diff --git a/src/clusterfuzz/_internal/metrics/fuzzer_logs.py b/src/clusterfuzz/_internal/metrics/fuzzer_logs.py index b9e6149a945..1517d1748be 100644 --- a/src/clusterfuzz/_internal/metrics/fuzzer_logs.py +++ b/src/clusterfuzz/_internal/metrics/fuzzer_logs.py @@ -27,12 +27,13 @@ LOG_PATH_FORMAT = DATE_FORMAT + '/' + TIME_FORMAT -def get_bucket(): +def get_bucket() -> str | None: """Return path to fuzzer logs bucket.""" return local_config.ProjectConfig().get('logs.fuzzer.bucket') -def get_log_relative_path(log_time, file_extension=None): +def get_log_relative_path(log_time: datetime.datetime, + file_extension: str | None = None) -> str: """Generate a relative path for a log using the given time. Args: log_time: A datetime object. @@ -47,7 +48,10 @@ def get_log_relative_path(log_time, file_extension=None): return log_time.strftime(LOG_PATH_FORMAT) + file_extension -def get_logs_directory(bucket_name, fuzzer_name, job_type=None, logs_date=None): +def get_logs_directory(bucket_name: str | None, + fuzzer_name: str | None, + job_type: str | None = None, + logs_date: datetime.date | str | None = None) -> str: """Get directory path of logs for a given fuzzer/job. Args: bucket_name: Bucket logs are stored in. @@ -70,11 +74,11 @@ def get_logs_directory(bucket_name, fuzzer_name, job_type=None, logs_date=None): return path -def get_logs_gcs_path(bucket_name=None, - time=None, - fuzzer_name=None, - job_type=None, - file_extension=None): +def get_logs_gcs_path(bucket_name: str | None = None, + time: datetime.datetime | None = None, + fuzzer_name: str | None = None, + job_type: str | None = None, + file_extension: str | None = None) -> str: """Determines the GCS path to upload a log file to. Args: bucket_name: Bucket logs are stored in. @@ -105,13 +109,13 @@ def get_logs_gcs_path(bucket_name=None, return log_path -def upload_to_logs(bucket_name, - contents, - time=None, - fuzzer_name=None, - job_type=None, - file_extension=None, - signed_upload_url=None): +def upload_to_logs(bucket_name: str | None, + contents: str | bytes, + time: datetime.datetime | None = None, + fuzzer_name: str | None = None, + job_type: str | None = None, + file_extension: str | None = None, + signed_upload_url: str | None = None) -> None: """Uploads file contents to log directory in GCS bucket. Args: bucket_name: Bucket logs are stored in. @@ -150,10 +154,10 @@ def upload_to_logs(bucket_name, logs.error('Failed to write file to logs bucket.', log_path=log_path) -def upload_script_log(log_contents, - fuzzer_name=None, - job_type=None, - signed_upload_url=None): +def upload_script_log(log_contents: str, + fuzzer_name: str | None = None, + job_type: str | None = None, + signed_upload_url: str | None = None) -> None: """Uploads logs to script logs GCS bucket. Args: logs_bucket: Bucket logs are stored in. diff --git a/src/clusterfuzz/_internal/metrics/fuzzer_stats.py b/src/clusterfuzz/_internal/metrics/fuzzer_stats.py index 63d86cf214a..fc97df61304 100644 --- a/src/clusterfuzz/_internal/metrics/fuzzer_stats.py +++ b/src/clusterfuzz/_internal/metrics/fuzzer_stats.py @@ -20,6 +20,17 @@ import os import random import re +from typing import Any +from typing import Callable +from typing import cast +from typing import ClassVar +from typing import Dict +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Type +from typing import Union from clusterfuzz._internal.base import memoize from clusterfuzz._internal.base import utils @@ -33,9 +44,9 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import shell -STATS_FILE_EXTENSION = '.stats2' +STATS_FILE_EXTENSION: str = '.stats2' -JOB_RUN_SCHEMA = { +JOB_RUN_SCHEMA: Dict[str, Any] = { 'fields': [{ 'name': 'testcases_executed', 'type': 'INTEGER', @@ -120,7 +131,8 @@ class FuzzerStatsError(ValueError): """Fuzzer stats exception.""" -def _timedelta_to_duration_string(time_delta): +def _timedelta_to_duration_string( + time_delta: Union[datetime.timedelta, str]) -> str: """Converts a datetime.timedelta to ISO8601 duration string. BigQuery Load API requires the ISO8601 duration string rather than an INTERVAL @@ -144,65 +156,68 @@ def _timedelta_to_duration_string(time_delta): class BaseRun: """Base run.""" - VALID_FIELDNAME_PATTERN = re.compile(r'[a-zA-Z][a-zA-Z0-9_]*') + VALID_FIELDNAME_PATTERN: ClassVar[re.Pattern[str]] = re.compile( + r'[a-zA-Z][a-zA-Z0-9_]*') - def __init__(self, fuzzer, job, build_revision, timestamp): - self._stats_data = { + def __init__(self, fuzzer: Optional[str], job: Optional[str], + build_revision: Optional[int], + timestamp: Optional[float]) -> None: + self._stats_data: Dict[str, Any] = { 'fuzzer': fuzzer, 'job': job, 'build_revision': build_revision, 'timestamp': timestamp, } - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return self._stats_data.__getitem__(key) - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: if not re.compile(self.VALID_FIELDNAME_PATTERN): raise ValueError('Invalid key name.') return self._stats_data.__setitem__(key, value) - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: return self._stats_data.__delitem__(key) - def __contains__(self, key): + def __contains__(self, key: object) -> bool: return self._stats_data.__contains__(key) - def to_json(self): + def to_json(self) -> str: """Return JSON representation of the stats.""" return json.dumps(self._stats_data) - def update(self, other): + def update(self, other: Dict[str, Any]) -> None: """Update stats with a dict.""" self._stats_data.update(other) @property - def data(self): + def data(self) -> Dict[str, Any]: return self._stats_data @property - def kind(self): + def kind(self) -> str: return self._stats_data['kind'] @property - def fuzzer(self): + def fuzzer(self) -> str: return self._stats_data['fuzzer'] @property - def job(self): + def job(self) -> str: return self._stats_data['job'] @property - def build_revision(self): + def build_revision(self) -> int: return self._stats_data['build_revision'] @property - def timestamp(self): + def timestamp(self) -> float: return self._stats_data['timestamp'] @staticmethod - def from_json(json_data): + def from_json(json_data: str) -> Optional['BaseRun']: """Convert json to the run.""" try: data = json.loads(json_data) @@ -237,22 +252,23 @@ def from_json(json_data): class JobRun(BaseRun): """Represents stats for a particular job run.""" - SCHEMA = JOB_RUN_SCHEMA + SCHEMA: ClassVar[Optional[Dict[str, Any]]] = JOB_RUN_SCHEMA # `crashes` is a new field that will replace `new_crashes` and `old_crashes`. - def __init__(self, - fuzzer, - job, - build_revision, - timestamp, - number_of_testcases, - new_crashes, - known_crashes, - crashes, - testcases_generated=None, - testcase_generation_duration=None, - testcase_execution_duration=None, - fuzzing_duration=None): + def __init__( + self, + fuzzer: Optional[str], + job: Optional[str], + build_revision: Optional[int], + timestamp: Optional[float], + number_of_testcases: int, + new_crashes: int, + known_crashes: int, + crashes: Any, + testcases_generated: Optional[int] = None, + testcase_generation_duration: Union[datetime.timedelta, str, None] = None, + testcase_execution_duration: Union[datetime.timedelta, str, None] = None, + fuzzing_duration: Union[datetime.timedelta, str, None] = None) -> None: super().__init__(fuzzer, job, build_revision, timestamp) self._stats_data.update({ 'kind': 'JobRun', @@ -279,9 +295,11 @@ def __init__(self, class TestcaseRun(BaseRun): """Represents stats for a particular testcase run.""" - SCHEMA = None + SCHEMA: ClassVar[Optional[Dict[str, Any]]] = None - def __init__(self, fuzzer, job, build_revision, timestamp): + def __init__(self, fuzzer: Optional[str], job: Optional[str], + build_revision: Optional[int], + timestamp: Optional[float]) -> None: super().__init__(fuzzer, job, build_revision, timestamp) self._stats_data.update({ 'kind': 'TestcaseRun', @@ -292,12 +310,13 @@ def __init__(self, fuzzer, job, build_revision, timestamp): self._stats_data['source'] = source @staticmethod - def get_stats_filename(testcase_file_path): + def get_stats_filename(testcase_file_path: str) -> str: """Get stats filename for the given testcase.""" return testcase_file_path + STATS_FILE_EXTENSION @staticmethod - def read_from_disk(testcase_file_path, delete=False): + def read_from_disk(testcase_file_path: str, + delete: bool = False) -> Optional[BaseRun]: """Read the TestcaseRun for the given testcase.""" stats_file_path = TestcaseRun.get_stats_filename(testcase_file_path) if not os.path.exists(stats_file_path): @@ -313,7 +332,8 @@ def read_from_disk(testcase_file_path, delete=False): return fuzzer_run @staticmethod - def write_to_disk(testcase_run, testcase_file_path): + def write_to_disk(testcase_run: Optional[BaseRun], + testcase_file_path: str) -> None: """Write the given TestcaseRun for |testcase_file_path| to disk.""" if not testcase_run: return @@ -326,15 +346,15 @@ def write_to_disk(testcase_run, testcase_file_path): class QueryGroupBy: """GroupBy enum.""" - GROUP_BY_NONE = 0 - GROUP_BY_REVISION = 1 - GROUP_BY_DAY = 2 - GROUP_BY_TIME = 3 - GROUP_BY_JOB = 4 - GROUP_BY_FUZZER = 5 + GROUP_BY_NONE: ClassVar[int] = 0 + GROUP_BY_REVISION: ClassVar[int] = 1 + GROUP_BY_DAY: ClassVar[int] = 2 + GROUP_BY_TIME: ClassVar[int] = 3 + GROUP_BY_JOB: ClassVar[int] = 4 + GROUP_BY_FUZZER: ClassVar[int] = 5 -def group_by_to_field_name(group_by): +def group_by_to_field_name(group_by: Optional[int]) -> Optional[str]: """Convert QueryGroupBy value to its corresponding field name.""" if group_by == QueryGroupBy.GROUP_BY_REVISION: return 'build_revision' @@ -357,7 +377,10 @@ def group_by_to_field_name(group_by): class BuiltinFieldData: """Represents a cell value for a builtin field.""" - def __init__(self, value, sort_key=None, link=None): + def __init__(self, + value: Any, + sort_key: Any = None, + link: Optional[str] = None) -> None: self.value = value self.sort_key = sort_key self.link = link @@ -366,11 +389,12 @@ def __init__(self, value, sort_key=None, link=None): class BuiltinFieldSpecifier: """Represents a builtin field.""" - def __init__(self, name, alias=None): + def __init__(self, name: str, alias: Optional[str] = None) -> None: self.name = name self.alias = alias - def create(self, ctx=None): + def create(self, ctx: Optional['BuiltinFieldContext'] = None + ) -> Optional['BuiltinField']: """Create the actual BuiltinField.""" constructor = BUILTIN_FIELD_CONSTRUCTORS.get(self.name) if not constructor: @@ -378,37 +402,43 @@ def create(self, ctx=None): return constructor(ctx) - def field_class(self): + def field_class(self) -> Optional[Type['BuiltinField']]: """Return the class for the field.""" constructor = BUILTIN_FIELD_CONSTRUCTORS.get(self.name) if not constructor: return None if isinstance(constructor, functools.partial): - return constructor.func + return cast(Type['BuiltinField'], constructor.func) - return constructor + return cast(Type['BuiltinField'], constructor) class BuiltinField: """Base Builtin field.""" - def __init__(self, ctx=None): + CONTEXT_CLASS: ClassVar[Optional[Type['BuiltinFieldContext']]] = None + VALUE_TYPE: ClassVar[Optional[Type[Any]]] = None + + def __init__(self, ctx: Optional['BuiltinFieldContext'] = None) -> None: self.ctx = ctx - def get(self, group_by, group_by_value): # pylint: disable=unused-argument + def get(self, group_by: int, + group_by_value: Any) -> Optional[BuiltinFieldData]: """Return BuiltinFieldData.""" - return None + del group_by, group_by_value class BuiltinFieldContext: """Context for builtin fields.""" - def __init__(self, fuzzer=None, jobs=None): + def __init__(self, + fuzzer: Optional[str] = None, + jobs: Optional[Sequence[str]] = None) -> None: self.fuzzer = fuzzer self.jobs = jobs - def single_job_or_none(self): + def single_job_or_none(self) -> Optional[str]: """Return the job if only 1 is specified, or None.""" if self.jobs and len(self.jobs) == 1: return self.jobs[0] @@ -419,35 +449,44 @@ def single_job_or_none(self): class CoverageFieldContext(BuiltinFieldContext): """Coverage field context. Acts as a cache.""" - def __init__(self, fuzzer=None, jobs=None): + def __init__(self, + fuzzer: Optional[str] = None, + jobs: Optional[Sequence[str]] = None) -> None: super().__init__(fuzzer=fuzzer, jobs=jobs) @memoize.wrap(memoize.FifoInMemory(256)) - def get_coverage_info(self, fuzzer, date=None): + def get_coverage_info(self, + fuzzer: Optional[str], + date: Optional[Union[datetime.date, str]] = None + ) -> Optional[data_types.CoverageInformation]: """Return coverage info of child fuzzers.""" if fuzzer in data_types.BUILTIN_FUZZERS: # Get coverage info for a job (i.e. a project). job = self.single_job_or_none() - project = data_handler.get_project_name(job) + project = data_handler.get_project_name(cast(str, job)) return get_coverage_info(project, date) - fuzz_target = data_handler.get_fuzz_target(fuzzer) + fuzz_target = data_handler.get_fuzz_target(cast(str, fuzzer)) if fuzz_target: fuzzer = fuzz_target.project_qualified_name() return get_coverage_info(fuzzer, date) -class BaseCoverageField: +class BaseCoverageField(BuiltinField): """Base builtin field class for coverage related fields.""" - CONTEXT_CLASS = CoverageFieldContext + CONTEXT_CLASS: ClassVar[Optional[Type[ + BuiltinFieldContext]]] = CoverageFieldContext - def __init__(self, ctx): - self.ctx = ctx + def __init__(self, ctx: Optional[CoverageFieldContext] = None) -> None: + super().__init__(ctx) + self.ctx: Optional[CoverageFieldContext] = ctx - def get_coverage_info(self, group_by, group_by_value): + def get_coverage_info(self, group_by: int, group_by_value: Any + ) -> Optional[data_types.CoverageInformation]: """Return coverage information.""" + assert self.ctx is not None coverage_info = None if group_by == QueryGroupBy.GROUP_BY_DAY: # Return coverage data for the fuzzer and the day. @@ -470,15 +509,18 @@ def get_coverage_info(self, group_by, group_by_value): class CoverageField(BaseCoverageField): """Coverage field.""" - EDGE = 0 - FUNCTION = 1 - VALUE_TYPE = float + EDGE: ClassVar[int] = 0 + FUNCTION: ClassVar[int] = 1 + VALUE_TYPE: ClassVar[Optional[Type[Any]]] = float - def __init__(self, coverage_type, ctx=None): + def __init__(self, + coverage_type: int, + ctx: Optional[CoverageFieldContext] = None) -> None: super().__init__(ctx) self.coverage_type = coverage_type - def get(self, group_by, group_by_value): + def get(self, group_by: int, + group_by_value: Any) -> Optional[BuiltinFieldData]: """Return data.""" coverage_info = self.get_coverage_info(group_by, group_by_value) if not coverage_info: @@ -495,8 +537,8 @@ def get(self, group_by, group_by_value): return None if not total: - logs.error( - 'Invalid coverage info: total equals 0 for "%s".' % self.ctx.fuzzer) + logs.error('Invalid coverage info: total equals 0 for "%s".' % cast( + CoverageFieldContext, self.ctx).fuzzer) return BuiltinFieldData('No coverage', sort_key=0.0) percentage = 100.0 * float(covered) / total @@ -506,12 +548,10 @@ def get(self, group_by, group_by_value): class CorpusBackupField(BaseCoverageField): """Link to the latest corpus backup archive.""" - VALUE_TYPE = str + VALUE_TYPE: ClassVar[Optional[Type[Any]]] = str - def __init__(self, ctx=None): - super().__init__(ctx) - - def get(self, group_by, group_by_value): + def get(self, group_by: int, + group_by_value: Any) -> Optional[BuiltinFieldData]: """Return data.""" coverage_info = self.get_coverage_info(group_by, group_by_value) if not coverage_info: @@ -532,17 +572,21 @@ def get(self, group_by, group_by_value): class CorpusSizeField(BaseCoverageField): """Corpus size field.""" - CORPUS = 0 - QUARANTINE = 1 - VALUE_TYPE = int + CORPUS: ClassVar[int] = 0 + QUARANTINE: ClassVar[int] = 1 + VALUE_TYPE: ClassVar[Optional[Type[Any]]] = int - def __init__(self, corpus_type, ctx=None): + def __init__(self, + corpus_type: int, + ctx: Optional[CoverageFieldContext] = None) -> None: super().__init__(ctx) self.corpus_type = corpus_type - def get(self, group_by, group_by_value): + def get(self, group_by: int, + group_by_value: Any) -> Optional[BuiltinFieldData]: """Return data.""" - if (self.ctx.fuzzer in data_types.BUILTIN_FUZZERS and + if (cast(CoverageFieldContext, + self.ctx).fuzzer in data_types.BUILTIN_FUZZERS and group_by == QueryGroupBy.GROUP_BY_DAY): # Explicitly return None here, as coverage_info below might exist and have # default corpus size of 0, which might look confusing on the stats page. @@ -575,12 +619,10 @@ def get(self, group_by, group_by_value): class CoverageReportField(BaseCoverageField): """Coverage report field.""" - VALUE_TYPE = str - - def __init__(self, ctx=None): - super().__init__(ctx) + VALUE_TYPE: ClassVar[Optional[Type[Any]]] = str - def get(self, group_by, group_by_value): + def get(self, group_by: int, + group_by_value: Any) -> Optional[BuiltinFieldData]: """Return data.""" coverage_info = self.get_coverage_info(group_by, group_by_value) if not coverage_info or not coverage_info.html_report_url: @@ -590,26 +632,30 @@ def get(self, group_by, group_by_value): return BuiltinFieldData(display_value, link=coverage_info.html_report_url) -def _logs_bucket_key_fn(func, args, kwargs): # pylint: disable=unused-argument - return 'fuzzer_logs_bucket:' + args[1] +def _logs_bucket_key_fn(func: Any, args: Tuple[Any, ...], + kwargs: Dict[str, Any]) -> str: + del func, kwargs + return 'fuzzer_logs_bucket:' + str(args[1]) class FuzzerRunLogsContext(BuiltinFieldContext): """Fuzzer logs context.""" - MEMCACHE_TTL = 30 * 60 + MEMCACHE_TTL: ClassVar[int] = 30 * 60 - def __init__(self, fuzzer=None, jobs=None): + def __init__(self, + fuzzer: Optional[str] = None, + jobs: Optional[Sequence[str]] = None) -> None: super().__init__(fuzzer=fuzzer, jobs=jobs) @memoize.wrap(memoize.FifoInMemory(1024)) - def _get_logs_bucket_from_job(self, job_type): + def _get_logs_bucket_from_job(self, job_type: Optional[str]) -> Optional[str]: """Get logs bucket from job.""" return data_handler.get_value_from_job_definition_or_environment( - job_type, 'FUZZ_LOGS_BUCKET') + cast(str, job_type), 'FUZZ_LOGS_BUCKET') @memoize.wrap(memoize.Memcache(MEMCACHE_TTL, key_fn=_logs_bucket_key_fn)) - def _get_logs_bucket_from_fuzzer(self, fuzzer_name): + def _get_logs_bucket_from_fuzzer(self, fuzzer_name: str) -> Optional[str]: """Get logs bucket from fuzzer (child fuzzers only).""" jobs = [ mapping.job for mapping in fuzz_target_utils.get_fuzz_target_jobs( @@ -625,7 +671,9 @@ def _get_logs_bucket_from_fuzzer(self, fuzzer_name): return None - def get_logs_bucket(self, fuzzer_name=None, job_type=None): + def get_logs_bucket(self, + fuzzer_name: Optional[str] = None, + job_type: Optional[str] = None) -> Optional[str]: """Return logs bucket for the job.""" if job_type: return self._get_logs_bucket_from_job(job_type) @@ -639,11 +687,18 @@ def get_logs_bucket(self, fuzzer_name=None, job_type=None): class FuzzerRunLogsField(BuiltinField): """Fuzzer logs field.""" - CONTEXT_CLASS = FuzzerRunLogsContext - VALUE_TYPE = str + CONTEXT_CLASS: ClassVar[Optional[Type[ + BuiltinFieldContext]]] = FuzzerRunLogsContext + VALUE_TYPE: ClassVar[Optional[Type[Any]]] = str + + def __init__(self, ctx: Optional[FuzzerRunLogsContext] = None) -> None: + super().__init__(ctx) + self.ctx: Optional[FuzzerRunLogsContext] = ctx - def _get_logs_bucket_path(self, group_by, group_by_value): + def _get_logs_bucket_path(self, group_by: int, + group_by_value: Any) -> Optional[str]: """Return logs bucket path.""" + assert self.ctx is not None fuzzer = self.ctx.fuzzer job = self.ctx.single_job_or_none() date = None @@ -676,7 +731,8 @@ def _get_logs_bucket_path(self, group_by, group_by_value): return 'gs:/' + fuzzer_logs.get_logs_directory(logs_bucket, fuzzer, job, date) - def get(self, group_by, group_by_value): + def get(self, group_by: int, + group_by_value: Any) -> Optional[BuiltinFieldData]: """Return data.""" logs_path = self._get_logs_bucket_path(group_by, group_by_value) if not logs_path: @@ -689,23 +745,23 @@ class QueryField: """Represents a query field.""" def __init__(self, - table_alias, - field_name, - aggregate_function, - select_alias=None): + table_alias: str, + field_name: str, + aggregate_function: Optional[str], + select_alias: Optional[str] = None) -> None: self.table_alias = table_alias self.name = field_name self.aggregate_function = aggregate_function self.select_alias = select_alias or field_name - def is_custom(self): + def is_custom(self) -> bool: """Return true if this field uses complex query. This field won't appear in the SELECT's fields automatically. We will need to define how to get the data.""" - return (self.aggregate_function and - self.aggregate_function.lower() == 'custom') + return bool(self.aggregate_function and + self.aggregate_function.lower() == 'custom') - def __str__(self): + def __str__(self) -> str: if self.aggregate_function: result = '%s(%s.%s)' % (self.aggregate_function, self.table_alias, self.name) @@ -721,13 +777,16 @@ def __str__(self): class Query: """Represents a stats query.""" - def _ensure_valid_name(self, name, regex): + def _ensure_valid_name(self, name: Optional[str], + regex: re.Pattern[str]) -> None: """Ensure that the given name is valid for fuzzer/jobs.""" if name and not regex.match(name): raise FuzzerStatsError('Invalid fuzzer or job name.') - def __init__(self, fuzzer_name, job_types, query_fields, group_by, date_start, - date_end, base_table, alias): + def __init__(self, fuzzer_name: str, job_types: Optional[Sequence[str]], + query_fields: Sequence[QueryField], group_by: int, + date_start: datetime.date, date_end: datetime.date, + base_table: str, alias: str) -> None: assert group_by is not None self._ensure_valid_name(fuzzer_name, data_types.Fuzzer.VALID_NAME_REGEX) @@ -747,7 +806,7 @@ def __init__(self, fuzzer_name, job_types, query_fields, group_by, date_start, self.fuzzer_or_engine_name = get_fuzzer_or_engine_name(fuzzer_name) - def _group_by_select(self): + def _group_by_select(self) -> Optional[str]: """Return a group by field.""" if self.group_by == QueryGroupBy.GROUP_BY_DAY: return ('TIMESTAMP_TRUNC(TIMESTAMP_SECONDS(CAST(' @@ -758,11 +817,11 @@ def _group_by_select(self): return group_by_to_field_name(self.group_by) - def _group_by(self): + def _group_by(self) -> Optional[str]: """Return the group by part of the query.""" return group_by_to_field_name(self.group_by) - def _select_fields(self): + def _select_fields(self) -> str: """Return fields for the query.""" group_by_select = self._group_by_select() fields = [group_by_select] if group_by_select else [] @@ -778,7 +837,7 @@ def _select_fields(self): return ', '.join(fields) - def _table_name(self): + def _table_name(self) -> str: """Return the table name for the query.""" app_id = utils.get_application_id() @@ -786,19 +845,19 @@ def _table_name(self): return '`%s`.%s.%s' % (app_id, dataset, self.base_table) - def _where(self): + def _where(self) -> str: """Return the where part of the query.""" result = [] result.extend(self._partition_selector()) result.extend(self._job_and_fuzzer_selector()) - result = ' AND '.join(result) - if result: - return 'WHERE ' + result + where_clause = ' AND '.join(result) + if where_clause: + return 'WHERE ' + where_clause return '' - def _job_and_fuzzer_selector(self): + def _job_and_fuzzer_selector(self) -> List[str]: """Return the job filter condition.""" result = [] if self.job_types: @@ -810,7 +869,7 @@ def _job_and_fuzzer_selector(self): return result - def _partition_selector(self): + def _partition_selector(self) -> List[str]: """Return the partition filter condition.""" result = ('(_PARTITIONTIME BETWEEN TIMESTAMP_SECONDS(%d) ' 'AND TIMESTAMP_SECONDS(%d))') @@ -820,7 +879,7 @@ def _partition_selector(self): int(utils.utc_date_to_timestamp(self.date_end))) ] - def build(self): + def build(self) -> str: """Return query.""" query_parts = [ 'SELECT', @@ -830,8 +889,9 @@ def build(self): self._where(), ] - if self._group_by(): - query_parts += ['GROUP BY', self._group_by()] + group_by = self._group_by() + if group_by: + query_parts += ['GROUP BY', group_by] return ' '.join(query_parts) @@ -839,10 +899,11 @@ def build(self): class TestcaseQuery(Query): """The query class for TestcaseRun Query.""" - ALIAS = 't' + ALIAS: ClassVar[str] = 't' - def __init__(self, fuzzer_name, job_types, query_fields, group_by, date_start, - date_end): + def __init__(self, fuzzer_name: str, job_types: Optional[Sequence[str]], + query_fields: Sequence[QueryField], group_by: int, + date_start: datetime.date, date_end: datetime.date) -> None: super().__init__( fuzzer_name=fuzzer_name, job_types=job_types, @@ -857,13 +918,13 @@ def __init__(self, fuzzer_name, job_types, query_fields, group_by, date_start, class JobQuery(Query): """The query class for JobRun Query.""" - DEFAULT_FIELDS = """ + DEFAULT_FIELDS: ClassVar[str] = """ sum(j.testcases_executed) as testcases_executed, custom(j.total_crashes) as total_crashes, custom(j.new_crashes) as new_crashes, custom(j.known_crashes) as known_crashes """ - SQL = """ + SQL: ClassVar[str] = """ WITH JobRunWithConcatedCrashes AS ( SELECT @@ -918,10 +979,11 @@ class JobQuery(Query): FROM JobRunWithSummary """ - ALIAS = 'j' + ALIAS: ClassVar[str] = 'j' - def __init__(self, fuzzer_name, job_types, query_fields, group_by, date_start, - date_end): + def __init__(self, fuzzer_name: str, job_types: Optional[Sequence[str]], + query_fields: Sequence[QueryField], group_by: int, + date_start: datetime.date, date_end: datetime.date) -> None: super().__init__( fuzzer_name=fuzzer_name, @@ -933,7 +995,7 @@ def __init__(self, fuzzer_name, job_types, query_fields, group_by, date_start, base_table='JobRun', alias=JobQuery.ALIAS) - def build(self): + def build(self) -> str: """Return query.""" sql = JobQuery.SQL.format( table_name=self._table_name(), @@ -947,8 +1009,9 @@ def build(self): class TableQuery: """Query for generating results in a table.""" - def __init__(self, fuzzer_name, job_types, stats_columns, group_by, - date_start, date_end): + def __init__(self, fuzzer_name: str, job_types: Optional[Sequence[str]], + stats_columns: str, group_by: int, date_start: datetime.date, + date_end: datetime.date) -> None: assert group_by self.fuzzer_name = fuzzer_name @@ -956,11 +1019,11 @@ def __init__(self, fuzzer_name, job_types, stats_columns, group_by, self.group_by = group_by self.date_start = date_start self.date_end = date_end - self.job_run_query = None - self.testcase_run_query = None + self.job_run_query: Optional[JobQuery] = None + self.testcase_run_query: Optional[TestcaseQuery] = None - job_run_fields = [] - testcase_run_fields = [] + job_run_fields: List[QueryField] = [] + testcase_run_fields: List[QueryField] = [] fields = parse_stats_column_fields(stats_columns) for field in fields: @@ -992,8 +1055,10 @@ def __init__(self, fuzzer_name, job_types, stats_columns, group_by, assert self.job_run_query or self.testcase_run_query, ( 'Unable to create query.') - def _join_subqueries(self): + def _join_subqueries(self) -> str: """Create an inner join for subqueries.""" + assert self.job_run_query is not None + assert self.testcase_run_query is not None result = [ '(%s) as %s' % (self.job_run_query.build(), self.job_run_query.alias), 'INNER JOIN', @@ -1006,14 +1071,16 @@ def _join_subqueries(self): ] return ' '.join(result) - def _single_subquery(self): + def _single_subquery(self) -> str: """Create a single subquery.""" query = self.job_run_query or self.testcase_run_query + assert query is not None return '(%s) as %s' % (query.build(), query.alias) - def build(self): + def build(self) -> str: """Build the table query.""" valid_run_query = self.job_run_query or self.testcase_run_query + assert valid_run_query is not None result = [ # We need to do the below to avoid the duplicate column name error. 'SELECT {0}.{1}, * EXCEPT({1}) FROM'.format( @@ -1029,7 +1096,9 @@ def build(self): return ' '.join(result) -def get_coverage_info(fuzzer, date=None): +def get_coverage_info(fuzzer: Optional[str], + date: Optional[Union[datetime.date, str]] = None + ) -> Optional[data_types.CoverageInformation]: """Returns a CoverageInformation entity for a given fuzzer and date. If date is not specified, returns the latest entity available.""" query = data_types.CoverageInformation.query( @@ -1044,7 +1113,8 @@ def get_coverage_info(fuzzer, date=None): return query.get() -def get_gcs_stats_path(kind, fuzzer, timestamp): +def get_gcs_stats_path(kind: str, fuzzer: str, + timestamp: float) -> Optional[str]: """Return gcs path in the format "/bucket/path/to/containing_dir/" for the given fuzzer, job, and timestamp or revision.""" bucket_name = big_query.get_bucket() @@ -1059,7 +1129,8 @@ def get_gcs_stats_path(kind, fuzzer, timestamp): @environment.local_noop -def upload_stats(stats_list, filename=None): +def upload_stats(stats_list: List[BaseRun], + filename: Optional[str] = None) -> None: """Upload the fuzzer run to the bigquery bucket. Assumes that all the stats given are for the same fuzzer/job run.""" if not stats_list: @@ -1084,7 +1155,7 @@ def upload_stats(stats_list, filename=None): filename = '%016x' % random.randint(0, (1 << 64) - 1) + '.json' # Handle runs that bleed into the next day. - def timestamp_start_of_day(s): + def timestamp_start_of_day(s: BaseRun) -> float: return utils.utc_date_to_timestamp( datetime.datetime.utcfromtimestamp(s.timestamp).date()) @@ -1093,8 +1164,10 @@ def timestamp_start_of_day(s): for timestamp, stats in itertools.groupby(stats_list, timestamp_start_of_day): upload_data = '\n'.join(stat.to_json() for stat in stats) - day_path = 'gs:/' + get_gcs_stats_path( - kind, fuzzer_or_engine_name, timestamp=timestamp) + filename + gcs_stats_path = get_gcs_stats_path( + kind, fuzzer_or_engine_name, timestamp=timestamp) + assert gcs_stats_path is not None + day_path = 'gs:/' + gcs_stats_path + filename if storage.write_data(upload_data.encode('utf-8'), day_path): logs.info(f'Uploaded {kind} stats for {fuzzer} to {day_path}.') @@ -1102,7 +1175,8 @@ def timestamp_start_of_day(s): logs.error(f'Failed to upload {kind} stats for {fuzzer} to {day_path}.') -def parse_stats_column_fields(column_fields): +def parse_stats_column_fields( + column_fields: str) -> List[Union[QueryField, BuiltinFieldSpecifier]]: """Parse the stats column fields.""" # e.g. 'sum(t.field_name) as display_name'. aggregate_regex = re.compile(r'^(\w+)\(([a-z])\.([^\)]+)\)(\s*as\s*(\w+))?$') @@ -1110,7 +1184,7 @@ def parse_stats_column_fields(column_fields): # e.g. '_EDGE_COV as blah'. builtin_regex = re.compile(r'^(_\w+)(\s*as\s*(\w+))?$') - fields = [] + fields: List[Union[QueryField, BuiltinFieldSpecifier]] = [] parts = [field.strip() for field in column_fields.split(',')] for part in parts: match = aggregate_regex.match(part) @@ -1138,21 +1212,21 @@ def parse_stats_column_fields(column_fields): return fields -def get_fuzzer_or_engine_name(fuzzer_name): +def get_fuzzer_or_engine_name(fuzzer_name: str) -> str: """Return fuzzing engine name if it exists, or |fuzzer_name|.""" fuzz_target = data_handler.get_fuzz_target(fuzzer_name) if fuzz_target: - return fuzz_target.engine + return cast(str, fuzz_target.engine) return fuzzer_name -def dataset_name(fuzzer_name): +def dataset_name(fuzzer_name: str) -> str: """Get the stats dataset name for the given |fuzzer_name|.""" return fuzzer_name.replace('-', '_') + '_stats' -BUILTIN_FIELD_CONSTRUCTORS = { +BUILTIN_FIELD_CONSTRUCTORS: Dict[str, Callable[..., BuiltinField]] = { '_EDGE_COV': functools.partial(CoverageField, CoverageField.EDGE), '_FUNC_COV': diff --git a/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py b/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py index c2a9df8d07b..04ab6300026 100644 --- a/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py +++ b/src/clusterfuzz/_internal/metrics/fuzzer_stats_schema.py @@ -13,7 +13,7 @@ # limitations under the License. """Fuzzer stats schema.""" -_COMMON_COLUMNS = [{ +_COMMON_COLUMNS: list[dict[str, str]] = [{ 'mode': 'NULLABLE', 'name': 'kind', 'type': 'STRING' @@ -43,7 +43,7 @@ 'type': 'STRING' }] -_AFL_SCHEMA = [{ +_AFL_SCHEMA: list[dict[str, str]] = [{ 'mode': 'NULLABLE', 'name': 'dict_used', 'type': 'INTEGER' @@ -157,7 +157,7 @@ 'type': 'STRING' }] + _COMMON_COLUMNS -_LIBFUZZER_SCHEMA = [{ +_LIBFUZZER_SCHEMA: list[dict[str, str]] = [{ 'mode': 'NULLABLE', 'name': 'max_len', 'type': 'INTEGER' @@ -361,7 +361,7 @@ 'type': 'INTEGER' }] + _COMMON_COLUMNS -_HONGGFUZZ_SCHEMA = [{ +_HONGGFUZZ_SCHEMA: list[dict[str, str]] = [{ 'mode': 'NULLABLE', 'name': 'timeout_count', 'type': 'INTEGER' @@ -403,7 +403,7 @@ 'type': 'INTEGER' }] + _COMMON_COLUMNS -_CENTIPEDE_SCHEMA = [{ +_CENTIPEDE_SCHEMA: list[dict[str, str]] = [{ 'mode': 'NULLABLE', 'name': 'NumCoveredPcs_Min', 'type': 'INTEGER' @@ -861,7 +861,7 @@ 'type': 'INTEGER' }] + _COMMON_COLUMNS -_SCHEMA = { +_SCHEMA: dict[str, list[dict[str, str]]] = { 'afl': _AFL_SCHEMA, 'honggfuzz': _HONGGFUZZ_SCHEMA, 'libFuzzer': _LIBFUZZER_SCHEMA, @@ -869,7 +869,7 @@ } -def get(engine_name): +def get(engine_name: str) -> dict[str, list[dict[str, str]]] | None: """Get the schema for an engine name.""" schema = _SCHEMA.get(engine_name) if not schema: diff --git a/src/clusterfuzz/_internal/metrics/logs.py b/src/clusterfuzz/_internal/metrics/logs.py index a66fa138ba4..a071238fd3e 100644 --- a/src/clusterfuzz/_internal/metrics/logs.py +++ b/src/clusterfuzz/_internal/metrics/logs.py @@ -13,6 +13,8 @@ # limitations under the License. """Logging functions.""" +from collections.abc import Callable +from collections.abc import Generator import contextlib import dataclasses import datetime @@ -29,8 +31,10 @@ import time import traceback from typing import Any +from typing import cast from typing import NamedTuple from typing import TYPE_CHECKING +from typing import TypeVar from clusterfuzz._internal.system import environment @@ -45,16 +49,16 @@ # accommodate both the primary log message and potential traceback exceptions. # This reserves roughly up to 200 KB for message content, leaving sufficient # space for structured logging metadata within the 256 KB total limit. -STACKDRIVER_LOG_MESSAGE_LIMIT = 80000 -NON_TRUNCATABLE_TYPES = (int, float, bool, type(None)) -LOCAL_LOG_MESSAGE_LIMIT = 100000 -LOCAL_LOG_LIMIT = 500000 -_logger = None -_is_already_handling_uncaught = False -_default_extras = {} +STACKDRIVER_LOG_MESSAGE_LIMIT: int = 80000 +NON_TRUNCATABLE_TYPES: tuple[type, ...] = (int, float, bool, type(None)) +LOCAL_LOG_MESSAGE_LIMIT: int = 100000 +LOCAL_LOG_LIMIT: int = 500000 +_logger: logging.Logger | None = None +_is_already_handling_uncaught: bool = False +_default_extras: dict[str, Any] = {} -def _increment_error_count(): +def _increment_error_count() -> None: """"Increment the error count metric.""" if environment.is_running_on_swarming(): task_name = 'swarming' @@ -69,13 +73,13 @@ def _increment_error_count(): monitoring_metrics.LOG_ERROR_COUNT.increment({'task_name': task_name}) -def _is_local(): +def _is_local() -> bool: """Return whether or not in a local development environment.""" return (bool(os.getenv('LOCAL_DEVELOPMENT')) or os.getenv('SERVER_SOFTWARE', '').startswith('Development/')) -def _console_logging_enabled(): +def _console_logging_enabled() -> bool: """Return bool on where console logging is enabled, usually for tests.""" return bool(os.getenv('LOG_TO_CONSOLE')) @@ -95,21 +99,21 @@ def _cloud_logging_enabled() -> bool: not environment.is_running_unit_tests() and not _is_local()) -def suppress_unwanted_warnings(): +def suppress_unwanted_warnings() -> None: """Suppress unwanted warnings.""" # See https://github.com/googleapis/google-api-python-client/issues/299 logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR) -def set_logger(logger): +def set_logger(logger: logging.Logger | None) -> None: """Set the logger.""" global _logger _logger = logger -def get_handler_config(filename, backup_count): +def get_handler_config(filename: str, backup_count: int) -> dict[str, Any]: """Get handler config.""" - root_directory = os.getenv('ROOT_DIR') + root_directory = cast(str, os.getenv('ROOT_DIR')) file_path = os.path.join(root_directory, filename) max_bytes = 0 if _is_local() else LOCAL_LOG_LIMIT @@ -124,7 +128,7 @@ def get_handler_config(filename, backup_count): } -def get_logging_config_dict(name): +def get_logging_config_dict(name: str) -> dict[str, Any]: """Get config dict for the logger `name`.""" logging_handler = { 'run_bot': @@ -205,7 +209,7 @@ def truncate( # Check if it's a NamedTuple if isinstance(msg, tuple) and hasattr(msg, '_fields'): - msg = msg._asdict() # type: ignore + msg = cast(Any, msg)._asdict() if isinstance(msg, dict): return {k: truncate(v, limit) for k, v in msg.items()} @@ -311,7 +315,7 @@ def _is_json_serializable(obj: Any) -> bool: return False -def update_entry_with_exc(entry, exc_info): +def update_entry_with_exc(entry: dict[str, Any], exc_info: Any) -> None: """Update the dict `entry` with exc_info.""" if not exc_info: return @@ -351,8 +355,11 @@ def update_entry_with_exc(entry, exc_info): } -def uncaught_exception_handler(exception_type, exception_value, - exception_traceback): +def uncaught_exception_handler( + exception_type: type[BaseException] | None, + exception_value: BaseException | None, + exception_traceback: Any, +) -> None: """Handles any exception that are uncaught by logging an error and calling the sys.__excepthook__.""" # Ensure that we are not calling ourself. This shouldn't be needed since we @@ -373,10 +380,12 @@ def uncaught_exception_handler(exception_type, exception_value, 'Uncaught exception', exc_info=(exception_type, exception_value, exception_traceback)) - sys.__excepthook__(exception_type, exception_value, exception_traceback) + sys.__excepthook__( + cast(type[BaseException], exception_type), + cast(BaseException, exception_value), exception_traceback) -def json_fields_filter(record): +def json_fields_filter(record: logging.LogRecord) -> bool: """Add logs `extras` argument to `json_fields` metadata for cloud logging.""" # TODO(vtcosta): This is a workaround to allow structured logs for # cleanup/triage cronjobs in GKE/GAE. We should try to refactor and @@ -390,11 +399,11 @@ def json_fields_filter(record): val if _is_json_serializable(val) else _handle_unserializable(val)) json_extras[key] = truncate(valid_value, STACKDRIVER_LOG_MESSAGE_LIMIT) - record.json_fields.update({'extras': json_extras}) + record.json_fields.update({'extras': json_extras}) # type: ignore return True -def configure_appengine(): +def configure_appengine() -> None: """Configure logging for App Engine.""" logging.getLogger().setLevel(logging.INFO) @@ -408,7 +417,7 @@ def configure_appengine(): logging.getLogger().addHandler(handler) -def configure_k8s(): +def configure_k8s() -> None: """Configure logging for K8S.""" import google.cloud.logging from google.cloud.logging.handlers import CloudLoggingHandler @@ -422,7 +431,7 @@ def configure_k8s(): class FlushIntervalTransport(BackgroundThreadTransport): - def __init__(self, client, name, **kwargs): + def __init__(self, client: Any, name: str, **kwargs: Any) -> None: super().__init__( client, name, @@ -436,8 +445,8 @@ def __init__(self, client, name, **kwargs): labels=labels, transport=FlushIntervalTransport) - def k8s_label_filter(record): - handler.labels.update({ + def k8s_label_filter(record: logging.LogRecord) -> bool: + cast(dict[str, Any], handler.labels).update({ 'task_payload': os.getenv('TASK_PAYLOAD', 'null'), 'fuzz_target': @@ -466,7 +475,7 @@ def k8s_label_filter(record): logging.getLogger().setLevel(logging.INFO) -def configure_cloud_logging(): +def configure_cloud_logging() -> None: """ Configure Google cloud logging, for bots not running on appengine nor k8s. """ import google.cloud.logging @@ -499,14 +508,14 @@ def __init__(self, client, name, **kwargs): handler = CloudLoggingHandler( client=client, labels=labels, transport=FlushIntervalTransport) - def cloud_label_filter(record): + def cloud_label_filter(record: logging.LogRecord) -> bool: # Update the labels with additional information. # Ideally we would use json_fields as done in configure_k8s(), but since # src/Pipfile forces google-cloud-logging = "==1.15.0", we have fairly # limited options to format the output, see: # https://github.com/googleapis/python-logging/blob/6236537b197422d3dcfff38fe7729dee7f361ca9/google/cloud/logging/handlers/handlers.py#L98 # pylint: disable=line-too-long # https://github.com/googleapis/python-logging/blob/6236537b197422d3dcfff38fe7729dee7f361ca9/google/cloud/logging/handlers/transports/background_thread.py#L233 # pylint: disable=line-too-long - handler.labels.update({ + cast(dict[str, Any], handler.labels).update({ 'task_payload': os.getenv('TASK_PAYLOAD', 'null'), 'fuzz_target': @@ -534,7 +543,7 @@ def cloud_label_filter(record): logging.getLogger().addHandler(handler) -def configure_swarming(name: str, extras: dict[str, str] | None = None) -> None: +def configure_swarming(name: str, extras: dict[str, Any] | None = None) -> None: """Configure logging for swarming bots.""" if extras is None: extras = {} @@ -555,7 +564,7 @@ def configure_swarming(name: str, extras: dict[str, str] | None = None) -> None: sys.excepthook = uncaught_exception_handler -def configure(name, extras=None): +def configure(name: str, extras: dict[str, Any] | None = None) -> None: """Set logger. See the list of loggers in bot/config/logging.yaml. Also configures the process to log any uncaught exceptions as an error. |extras| will be included by emit() in log messages.""" @@ -594,7 +603,7 @@ def configure(name, extras=None): sys.excepthook = uncaught_exception_handler -def get_logger(): +def get_logger() -> logging.Logger | None: """Return logger. We need this method because we need to mock logger.""" if _logger: return _logger @@ -610,7 +619,7 @@ def get_logger(): return _logger -def get_source_location(): +def get_source_location() -> tuple[str, str | int, str]: """Return the caller file, lineno, and funcName.""" try: raise RuntimeError() @@ -619,7 +628,7 @@ def get_source_location(): # to leave emit(..). # The code is adapted from: # https://github.com/python/cpython/blob/2.7/Lib/logging/__init__.py#L1244 - frame = sys.exc_info()[2].tb_frame.f_back + frame = cast(Any, sys.exc_info()[2]).tb_frame.f_back while frame and hasattr(frame, 'f_code'): if not frame.f_code.co_filename.endswith('logs.py'): @@ -629,7 +638,7 @@ def get_source_location(): return 'Unknown', '-1', 'Unknown' -def _add_appengine_trace(extras): +def _add_appengine_trace(extras: dict[str, Any]) -> None: """Add App Engine tracing information.""" if not environment.is_running_on_app_engine(): return @@ -637,7 +646,7 @@ def _add_appengine_trace(extras): from libs import auth try: - request = auth.get_current_request() + request = auth.get_current_request() # type: ignore if not request: return except Exception: @@ -656,13 +665,16 @@ def _add_appengine_trace(extras): project_id=project_id, trace_id=trace_id) -def intercept_log_context(func): +F = TypeVar('F', bound=Callable[..., Any]) + + +def intercept_log_context(func: F) -> F: """Intercepts the wrapped function and injects metadata into the kwargs for a given log context """ @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: if not kwargs.get('ignore_context'): for context in log_contexts.contexts: context.setup() @@ -672,10 +684,11 @@ def wrapper(*args, **kwargs): del kwargs["ignore_context"] return func(*args, **kwargs) - return wrapper + return cast(F, wrapper) -def _parse_symmetric_logs(extras): +def _parse_symmetric_logs( + extras: dict[str, Any]) -> list[dict[str, Any]] | None: """Return a list containing the fields of each symmetrics log entry. Checks if the symmetric logs label was passed and formatted as expected, @@ -698,7 +711,7 @@ def _parse_symmetric_logs(extras): @intercept_log_context -def emit(level, message, exc_info=None, **extras): +def emit(level: int, message: str, exc_info: Any = None, **extras: Any) -> None: """Log in JSON.""" logger = get_logger() if not logger: @@ -759,17 +772,17 @@ def emit(level, message, exc_info=None, **extras): }) -def info(message, **extras): +def info(message: str, **extras: Any) -> None: """Logs the message to a given log file.""" emit(logging.INFO, message, **extras) -def warning(message, **extras): +def warning(message: str, **extras: Any) -> None: """Logs the warning message.""" emit(logging.WARN, message, exc_info=sys.exc_info(), **extras) -def error(message, **extras): +def error(message: str, **extras: Any) -> None: """Logs the error in the error log file.""" exception = extras.pop('exception', None) if exception: @@ -782,7 +795,7 @@ def error(message, **extras): _increment_error_count() -def log_fatal_and_exit(message, **extras): +def log_fatal_and_exit(message: str, **extras: Any) -> None: """Logs a fatal error and exits.""" wait_before_exit = extras.pop('wait_before_exit', None) emit(logging.CRITICAL, message, exc_info=sys.exc_info(), **extras) @@ -834,7 +847,7 @@ def get_testcase_id( from google.cloud import ndb if isinstance(testcase, ndb.Model): - return testcase.key.id() # type: ignore + return testcase.key.id() return getattr(testcase, 'id', None) @@ -880,7 +893,7 @@ class TestcaseLogStruct(NamedTuple): class GrouperStruct(NamedTuple): # Represents the TestcaseLogStruct for each testcase being grouped. - symmetric_logs: list[dict] + symmetric_logs: list[dict[str, Any]] class LogContextType(enum.Enum): @@ -1022,10 +1035,10 @@ class Stage(enum.Enum): class Singleton(type): - _instances = {} - _lock = threading.Lock() + _instances: dict[type, Any] = {} + _lock: threading.Lock = threading.Lock() - def __call__(cls, *args, **kwargs): + def __call__(cls, *args: Any, **kwargs: Any) -> Any: with cls._lock: if cls not in cls._instances: cls._instances[cls] = super().__call__(*args, **kwargs) @@ -1035,39 +1048,44 @@ def __call__(cls, *args, **kwargs): class LogContexts(metaclass=Singleton): """Class to keep the log contexts and metadata.""" - def __init__(self): + contexts: list[LogContextType] + meta: dict[Any, Any] + _data_lock: threading.Lock + + def __init__(self) -> None: self.contexts: list[LogContextType] = [LogContextType.COMMON] self.meta: dict[Any, Any] = {} self._data_lock = threading.Lock() - def add(self, new_contexts: list[LogContextType]): + def add(self, new_contexts: list[LogContextType]) -> None: with self._data_lock: self.contexts += new_contexts - def add_metadata(self, key: Any, value: Any): + def add_metadata(self, key: Any, value: Any) -> None: with self._data_lock: self.meta[key] = value - def delete(self, contexts: list[LogContextType]): + def delete(self, contexts: list[LogContextType]) -> None: with self._data_lock: for ctx in contexts: self.contexts.remove(ctx) - def delete_metadata(self, key: Any): + def delete_metadata(self, key: Any) -> None: with self._data_lock: if key in self.meta: del self.meta[key] - def clear(self): + def clear(self) -> None: with self._data_lock: self.contexts = [LogContextType.COMMON] -log_contexts = LogContexts() +log_contexts: LogContexts = LogContexts() @contextlib.contextmanager -def wrap_log_context(contexts: list[LogContextType]): +def wrap_log_context( + contexts: list[LogContextType]) -> Generator[None, None, None]: try: log_contexts.add(contexts) yield @@ -1076,7 +1094,7 @@ def wrap_log_context(contexts: list[LogContextType]): @contextlib.contextmanager -def task_stage_context(stage: Stage): +def task_stage_context(stage: Stage) -> Generator[None, None, None]: """Creates a task context for a given stage.""" with wrap_log_context(contexts=[LogContextType.TASK]): try: @@ -1093,8 +1111,9 @@ def task_stage_context(stage: Stage): @contextlib.contextmanager -def fuzzer_log_context(fuzzer_name: str, job_type: str, - fuzz_target: 'FuzzTarget | None'): +def fuzzer_log_context( + fuzzer_name: str, job_type: str, + fuzz_target: 'FuzzTarget | None') -> Generator[None, None, None]: """Creates a fuzzer context for a given fuzzer, job, and target (optional).""" with wrap_log_context(contexts=[LogContextType.FUZZER]): try: @@ -1116,8 +1135,9 @@ def fuzzer_log_context(fuzzer_name: str, job_type: str, @contextlib.contextmanager -def testcase_log_context(testcase: 'Testcase | TestcaseAttributes', - fuzz_target: 'FuzzTarget | None'): +def testcase_log_context( + testcase: 'Testcase | TestcaseAttributes', + fuzz_target: 'FuzzTarget | None') -> Generator[None, None, None]: """Creates a testcase-based context for a given testcase. Fuzz target as an argument is needed since retrieving this entity depends on @@ -1158,7 +1178,7 @@ def testcase_log_context(testcase: 'Testcase | TestcaseAttributes', @contextlib.contextmanager -def cron_log_context(): +def cron_log_context() -> Generator[None, None, None]: """Creates a cronjob log context, mainly for triage/cleanup tasks.""" with wrap_log_context(contexts=[LogContextType.CRON]): try: @@ -1169,8 +1189,9 @@ def cron_log_context(): @contextlib.contextmanager -def grouper_log_context(testcase_1: 'Testcase | TestcaseAttributes', - testcase_2: 'Testcase | TestcaseAttributes'): +def grouper_log_context( + testcase_1: 'Testcase | TestcaseAttributes', + testcase_2: 'Testcase | TestcaseAttributes') -> Generator[None, None, None]: """Creates a grouper context for a given pair of testcases.""" with wrap_log_context(contexts=[LogContextType.GROUPER]): try: diff --git a/src/clusterfuzz/_internal/metrics/monitor.py b/src/clusterfuzz/_internal/metrics/monitor.py index 8752c24d610..8c00a92c687 100644 --- a/src/clusterfuzz/_internal/metrics/monitor.py +++ b/src/clusterfuzz/_internal/metrics/monitor.py @@ -17,7 +17,7 @@ # stackdriver. import bisect -import collections +from collections.abc import Iterator import contextlib import functools import itertools @@ -25,14 +25,21 @@ import signal import threading import time -from typing import List +from typing import Any +from typing import Callable +from typing import cast +from typing import NamedTuple +from typing import Sequence +from typing import TypeVar import requests try: from google.cloud import monitoring_v3 + from google.cloud.monitoring_v3 import types as monitoring_v3_types except (ImportError, RuntimeError): - monitoring_v3 = None + monitoring_v3: Any = None + monitoring_v3_types: Any = None from google.api import label_pb2 from google.api import metric_pb2 @@ -49,21 +56,26 @@ from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -CUSTOM_METRIC_PREFIX = 'custom.googleapis.com/' -FLUSH_INTERVAL_SECONDS = 10 * 60 # 10 minutes. -PREEMPTION_CHECK_INTERVAL = 15 # seconds -RETRY_DEADLINE_SECONDS = 5 * 60 # 5 minutes. -INITIAL_DELAY_SECONDS = 16 -MAXIMUM_DELAY_SECONDS = 2 * 60 # 2 minutes. -MAX_TIME_SERIES_PER_CALL = 200 +CUSTOM_METRIC_PREFIX: str = 'custom.googleapis.com/' +FLUSH_INTERVAL_SECONDS: int = 10 * 60 # 10 minutes. +PREEMPTION_CHECK_INTERVAL: int = 15 # seconds +RETRY_DEADLINE_SECONDS: int = 5 * 60 # 5 minutes. +INITIAL_DELAY_SECONDS: int = 16 +MAXIMUM_DELAY_SECONDS: int = 2 * 60 # 2 minutes. +MAX_TIME_SERIES_PER_CALL: int = 200 # Since `monitoring_v3` is only conditionally imported, pylint complains about # any accesses to its members. Define type aliases here once and for all, for # use below. -if monitoring_v3 is not None: - _TimeInterval = monitoring_v3.types.TimeInterval # pylint: disable=no-member - _Point = monitoring_v3.types.Point # pylint: disable=no-member - _TimeSeries = monitoring_v3.types.TimeSeries # pylint: disable=no-member +_TimeInterval: Any +_Point: Any +_TimeSeries: Any +_Timestamp: Any = getattr(timestamp_pb2, 'Timestamp') + +if monitoring_v3 is not None and monitoring_v3_types is not None: + _TimeInterval = monitoring_v3_types.TimeInterval # pylint: disable=no-member + _Point = monitoring_v3_types.Point # pylint: disable=no-member + _TimeSeries = monitoring_v3_types.TimeSeries # pylint: disable=no-member else: _TimeInterval = None _Point = None @@ -81,7 +93,7 @@ initial=INITIAL_DELAY_SECONDS, maximum=MAXIMUM_DELAY_SECONDS, deadline=RETRY_DEADLINE_SECONDS) -def _create_time_series(name: str, time_series: List[_TimeSeries]): +def _create_time_series(name: str, time_series: list[Any]) -> None: """Wraps `create_time_series()` from the monitoring client. Adds retries and logging. @@ -96,21 +108,24 @@ def _create_time_series(name: str, time_series: List[_TimeSeries]): class _MockMetric: """Mock metric object, used for when monitoring isn't available.""" - def _mock_method(self, *args, **kwargs): # pylint: disable=unused-argument + def _mock_method(self, *args: Any, **kwargs: Any) -> None: # pylint: disable=unused-argument pass - def __getattr__(self, _): + def __getattr__(self, _: str) -> Callable[..., None]: return self._mock_method -def _time_series_sort_key(ts): +def _time_series_sort_key(ts: Any) -> Any: return ts.points[-1].interval.start_time -def _flush_metrics(): +def _flush_metrics() -> None: """Flushes all metrics stored in _metrics_store""" + app_id = utils.get_application_id() + assert app_id is not None project_path = _monitoring_v3_client.common_project_path( # pylint: disable=no-member - utils.get_application_id()) + app_id) + assert _TimeSeries is not None try: time_series = [] end_time = time.time() @@ -138,15 +153,15 @@ def _flush_metrics(): logs.error(f'Failed to flush metrics: {e}') -_on_sigterm_callbacks = [] +_on_sigterm_callbacks: list[Callable[[], Any]] = [] -def register_on_sigterm(callback): +def register_on_sigterm(callback: Callable[[], Any]) -> None: """Register a callback to be called on SIGTERM.""" _on_sigterm_callbacks.append(callback) -def _trigger_shutdown_cleanup(): +def _trigger_shutdown_cleanup() -> None: """Runs registered callbacks and stops the monitoring daemon.""" for callback in _on_sigterm_callbacks: try: @@ -157,14 +172,14 @@ def _trigger_shutdown_cleanup(): stop() -def handle_sigterm(signo, stack_frame): #pylint: disable=unused-argument +def handle_sigterm(signo: int, stack_frame: Any) -> None: # pylint: disable=unused-argument logs.info('Handling SIGTERM.') _trigger_shutdown_cleanup() logs.info('SIGTERM handled, metrics flushed.') @contextlib.contextmanager -def wrap_with_monitoring(): +def wrap_with_monitoring() -> Iterator[None]: """Wraps execution so we flush metrics on exit""" try: initialize() @@ -177,17 +192,18 @@ def wrap_with_monitoring(): stop() -class _MonitoringDaemon(): +class _MonitoringDaemon: """Wrapper for the daemon threads responsible for flushing metrics.""" - def __init__(self, flush_function, tick_interval): - self._tick_interval = tick_interval - self._flush_function = flush_function - self._flushing_thread = threading.Thread( + def __init__(self, flush_function: Callable[[], Any], + tick_interval: float | int) -> None: + self._tick_interval: float | int = tick_interval + self._flush_function: Callable[[], Any] = flush_function + self._flushing_thread: threading.Thread = threading.Thread( name='flushing_thread', target=self._flush_loop, daemon=True) - self._flushing_thread_stop_event = threading.Event() + self._flushing_thread_stop_event: threading.Event = threading.Event() - def _flush_loop(self): + def _flush_loop(self) -> None: while True: should_stop = self._flushing_thread_stop_event.wait( timeout=self._tick_interval) @@ -195,24 +211,24 @@ def _flush_loop(self): if should_stop: break - def start(self): + def start(self) -> None: self._flushing_thread.start() - def stop(self): + def stop(self) -> None: self._flushing_thread_stop_event.set() self._flushing_thread.join() -class _PreemptionPoller(): +class _PreemptionPoller: """Background thread to poll GCP metadata for preemption.""" - def __init__(self, check_interval): - self._check_interval = check_interval - self._poller_thread = threading.Thread( + def __init__(self, check_interval: float | int) -> None: + self._check_interval: float | int = check_interval + self._poller_thread: threading.Thread = threading.Thread( name='preemption_poller', target=self._poll_loop, daemon=True) - self._poller_thread_stop_event = threading.Event() + self._poller_thread_stop_event: threading.Event = threading.Event() - def _poll_loop(self): + def _poll_loop(self) -> None: """Polls GCP metadata for preemption status.""" if not compute_metadata.is_gce(): return @@ -230,32 +246,37 @@ def _poll_loop(self): _trigger_shutdown_cleanup() break except requests.exceptions.HTTPError as e: - if e.response.status_code != 404: + if cast(requests.Response, e.response).status_code != 404: logs.warning(f'HTTP error checking preemption metadata: {e}') except Exception as e: logs.warning(f'Unknown error checking preemption metadata: {e}') - def start(self): + def start(self) -> None: self._poller_thread.start() - def stop(self): + def stop(self) -> None: self._poller_thread_stop_event.set() if threading.current_thread() != self._poller_thread: self._poller_thread.join() -_StoreValue = collections.namedtuple( - '_StoreValue', ['metric', 'labels', 'start_time', 'value']) +class _StoreValue(NamedTuple): + metric: 'Metric' + labels: dict[str, Any] | None + start_time: float + value: Any class _MetricsStore: """In-process metrics store.""" - def __init__(self): - self._store = {} - self._lock = threading.RLock() + def __init__(self) -> None: + self._store: dict[tuple[str, tuple[tuple[str, Any], ...] + | None], _StoreValue] = {} + self._lock: threading.RLock = threading.RLock() - def _get_key(self, metric_name, labels): + def _get_key(self, metric_name: str, labels: dict[str, Any] | None + ) -> tuple[str, tuple[tuple[str, Any], ...] | None]: """Get the key used for storing values.""" if labels: normalized_labels = tuple(sorted(labels.items())) @@ -264,17 +285,18 @@ def _get_key(self, metric_name, labels): return (metric_name, normalized_labels) - def iter_values(self): + def iter_values(self) -> Iterator[_StoreValue]: with self._lock: yield from self._store.values() - def get(self, metric, labels): + def get(self, metric: 'Metric', labels: dict[str, Any] | None) -> _StoreValue: """Get the stored value for the metric.""" with self._lock: key = self._get_key(metric.name, labels) return self._store[key] - def put(self, metric, labels, value): + def put(self, metric: 'Metric', labels: dict[str, Any] | None, + value: Any) -> None: """Store new value for the metric.""" with self._lock: key = self._get_key(metric.name, labels) @@ -285,7 +307,8 @@ def put(self, metric, labels, value): self._store[key] = _StoreValue(metric, labels, start_time, value) - def increment(self, metric, labels, delta): + def increment(self, metric: 'Metric', labels: dict[str, Any] | None, + delta: Any) -> None: """Increment a value by |delta|.""" with self._lock: key = self._get_key(metric.name, labels) @@ -299,7 +322,7 @@ def increment(self, metric, labels, delta): self._store[key] = _StoreValue(metric, labels, start_time, value) - def reset_for_testing(self): + def reset_for_testing(self) -> None: """Reset all data. Used for tests.""" with self._lock: self._store.clear() @@ -308,11 +331,11 @@ def reset_for_testing(self): class _Field: """_Field is the base class used for field specs.""" - def __init__(self, name): - self.name = name + def __init__(self, name: str) -> None: + self.name: str = name @property - def value_type(self): + def value_type(self) -> int: raise NotImplementedError @@ -320,7 +343,7 @@ class StringField(_Field): """StringField spec.""" @property - def value_type(self): + def value_type(self) -> int: return label_pb2.LabelDescriptor.ValueType.STRING # pylint: disable=no-member @@ -328,7 +351,7 @@ class BooleanField(_Field): """BooleanField spec.""" @property - def value_type(self): + def value_type(self) -> int: return label_pb2.LabelDescriptor.ValueType.BOOL # pylint: disable=no-member @@ -336,41 +359,46 @@ class IntegerField(_Field): """IntegerField spec.""" @property - def value_type(self): + def value_type(self) -> int: return label_pb2.LabelDescriptor.ValueType.INT64 # pylint: disable=no-member class Metric: """Base metric class.""" - def __init__(self, name, description, field_spec): - self.name = name - self.description = description - self.field_spec = field_spec or [] + def __init__(self, + name: str, + description: str, + field_spec: Sequence[_Field] | None = None) -> None: + self.name: str = name + self.description: str = description + self.field_spec: Sequence[_Field] = field_spec or [] @property - def value_type(self): + def value_type(self) -> int: raise NotImplementedError @property - def metric_kind(self): + def metric_kind(self) -> int: raise NotImplementedError @property - def default_value(self): + def default_value(self) -> Any: raise NotImplementedError - def _set_value(self, point, value): + def _set_value(self, point: Any, value: Any) -> None: raise NotImplementedError - def get(self, labels=None): + def get(self, labels: dict[str, Any] | None = None) -> Any: """Return the current value for the labels. Used for testing.""" try: return _metrics_store.get(self, labels).value except KeyError: return self.default_value - def monitoring_v3_metric(self, metric, labels=None): + def monitoring_v3_metric(self, + metric: Any, + labels: dict[str, Any] | None = None) -> Any: """Get the monitoring_v3 Metric.""" metric.type = CUSTOM_METRIC_PREFIX + self.name @@ -385,7 +413,7 @@ def monitoring_v3_metric(self, metric, labels=None): return metric - def monitoring_v3_metric_descriptor(self, descriptor): + def monitoring_v3_metric_descriptor(self, descriptor: Any) -> Any: """Get the monitoring_v3 MetricDescriptor.""" descriptor.name = self.name descriptor.type = CUSTOM_METRIC_PREFIX + self.name @@ -400,14 +428,16 @@ def monitoring_v3_metric_descriptor(self, descriptor): return descriptor - def monitoring_v3_time_series(self, time_series, labels, start_time, end_time, - value): + def monitoring_v3_time_series( + self, time_series: Any, labels: dict[str, Any] | None, start_time: float, + end_time: float, value: Any) -> Any: """Get the TimeSeries corresponding to the metric.""" self.monitoring_v3_metric(time_series.metric, labels) time_series.resource.CopyFrom(_monitored_resource) time_series.metric_kind = self.metric_kind time_series.value_type = self.value_type + assert _TimeInterval is not None and _Point is not None interval = _TimeInterval() point = _Point(interval=interval) @@ -425,24 +455,26 @@ class _CounterMetric(Metric): """Counter metric.""" @property - def value_type(self): + def value_type(self) -> int: return metric_pb2.MetricDescriptor.ValueType.INT64 # pylint: disable=no-member @property - def metric_kind(self): + def metric_kind(self) -> int: return metric_pb2.MetricDescriptor.MetricKind.CUMULATIVE # pylint: disable=no-member @property - def default_value(self): + def default_value(self) -> int: return 0 - def increment(self, labels=None): + def increment(self, labels: dict[str, Any] | None = None) -> None: self.increment_by(1, labels=labels) - def increment_by(self, count, labels=None): + def increment_by(self, + count: int | float, + labels: dict[str, Any] | None = None) -> None: _metrics_store.increment(self, labels, count) - def _set_value(self, point, value): + def _set_value(self, point: Any, value: int) -> None: """Get Point.""" point.int64_value = value @@ -451,21 +483,21 @@ class _GaugeMetric(Metric): """Gauge metric.""" @property - def value_type(self): + def value_type(self) -> int: return metric_pb2.MetricDescriptor.ValueType.INT64 # pylint: disable=no-member @property - def metric_kind(self): + def metric_kind(self) -> int: return metric_pb2.MetricDescriptor.MetricKind.GAUGE # pylint: disable=no-member @property - def default_value(self): + def default_value(self) -> int: return 0 - def set(self, value, labels=None): + def set(self, value: int, labels: dict[str, Any] | None = None) -> None: _metrics_store.put(self, labels, value) - def _set_value(self, point, value): + def _set_value(self, point: Any, value: int) -> None: """Get Point.""" point.int64_value = value @@ -473,25 +505,25 @@ def _set_value(self, point, value): class _Bucketer: """Bucketer.""" - def __init__(self): - self._lower_bounds = None + def __init__(self) -> None: + self._lower_bounds: list[float] = [] - def bucket_for_value(self, value): + def bucket_for_value(self, value: float | int) -> int: """Get the bucket index for the given value.""" return bisect.bisect(self._lower_bounds, value) - 1 @property - def num_buckets(self): + def num_buckets(self) -> int: return len(self._lower_bounds) class FixedWidthBucketer(_Bucketer): """Fixed width bucketer.""" - def __init__(self, width, num_finite_buckets=100): + def __init__(self, width: float | int, num_finite_buckets: int = 100) -> None: super().__init__() - self.width = width - self.num_finite_buckets = num_finite_buckets + self.width: float | int = width + self.num_finite_buckets: int = num_finite_buckets # [-Inf, 0), [0, width), [width, 2*width], ... , [n*width, Inf) self._lower_bounds = [float('-Inf')] @@ -502,11 +534,14 @@ def __init__(self, width, num_finite_buckets=100): class GeometricBucketer(_Bucketer): """Geometric bucketer.""" - def __init__(self, growth_factor=10**0.2, num_finite_buckets=100, scale=1.0): + def __init__(self, + growth_factor: float = 10**0.2, + num_finite_buckets: int = 100, + scale: float = 1.0) -> None: super().__init__() - self.growth_factor = growth_factor - self.num_finite_buckets = num_finite_buckets - self.scale = scale + self.growth_factor: float = growth_factor + self.num_finite_buckets: int = num_finite_buckets + self.scale: float = scale # [-Inf, scale), [scale, scale*growth), # [scale*growth^i, scale*growth^(i+1)), ..., [scale*growth^n, Inf) @@ -518,13 +553,13 @@ def __init__(self, growth_factor=10**0.2, num_finite_buckets=100, scale=1.0): class _Distribution: """Holds a distribution.""" - def __init__(self, bucketer): - self.bucketer = bucketer - self.buckets = [0 for _ in range(bucketer.num_buckets)] - self.sum = 0 - self.count = 0 + def __init__(self, bucketer: _Bucketer) -> None: + self.bucketer: _Bucketer = bucketer + self.buckets: list[int] = [0 for _ in range(bucketer.num_buckets)] + self.sum: float | int = 0 + self.count: int = 0 - def add(self, value): + def add(self, value: float | int) -> '_Distribution': self.buckets[self.bucketer.bucket_for_value(value)] += 1 self.count += 1 self.sum += value @@ -532,7 +567,7 @@ def add(self, value): __add__ = add - def monitoring_v3_distribution(self, distribution): + def monitoring_v3_distribution(self, distribution: Any) -> None: """Set the monitoring_v3 Distribution value.""" distribution.count = self.count if self.count: @@ -561,67 +596,76 @@ def monitoring_v3_distribution(self, distribution): class _CumulativeDistributionMetric(Metric): """Cumulative distribution metric.""" - def __init__(self, name, description, bucketer, field_spec=None): + def __init__(self, + name: str, + description: str, + bucketer: _Bucketer | None = None, + field_spec: Sequence[_Field] | None = None) -> None: super().__init__(name, description=description, field_spec=field_spec) - self.bucketer = bucketer + self.bucketer: _Bucketer | None = bucketer @property - def value_type(self): + def value_type(self) -> int: return metric_pb2.MetricDescriptor.ValueType.DISTRIBUTION # pylint: disable=no-member @property - def metric_kind(self): + def metric_kind(self) -> int: return metric_pb2.MetricDescriptor.MetricKind.CUMULATIVE # pylint: disable=no-member @property - def default_value(self): + def default_value(self) -> _Distribution: + assert self.bucketer is not None return _Distribution(self.bucketer) - def add(self, value, labels=None): + def add(self, value: float | int, + labels: dict[str, Any] | None = None) -> None: _metrics_store.increment(self, labels, value) - def _set_value(self, point, value): + def _set_value(self, point: Any, value: _Distribution) -> None: value.monitoring_v3_distribution(point.distribution_value) # Global state. -_metrics_store = _MetricsStore() -_monitoring_v3_client = None -_monitoring_daemon = None -_preemption_poller = None -_monitored_resource = None +_metrics_store: _MetricsStore = _MetricsStore() +_monitoring_v3_client: Any = None +_monitoring_daemon: _MonitoringDaemon | None = None +_preemption_poller: _PreemptionPoller | None = None +_monitored_resource: Any = None # Add fields very conservatively here. There is a limit of 10 labels per metric # descriptor, and metrics should be low in cardinality. That is, only add fields # which have a small number of possible values. -DEFAULT_FIELDS = [ +DEFAULT_FIELDS: list[_Field] = [ StringField('region'), ] -def check_module_loaded(module): +def check_module_loaded(module: Any) -> bool: """Used for mocking.""" return module is not None -def stub_unavailable(module): +_F = TypeVar('_F', bound=Callable[..., Any]) + + +def stub_unavailable(module: Any) -> Callable[[_F], _F]: """Decorator to stub out functions on failed imports.""" - def decorator(func): + def decorator(func: _F) -> _F: @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: if check_module_loaded(module): return func(*args, **kwargs) return _MockMetric() - return wrapper + return cast(_F, wrapper) return decorator -def _initialize_monitored_resource(): +def _initialize_monitored_resource() -> None: """Monitored resources.""" global _monitored_resource _monitored_resource = monitored_resource_pb2.MonitoredResource() # pylint: disable=no-member @@ -631,7 +675,9 @@ def _initialize_monitored_resource(): # The project ID must be the same as the one we write metrics to, not the ID # where the instance lives. - _monitored_resource.labels['project_id'] = utils.get_application_id() + app_id = utils.get_application_id() + assert app_id is not None + _monitored_resource.labels['project_id'] = app_id _monitored_resource.labels['instance_id'] = utils.get_instance_name() @@ -644,15 +690,16 @@ def _initialize_monitored_resource(): _monitored_resource.labels['zone'] = 'us-central1-f' -def _time_to_timestamp(interval, attr, time_seconds): +def _time_to_timestamp(interval: Any, attr: str, + time_seconds: float | int) -> None: """Convert result of time.time() to Timestamp.""" seconds = int(time_seconds) nanos = int((time_seconds - seconds) * 10**9) - timestamp = timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) # pylint: disable=no-member + timestamp = _Timestamp(seconds=seconds, nanos=nanos) setattr(interval, attr, timestamp) -def initialize(): +def initialize() -> None: """Initialize if monitoring is enabled for this bot.""" global _monitoring_v3_client global _monitoring_daemon @@ -665,6 +712,7 @@ def initialize(): return if check_module_loaded(monitoring_v3): + assert monitoring_v3 is not None _initialize_monitored_resource() _monitoring_v3_client = monitoring_v3.MetricServiceClient( credentials=credentials.get_default()[0]) @@ -677,7 +725,7 @@ def initialize(): _preemption_poller.start() -def stop(): +def stop() -> None: """Stops monitoring and cleans up (only if monitoring is enabled).""" if _monitoring_daemon: _monitoring_daemon.stop() @@ -685,12 +733,12 @@ def stop(): _preemption_poller.stop() -def metrics_store(): +def metrics_store() -> _MetricsStore: """Get the per-process metrics store.""" return _metrics_store -def _get_region(bot_name): +def _get_region(bot_name: str | None) -> str: """Get bot region.""" if not bot_name: return 'unknown' @@ -708,19 +756,27 @@ def _get_region(bot_name): @stub_unavailable(monitoring_v3) -def CounterMetric(name, description, field_spec): +def CounterMetric(name: str, + description: str, + field_spec: Sequence[_Field] | None = None) -> _CounterMetric: """Build _CounterMetric.""" return _CounterMetric(name, field_spec=field_spec, description=description) @stub_unavailable(monitoring_v3) -def GaugeMetric(name, description, field_spec): +def GaugeMetric(name: str, + description: str, + field_spec: Sequence[_Field] | None = None) -> _GaugeMetric: """Build _CounterMetric.""" return _GaugeMetric(name, field_spec=field_spec, description=description) @stub_unavailable(monitoring_v3) -def CumulativeDistributionMetric(name, description, bucketer, field_spec): +def CumulativeDistributionMetric(name: str, + description: str, + bucketer: _Bucketer | None = None, + field_spec: Sequence[_Field] | None = None + ) -> _CumulativeDistributionMetric: """Build _CounterMetric.""" return _CumulativeDistributionMetric( name, description=description, bucketer=bucketer, field_spec=field_spec) diff --git a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py index ba935f46802..767ee7dda6d 100644 --- a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py +++ b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py @@ -16,7 +16,7 @@ from clusterfuzz._internal.metrics import monitor # Fuzz task metrics. -UPLOAD_TESTCASE_COUNT = monitor.CounterMetric( +UPLOAD_TESTCASE_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/uploaded_testcase_count', description='The number of uploaded testcases in GAE.', field_spec=[ @@ -27,14 +27,14 @@ ]) # Fuzz task metrics. -BIG_QUERY_WRITE_COUNT = monitor.CounterMetric( +BIG_QUERY_WRITE_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'debug/big_query/write_count', description='The number of BigQuery writes', field_spec=[ monitor.BooleanField('success'), ]) -CF_TIP_BOOT_FAILED_COUNT = monitor.CounterMetric( +CF_TIP_BOOT_FAILED_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'tip_boot_failure', description= 'Count of failure in booting up cuttlefish with tip-of-the-tree build ', @@ -45,7 +45,7 @@ monitor.BooleanField('is_succeeded'), ]) -JOB_BAD_BUILD_COUNT = monitor.CounterMetric( +JOB_BAD_BUILD_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/job/bad_build_count', description=("Count of fuzz task's bad build count " '(grouped by job type)'), @@ -54,19 +54,20 @@ monitor.BooleanField('bad_build'), ]) -JOB_BUILD_AGE = monitor.CumulativeDistributionMetric( - 'job/build_age', - bucketer=monitor.GeometricBucketer(), - description=('Distribution of latest build\'s age in hours. ' - '(grouped by fuzzer/job)'), - field_spec=[ - monitor.StringField('job'), - monitor.StringField('platform'), - monitor.StringField('task'), - ], -) - -JOB_BUILD_REVISION = monitor.GaugeMetric( +JOB_BUILD_AGE: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'job/build_age', + bucketer=monitor.GeometricBucketer(), + description=('Distribution of latest build\'s age in hours. ' + '(grouped by fuzzer/job)'), + field_spec=[ + monitor.StringField('job'), + monitor.StringField('platform'), + monitor.StringField('task'), + ], + )) + +JOB_BUILD_REVISION: monitor._GaugeMetric = monitor.GaugeMetric( 'job/build_revision', description=('Gauge for revision of trunk build ' '(grouped by job/platform/task).'), @@ -77,20 +78,21 @@ ], ) -JOB_BUILD_RETRIEVAL_TIME = monitor.CumulativeDistributionMetric( - 'task/build_retrieval_time', - bucketer=monitor.GeometricBucketer(), - description=('Distribution of fuzz task\'s build retrieval times. ' - '(grouped by fuzzer/job, in minutes).'), - field_spec=[ - monitor.StringField('job'), - monitor.StringField('step'), - monitor.StringField('platform'), - monitor.StringField('build_type'), - ], -) - -FUZZER_KNOWN_CRASH_COUNT = monitor.CounterMetric( +JOB_BUILD_RETRIEVAL_TIME: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'task/build_retrieval_time', + bucketer=monitor.GeometricBucketer(), + description=('Distribution of fuzz task\'s build retrieval times. ' + '(grouped by fuzzer/job, in minutes).'), + field_spec=[ + monitor.StringField('job'), + monitor.StringField('step'), + monitor.StringField('platform'), + monitor.StringField('build_type'), + ], + )) + +FUZZER_KNOWN_CRASH_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/fuzzer/known_crash_count', description=('Count of fuzz task\'s known crash count ' '(grouped by fuzzer)'), @@ -100,7 +102,7 @@ monitor.StringField('runtime') ]) -FUZZER_NEW_CRASH_COUNT = monitor.CounterMetric( +FUZZER_NEW_CRASH_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/fuzzer/new_crash_count', description=('Count of fuzz task\'s new crash count ' '(grouped by fuzzer)'), @@ -110,7 +112,7 @@ monitor.StringField('runtime') ]) -JOB_KNOWN_CRASH_COUNT = monitor.CounterMetric( +JOB_KNOWN_CRASH_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/job/known_crash_count', description=('Count of fuzz task\'s known crash count ' '(grouped by job)'), @@ -120,7 +122,7 @@ monitor.StringField('runtime') ]) -JOB_NEW_CRASH_COUNT = monitor.CounterMetric( +JOB_NEW_CRASH_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/job/new_crash_count', description=('Count of fuzz task\'s new crash count ' '(grouped by job)'), @@ -130,7 +132,7 @@ monitor.StringField('runtime') ]) -FUZZER_RETURN_CODE_COUNT = monitor.CounterMetric( +FUZZER_RETURN_CODE_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/fuzzer/return_code_count', description=("Count of fuzz task's return codes " '(grouped by fuzzer)'), @@ -143,7 +145,7 @@ ], ) -FUZZER_TOTAL_FUZZ_TIME = monitor.CounterMetric( +FUZZER_TOTAL_FUZZ_TIME: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/fuzzer/total_time', description=('The total fuzz time in seconds ' '(grouped by fuzzer)'), @@ -158,19 +160,20 @@ ], ) -FUZZER_PREEMPTED_TOTAL_FUZZ_TIME = monitor.CounterMetric( - 'task/fuzz/fuzzer/preempted_total_time', - description=('The total fuzz time wasted due to preemption in seconds ' - '(grouped by fuzzer).'), - field_spec=[ - monitor.StringField('fuzzer'), - monitor.StringField('platform'), - monitor.StringField('is_batch'), - monitor.StringField('runtime') - ], -) - -JOB_PREEMPTED_TOTAL_FUZZ_TIME = monitor.CounterMetric( +FUZZER_PREEMPTED_TOTAL_FUZZ_TIME: monitor._CounterMetric = ( + monitor.CounterMetric( + 'task/fuzz/fuzzer/preempted_total_time', + description=('The total fuzz time wasted due to preemption in seconds ' + '(grouped by fuzzer).'), + field_spec=[ + monitor.StringField('fuzzer'), + monitor.StringField('platform'), + monitor.StringField('is_batch'), + monitor.StringField('runtime') + ], + )) + +JOB_PREEMPTED_TOTAL_FUZZ_TIME: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/job/preempted_total_time', description=('The total fuzz time wasted due to preemption in seconds ' '(grouped by job).'), @@ -184,19 +187,20 @@ # This metric tracks fuzzer setup and data bundle update, # fuzzing time and crash processing on the worker -FUZZING_SESSION_DURATION = monitor.CumulativeDistributionMetric( - 'task/fuzz/session/duration', - bucketer=monitor.GeometricBucketer(), - description=('Total duration of fuzzing session.'), - field_spec=[ - monitor.StringField('fuzzer'), - monitor.StringField('job'), - monitor.StringField('platform'), - monitor.StringField('runtime') - ], -) - -JOB_TOTAL_FUZZ_TIME = monitor.CounterMetric( +FUZZING_SESSION_DURATION: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'task/fuzz/session/duration', + bucketer=monitor.GeometricBucketer(), + description=('Total duration of fuzzing session.'), + field_spec=[ + monitor.StringField('fuzzer'), + monitor.StringField('job'), + monitor.StringField('platform'), + monitor.StringField('runtime') + ], + )) + +JOB_TOTAL_FUZZ_TIME: monitor._CounterMetric = monitor.CounterMetric( 'task/fuzz/job/total_time', description=('The total fuzz time in seconds ' '(grouped by job)'), @@ -211,32 +215,34 @@ ], ) -TESTCASE_GENERATION_AVERAGE_TIME = monitor.CumulativeDistributionMetric( - 'task/fuzz/fuzzer/testcase_generation_duration', - bucketer=monitor.GeometricBucketer(), - description=('Distribution of blackbox fuzzer average testcase ' - ' generation time, in seconds ' - '(grouped by fuzzer, job and platform).'), - field_spec=[ - monitor.StringField('platform'), - monitor.StringField('job'), - monitor.StringField('fuzzer'), - ], -) - -FUZZER_TESTCASE_COUNT_RATIO = monitor.CumulativeDistributionMetric( - 'task/fuzz/fuzzer/testcase_count_ratio', - bucketer=monitor.FixedWidthBucketer(width=0.05, num_finite_buckets=20), - description=('Distribution of fuzz task\'s generated testcase ' - 'counts divided by expected testcase counts ' - '(grouped by fuzzers)'), - field_spec=[ - monitor.StringField('fuzzer'), - ], -) +TESTCASE_GENERATION_AVERAGE_TIME: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'task/fuzz/fuzzer/testcase_generation_duration', + bucketer=monitor.GeometricBucketer(), + description=('Distribution of blackbox fuzzer average testcase ' + ' generation time, in seconds ' + '(grouped by fuzzer, job and platform).'), + field_spec=[ + monitor.StringField('platform'), + monitor.StringField('job'), + monitor.StringField('fuzzer'), + ], + )) + +FUZZER_TESTCASE_COUNT_RATIO: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'task/fuzz/fuzzer/testcase_count_ratio', + bucketer=monitor.FixedWidthBucketer(width=0.05, num_finite_buckets=20), + description=('Distribution of fuzz task\'s generated testcase ' + 'counts divided by expected testcase counts ' + '(grouped by fuzzers)'), + field_spec=[ + monitor.StringField('fuzzer'), + ], + )) # Global error count. -LOG_ERROR_COUNT = monitor.CounterMetric( +LOG_ERROR_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'errors/count', description='Error count.', field_spec=[ @@ -244,19 +250,19 @@ ]) # Untrusted host metrics. -HOST_INCONSISTENT_COUNT = monitor.CounterMetric( +HOST_INCONSISTENT_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'untrusted_runner/host/inconsistent_count', description='Inconsistent worker state count.', field_spec=None) -HOST_ERROR_COUNT = monitor.CounterMetric( +HOST_ERROR_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'untrusted_runner/host/error_count', description='Error count.', field_spec=[ monitor.IntegerField('return_code'), ]) -TRY_COUNT = monitor.CounterMetric( +TRY_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'utils/retry/count', description='Success/Failure count when utils.retry fails', field_spec=[ @@ -264,7 +270,7 @@ monitor.BooleanField('is_succeeded'), ]) -BOT_COUNT = monitor.GaugeMetric( +BOT_COUNT: monitor._GaugeMetric = monitor.GaugeMetric( 'bot_count', description='Bot count', field_spec=[ @@ -274,7 +280,7 @@ monitor.StringField('os_version') ]) -TASK_COUNT = monitor.CounterMetric( +TASK_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/count', description='The number of started tasks', field_spec=[ @@ -282,7 +288,7 @@ monitor.StringField('job'), ]) -TASK_TOTAL_RUN_TIME = monitor.CounterMetric( +TASK_TOTAL_RUN_TIME: monitor._CounterMetric = monitor.CounterMetric( 'task/total_time', description=('The task run time in seconds'), field_spec=[ @@ -291,21 +297,22 @@ ], ) -TESTCASE_TRIAGE_DURATION = monitor.CumulativeDistributionMetric( - 'testcase_analysis/triage_duration_hours', - description=('Time elapsed between testcase upload and completion' - ' of relevant tasks in the testcase lifecycle.' - ' Origin can be either from a fuzzer, or a manual' - ' upload. Measured in hours.'), - bucketer=monitor.GeometricBucketer(), - field_spec=[ - monitor.StringField('step'), - monitor.StringField('job'), - monitor.BooleanField('from_fuzzer'), - ], -) - -TASK_RATE_LIMIT_COUNT = monitor.CounterMetric( +TESTCASE_TRIAGE_DURATION: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'testcase_analysis/triage_duration_hours', + description=('Time elapsed between testcase upload and completion' + ' of relevant tasks in the testcase lifecycle.' + ' Origin can be either from a fuzzer, or a manual' + ' upload. Measured in hours.'), + bucketer=monitor.GeometricBucketer(), + field_spec=[ + monitor.StringField('step'), + monitor.StringField('job'), + monitor.BooleanField('from_fuzzer'), + ], + )) + +TASK_RATE_LIMIT_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/rate_limit', description=('Counter for rate limit events.'), field_spec=[ @@ -314,7 +321,7 @@ monitor.StringField('argument'), ]) -TASK_OUTCOME_COUNT = monitor.CounterMetric( +TASK_OUTCOME_COUNT: monitor._CounterMetric = monitor.CounterMetric( 'task/outcome', description=('Counter metric for task outcome (success/failure).'), field_spec=[ @@ -326,55 +333,58 @@ monitor.BooleanField('task_succeeded'), ]) -TASK_OUTCOME_COUNT_BY_ERROR_TYPE = monitor.CounterMetric( - 'task/outcome_by_error_type', - description=('Counter metric for task outcome, with error type.'), - field_spec=[ - monitor.StringField('task'), - monitor.StringField('subtask'), - monitor.StringField('mode'), - monitor.StringField('platform'), - monitor.BooleanField('task_succeeded'), - monitor.StringField('error_condition'), - ]) - -UTASK_SUBTASK_E2E_DURATION_SECS = monitor.CumulativeDistributionMetric( - 'utask/subtask_e2e_duration_secs', - description=( - 'Time elapsed since preprocess started for this task, in ' + - 'seconds, per subtask ("preprocess", "uworker_main" and ' + - '"postprocess"). Subtask "postprocess" being the last, that ' + - 'measures total e2e task duration. Mode is either "batch" or ' + - '"queue" depending on whether uworker_main was scheduled and ' + - 'executed on Cloud Batch or not, respectively.'), - bucketer=monitor.GeometricBucketer(), - field_spec=[ - monitor.StringField('task'), - monitor.StringField('job'), - monitor.StringField('subtask'), - monitor.StringField('mode'), - monitor.StringField('platform'), - ], -) - -UTASK_SUBTASK_DURATION_SECS = monitor.CumulativeDistributionMetric( - 'utask/subtask_duration_secs', - description=( - 'Duration of each subtask ("preprocess", "uworker_main" and ' + - '"postprocess"). Mode is either "batch" or "queue" depending on ' + - 'whether uworker_main was scheduled and executed on Cloud Batch or ' + - 'not, respectively.'), - bucketer=monitor.GeometricBucketer(), - field_spec=[ - monitor.StringField('task'), - monitor.StringField('job'), - monitor.StringField('subtask'), - monitor.StringField('mode'), - monitor.StringField('platform'), - ], -) - -ANDROID_UPTIME = monitor.CounterMetric( +TASK_OUTCOME_COUNT_BY_ERROR_TYPE: monitor._CounterMetric = ( + monitor.CounterMetric( + 'task/outcome_by_error_type', + description=('Counter metric for task outcome, with error type.'), + field_spec=[ + monitor.StringField('task'), + monitor.StringField('subtask'), + monitor.StringField('mode'), + monitor.StringField('platform'), + monitor.BooleanField('task_succeeded'), + monitor.StringField('error_condition'), + ])) + +UTASK_SUBTASK_E2E_DURATION_SECS: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'utask/subtask_e2e_duration_secs', + description=( + 'Time elapsed since preprocess started for this task, in ' + + 'seconds, per subtask ("preprocess", "uworker_main" and ' + + '"postprocess"). Subtask "postprocess" being the last, that ' + + 'measures total e2e task duration. Mode is either "batch" or ' + + '"queue" depending on whether uworker_main was scheduled and ' + + 'executed on Cloud Batch or not, respectively.'), + bucketer=monitor.GeometricBucketer(), + field_spec=[ + monitor.StringField('task'), + monitor.StringField('job'), + monitor.StringField('subtask'), + monitor.StringField('mode'), + monitor.StringField('platform'), + ], + )) + +UTASK_SUBTASK_DURATION_SECS: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'utask/subtask_duration_secs', + description=( + 'Duration of each subtask ("preprocess", "uworker_main" and ' + + '"postprocess"). Mode is either "batch" or "queue" depending on ' + + 'whether uworker_main was scheduled and executed on Cloud Batch or ' + + 'not, respectively.'), + bucketer=monitor.GeometricBucketer(), + field_spec=[ + monitor.StringField('task'), + monitor.StringField('job'), + monitor.StringField('subtask'), + monitor.StringField('mode'), + monitor.StringField('platform'), + ], + )) + +ANDROID_UPTIME: monitor._CounterMetric = monitor.CounterMetric( 'android_device_uptime', description='Android device uptime', field_spec=[ @@ -383,14 +393,14 @@ ], ) -CHROME_TEST_SYNCER_SUCCESS = monitor.CounterMetric( +CHROME_TEST_SYNCER_SUCCESS: monitor._CounterMetric = monitor.CounterMetric( 'chrome_test_syncer_success', description='Counter for successful test syncer exits.', field_spec=[]) # Metrics related to issue lifecycle -ISSUE_FILING = monitor.CounterMetric( +ISSUE_FILING: monitor._CounterMetric = monitor.CounterMetric( 'issues/filing', description='Bugs opened through triage task.', field_spec=[ @@ -398,7 +408,7 @@ monitor.StringField('status'), ]) -ISSUE_CLOSING = monitor.CounterMetric( +ISSUE_CLOSING: monitor._CounterMetric = monitor.CounterMetric( 'issues/closing/success', description='Bugs closed during cleanup task.', field_spec=[ @@ -406,28 +416,31 @@ monitor.StringField('status'), ]) -BUG_FILING_FROM_TESTCASE_ELAPSED_TIME = monitor.CumulativeDistributionMetric( - 'fuzzed_testcase_analysis/triage_duration_secs', - description='Time elapsed between testcase and bug creation, in minutes.', - bucketer=monitor.GeometricBucketer(), - field_spec=[ - monitor.StringField('job'), - monitor.StringField('platform'), - ]) - -UNTRIAGED_TESTCASE_AGE = monitor.CumulativeDistributionMetric( - 'issues/untriaged_testcase_age', - description='Age of testcases that were not yet triaged ' - '(have not yet completed analyze, regression,' - ' minimization, impact task), in hours.', - bucketer=monitor.GeometricBucketer(), - field_spec=[ - monitor.StringField('job'), - monitor.StringField('platform'), - monitor.StringField('step'), - ]) - -UNTRIAGED_TESTCASE_COUNT = monitor.GaugeMetric( +BUG_FILING_FROM_TESTCASE_ELAPSED_TIME: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'fuzzed_testcase_analysis/triage_duration_secs', + description=( + 'Time elapsed between testcase and bug creation, in minutes.'), + bucketer=monitor.GeometricBucketer(), + field_spec=[ + monitor.StringField('job'), + monitor.StringField('platform'), + ])) + +UNTRIAGED_TESTCASE_AGE: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'issues/untriaged_testcase_age', + description='Age of testcases that were not yet triaged ' + '(have not yet completed analyze, regression,' + ' minimization, impact task), in hours.', + bucketer=monitor.GeometricBucketer(), + field_spec=[ + monitor.StringField('job'), + monitor.StringField('platform'), + monitor.StringField('step'), + ])) + +UNTRIAGED_TESTCASE_COUNT: monitor._GaugeMetric = monitor.GaugeMetric( 'issues/untriaged_testcase_count', description='Number of testcases that were not yet triaged ' '(have not yet completed analyze, regression,' @@ -440,7 +453,7 @@ # Grouper-related metrics -TESTCASE_GROUP_OVERFLOW_COUNT = monitor.GaugeMetric( +TESTCASE_GROUP_OVERFLOW_COUNT: monitor._GaugeMetric = monitor.GaugeMetric( 'grouper/testcase_group_overflow_count', description='The number of testcases that were marked as closed or deleted ' 'due to group overflow.', @@ -449,13 +462,14 @@ monitor.StringField('fuzzer_name'), ]) -TESTCASE_GROUPS_SIZES = monitor.CumulativeDistributionMetric( - 'grouper/testcase_groups_sizes', - description='Distribution of testcase groups sizes (before shrinkage).', - bucketer=monitor.GeometricBucketer(), - field_spec=None) +TESTCASE_GROUPS_SIZES: monitor._CumulativeDistributionMetric = ( + monitor.CumulativeDistributionMetric( + 'grouper/testcase_groups_sizes', + description='Distribution of testcase groups sizes (before shrinkage).', + bucketer=monitor.GeometricBucketer(), + field_spec=None)) -ANALYZE_TASK_REPRODUCIBILITY = monitor.CounterMetric( +ANALYZE_TASK_REPRODUCIBILITY: monitor._CounterMetric = monitor.CounterMetric( 'task/analyze/reproducibility', description='Outcome count for analyze task.', field_spec=[ @@ -465,7 +479,7 @@ monitor.BooleanField('crashes'), ]) -PRODUCTION_DEPLOYMENT = monitor.CounterMetric( +PRODUCTION_DEPLOYMENT: monitor._CounterMetric = monitor.CounterMetric( 'debug/deployment/count', description='The number of deployments', field_spec=[ diff --git a/src/clusterfuzz/_internal/metrics/profiler.py b/src/clusterfuzz/_internal/metrics/profiler.py index c026ffe5186..48208fa2a32 100644 --- a/src/clusterfuzz/_internal/metrics/profiler.py +++ b/src/clusterfuzz/_internal/metrics/profiler.py @@ -18,7 +18,7 @@ from clusterfuzz._internal.system import environment -def start_if_needed(service): +def start_if_needed(service: str) -> bool: """Start Google Cloud Profiler if profiling key in project config is enabled.""" config = local_config.ProjectConfig() diff --git a/src/clusterfuzz/_internal/platforms/android/adb.py b/src/clusterfuzz/_internal/platforms/android/adb.py index 7d3c712206b..ee954a3c95c 100755 --- a/src/clusterfuzz/_internal/platforms/android/adb.py +++ b/src/clusterfuzz/_internal/platforms/android/adb.py @@ -13,7 +13,6 @@ # limitations under the License. """ADB shell related functions.""" -import collections import os import re import signal @@ -22,6 +21,11 @@ import tempfile import threading import time +from typing import Any +from typing import cast +from typing import NamedTuple +from typing import Optional +from typing import Union from clusterfuzz._internal.base import persistent_cache from clusterfuzz._internal.base import utils @@ -30,48 +34,55 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import shell -ADB_TIMEOUT = 1200 # Should be lower than |REBOOT_TIMEOUT|. -BAD_STATE_WAIT = 60 * 60 # 60 mins -BOOT_WAIT_INTERVAL = 60 * 2 -CUTTLEFISH_USER = 'vsoc-01' -CUTTLEFISH_CVD_PORT = 6520 -DEFAULT_DEVICE_MEMORY_MB = 2048 -DEVICE = collections.namedtuple('Device', ['serial', 'path']) -DEVICE_HANG_STRING = None -DEVICE_NOT_FOUND_STRING = 'error: device \'{serial}\' not found' -DEVICE_OFFLINE_STRING = 'error: device offline' -FACTORY_RESET_WAIT = 60 -KERNEL_LOG_FILES = [ +ADB_TIMEOUT: int = 1200 # Should be lower than |REBOOT_TIMEOUT|. +BAD_STATE_WAIT: int = 60 * 60 # 60 mins +BOOT_WAIT_INTERVAL: int = 60 * 2 +CUTTLEFISH_USER: str = 'vsoc-01' +CUTTLEFISH_CVD_PORT: int = 6520 +DEFAULT_DEVICE_MEMORY_MB: int = 2048 + + +class Device(NamedTuple): + serial: str + path: str + + +DEVICE = Device +DEVICE_HANG_STRING: Optional[str] = None +DEVICE_NOT_FOUND_STRING: str = 'error: device \'{serial}\' not found' +DEVICE_OFFLINE_STRING: str = 'error: device offline' +FACTORY_RESET_WAIT: int = 60 +KERNEL_LOG_FILES: list[str] = [ '/proc/last_kmsg', '/sys/fs/pstore/console-ramoops', ] -MONKEY_PROCESS_NAME = 'monkey' -WAIT_FOR_DEVICE_TIMEOUT = 600 -REBOOT_TIMEOUT = 3600 -RECOVERY_CMD_TIMEOUT = 60 * 2 -GET_DEVICE_STATE_TIMEOUT = 20 -STOP_CVD_WAIT = 20 -LAUNCH_CVD_TIMEOUT = 2700 -CMD_KILL_CROSVM = 'pkill crosvm' +MONKEY_PROCESS_NAME: str = 'monkey' +WAIT_FOR_DEVICE_TIMEOUT: int = 600 +REBOOT_TIMEOUT: int = 3600 +RECOVERY_CMD_TIMEOUT: int = 60 * 2 +GET_DEVICE_STATE_TIMEOUT: int = 20 +STOP_CVD_WAIT: int = 20 +LAUNCH_CVD_TIMEOUT: int = 2700 +CMD_KILL_CROSVM: str = 'pkill crosvm' # Output patterns to parse "lsusb" output. -LSUSB_BUS_RE = re.compile(r'Bus\s+(\d+)\s+Device\s+(\d+):.*') -LSUSB_SERIAL_RE = re.compile(r'\s+iSerial\s+\d\s+(.*)') +LSUSB_BUS_RE: re.Pattern = re.compile(r'Bus\s+(\d+)\s+Device\s+(\d+):.*') +LSUSB_SERIAL_RE: re.Pattern = re.compile(r'\s+iSerial\s+\d\s+(.*)') # This is a constant value defined in usbdevice_fs.h in Linux system. -USBDEVFS_RESET = ord('U') << 8 | 20 +USBDEVFS_RESET: int = ord('U') << 8 | 20 -RAMOOPS_READER_GCS_PATH = 'gs://haiku-storage/trusty/ramoops_reader.py' +RAMOOPS_READER_GCS_PATH: str = 'gs://haiku-storage/trusty/ramoops_reader.py' -def bad_state_reached(): +def bad_state_reached() -> None: """Wait when device is in a bad state and exit.""" persistent_cache.clear_values() logs.log_fatal_and_exit( 'Device in bad state.', wait_before_exit=BAD_STATE_WAIT) -def connect_to_cuttlefish_device(): +def connect_to_cuttlefish_device() -> Optional[str]: """Connect to Cuttlefish cvd.""" logs.info('Connect to cuttlefish device.') device_serial = environment.get_value('ANDROID_SERIAL') @@ -79,51 +90,57 @@ def connect_to_cuttlefish_device(): return execute_command(connect_cmd, timeout=RECOVERY_CMD_TIMEOUT) -def copy_local_directory_to_remote(local_directory, remote_directory): +def copy_local_directory_to_remote(local_directory: str, + remote_directory: str) -> None: """Copies local directory contents to a device directory.""" create_directory_if_needed(remote_directory) if os.listdir(local_directory): run_command(['push', '%s/.' % local_directory, remote_directory], True) -def copy_local_file_to_remote(local_file_path, remote_file_path): +def copy_local_file_to_remote(local_file_path: str, + remote_file_path: str) -> None: """Copies local file to a device file.""" create_directory_if_needed(os.path.dirname(remote_file_path)) run_command(['push', local_file_path, remote_file_path], True) -def copy_remote_directory_to_local(remote_directory, local_directory): +def copy_remote_directory_to_local(remote_directory: str, + local_directory: str) -> None: """Copies local directory contents to a device directory.""" run_command(['pull', '%s/.' % remote_directory, local_directory]) -def copy_remote_file_to_local(remote_file_path, local_file_path): +def copy_remote_file_to_local(remote_file_path: str, + local_file_path: str) -> None: """Copies device file to a local file.""" shell.create_directory( os.path.dirname(local_file_path), create_intermediates=True) run_command(['pull', remote_file_path, local_file_path]) -def create_directory_if_needed(device_directory): +def create_directory_if_needed(device_directory: str) -> None: """Creates a directory on the device if it doesn't already exist.""" run_shell_command(['mkdir', '-p', device_directory]) -def directory_exists(directory_path): +def directory_exists(directory_path: str) -> bool: """Return whether a directory exists or not.""" expected = '0' result = run_shell_command('\'test -d "%s"; echo $?\'' % directory_path) return result == expected -def execute_command(cmd, timeout=None, on_cuttlefish_host=False): +def execute_command(cmd: str, + timeout: Optional[Union[int, float]] = None, + on_cuttlefish_host: bool = False) -> Optional[str]: """Spawns a subprocess to run the given shell command.""" if on_cuttlefish_host and environment.is_android_cuttlefish(): # Auto accept key fingerprint for ssh command. cmd = ('ssh -o StrictHostKeyChecking=no ' f'{get_cuttlefish_ssh_target()} "{cmd}"') - so = [] + so: list[bytes] = [] # pylint: disable=consider-using-with output_dest = tempfile.TemporaryFile() # pylint: disable=subprocess-popen-preexec-fn,consider-using-with @@ -136,8 +153,9 @@ def execute_command(cmd, timeout=None, on_cuttlefish_host=False): preexec_fn=lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL), bufsize=0) - def run(): + def run() -> None: """Thread target function that waits for subprocess to complete.""" + output = None try: pipe.communicate() output_dest.seek(0) @@ -168,7 +186,10 @@ def run(): return bytes_output.strip().decode('utf-8', errors='ignore') -def copy_to_cuttlefish(src_path, dest_path, timeout=None): +def copy_to_cuttlefish(src_path: str, + dest_path: str, + timeout: Optional[Union[int, float]] = None + ) -> Optional[str]: """Copy file to cuttlefish device.""" cvd_address = get_cuttlefish_ssh_target() return execute_command( @@ -177,7 +198,7 @@ def copy_to_cuttlefish(src_path, dest_path, timeout=None): timeout=timeout) -def factory_reset(): +def factory_reset() -> None: """Reset device to factory state.""" logs.info('reached factory_reset') if environment.is_android_cuttlefish(): @@ -200,30 +221,30 @@ def factory_reset(): time.sleep(FACTORY_RESET_WAIT) -def file_exists(file_path): +def file_exists(file_path: str) -> bool: """Return whether a file exists or not.""" expected = '0' result = run_shell_command('\'test -f "%s"; echo $?\'' % file_path) return result == expected -def get_adb_command_line(adb_cmd): +def get_adb_command_line(adb_cmd: str) -> str: """Return adb command line for running an adb command.""" device_serial = environment.get_value('ANDROID_SERIAL') adb_cmd_line = '%s -s %s %s' % (get_adb_path(), device_serial, adb_cmd) return adb_cmd_line -def get_adb_path(): +def get_adb_path() -> str: """Return path to ADB binary.""" - adb_path = environment.get_value('ADB') + adb_path: Optional[str] = environment.get_value('ADB') if adb_path: return adb_path return os.path.join(environment.get_platform_resources_directory(), 'adb') -def get_device_state(): +def get_device_state() -> Optional[str]: """Return the device status.""" if environment.is_android_emulator(): fastboot_state = run_fastboot_command( @@ -235,15 +256,15 @@ def get_device_state(): return execute_command(state_cmd, timeout=RECOVERY_CMD_TIMEOUT) -def get_fastboot_command_line(fastboot_cmd): +def get_fastboot_command_line(fastboot_cmd: str) -> str: """Return fastboot command line for running a fastboot command.""" fastboot_cmd_line = '%s %s' % (get_fastboot_path(), fastboot_cmd) return fastboot_cmd_line -def get_fastboot_path(): +def get_fastboot_path() -> str: """Return path to fastboot binary.""" - fastboot_path = environment.get_value('FASTBOOT') + fastboot_path: Optional[str] = environment.get_value('FASTBOOT') if fastboot_path: return fastboot_path @@ -251,7 +272,7 @@ def get_fastboot_path(): 'fastboot') -def get_file_checksum(file_path): +def get_file_checksum(file_path: str) -> Optional[str]: """Return file's md5 checksum.""" if not file_exists(file_path): return None @@ -259,15 +280,15 @@ def get_file_checksum(file_path): return run_shell_command(['md5sum', '-b', file_path]) -def get_file_size(file_path): +def get_file_size(file_path: str) -> Optional[int]: """Return file's size.""" if not file_exists(file_path): return None - return int(run_shell_command(['stat', '-c%s', file_path])) + return int(cast(str, run_shell_command(['stat', '-c%s', file_path]))) -def get_kernel_log_content(): +def get_kernel_log_content() -> str: """Return content of kernel logs.""" kernel_log_content = '' for kernel_log_file in KERNEL_LOG_FILES: @@ -276,7 +297,7 @@ def get_kernel_log_content(): return kernel_log_content -def extract_logcat_from_ramdump_and_reboot(): +def extract_logcat_from_ramdump_and_reboot() -> str: """Extracts logcat from ramdump kernel log and reboots.""" run_fastboot_command( ['oem', 'ramdump', 'stage_file', 'kernel.log'], @@ -302,15 +323,15 @@ def extract_logcat_from_ramdump_and_reboot(): return logcat -def get_ps_output(): +def get_ps_output() -> Optional[str]: """Return ps output for all processes.""" return run_shell_command(['ps', '-A']) -def get_process_and_child_pids(process_name): +def get_process_and_child_pids(process_name: str) -> list[int]: """Return process and child pids matching a process name.""" - pids = [] - ps_output_lines = get_ps_output().splitlines() + pids: list[int] = [] + ps_output_lines = cast(str, get_ps_output()).splitlines() while True: old_pids_length = len(pids) @@ -323,7 +344,7 @@ def get_process_and_child_pids(process_name): line_process_pid = int(data[1]) # Parent PID is in the third column. line_parent_pid = int(data[2]) - except: + except Exception: continue # If we have already processed this pid, no more work to do. @@ -353,12 +374,12 @@ def get_process_and_child_pids(process_name): return pids -def get_property(property_name): +def get_property(property_name: str) -> Optional[str]: """Return property's value.""" return run_shell_command(['getprop', property_name]) -def hard_reset(): +def hard_reset() -> None: """Perform a hard reset of the device.""" if environment.is_android_cuttlefish(): # There is no recovery step at this point for a cuttlefish bot, so just exit @@ -387,17 +408,17 @@ def hard_reset(): run_shell_command('recovery --wipe_data') -def kill_processes_and_children_matching_name(process_name): +def kill_processes_and_children_matching_name(process_name: str) -> None: """Kills process along with children matching names.""" process_and_child_pids = get_process_and_child_pids(process_name) if not process_and_child_pids: return - kill_command = ['kill', '-9'] + process_and_child_pids + kill_command: list[Union[str, int]] = ['kill', '-9'] + process_and_child_pids run_shell_command(kill_command) -def read_data_from_file(file_path): +def read_data_from_file(file_path: str) -> Optional[str]: """Return device's file content.""" if not file_exists(file_path): return None @@ -405,12 +426,12 @@ def read_data_from_file(file_path): return run_shell_command(['cat', '"%s"' % file_path]) -def reboot(): +def reboot() -> None: """Reboots device.""" run_command('reboot') -def start_cuttlefish_device(): +def start_cuttlefish_device() -> None: """Start the cuttlefish device.""" cvd_dir = environment.get_value('CVD_DIR') cvd_bin_dir = os.path.join(cvd_dir, 'bin') @@ -425,7 +446,7 @@ def start_cuttlefish_device(): execute_command(launch_cvd_command_line, on_cuttlefish_host=True) -def copy_images_to_cuttlefish(): +def copy_images_to_cuttlefish() -> None: """Copy and Combine cvd host package and OTA images.""" image_directory = environment.get_value('IMAGES_DIR') cvd_dir = environment.get_value('CVD_DIR') @@ -437,7 +458,7 @@ def copy_images_to_cuttlefish(): copy_to_cuttlefish(image_src, image_dest) -def stop_cuttlefish_device(): +def stop_cuttlefish_device() -> None: """Stops the cuttlefish device.""" cvd_dir = environment.get_value('CVD_DIR') cvd_bin_dir = os.path.join(cvd_dir, 'bin') @@ -461,7 +482,7 @@ def stop_cuttlefish_device(): kill_helpers_cmd, timeout=RECOVERY_CMD_TIMEOUT, on_cuttlefish_host=True) -def restart_cuttlefish_device(): +def restart_cuttlefish_device() -> None: """Restarts the cuttlefish device.""" cvd_dir = environment.get_value('CVD_DIR') cvd_bin_dir = os.path.join(cvd_dir, 'bin') @@ -470,7 +491,7 @@ def restart_cuttlefish_device(): execute_command(restart_cvd_cmd, on_cuttlefish_host=True) -def recreate_cuttlefish_device(): +def recreate_cuttlefish_device() -> None: """Recreate cuttlefish device, restoring from backup images.""" logs.info('Reimaging cuttlefish device.') cvd_dir = environment.get_value('CVD_DIR') @@ -487,7 +508,7 @@ def recreate_cuttlefish_device(): start_cuttlefish_device() -def remount(): +def remount() -> None: """Remount /system as read/write.""" run_as_root() run_command('remount') @@ -495,24 +516,24 @@ def remount(): run_as_root() -def restart_adb(): +def restart_adb() -> None: run_command('kill-server') run_command('start-server') -def remove_directory(device_directory, recreate=False): +def remove_directory(device_directory: str, recreate: bool = False) -> None: """Delete everything inside of a device directory and recreate if needed.""" run_shell_command('rm -rf %s' % device_directory, root=True) if recreate: create_directory_if_needed(device_directory) -def remove_file(file_path): +def remove_file(file_path: str) -> None: """Remove file.""" run_shell_command('rm -f %s' % file_path, root=True) -def reset_device_connection(): +def reset_device_connection() -> bool: """Reset the connection to the physical device through USB. Returns whether or not the reset succeeded.""" if environment.is_android_cuttlefish(): @@ -531,7 +552,7 @@ def reset_device_connection(): return True -def get_cuttlefish_device_ip(): +def get_cuttlefish_device_ip() -> Optional[str]: """Return the ip address of cuttlefish device.""" try: return socket.gethostbyname('cuttlefish') @@ -540,22 +561,22 @@ def get_cuttlefish_device_ip(): return None -def set_cuttlefish_device_serial(): +def set_cuttlefish_device_serial() -> None: """Set the ANDROID_SERIAL to cuttlefish ip and port.""" device_serial = '%s:%d' % (get_cuttlefish_device_ip(), CUTTLEFISH_CVD_PORT) environment.set_value('ANDROID_SERIAL', device_serial) logs.info('Set cuttlefish device serial: %s' % device_serial) -def get_cuttlefish_ssh_target(): +def get_cuttlefish_ssh_target() -> str: """Return the target for cuttlefish ssh connection.""" return f'{CUTTLEFISH_USER}@{get_cuttlefish_device_ip()}' -def get_device_path(): +def get_device_path() -> Optional[str]: """Gets a device path to be cached and used by reset_usb.""" - def _get_usb_devices(): + def _get_usb_devices() -> list[Device]: """Returns a list of device objects containing a serial and USB path.""" usb_list_cmd = 'lsusb -v' output = execute_command(usb_list_cmd, timeout=RECOVERY_CMD_TIMEOUT) @@ -564,8 +585,9 @@ def _get_usb_devices(): 'host restart might be needed.') bad_state_reached() - devices = [] - path = None + devices: list[Device] = [] + path: Optional[str] = None + assert output is not None for line in output.splitlines(): match = LSUSB_BUS_RE.match(line) if match: @@ -579,7 +601,7 @@ def _get_usb_devices(): return devices - def _get_device_path_for_serial(): + def _get_device_path_for_serial() -> Optional[str]: """Return device path. Assumes a simple ANDROID_SERIAL.""" devices = _get_usb_devices() for device in devices: @@ -588,11 +610,12 @@ def _get_device_path_for_serial(): return None - def _get_device_path_for_usb(): + def _get_device_path_for_usb() -> str: """Returns a device path. Assumes ANDROID_SERIAL in the form "usb:".""" # Android serial may reference a usb device rather than a serial number. + assert device_serial is not None device_id = device_serial[len('usb:'):] bus_number = int( open('/sys/bus/usb/devices/%s/busnum' % device_id).read().strip()) @@ -604,13 +627,13 @@ def _get_device_path_for_usb(): return None device_serial = environment.get_value('ANDROID_SERIAL') - if device_serial.startswith('usb:'): + if cast(str, device_serial).startswith('usb:'): return _get_device_path_for_usb() return _get_device_path_for_serial() -def reset_usb(): +def reset_usb() -> bool: """Reset USB bus for a device serial.""" if environment.is_android_cuttlefish(): # Nothing to do here. @@ -637,7 +660,7 @@ def reset_usb(): try: with open(device_path, 'w') as f: fcntl.ioctl(f, USBDEVFS_RESET) - except: + except Exception: logs.warning('Failed to reset usb.') return False @@ -646,7 +669,7 @@ def reset_usb(): return True -def revert_asan_device_setup_if_needed(): +def revert_asan_device_setup_if_needed() -> None: """Reverts ASan device setup if installed.""" if not environment.get_value('ASAN_DEVICE_SETUP'): return @@ -663,7 +686,7 @@ def revert_asan_device_setup_if_needed(): execute_command(command, timeout=RECOVERY_CMD_TIMEOUT) -def run_as_root(): +def run_as_root() -> None: """Restart adbd and runs as root.""" # Check if we are already running as root. If yes bail out. if get_property('service.adb.root') == '1': @@ -674,7 +697,10 @@ def run_as_root(): wait_for_device() -def run_command(cmd, log_output=False, timeout=None, recover=True): +def run_command(cmd: Union[str, list[Any]], + log_output: bool = False, + timeout: Optional[Union[int, float]] = None, + recover: bool = True) -> Optional[str]: """Run a command in adb shell.""" if isinstance(cmd, list): cmd = ' '.join([str(i) for i in cmd]) @@ -719,14 +745,14 @@ def run_command(cmd, log_output=False, timeout=None, recover=True): return output -def run_shell_command(cmd, - log_output=False, - root=False, - timeout=None, - recover=True): +def run_shell_command(cmd: Union[str, list[Any]], + log_output: bool = False, + root: bool = False, + timeout: Optional[Union[int, float]] = None, + recover: bool = True) -> Optional[str]: """Run adb shell command (with root if needed).""" - def _escape_specials(command): + def _escape_specials(command: str) -> str: return command.replace('\\', '\\\\').replace('"', '\\"') if isinstance(cmd, list): @@ -748,7 +774,10 @@ def _escape_specials(command): full_cmd, log_output=log_output, timeout=timeout, recover=recover) -def run_fastboot_command(cmd, log_output=True, timeout=None): +def run_fastboot_command( + cmd: Union[str, list[Any]], + log_output: bool = True, + timeout: Optional[Union[int, float]] = None) -> Optional[str]: """Run a command in fastboot shell.""" if environment.is_android_cuttlefish(): # We can't run fastboot commands on Android cuttlefish instances. @@ -765,7 +794,7 @@ def run_fastboot_command(cmd, log_output=True, timeout=None): return output -def setup_adb(): +def setup_adb() -> None: """Sets up ADB binary for use.""" adb_binary_path = get_adb_path() @@ -774,7 +803,7 @@ def setup_adb(): environment.set_value('ADB', adb_binary_path) -def start_shell(): +def start_shell() -> None: """Stops shell.""" # Make sure we are running as root. run_as_root() @@ -783,16 +812,16 @@ def start_shell(): wait_until_fully_booted() -def stop_shell(): +def stop_shell() -> None: """Stops shell.""" # Make sure we are running as root. run_as_root() run_shell_command('stop') -def time_since_last_reboot(): +def time_since_last_reboot() -> float: """Return time in seconds since last reboot.""" - uptime_string = run_shell_command(['cat', '/proc/uptime']).split( + uptime_string = cast(str, run_shell_command(['cat', '/proc/uptime'])).split( ' ', maxsplit=1)[0] try: return float(uptime_string) @@ -800,28 +829,30 @@ def time_since_last_reboot(): # Sometimes, adb can just hang or return null output. In these cases, just # return infinity uptime value. return float('inf') + except AttributeError: + return float('inf') -def wait_for_device(recover=True): +def wait_for_device(recover: bool = True) -> None: """Waits indefinitely for the device to come online.""" run_command( 'wait-for-device', timeout=WAIT_FOR_DEVICE_TIMEOUT, recover=recover) -def wait_until_fully_booted(): +def wait_until_fully_booted() -> bool: """Wait until device is fully booted or timeout expires.""" - def boot_completed(): + def boot_completed() -> bool: expected = '1' result = run_shell_command('getprop sys.boot_completed') return result == expected - def drive_ready(): + def drive_ready() -> bool: expected = '0' result = run_shell_command('\'test -d "/"; echo $?\'') return result == expected - def package_manager_ready(): + def package_manager_ready() -> bool: expected = 'package:/system/framework/framework-res.apk' result = run_shell_command('pm path android') if not result: @@ -868,7 +899,7 @@ def package_manager_ready(): return False -def write_command_line_file(command_line, app_path): +def write_command_line_file(command_line: str, app_path: str) -> None: """Write command line file with command line argument for the application.""" command_line_path = environment.get_value('COMMAND_LINE_PATH') if not command_line_path: @@ -885,7 +916,9 @@ def write_command_line_file(command_line, app_path): write_data_to_file(command_line_file_contents, command_line_path) -def write_data_to_file(contents, file_path, should_reboot=True): +def write_data_to_file(contents: str, + file_path: str, + should_reboot: bool = True) -> None: """Writes content to file. Args: diff --git a/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py b/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py index cd605efe6d6..e5f95a5aa4d 100644 --- a/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py +++ b/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py @@ -15,6 +15,8 @@ import json import os +from typing import Any +from typing import cast from urllib.parse import quote import google.auth.exceptions @@ -26,18 +28,18 @@ from clusterfuzz._internal.metrics import logs # HTTP request timeout in seconds. -HTTP_TIMEOUT = 60 +HTTP_TIMEOUT: int = 60 # 20 MB default chunk size for file downloads. -DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 +DEFAULT_CHUNK_SIZE: int = 20 * 1024 * 1024 class AndroidBuildV4Api: """HTTP client for Android Build API V4 REST endpoints.""" - BASE_URL = 'https://androidbuild-pa.googleapis.com' + BASE_URL: str = 'https://androidbuild-pa.googleapis.com' - def __init__(self, token: str): + def __init__(self, token: str) -> None: """Initializes the Android Build API V4 client. Do not call this directly. Use AndroidBuildV4Api.create_authenticated() @@ -46,7 +48,7 @@ def __init__(self, token: str): Args: token: OAuth2 authorization bearer token. """ - self.token = token + self.token: str = token @staticmethod def create_authenticated( @@ -65,7 +67,7 @@ def create_authenticated( """ if not credentials.valid: credentials.refresh(Request()) - return AndroidBuildV4Api(credentials.token) + return AndroidBuildV4Api(cast(str, credentials.token)) def _get_headers(self) -> dict[str, str]: """Returns headers required for Android Build API V4 requests. @@ -98,13 +100,14 @@ def _download_file(self, url: str, output_path: str) -> None: request_timeout=HTTP_TIMEOUT, raise_for_not_found=True, stream=True) + response_obj = cast(requests.Response, response) with open(output_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=DEFAULT_CHUNK_SIZE): + for chunk in response_obj.iter_content(chunk_size=DEFAULT_CHUNK_SIZE): if chunk: - f.write(chunk) + f.write(cast(bytes, chunk)) def list_builds(self, branch: str, target: str, - signed: bool = False) -> dict | None: + signed: bool = False) -> dict[str, Any] | None: """List builds for a branch and target. Args: @@ -115,7 +118,7 @@ def list_builds(self, branch: str, target: str, Returns: JSON response dictionary containing builds, or None on failure. """ - params = { + params: dict[str, Any] = { 'buildType': 'submitted', 'branches': branch, 'targets': target, @@ -133,9 +136,9 @@ def list_builds(self, branch: str, target: str, headers=self._get_headers(), request_timeout=HTTP_TIMEOUT, raise_for_not_found=True) - data = json.loads(response_text) + data = json.loads(cast(str, response_text)) if data.get('builds') and len(data['builds']) > 0: - return data + return cast(dict[str, Any], data) return None except (requests.exceptions.RequestException, google.auth.exceptions.GoogleAuthError, ValueError) as e: @@ -148,7 +151,7 @@ def list_artifacts(self, target: str, attempt_id: str = 'latest', regexp: str | None = None, - page_size: int = 100) -> list: + page_size: int = 100) -> list[dict[str, Any]]: """List artifacts for a given build. Args: @@ -161,15 +164,15 @@ def list_artifacts(self, Returns: List of artifact dictionary objects. """ - params = {'pageSize': page_size} + params: dict[str, Any] = {'pageSize': page_size} if regexp: params['nameRegexp'] = regexp path = f'/v4/builds/{bid}/{target}/attempts/{attempt_id}/artifacts' url = f'{self.BASE_URL}{path}' - artifacts = [] + artifacts: list[dict[str, Any]] = [] - page_token = None + page_token: str | None = None while True: if page_token: params['pageToken'] = page_token @@ -181,7 +184,7 @@ def list_artifacts(self, headers=self._get_headers(), request_timeout=HTTP_TIMEOUT, raise_for_not_found=True) - result = json.loads(response_text) + result = json.loads(cast(str, response_text)) except (requests.exceptions.RequestException, google.auth.exceptions.GoogleAuthError, ValueError) as e: logs.error( @@ -198,7 +201,7 @@ def list_artifacts(self, return artifacts def get_artifact_metadata(self, bid: str, target: str, attempt_id: str, - name: str) -> dict | None: + name: str) -> dict[str, Any] | None: """Get artifact metadata. Args: @@ -220,8 +223,9 @@ def get_artifact_metadata(self, bid: str, target: str, attempt_id: str, headers=self._get_headers(), request_timeout=HTTP_TIMEOUT, raise_for_not_found=True) - data = json.loads(response_text) - return data.get('buildArtifactMetadata', data) + data = json.loads(cast(str, response_text)) + return cast(dict[str, Any] | None, data.get('buildArtifactMetadata', + data)) except (requests.exceptions.RequestException, google.auth.exceptions.GoogleAuthError, ValueError) as e: logs.error(f'V4 get_artifact_metadata failed for artifact {name}: {e}') @@ -251,7 +255,7 @@ def download_artifact_file(self, bid: str, target: str, attempt_id: str, headers=self._get_headers(), request_timeout=HTTP_TIMEOUT, raise_for_not_found=True) - signed_url = json.loads(response_text).get('signedUrl') + signed_url = json.loads(cast(str, response_text)).get('signedUrl') except (requests.exceptions.RequestException, google.auth.exceptions.GoogleAuthError, ValueError) as e: logs.error(f'Error getting V4 download url for artifact {name}: {e}') diff --git a/src/clusterfuzz/_internal/platforms/android/app.py b/src/clusterfuzz/_internal/platforms/android/app.py index 396782f5e9e..77fa96552c1 100644 --- a/src/clusterfuzz/_internal/platforms/android/app.py +++ b/src/clusterfuzz/_internal/platforms/android/app.py @@ -16,6 +16,8 @@ import os import re import time +from typing import Any +from typing import cast from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment @@ -23,26 +25,27 @@ from . import adb from . import constants -AAPT_CMD_TIMEOUT = 60 -CHROME_CACHE_DIRS = [ +AAPT_CMD_TIMEOUT: int = 60 +CHROME_CACHE_DIRS: list[str] = [ 'app_chrome/*', 'app_tabs/*', 'app_textures/*', 'cache/*', 'files/*', 'shared_prefs/*' ] -PACKAGES_THAT_CRASH_WITH_GESTURES = [ +PACKAGES_THAT_CRASH_WITH_GESTURES: list[str] = [ 'com.android.printspooler', 'com.android.settings', ] -PACKAGE_OPTIMIZATION_INTERVAL = 30 -PACKAGE_OPTIMIZATION_TIMEOUT = 30 * 60 +PACKAGE_OPTIMIZATION_INTERVAL: int = 30 +PACKAGE_OPTIMIZATION_TIMEOUT: int = 30 * 60 -def disable_packages_that_crash_with_gestures(): +def disable_packages_that_crash_with_gestures() -> None: """Disable known packages that crash on gesture fuzzing.""" for package in PACKAGES_THAT_CRASH_WITH_GESTURES: adb.run_shell_command(['pm', 'disable-user', package]) -def get_launch_command(app_args, testcase_path, testcase_file_url): +def get_launch_command(app_args: str, testcase_path: str, + testcase_file_url: str) -> str: """Get command to launch application with an optional testcase path.""" application_launch_command = environment.get_value('APP_LAUNCH_COMMAND') if not application_launch_command: @@ -64,7 +67,7 @@ def get_launch_command(app_args, testcase_path, testcase_file_url): return application_launch_command -def get_package_name(apk_path=None): +def get_package_name(apk_path: str | None = None) -> str | None: """Return package name.""" # See if our environment is already set with this info. package_name = environment.get_value('PKG_NAME') @@ -87,13 +90,13 @@ def get_package_name(apk_path=None): environment.get_platform_resources_directory(), 'aapt') aapt_command = '%s dump badging %s' % (aapt_binary_path, apk_path) output = adb.execute_command(aapt_command, timeout=AAPT_CMD_TIMEOUT) - match = re.match('.*package: name=\'([^\']+)\'', output, re.DOTALL) + match = re.match('.*package: name=\'([^\']+)\'', cast(str, output), re.DOTALL) if not match: return None return match.group(1) -def install(package_apk_path: str, **kwargs): +def install(package_apk_path: str, **kwargs: Any) -> Any: """Install a package from an apk path. Args: @@ -114,15 +117,17 @@ def install(package_apk_path: str, **kwargs): return adb.run_command(cmd) -def is_installed(package_name): +def is_installed(package_name: str) -> bool: """Checks if the app is installed.""" output = adb.run_shell_command(['pm', 'list', 'packages']) - package_names = [line.split(':')[-1] for line in output.splitlines()] + package_names = [ + line.split(':')[-1] for line in cast(str, output).splitlines() + ] return package_name in package_names -def reset(): +def reset() -> None: """Reset to original clean state and kills pending instances.""" package_name = get_package_name() if not package_name: @@ -143,7 +148,7 @@ def reset(): ]) -def stop(): +def stop() -> None: """Stop application and cleanup state.""" package_name = get_package_name() if not package_name: @@ -164,17 +169,18 @@ def stop(): ['rm', '-rf', ' '.join(cache_dirs_absolute_paths)], root=True) -def uninstall(package_name): +def uninstall(package_name: str) -> Any: """Uninstall a package given a name.""" return adb.run_command(['uninstall', package_name]) -def wait_until_optimization_complete(): +def wait_until_optimization_complete() -> None: """Waits for package optimization to finish.""" start_time = time.time() while time.time() - start_time < PACKAGE_OPTIMIZATION_TIMEOUT: - package_optimization_finished = 'dex2oat' not in adb.get_ps_output() + ps_output = cast(str, adb.get_ps_output()) + package_optimization_finished = 'dex2oat' not in ps_output if package_optimization_finished: return diff --git a/src/clusterfuzz/_internal/platforms/android/battery.py b/src/clusterfuzz/_internal/platforms/android/battery.py index 922f41f2c97..8a86d941e73 100644 --- a/src/clusterfuzz/_internal/platforms/android/battery.py +++ b/src/clusterfuzz/_internal/platforms/android/battery.py @@ -25,19 +25,23 @@ from . import adb from . import settings -BATTERY_CHARGE_INTERVAL = 30 * 60 # 0.5 hour. -BATTERY_CHECK_INTERVAL = 15 * 60 # 15 minutes. -EXPECTED_BATTERY_LEVEL = 60 # A percentage. -EXPECTED_BATTERY_TEMPERATURE = 35.0 # Degrees Celsius. -LOW_BATTERY_LEVEL_THRESHOLD = 30 # A percentage. -MAX_BATTERY_TEMPERATURE_THRESHOLD = 37.0 # Don't change this or battery swells. +BATTERY_CHARGE_INTERVAL: int = 30 * 60 # 0.5 hour. +BATTERY_CHECK_INTERVAL: int = 15 * 60 # 15 minutes. +EXPECTED_BATTERY_LEVEL: int = 60 # A percentage. +EXPECTED_BATTERY_TEMPERATURE: float = 35.0 # Degrees Celsius. +LOW_BATTERY_LEVEL_THRESHOLD: int = 30 # A percentage. +# Don't change this or battery swells. +MAX_BATTERY_TEMPERATURE_THRESHOLD: float = 37.0 -LAST_BATTERY_CHECK_TIME_KEY = 'android_last_battery_check' +LAST_BATTERY_CHECK_TIME_KEY: str = 'android_last_battery_check' -def get_battery_level_and_temperature(): +def get_battery_level_and_temperature() -> dict[str, float] | None: """Return device's battery and temperature levels.""" output = adb.run_shell_command(['dumpsys', 'battery']) + if not output: + logs.error('Error occurred while getting battery status.') + return None # Get battery level. m_battery_level = re.match(r'.*\n[\t ]*level: (\d+).*', output, re.DOTALL) @@ -56,7 +60,7 @@ def get_battery_level_and_temperature(): return {'level': level, 'temperature': temperature} -def wait_until_good_state(): +def wait_until_good_state() -> None: """Check battery and make sure it is charged beyond minimum level and temperature thresholds.""" # Battery levels are not applicable on GCE. @@ -76,11 +80,11 @@ def wait_until_good_state(): return # Initialize variables. - battery_level_threshold = environment.get_value('LOW_BATTERY_LEVEL_THRESHOLD', - LOW_BATTERY_LEVEL_THRESHOLD) - battery_temperature_threshold = environment.get_value( + battery_level_threshold: float = environment.get_value( + 'LOW_BATTERY_LEVEL_THRESHOLD', LOW_BATTERY_LEVEL_THRESHOLD) + battery_temperature_threshold: float = environment.get_value( 'MAX_BATTERY_TEMPERATURE_THRESHOLD', MAX_BATTERY_TEMPERATURE_THRESHOLD) - device_restarted = False + device_restarted: bool = False while True: battery_information = get_battery_level_and_temperature() diff --git a/src/clusterfuzz/_internal/platforms/android/constants.py b/src/clusterfuzz/_internal/platforms/android/constants.py index a797b3a5a0e..06c65b0e769 100644 --- a/src/clusterfuzz/_internal/platforms/android/constants.py +++ b/src/clusterfuzz/_internal/platforms/android/constants.py @@ -15,38 +15,38 @@ import re -DEVICE_DOWNLOAD_DIR = '/sdcard/Download' +DEVICE_DOWNLOAD_DIR: str = '/sdcard/Download' -DEVICE_TESTCASES_DIR = '/sdcard/fuzzer-testcases' +DEVICE_TESTCASES_DIR: str = '/sdcard/fuzzer-testcases' -DEVICE_TMP_DIR = '/data/local/tmp' +DEVICE_TMP_DIR: str = '/data/local/tmp' # Directory to keep fuzzing artifacts for grey-box fuzzers e.g. corpus. -DEVICE_FUZZING_DIR = '/data/fuzz' +DEVICE_FUZZING_DIR: str = '/data/fuzz' -DEVICE_MTE_DIR = '/system/lib64' -DEVICE_SANITIZER_DIR = '/system/lib64' +DEVICE_MTE_DIR: str = '/system/lib64' +DEVICE_SANITIZER_DIR: str = '/system/lib64' -MTE_STACKTRACE_BEGIN = 'Build fingerprint:' -MTE_STACKTRACE_END = 'mte-reports' +MTE_STACKTRACE_BEGIN: str = 'Build fingerprint:' +MTE_STACKTRACE_END: str = 'mte-reports' -TRUSTY_STACKTRACE_BEGIN = 'panic notifier - trusty version' -TRUSTY_STACKTRACE_END = 'Built:' +TRUSTY_STACKTRACE_BEGIN: str = 'panic notifier - trusty version' +TRUSTY_STACKTRACE_END: str = 'Built:' # The format of logcat when lowmemorykiller kills a process. See: # https://android.googlesource.com/platform/system/core/+/master/lmkd/lmkd.c#586 -LOW_MEMORY_REGEX = re.compile( +LOW_MEMORY_REGEX: re.Pattern[str] = re.compile( r'Low on memory:|' r'lowmemorykiller: Killing|' r'to\s+free.*because\s+cache.*is\s+below\s+limit.*for\s+oom_', re.DOTALL) # Various persistent cached values. -BUILD_PROP_MD5_KEY = 'android_build_prop_md5' -LAST_TEST_ACCOUNT_CHECK_KEY = 'android_last_test_account_check' -LAST_FLASH_BUILD_KEY = 'android_last_flash' -LAST_FLASH_TIME_KEY = 'android_last_flash_time' +BUILD_PROP_MD5_KEY: str = 'android_build_prop_md5' +LAST_TEST_ACCOUNT_CHECK_KEY: str = 'android_last_test_account_check' +LAST_FLASH_BUILD_KEY: str = 'android_last_flash' +LAST_FLASH_TIME_KEY: str = 'android_last_flash_time' -PRODUCT_TO_KERNEL = { +PRODUCT_TO_KERNEL: dict[str, str] = { 'blueline': 'bluecross', 'crosshatch': 'bluecross', 'flame': 'floral', @@ -56,17 +56,17 @@ 'taimen': 'wahoo', } -RELEASE_CONFIGURATION = 'next' +RELEASE_CONFIGURATION: str = 'next' -AUTOMOTIVE_RELEASE_CONFIGURATION = 'trunk_staging' +AUTOMOTIVE_RELEASE_CONFIGURATION: str = 'trunk_staging' -AUTOMOTIVE_TARGET_LIST = ['seahawk_hwasan'] +AUTOMOTIVE_TARGET_LIST: list[str] = ['seahawk_hwasan'] -NO_RELEASE_CONFIGURATION_TARGET_LIST = [ +NO_RELEASE_CONFIGURATION_TARGET_LIST: list[str] = [ 'shiba_fullmte', 'husky_fullmte', 'komodo_fullmte' ] -DEPRECATED_DEVICE_LIST = [ +DEPRECATED_DEVICE_LIST: list[str] = [ 'sailfish', # Pixel 'marlin', # Pixel XL 'walleye', # Pixel 2 @@ -85,4 +85,4 @@ # Restrict pixel6 from picking up generic Android jobs to avoid # Binary Mismatch: Hence, 'ANDROID:PIXEL6' is added to the list. -DEVICES_WITH_NO_FALLBACK_QUEUE_LIST = ['ANDROID:PIXEL6'] +DEVICES_WITH_NO_FALLBACK_QUEUE_LIST: list[str] = ['ANDROID:PIXEL6'] diff --git a/src/clusterfuzz/_internal/platforms/android/device.py b/src/clusterfuzz/_internal/platforms/android/device.py index 0fd828f0527..bcb5d7099fd 100755 --- a/src/clusterfuzz/_internal/platforms/android/device.py +++ b/src/clusterfuzz/_internal/platforms/android/device.py @@ -17,6 +17,7 @@ import datetime import os import time +from typing import cast from clusterfuzz._internal.base import dates from clusterfuzz._internal.base import persistent_cache @@ -34,16 +35,16 @@ from . import wifi # Variables related to adding test account on device. -ADD_TEST_ACCOUNT_APK_NAME = 'user_account_setup.apk' -ADD_TEST_ACCOUNT_CHECK_INTERVAL = 1 * 24 * 60 * 60 -ADD_TEST_ACCOUNT_PKG_NAME = 'com.google.android.tests.utilities' -ADD_TEST_ACCOUNT_CALL_PATH = '%s/.AddAccount' % ADD_TEST_ACCOUNT_PKG_NAME -ADD_TEST_ACCOUNT_TIMEOUT = 20 +ADD_TEST_ACCOUNT_APK_NAME: str = 'user_account_setup.apk' +ADD_TEST_ACCOUNT_CHECK_INTERVAL: int = 1 * 24 * 60 * 60 +ADD_TEST_ACCOUNT_PKG_NAME: str = 'com.google.android.tests.utilities' +ADD_TEST_ACCOUNT_CALL_PATH: str = '%s/.AddAccount' % ADD_TEST_ACCOUNT_PKG_NAME +ADD_TEST_ACCOUNT_TIMEOUT: int = 20 # System build properties related vars. -BUILD_PROP_PATH = '/system/build.prop' -BUILD_PROP_BACKUP_PATH = BUILD_PROP_PATH + '.bak' -BUILD_PROPERTIES = { +BUILD_PROP_PATH: str = '/system/build.prop' +BUILD_PROP_BACKUP_PATH: str = BUILD_PROP_PATH + '.bak' +BUILD_PROPERTIES: dict[str, str] = { # Disable boot animation. 'debug.sf.nobootanimation': '1', # Disable privileged app permissions enforcement. @@ -53,8 +54,8 @@ } # Local development settings related vars. -LOCAL_PROP_PATH = '/data/local.prop' -LOCAL_PROP_SETTINGS = [ +LOCAL_PROP_PATH: str = '/data/local.prop' +LOCAL_PROP_SETTINGS: list[str] = [ 'ro.audio.silent=1', 'ro.monkey=1', 'ro.setupwizard.mode=DISABLED', @@ -63,11 +64,11 @@ ] # Lockscreen database settings related vars. -LOCKSCREEN_DB = '/data/system/locksettings.db' -LOCKSCREEN_TABLE_NAME = 'locksettings' +LOCKSCREEN_DB: str = '/data/system/locksettings.db' +LOCKSCREEN_TABLE_NAME: str = 'locksettings' -def add_test_accounts_if_needed(): +def add_test_accounts_if_needed() -> None: """Add test account to work with GmsCore, etc.""" # Short-circuit: UWORKERs do not have Datastore access to retrieve # credentials. @@ -116,19 +117,19 @@ def add_test_accounts_if_needed(): persistent_cache.set_value(constants.LAST_TEST_ACCOUNT_CHECK_KEY, time.time()) -def clear_temp_directories(): +def clear_temp_directories() -> None: """Clear temp directories.""" adb.remove_directory(constants.DEVICE_DOWNLOAD_DIR, recreate=True) adb.remove_directory(constants.DEVICE_TMP_DIR, recreate=True) adb.remove_directory(constants.DEVICE_FUZZING_DIR, recreate=True) -def clear_testcase_directory(): +def clear_testcase_directory() -> None: """Clears testcase directory.""" adb.remove_directory(constants.DEVICE_TESTCASES_DIR, recreate=True) -def configure_device_settings(): +def configure_device_settings() -> None: """Configures device settings for test environment.""" adb.run_as_root() @@ -188,7 +189,7 @@ def configure_device_settings(): adb.write_data_to_file(local_properties_file_contents, LOCAL_PROP_PATH) -def configure_system_build_properties(): +def configure_system_build_properties() -> None: """Modifies system build properties in /system/build.prop for better boot speed and power use.""" adb.run_as_root() @@ -205,7 +206,7 @@ def configure_system_build_properties(): return # Pull to tmp file. - bot_tmp_directory = environment.get_value('BOT_TMPDIR') + bot_tmp_directory = cast(str, environment.get_value('BOT_TMPDIR')) old_build_prop_path = os.path.join(bot_tmp_directory, 'old.prop') adb.run_command(['pull', BUILD_PROP_PATH, old_build_prop_path]) if not os.path.exists(old_build_prop_path): @@ -243,8 +244,10 @@ def configure_system_build_properties(): # Remove seccomp policies (on N and higher) as ASan requires extra syscalls. if is_build_at_least(build_version, 'N'): - policy_files = adb.run_shell_command( - ['find', '/system/etc/seccomp_policy/', '-type', 'f']) + policy_files = cast( + str, + adb.run_shell_command( + ['find', '/system/etc/seccomp_policy/', '-type', 'f'])) for policy_file in policy_files.splitlines(): adb.run_shell_command(['rm', policy_file.strip()]) @@ -259,7 +262,7 @@ def configure_system_build_properties(): persistent_cache.set_value(constants.BUILD_PROP_MD5_KEY, current_md5) -def get_debug_props_and_values(): +def get_debug_props_and_values() -> list[str]: """Return debug property names and values based on |ENABLE_DEBUG_CHECKS| flag.""" debug_props_and_values_list = [] @@ -299,7 +302,7 @@ def get_debug_props_and_values(): return debug_props_and_values_list -def initialize_device(): +def initialize_device() -> None: """Prepares android device for app install.""" if environment.is_engine_fuzzer_job(): # These steps are not applicable to libFuzzer and syzkaller jobs and can @@ -335,7 +338,7 @@ def initialize_device(): # FIXME: Should we should revert back to regular user permission ? -def initialize_environment(): +def initialize_environment() -> None: """Set common environment variables for easy access.""" environment.set_value('BUILD_FINGERPRINT', settings.get_build_fingerprint()) environment.set_value('BUILD_VERSION', settings.get_build_version()) @@ -373,7 +376,8 @@ def _needs_no_streaming_for_asan() -> bool: return is_asan_job or is_asan_device -def install_application_if_needed(apk_path, force_update): +def install_application_if_needed(apk_path: str | None, + force_update: bool) -> None: """Install application package if it does not exist on device or if force_update is set.""" # Make sure that apk exists and has non-zero size. Otherwise, it means we @@ -410,7 +414,7 @@ def install_application_if_needed(apk_path, force_update): app.reset() -def is_build_at_least(current_version, other_version): +def is_build_at_least(current_version: str | None, other_version: str) -> bool: """Returns whether or not |current_version| is at least as new as |other_version|.""" if current_version is None: @@ -429,13 +433,13 @@ def is_build_at_least(current_version, other_version): return current_version >= other_version -def push_testcases_to_device(): +def push_testcases_to_device() -> None: """Pushes testcases from local fuzz directory onto device.""" # Attempt to ensure that the local state is the same as the state on the # device by clearing existing files on device before pushing. clear_testcase_directory() - local_testcases_directory = environment.get_value('FUZZ_INPUTS') + local_testcases_directory = cast(str, environment.get_value('FUZZ_INPUTS')) if not os.listdir(local_testcases_directory): # Directory is empty, nothing to push. logs.info('No testcases to copy to device, skipping.') @@ -445,7 +449,7 @@ def push_testcases_to_device(): constants.DEVICE_TESTCASES_DIR) -def reboot(): +def reboot() -> None: """Reboots device and clear config state.""" # Make sure to clear logcat before reboot occurs. In case of kernel crashes, # we use the log before reboot, so it is good to clear it when we are doing @@ -460,7 +464,7 @@ def reboot(): adb.wait_until_fully_booted() -def setup_host_and_device_forwarder_if_needed(): +def setup_host_and_device_forwarder_if_needed() -> None: """Sets up http(s) forwarding between device and host.""" # Get list of ports to map. http_port_1 = environment.get_value('HTTP_PORT_1', 8000) @@ -473,7 +477,9 @@ def setup_host_and_device_forwarder_if_needed(): adb.run_command(['reverse', port_string, port_string]) -def update_build(apk_path, force_update=True, should_initialize_device=True): +def update_build(apk_path: str | None, + force_update: bool = True, + should_initialize_device: bool = True) -> None: """Prepares the device and updates the build if necessary.""" # Prepare device for app install. if should_initialize_device: diff --git a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py index 066269ee4d8..e7df101d921 100644 --- a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py +++ b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py @@ -18,6 +18,9 @@ import json import os import re +from typing import Any +from typing import cast +from typing import Dict from typing import List from typing import Optional @@ -32,26 +35,26 @@ from clusterfuzz._internal.system import environment # Maximum number of retries for artifact access. -MAX_RETRIES = 5 +MAX_RETRIES: int = 5 -ANDROID_BUILD_API_SCOPES = [ +ANDROID_BUILD_API_SCOPES: list[str] = [ 'https://www.googleapis.com/auth/androidbuild.internal', 'https://www.googleapis.com/auth/cloud-platform' ] -STABLE_CUTTLEFISH_BUILD = { +STABLE_CUTTLEFISH_BUILD: dict[str, str] = { 'bid': '11655237', 'branch': 'git_main', 'target': 'cf_x86_64_phone-next-userdebug' } -DEFAULT_STABLE_CUTTLEFISH_BUILD_INFO = ( +DEFAULT_STABLE_CUTTLEFISH_BUILD_INFO: str = ( "gs://android-haiku/target-cuttlefish/stable_build_info.json") -API_VERSION = 'V4' +API_VERSION: str = 'V4' -def _call_android_api_enabled(): +def _call_android_api_enabled() -> bool: """Return True if we should call the Android Build API, enabled by default, Disabled always if invoked in a uworker """ @@ -60,11 +63,12 @@ def _call_android_api_enabled(): return False flag = feature_flags.FeatureFlags.CALL_ANDROID_API.flag - return flag.enabled if flag else True + return cast(bool, flag.enabled) if flag else True -def _download_artifact(client, bid, target, attempt_id, name, output_directory, - output_filename): +def _download_artifact(client: Any, bid: str, target: str, attempt_id: str, + name: str, output_directory: str, + output_filename: str | None) -> str | None: """Download one artifact.""" logs.info('reached download_artifact') logs.info('artifact to download: %s' % name) @@ -172,11 +176,12 @@ def _download_artifact(client, bid, target, attempt_id, name, output_directory, return output_path -def _get_artifacts_for_build(client, - bid: str, - target: str, - attempt_id: str = 'latest', - regexp: Optional[str] = None) -> List[str]: +def _get_artifacts_for_build( + client: Any, + bid: str, + target: str, + attempt_id: str = 'latest', + regexp: Optional[str] = None) -> List[Dict[str, Any]]: """Return list of artifacts for a given build.""" if not regexp: logs.warning( @@ -213,7 +218,7 @@ def _get_artifacts_for_build(client, return artifacts -def _get_client(): +def _get_client() -> Optional[Any]: """Return client with connection to build apiary.""" # Connect using build apiary service account credentials. build_apiary_service_account_private_key = db_config.get_value( @@ -239,7 +244,7 @@ def _get_client(): return client -def _get_stable_build_info(): +def _get_stable_build_info() -> Dict[str, Any]: """Return stable artifact for cuttlefish branch and target.""" logs.info('Reached get_stable_build_info') stable_build_info = STABLE_CUTTLEFISH_BUILD @@ -257,7 +262,11 @@ def _get_stable_build_info(): return stable_build_info -def get_latest_artifact_info(branch, target, signed=False, stable_build=False): +def get_latest_artifact_info(branch: str, + target: str, + signed: bool = False, + stable_build: bool = False + ) -> Optional[Dict[str, Any]]: """Return latest artifact for a branch and target.""" if not _call_android_api_enabled(): logs.warning( @@ -316,7 +325,11 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): return {'bid': bid, 'branch': branch, 'target': target} -def get(bid, target, regex, output_directory, output_filename=None): +def get(bid: str, + target: str, + regex: str, + output_directory: str, + output_filename: Optional[str] = None) -> Optional[List[str]]: """Return artifact for a given build id, target and file regex.""" if not _call_android_api_enabled(): logs.warning( @@ -337,7 +350,9 @@ def get(bid, target, regex, output_directory, output_filename=None): output_filename=output_filename) -def _run_script(client, bid, target, regex, output_directory, output_filename): +def _run_script(client: Any, bid: str, target: str, regex: str, + output_directory: str, + output_filename: Optional[str]) -> Optional[List[str]]: """Download artifacts as specified.""" artifacts = _get_artifacts_for_build( client=client, bid=bid, target=target, attempt_id='latest', regexp=regex) @@ -346,7 +361,7 @@ def _run_script(client, bid, target, regex, output_directory, output_filename): f'build id {bid}.') return None - regex = re.compile(regex) + compiled_regex = re.compile(regex) result = [] for artifact in artifacts: @@ -356,7 +371,7 @@ def _run_script(client, bid, target, regex, output_directory, output_filename): if artifact_name.endswith('.SIGN_INFO'): continue - if regex.match(artifact_name): + if compiled_regex.match(artifact_name): loop_result = _download_artifact( client=client, bid=bid, diff --git a/src/clusterfuzz/_internal/platforms/android/flash.py b/src/clusterfuzz/_internal/platforms/android/flash.py index 283a80120d3..5e3d31abbee 100644 --- a/src/clusterfuzz/_internal/platforms/android/flash.py +++ b/src/clusterfuzz/_internal/platforms/android/flash.py @@ -16,6 +16,9 @@ import os import socket import time +from typing import Any +from typing import cast +from typing import Sequence from clusterfuzz._internal.base import dates from clusterfuzz._internal.base import persistent_cache @@ -33,15 +36,15 @@ from . import fetch_artifact from . import settings -FLASH_IMAGE_REGEXES = [ +FLASH_IMAGE_REGEXES: list[str] = [ r'.*[.]img', r'.*-img-.*[.]zip', ] -FLASH_CUTTLEFISH_REGEXES = [ +FLASH_CUTTLEFISH_REGEXES: list[str] = [ r'.*-img-.*[.]zip', r'cvd-host_package.tar.gz', ] -FLASH_IMAGE_FILES = [ +FLASH_IMAGE_FILES: list[tuple[str, str]] = [ # Order is important here. ('bootloader', 'bootloader*.img'), ('radio', 'radio*.img'), @@ -54,16 +57,18 @@ ('dtbo', 'dtbo.img'), ('userdata', 'userdata.img'), ] -FLASH_DEFAULT_BUILD_TARGET = '-next-userdebug' -FLASH_DEFAULT_IMAGES_DIR = os.path.join( +FLASH_DEFAULT_BUILD_TARGET: str = '-next-userdebug' +FLASH_DEFAULT_IMAGES_DIR: str = os.path.join( environment.get_value('ROOT_DIR', ''), 'bot', 'inputs', 'images') -FLASH_INTERVAL = 1 * 24 * 60 * 60 -FLASH_RETRIES = 3 -FLASH_REBOOT_BOOTLOADER_WAIT = 15 -FLASH_REBOOT_WAIT = 5 * 60 +FLASH_INTERVAL: int = 1 * 24 * 60 * 60 +FLASH_RETRIES: int = 3 +FLASH_REBOOT_BOOTLOADER_WAIT: int = 15 +FLASH_REBOOT_WAIT: int = 5 * 60 -def download_latest_build(build_info, image_regexes, image_directory): +def download_latest_build(build_info: dict[str, Any], + image_regexes: Sequence[str], + image_directory: str) -> None: """Download the latest build artifact for the given branch and target.""" # Check if our local build matches the latest build. If not, we will # download it. @@ -87,13 +92,15 @@ def download_latest_build(build_info, image_regexes, image_directory): build_info['branch'], target)) adb.bad_state_reached() + assert image_file_paths is not None for file_path in image_file_paths: if file_path.endswith('.zip') or file_path.endswith('.tar.gz'): with archive.open(file_path) as reader: reader.extract_all(image_directory) -def boot_stable_build_cuttlefish(branch, target, image_directory): +def boot_stable_build_cuttlefish(branch: str, target: str, + image_directory: str) -> None: """Boot cuttlefish instance using stable build id fetched from gcs.""" build_info = fetch_artifact.get_latest_artifact_info( branch, target, stable_build=True) @@ -105,7 +112,7 @@ def boot_stable_build_cuttlefish(branch, target, image_directory): adb.connect_to_cuttlefish_device() -def flash_to_latest_build_if_needed(): +def flash_to_latest_build_if_needed() -> None: """Wipes user data, resetting the device to original factory state.""" if environment.get_value('LOCAL_DEVELOPMENT'): # Don't reimage local development devices. @@ -155,10 +162,11 @@ def flash_to_latest_build_if_needed(): if build_params: logs.info('build_params found on device: %s.' % build_params) if environment.is_android_cuttlefish(): - target = build_params.get('target') + FLASH_DEFAULT_BUILD_TARGET + target = cast(str, + build_params.get('target')) + FLASH_DEFAULT_BUILD_TARGET logs.info('is_android_cuttlefish() returned True. Target: %s.' % target) else: - target = build_params.get('target') + '-userdebug' + target = cast(str, build_params.get('target')) + '-userdebug' logs.info( 'is_android_cuttlefish() returned False. Target: %s.' % target) diff --git a/src/clusterfuzz/_internal/platforms/android/gestures.py b/src/clusterfuzz/_internal/platforms/android/gestures.py index 0086e15c801..092692e1f5a 100644 --- a/src/clusterfuzz/_internal/platforms/android/gestures.py +++ b/src/clusterfuzz/_internal/platforms/android/gestures.py @@ -15,17 +15,18 @@ import os import random +from typing import Any from . import adb # Fixed delay in milliseconds between consecutive monkey events. -MONKEY_THROTTLE_DELAY = 100 +MONKEY_THROTTLE_DELAY: int = 100 # Maximum number of monkey events per testcase. -NUM_MONKEY_EVENTS = 25 +NUM_MONKEY_EVENTS: int = 25 -def get_random_gestures(_): +def get_random_gestures(_: Any) -> list[str]: """Return a random gesture seed from monkey framework natively supported by Android OS.""" random_seed = random.getrandbits(32) @@ -33,7 +34,7 @@ def get_random_gestures(_): return [gesture] -def run_gestures(gestures, *_): +def run_gestures(gestures: list[str], *_: Any) -> None: """Run the provided interaction gestures.""" package_name = os.getenv('PKG_NAME') if not package_name: diff --git a/src/clusterfuzz/_internal/platforms/android/kernel_utils.py b/src/clusterfuzz/_internal/platforms/android/kernel_utils.py index a68a05691d3..900f88bfada 100644 --- a/src/clusterfuzz/_internal/platforms/android/kernel_utils.py +++ b/src/clusterfuzz/_internal/platforms/android/kernel_utils.py @@ -35,7 +35,7 @@ r'(:\d+)?') -def _get_clean_kernel_path(path): +def _get_clean_kernel_path(path: str) -> str: """Sometimes kernel paths start with /buildbot/src/partner-android/BRANCH/private/PROJ. We want to remove all of this.""" @@ -49,7 +49,8 @@ def _get_clean_kernel_path(path): return path -def get_kernel_stack_frame_link(stack_frame, kernel_prefix, kernel_hash): +def get_kernel_stack_frame_link(stack_frame: str, kernel_prefix: str, + kernel_hash: str) -> str: """Add source links data to kernel stack frames.""" match = LLVM_SYMBOLIZER_STACK_FRAME_PATH_LINE_REGEX.search(stack_frame) if not match: @@ -79,7 +80,8 @@ def get_kernel_stack_frame_link(stack_frame, kernel_prefix, kernel_hash): return link_added_stack_frame -def _get_prefix_and_full_hash(repo_data, kernel_partial_hash): +def _get_prefix_and_full_hash(repo_data: str, kernel_partial_hash: str | None + ) -> tuple[str | None, str | None]: """Find the prefix and full hash in the repo_data based on the partial.""" kernel_partial_hash_lookup = 'u\'%s' % kernel_partial_hash for line in repo_data.splitlines(): @@ -90,7 +92,7 @@ def _get_prefix_and_full_hash(repo_data, kernel_partial_hash): return None, None -def _get_repo_prop_data(build_id, target): +def _get_repo_prop_data(build_id: str, target: str) -> str | None: """Downloads repo.prop and returns the data based on build_id and target.""" symbols_directory = os.path.join( environment.get_value('SYMBOLS_DIR'), 'kernel') @@ -107,7 +109,7 @@ def _get_repo_prop_data(build_id, target): return None -def get_kernel_prefix_and_full_hash(): +def get_kernel_prefix_and_full_hash() -> tuple[str | None, str | None]: """Download repo.prop and return the full hash and prefix.""" kernel_partial_hash, build_id = get_kernel_hash_and_build_id() @@ -124,20 +126,23 @@ def get_kernel_prefix_and_full_hash(): return None, None -def get_kernel_name(): +def get_kernel_name() -> str | None: """Returns the kernel name for the device, since some kernels are shared.""" product_name = settings.get_product_name() build_product = settings.get_build_product() # Strip _kasan off of the end as we will add it later if needed. - utils.strip_from_right(product_name, '_kasan') + if product_name: + utils.strip_from_right(product_name, '_kasan') # Some devices have a different kernel name than product_name, if so use the # kernel name. - return constants.PRODUCT_TO_KERNEL.get(build_product, product_name) + if build_product: + return constants.PRODUCT_TO_KERNEL.get(build_product, product_name) + return product_name -def get_kernel_hash_and_build_id(): +def get_kernel_hash_and_build_id() -> tuple[str | None, str | None]: """Returns the (partial_hash, build_id) of the kernel.""" version_string = settings.get_kernel_version_string() match = re.match(LINUX_VERSION_REGEX, version_string) diff --git a/src/clusterfuzz/_internal/platforms/android/logger.py b/src/clusterfuzz/_internal/platforms/android/logger.py index add0d7f0efc..1bb46c0c1eb 100644 --- a/src/clusterfuzz/_internal/platforms/android/logger.py +++ b/src/clusterfuzz/_internal/platforms/android/logger.py @@ -21,7 +21,7 @@ from . import adb -def clear_log(): +def clear_log() -> None: """Clear log.""" adb.run_as_root() adb.run_shell_command(['stop', 'logd']) @@ -30,7 +30,7 @@ def clear_log(): adb.run_command(['logcat', '-c']) -def is_line_valid(line): +def is_line_valid(line: str) -> bool: """Returns true if we consider this line in logs.""" if re.match(r'^[-]+ beginning of', line): return False @@ -44,13 +44,13 @@ def is_line_valid(line): return is_chromium_resource_load or at_least_info_level -def filter_log_output(output): +def filter_log_output(output: str | None) -> str: """Filters log output. Removes debug info, etc and normalize output.""" if not output: return '' filtered_output = '' - last_process_tuple = (None, None) + last_process_tuple: tuple[str | None, int | None] = (None, None) for line in output.splitlines(): if not is_line_valid(line): continue @@ -115,12 +115,12 @@ def filter_log_output(output): return filtered_output -def log_output(additional_flags=''): +def log_output(additional_flags: str = '') -> str: """Return log data without noise and some normalization.""" output = adb.run_command('logcat -d -v brief %s *:V' % additional_flags) return filter_log_output(output) -def log_output_before_last_reboot(): +def log_output_before_last_reboot() -> str: """Return log data from last reboot without noise and some normalization.""" return log_output(additional_flags='-L') diff --git a/src/clusterfuzz/_internal/platforms/android/sanitizer.py b/src/clusterfuzz/_internal/platforms/android/sanitizer.py index c602df87221..75ebec3b492 100644 --- a/src/clusterfuzz/_internal/platforms/android/sanitizer.py +++ b/src/clusterfuzz/_internal/platforms/android/sanitizer.py @@ -14,6 +14,9 @@ """Sanitizer related functions.""" import os +from typing import Any +from typing import cast +from typing import Optional from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment @@ -26,28 +29,28 @@ from clusterfuzz._internal.system import new_process except ImportError: # On App Engine. - new_process = None + new_process: Any = None -ASAN_SCRIPT_TIMEOUT = 15 * 60 -SANITIZER_TOOL_TO_FILE_MAPPINGS = { +ASAN_SCRIPT_TIMEOUT: int = 15 * 60 +SANITIZER_TOOL_TO_FILE_MAPPINGS: dict[str, str] = { 'asan': 'asan.options', } -def get_ld_library_path_for_memory_tools(): +def get_ld_library_path_for_memory_tools() -> Optional[str]: """Return LD_LIBRARY_PATH setting for memory tools, None otherwise.""" - tool = settings.get_sanitizer_tool_name() - if tool: + sanitizer_tool = settings.get_sanitizer_tool_name() + if sanitizer_tool: return constants.DEVICE_SANITIZER_DIR - tool = settings.is_mte_build() - if tool: + is_mte = settings.is_mte_build() + if is_mte: return constants.DEVICE_MTE_DIR return None -def get_options_file_path(sanitizer_tool_name): +def get_options_file_path(sanitizer_tool_name: str) -> Optional[str]: """Return path for the sanitizer options file.""" # If this a full sanitizer system build, then update the options file in # /system, else just put it in device temp directory. @@ -63,7 +66,7 @@ def get_options_file_path(sanitizer_tool_name): return os.path.join(sanitizer_directory, sanitizer_filename) -def set_options(sanitizer_tool_name, sanitizer_options): +def set_options(sanitizer_tool_name: str, sanitizer_options: str) -> None: """Set sanitizer options on the disk file.""" sanitizer_options_file_path = get_options_file_path(sanitizer_tool_name) if not sanitizer_options_file_path: @@ -76,7 +79,7 @@ def set_options(sanitizer_tool_name, sanitizer_options): def setup_asan_if_needed() -> bool: """Set up asan on device. - + Returns: True if the device was rebooted or restarted during setup, False otherwise. """ @@ -103,12 +106,14 @@ def setup_asan_if_needed() -> bool: logs.info('Executing ASan device setup script.') asan_device_setup_script_path = os.path.join(android_directory, 'third_party', 'asan_device_setup.sh') - extra_options_arg = 'include_if_exists=' + get_options_file_path('asan') + options_file_path = cast(str, get_options_file_path('asan')) + extra_options_arg = 'include_if_exists=' + options_file_path asan_device_setup_script_args = [ '--lib', app_directory, '--device', device_id, '--extra-options', extra_options_arg ] + assert new_process is not None process = new_process.ProcessRunner(asan_device_setup_script_path, asan_device_setup_script_args) result = process.run_and_wait() diff --git a/src/clusterfuzz/_internal/platforms/android/settings.py b/src/clusterfuzz/_internal/platforms/android/settings.py index 679b7c39a99..d9f44491c0b 100644 --- a/src/clusterfuzz/_internal/platforms/android/settings.py +++ b/src/clusterfuzz/_internal/platforms/android/settings.py @@ -11,40 +11,43 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Settings change related functions.""" +"""Android settings utilities.""" + import re +from typing import cast from clusterfuzz._internal.system import environment from . import adb -BUILD_FINGERPRINT_REGEX = re.compile( +BUILD_FINGERPRINT_REGEX: re.Pattern[str] = re.compile( r'(?P.+)\/(?P.+)' r'\/(?P.+)\/(?P.+)' r'\/(?P.+):(?P.+)\/(?P.+)') -def change_se_linux_to_permissive_mode(): +def change_se_linux_to_permissive_mode() -> None: """Switch SELinux to permissive mode for working around local file access and other issues.""" adb.run_shell_command(['setenforce', '0']) -def get_build_fingerprint(): +def get_build_fingerprint() -> str | None: """Return build's fingerprint.""" return adb.get_property('ro.build.fingerprint') -def get_build_flavor(): +def get_build_flavor() -> str | None: """Return the build flavor.""" return adb.get_property('ro.build.flavor') -def get_build_parameters(): +def get_build_parameters() -> dict[str, str | None] | None: """Return build_id, target and type from the device's fingerprint""" build_fingerprint = environment.get_value('BUILD_FINGERPRINT', get_build_fingerprint()) - build_fingerprint_match = BUILD_FINGERPRINT_REGEX.match(build_fingerprint) + build_fingerprint_match = BUILD_FINGERPRINT_REGEX.match( + cast(str, build_fingerprint)) if not build_fingerprint_match: return None @@ -54,7 +57,7 @@ def get_build_parameters(): return {'build_id': build_id, 'target': target, 'type': build_type} -def get_build_version(): +def get_build_version() -> str | None: """Return the build version of the system as a character. K = Kitkat, L = Lollipop, M = Marshmellow, MASTER = Master. """ @@ -72,17 +75,18 @@ def get_build_version(): return match.group(1) -def get_cpu_arch(): +def get_cpu_arch() -> str | None: """Return cpu architecture.""" return adb.get_property('ro.product.cpu.abi') -def get_device_codename(): +def get_device_codename() -> str: """Return the device codename.""" serial = environment.get_value('ANDROID_SERIAL') - devices_output = adb.run_command(['devices', '-l']) + devices_output = cast(str, adb.run_command(['devices', '-l'])) - serial_pattern = r'(^|\s){serial}\s'.format(serial=re.escape(serial)) + serial_pattern = r'(^|\s){serial}\s'.format( + serial=re.escape(cast(str, serial))) serial_regex = re.compile(serial_pattern) for line in devices_output.splitlines(): @@ -102,7 +106,7 @@ def get_device_codename(): return '' -def get_platform_id(): +def get_platform_id() -> str: """Return a string as |android:{codename}_{sanitizer}:{build_version}|.""" platform_id = 'android' @@ -120,29 +124,32 @@ def get_platform_id(): return platform_id -def get_product_brand(): +def get_product_brand() -> str | None: """Return product's brand.""" return adb.get_property('ro.product.brand') -def get_product_name(): +def get_product_name() -> str | None: """Return product's name.""" return adb.get_property('ro.product.name') -def get_product_model(): +def get_product_model() -> str | None: """Return product's model.""" return adb.get_property('ro.product.model') -def get_build_product(): +def get_build_product() -> str | None: """Return builds's product.""" return adb.get_property('ro.build.product') -def get_sanitizer_tool_name(): +def get_sanitizer_tool_name() -> str | None: """Return sanitizer tool name e.g. ASAN if found on device.""" build_flavor = get_build_flavor() + if not build_flavor: + return None + if 'hwasan' in build_flavor: return 'hwasan' if 'kasan' in build_flavor: @@ -153,22 +160,28 @@ def get_sanitizer_tool_name(): return None -def is_mte_build(): +def is_mte_build() -> bool: """Return True if device is using Memory Tagging Extension.""" build_flavor = get_build_flavor() + if not build_flavor: + return False + return 'mte' in build_flavor -def get_security_patch_level(): +def get_security_patch_level() -> str | None: """Return the security patch level reported by the device.""" return adb.get_property('ro.build.version.security_patch') -def get_kernel_version_string(): - return adb.run_shell_command('cat /proc/version').strip() +def get_kernel_version_string() -> str: + output = adb.run_shell_command('cat /proc/version') + if not output: + return '' + return output.strip() -def is_google_device(): +def is_google_device() -> bool | None: """Return true if this is a google branded device.""" # If a build branch is already set, then this is a Google device. No need to # query device which can fail if the device is failing on recovery mode. @@ -183,18 +196,18 @@ def is_google_device(): return product_brand in ('google', 'generic') -def is_automotive(): +def is_automotive() -> bool: """Returns if we are running in Android Automotive OS, currently only for Osprey or Seahawk.""" product_model = get_product_model() return product_model in ('Osprey', 'Seahawk') -def set_content_setting(table, key, value): +def set_content_setting(table: str, key: str, value: object) -> None: """Set a device content setting. The input is not sanitized, so make sure to use with trusted input key and value pair only.""" - def _get_type_binding(value): + def _get_type_binding(value: object) -> str: """Return binding type for content setting.""" if isinstance(value, bool): return 'b' @@ -213,7 +226,8 @@ def _get_type_binding(value): adb.run_shell_command(content_setting_command) -def set_database_setting(database_path, table, key, value): +def set_database_setting(database_path: str, table: str, key: str, + value: object) -> None: """Update a key in a database. The input is not sanitized, so make sure to use with trusted input key and value pair only.""" sql_command_string = ('"UPDATE %s SET value=\'%s\' WHERE name=\'%s\'"') % ( diff --git a/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py b/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py index 55a906b5498..c8f61e48b78 100644 --- a/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py +++ b/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py @@ -14,6 +14,7 @@ """Used to download Android symbols.""" import os +from typing import Any import zipfile from clusterfuzz._internal.base import utils @@ -29,22 +30,23 @@ from . import constants -def get_repo_prop_archive_filename(build_id, target): +def get_repo_prop_archive_filename(build_id: str, target: str) -> str: return f'{target}-{build_id}-repo.prop' -def should_download_symbols(): +def should_download_symbols() -> bool: """Return True if we should continue to download symbols.""" # For local testing, we do not have access to the cloud storage bucket with # the symbols. In this case, just bail out. We have archived symbols for # google builds only. - return (not environment.get_value('LOCAL_DEVELOPMENT') and - settings.is_google_device()) + return bool(not environment.get_value('LOCAL_DEVELOPMENT') and + settings.is_google_device()) def download_artifact_if_needed( - build_id, artifact_directory, artifact_archive_path, - targets_with_type_and_san, artifact_file_name, output_filename_override): + build_id: str, artifact_directory: str, artifact_archive_path: str, + targets_with_type_and_san: list[str], artifact_file_name: str, + output_filename_override: str | None) -> None: """Downloads artifact to actifacts_archive_path if needed""" # Delete existing symbols directory first. shell.remove_directory(artifact_directory, recreate=True) @@ -57,15 +59,17 @@ def download_artifact_if_needed( break -def check_symbols_cached(build_params_check_path, build_params): +def check_symbols_cached(build_params_check_path: str, + build_params: dict[str, Any] | None) -> bool: # Check if we already have the symbols locally. cached_build_params = utils.read_data_from_file( build_params_check_path, eval_data=True) - return cached_build_params and cached_build_params == build_params + return bool(cached_build_params and cached_build_params == build_params) -def download_repo_prop_if_needed(symbols_directory, build_id, cache_target, - targets_with_type_and_san, cache_type): +def download_repo_prop_if_needed( + symbols_directory: str, build_id: str, cache_target: str, + targets_with_type_and_san: list[str], cache_type: str) -> None: """Downloads the repo.prop for a branch""" artifact_file_name = 'repo.prop' symbols_archive_filename = get_repo_prop_archive_filename( @@ -95,7 +99,7 @@ def download_repo_prop_if_needed(symbols_directory, build_id, cache_target, utils.write_data_to_file(build_params, build_params_check_path) -def download_kernel_repo_prop_if_needed(symbols_directory): +def download_kernel_repo_prop_if_needed(symbols_directory: str) -> None: """Downloads the repo.prop for the kernel of a device""" if not should_download_symbols(): return @@ -122,7 +126,7 @@ def download_kernel_repo_prop_if_needed(symbols_directory): targets_with_type_and_san, 'kernel') -def download_system_symbols_if_needed(symbols_directory): +def download_system_symbols_if_needed(symbols_directory: str) -> None: """Download system libraries from |SYMBOLS_URL| and cache locally.""" if not should_download_symbols(): return @@ -142,6 +146,8 @@ def download_system_symbols_if_needed(symbols_directory): target = build_params.get('target') build_type = build_params.get('type') + assert build_id is not None and target is not None and build_type is not None + if environment.is_android() and \ target not in constants.NO_RELEASE_CONFIGURATION_TARGET_LIST and \ target not in constants.AUTOMOTIVE_TARGET_LIST: @@ -183,10 +189,14 @@ def download_system_symbols_if_needed(symbols_directory): utils.write_data_to_file(build_params, build_params_check_path) -def download_trusty_symbols_if_needed(symbols_directory, app_name, bid): +def download_trusty_symbols_if_needed(symbols_directory: str, app_name: str, + bid: str | None) -> None: """Downloads and extracts Trusted App ELF files""" ab_target = '' - device = settings.get_build_parameters().get('target') + build_params = settings.get_build_parameters() + assert build_params is not None + device = build_params.get('target') + assert device is not None if 'cheetah' in device or 'panther' in device: ab_target = 'cloudripper-fuzz-test-debug' elif 'oriole' in device or 'raven' in device or 'bluejay' in device: @@ -201,7 +211,9 @@ def download_trusty_symbols_if_needed(symbols_directory, app_name, bid): logs.error(f'Unable to fetch build info for branch {branch} ' f'and target {ab_target}.') return - bid = build_info['bid'] + bid = str(build_info['bid']) + + assert bid is not None artifact_filename = f'{ab_target}-{bid}.syms.zip' symbols_archive_path = os.path.join(symbols_directory, artifact_filename) @@ -223,7 +235,7 @@ def download_trusty_symbols_if_needed(symbols_directory, app_name, bid): symbols_zipfile.extract(filepath, symbols_directory) -def _get_binary_from_build_or_device(binary_path): +def _get_binary_from_build_or_device(binary_path: str) -> str | None: """Look for binary on build server or on device.""" # Initialize some helper variables. symbols_directory = environment.get_value('SYMBOLS_DIR') @@ -246,7 +258,7 @@ def _get_binary_from_build_or_device(binary_path): return None -def filter_binary_path(binary_path): +def filter_binary_path(binary_path: str) -> str: """Filter binary path to provide local copy.""" # LKL fuzzer name is not full path. if environment.is_android(): diff --git a/src/clusterfuzz/_internal/platforms/android/ui.py b/src/clusterfuzz/_internal/platforms/android/ui.py index 4bedcc3cff0..6d551b09e83 100644 --- a/src/clusterfuzz/_internal/platforms/android/ui.py +++ b/src/clusterfuzz/_internal/platforms/android/ui.py @@ -18,15 +18,16 @@ from . import adb -def clear_notifications(): +def clear_notifications() -> None: """Clear all pending notifications.""" adb.run_shell_command(['service', 'call', 'notification', '1']) -def unlock_screen(): +def unlock_screen() -> None: """Unlocks the screen if it is locked.""" window_dump_output = adb.run_shell_command(['dumpsys', 'window']) - if 'mShowingLockscreen=true' not in window_dump_output: + if (not window_dump_output or + 'mShowingLockscreen=true' not in window_dump_output): # Screen is not locked, no work to do. return diff --git a/src/clusterfuzz/_internal/platforms/android/util.py b/src/clusterfuzz/_internal/platforms/android/util.py index 3cab131b78a..ba829822d59 100644 --- a/src/clusterfuzz/_internal/platforms/android/util.py +++ b/src/clusterfuzz/_internal/platforms/android/util.py @@ -20,14 +20,14 @@ from clusterfuzz._internal.system import environment -def get_device_path(local_path): +def get_device_path(local_path: str) -> str: """Returns device path for the given local path.""" root_directory = environment.get_root_directory() return os.path.join(android.constants.DEVICE_FUZZING_DIR, os.path.relpath(local_path, root_directory)) -def get_local_path(device_path): +def get_local_path(device_path: str) -> str | None: """Returns local path for the given device path.""" if not device_path.startswith(android.constants.DEVICE_FUZZING_DIR + '/'): logs.error('Bad device path: ' + device_path) @@ -39,8 +39,10 @@ def get_local_path(device_path): os.path.relpath(device_path, android.constants.DEVICE_FUZZING_DIR)) -def is_testcase_deprecated(platform_id=None): +def is_testcase_deprecated(platform_id: str | None = None) -> bool: """Whether or not the Android device is deprecated.""" + if not platform_id: + return False # Platform ID for Android is of the form as shown below # |android:{codename}_{sanitizer}:{build_version}| @@ -64,7 +66,8 @@ def is_testcase_deprecated(platform_id=None): return False -def can_testcase_run_on_platform(testcase_platform_id, current_platform_id): +def can_testcase_run_on_platform(testcase_platform_id: str | None, + current_platform_id: str) -> bool: """Whether or not the testcase can run on the current Android device.""" del testcase_platform_id # Unused argument for now diff --git a/src/clusterfuzz/_internal/platforms/android/wifi.py b/src/clusterfuzz/_internal/platforms/android/wifi.py index 1c0e4f585d1..c3e47d327e8 100644 --- a/src/clusterfuzz/_internal/platforms/android/wifi.py +++ b/src/clusterfuzz/_internal/platforms/android/wifi.py @@ -28,17 +28,17 @@ WIFI_UTIL_CALL_PATH = '%s/.WifiUtil' % WIFI_UTIL_PACKAGE_NAME -def disable(): +def disable() -> None: """Disable wifi.""" adb.run_shell_command(['svc', 'wifi', 'disable']) -def enable(): +def enable() -> None: """Enable wifi.""" adb.run_shell_command(['svc', 'wifi', 'enable']) -def disable_airplane_mode(): +def disable_airplane_mode() -> None: """Disable airplane mode.""" adb.run_shell_command(['settings', 'put', 'global', 'airplane_mode_on', '0']) adb.run_shell_command([ @@ -47,7 +47,7 @@ def disable_airplane_mode(): ]) -def configure(force_enable=False): +def configure(force_enable: bool = False) -> None: """Configure airplane mode and wifi on device.""" # Short-circuit: UWORKERs do not have Datastore access for Wi-Fi credentials. if environment.is_uworker(): @@ -83,6 +83,7 @@ def configure(force_enable=False): wifi_password = '' else: config = db_config.get() + assert config is not None if not config.wifi_ssid: logs.info('No wifi ssid is set, skipping wifi config.') return @@ -100,5 +101,6 @@ def configure(force_enable=False): ssid=shlex.quote(wifi_ssid), password=shlex.quote(wifi_password), call_path=WIFI_UTIL_CALL_PATH)) + assert output is not None if 'result=true' not in output: logs.warning('Failed to connect to wifi.', output=output) diff --git a/src/clusterfuzz/_internal/platforms/fuchsia/undercoat.py b/src/clusterfuzz/_internal/platforms/fuchsia/undercoat.py index a822bfc7937..2f1e31e7752 100644 --- a/src/clusterfuzz/_internal/platforms/fuchsia/undercoat.py +++ b/src/clusterfuzz/_internal/platforms/fuchsia/undercoat.py @@ -28,34 +28,34 @@ # We only report the tail of the log because otherwise we would only end up # seeing the beginning of it once the logging library later truncates it to the # STACKDRIVER_LOG_MESSAGE_LIMIT. -QEMU_LOG_LIMIT = 64 * 1024 +QEMU_LOG_LIMIT: int = 64 * 1024 # In order to properly clean up any stale instances, we keep track of handles in # the persistent cache. We assume that only one instance of a bot will be # accessing this cache at a time. -HANDLE_CACHE_KEY = 'undercoat-handles' +HANDLE_CACHE_KEY: str = 'undercoat-handles' -def add_running_handle(handle): +def add_running_handle(handle: str) -> None: """Record a handle as potentially needing to be cleaned up on restart.""" new_handle_list = list(set(get_running_handles()) | {handle}) persistent_cache.set_value( HANDLE_CACHE_KEY, new_handle_list, persist_across_reboots=True) -def remove_running_handle(handle): +def remove_running_handle(handle: str) -> None: """Remove a handle from the tracked set.""" new_handle_list = list(set(get_running_handles()) - {handle}) persistent_cache.set_value( HANDLE_CACHE_KEY, new_handle_list, persist_across_reboots=True) -def get_running_handles(): +def get_running_handles() -> list[str]: """Get a list of potentially stale handles from previous runs.""" return persistent_cache.get_value(HANDLE_CACHE_KEY, default_value=[]) -def get_temp_dir(): +def get_temp_dir() -> str: """Define a tempdir for undercoat to store its data in. This tempdir needs to be of a scope that persists across invocations of the @@ -70,7 +70,10 @@ class UndercoatError(Exception): """Error for errors while running undercoat.""" -def undercoat_api_command(*args, timeout=None): +def undercoat_api_command( + *args: str, + timeout: float | None = None, +) -> new_process.ProcessResult: """Make an API call to the undercoat binary.""" logs.info(f'Running undercoat command {args}') bundle_dir = environment.get_value('FUCHSIA_RESOURCES_DIR') @@ -97,11 +100,13 @@ def undercoat_api_command(*args, timeout=None): return result -def undercoat_instance_command(command, - handle, - *args, - timeout=None, - abort_on_error=True): +def undercoat_instance_command( + command: str, + handle: str, + *args: str, + timeout: float | None = None, + abort_on_error: bool = True, +) -> new_process.ProcessResult: """Helper for the subset of undercoat commands that operate on an instance.""" try: return undercoat_api_command( @@ -115,7 +120,7 @@ def undercoat_instance_command(command, raise -def get_version(): +def get_version() -> tuple[int, int, int]: """Get undercoat API version as (major, minor, patch) tuple.""" version = undercoat_api_command('version', timeout=30).output @@ -127,12 +132,12 @@ def get_version(): raise UndercoatError('Invalid version reported: %s' % version) try: - return tuple(int(part) for part in parts) + return (int(parts[0]), int(parts[1]), int(parts[2])) except ValueError as e: raise UndercoatError('Invalid version reported: %s' % version) from e -def validate_api_version(): +def validate_api_version() -> None: """Check that the undercoat API version is supported. Raises an error if it is not.""" major, minor, patch = get_version() @@ -141,14 +146,14 @@ def validate_api_version(): 'Unsupported API version: %d.%d.%d' % (major, minor, patch)) -def dump_instance_logs(handle): +def dump_instance_logs(handle: str) -> None: """Dump logs from an undercoat instance.""" qemu_log = undercoat_instance_command( 'get_logs', handle, abort_on_error=False).output logs.warning(qemu_log[-QEMU_LOG_LIMIT:]) -def start_instance(): +def start_instance() -> str: """Start an instance via undercoat.""" handle = undercoat_api_command('start_instance').output.strip() logs.info('Started undercoat instance with handle %s' % handle) @@ -160,7 +165,7 @@ def start_instance(): return handle -def stop_all(): +def stop_all() -> None: """Attempt to stop any running undercoat instances that may have not been cleanly shut down.""" for handle in get_running_handles(): @@ -178,7 +183,7 @@ def stop_all(): shutil.rmtree(get_temp_dir()) -def stop_instance(handle): +def stop_instance(handle: str) -> new_process.ProcessResult: """Stop a running undercoat instance.""" result = undercoat_instance_command( 'stop_instance', handle, abort_on_error=False) @@ -189,17 +194,23 @@ def stop_instance(handle): return result -def list_fuzzers(handle): +def list_fuzzers(handle: str) -> list[str]: """List fuzzers available on an instance, via undercoat.""" return undercoat_instance_command('list_fuzzers', handle).output.split('\n') -def prepare_fuzzer(handle, fuzzer): +def prepare_fuzzer(handle: str, fuzzer: str) -> new_process.ProcessResult: """Prepare a fuzzer of the given name for use, via undercoat.""" return undercoat_instance_command('prepare_fuzzer', handle, '-fuzzer', fuzzer) -def run_fuzzer(handle, fuzzer, outdir, args, timeout=None): +def run_fuzzer( + handle: str, + fuzzer: str, + outdir: str | None, + args: list[str], + timeout: float | None = None, +) -> new_process.ProcessResult: """Run a fuzzer of the given name, via undercoat.""" # TODO(fxbug.dev/47490): Pass back raw return code from libFuzzer? undercoat_args = ['-fuzzer', fuzzer] @@ -210,7 +221,7 @@ def run_fuzzer(handle, fuzzer, outdir, args, timeout=None): 'run_fuzzer', handle, *args, timeout=timeout) -def put_data(handle, fuzzer, src, dst): +def put_data(handle: str, fuzzer: str, src: str, dst: str) -> str: """Put files for a fuzzer onto an instance, via undercoat. If src is a directory, it will be copied recursively. Standard globs are supported.""" @@ -218,7 +229,7 @@ def put_data(handle, fuzzer, src, dst): '-src', src, '-dst', dst).output -def get_data(handle, fuzzer, src, dst): +def get_data(handle: str, fuzzer: str, src: str, dst: str) -> str | None: """Get files from a fuzzer on an instance, via undercoat. If src is a directory, it will be copied recursively. Standard globs are supported.""" diff --git a/src/clusterfuzz/_internal/platforms/linux/gestures.py b/src/clusterfuzz/_internal/platforms/linux/gestures.py index 2c5aae3921a..0346114fb49 100644 --- a/src/clusterfuzz/_internal/platforms/linux/gestures.py +++ b/src/clusterfuzz/_internal/platforms/linux/gestures.py @@ -16,25 +16,26 @@ import random import shutil import time +from typing import Any from clusterfuzz._internal.base import utils from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import shell -MAX_CHARS_TO_TYPE = 20 +MAX_CHARS_TO_TYPE: int = 20 -COORDINATE_DELTA_MIN = -100 -COORDINATE_DELTA_MAX = 200 -SCREEN_WIDTH = 1280 -SCREEN_HEIGHT = 1024 +COORDINATE_DELTA_MIN: int = -100 +COORDINATE_DELTA_MAX: int = 200 +SCREEN_WIDTH: int = 1280 +SCREEN_HEIGHT: int = 1024 -def _xdotool_path(): +def _xdotool_path() -> str | None: """Return full path to xdotool.""" return shutil.which('xdotool') -def find_windows_for_process(process_id): +def find_windows_for_process(process_id: int) -> list[int]: """Return visible windows belonging to a process.""" pids = utils.get_process_ids(process_id) if not pids: @@ -45,7 +46,7 @@ def find_windows_for_process(process_id): logs.error('Xdotool not installed, cannot locate process windows.') return [] - visible_windows = [] + visible_windows: list[int] = [] for pid in pids: windows = ( shell.execute_command( @@ -60,7 +61,7 @@ def find_windows_for_process(process_id): return visible_windows -def get_random_gestures(gesture_count): +def get_random_gestures(gesture_count: int) -> list[str]: """Return list of random gesture command strings.""" gesture_types = [ 'click --repeat TIMES,mbutton', @@ -84,7 +85,7 @@ def get_random_gestures(gesture_count): if not random.randint(0, 3): gesture_types.append('windowsize,P P') - gestures = [] + gestures: list[str] = [] for _ in range(gesture_count): random_gesture = utils.random_element_from_list(gesture_types) if random_gesture == 'drag': @@ -139,7 +140,7 @@ def get_random_gestures(gesture_count): if 'Letters' in random_gesture: num_letters = random.randint(1, 10) - letters = [] + letters: list[str] = [] for _ in range(num_letters): letters.append( utils.random_element_from_list([ @@ -169,9 +170,9 @@ def get_random_gestures(gesture_count): return gestures -def get_text_to_type(): +def get_text_to_type() -> str: """Return text to type.""" - chars = [] + chars: list[str] = [] chars_to_type_count = random.randint(1, MAX_CHARS_TO_TYPE) meta_chars = [ '|', '&', ';', '(', ')', '<', '>', ' ', '\t', ',', '\'', '"', '`', '[', @@ -189,8 +190,8 @@ def get_text_to_type(): return ''.join(chars) -def run_gestures(gestures, process_id, process_status, start_time, timeout, - windows): +def run_gestures(gestures: list[str], process_id: int, process_status: Any, + start_time: float, timeout: float, windows: list[Any]) -> None: """Run the provided interaction gestures.""" xdotool_path = _xdotool_path() if not xdotool_path: diff --git a/src/clusterfuzz/_internal/platforms/linux/lkl/constants.py b/src/clusterfuzz/_internal/platforms/linux/lkl/constants.py index 850d481f484..e8fa52e4cc2 100644 --- a/src/clusterfuzz/_internal/platforms/linux/lkl/constants.py +++ b/src/clusterfuzz/_internal/platforms/linux/lkl/constants.py @@ -15,17 +15,18 @@ import re -LINUX_KERNEL_MODULE_STACK_TRACE = 'Linux Kernel Library Stack Trace:' +LINUX_KERNEL_MODULE_STACK_TRACE: str = 'Linux Kernel Library Stack Trace:' #hid-fuzzer: lib/posix-host.c:401: void panic(void): Assertion `0' failed. -LINUX_KERNEL_LIBRARY_ASSERT_REGEX = re.compile( +LINUX_KERNEL_LIBRARY_ASSERT_REGEX: re.Pattern[str] = re.compile( r'([^:]+): lib/posix-host\.c:\d+: void panic\(void\): Assertion .*') # Linux version 5.4.58+-ab6926695 where 6926695 is the build id. # Unlike in a normal linux version string, we do not know the build hash. -LINUX_VERSION_REGEX_LKL = re.compile(r'Linux version .+-(ab([0-9a-f]+)\s)') +LINUX_VERSION_REGEX_LKL: re.Pattern[str] = re.compile( + r'Linux version .+-(ab([0-9a-f]+)\s)') # This is the prefix in the repo.prop for the kernel for all # lkl fuzzers. -LKL_REPO_KERNEL_PREFIX = 'kernel/private/lkl' -LKL_BUILD_TARGET = 'kernel_kasan.lkl_fuzzers' +LKL_REPO_KERNEL_PREFIX: str = 'kernel/private/lkl' +LKL_BUILD_TARGET: str = 'kernel_kasan.lkl_fuzzers' diff --git a/src/clusterfuzz/_internal/platforms/linux/lkl/kernel_utils.py b/src/clusterfuzz/_internal/platforms/linux/lkl/kernel_utils.py index 7a30a1d4229..cb1ea582b78 100644 --- a/src/clusterfuzz/_internal/platforms/linux/lkl/kernel_utils.py +++ b/src/clusterfuzz/_internal/platforms/linux/lkl/kernel_utils.py @@ -21,14 +21,15 @@ from . import constants -def _should_download_symbols(): +def _should_download_symbols() -> bool: """Return True if we should continue to download symbols.""" # For local testing, we do not have access to the cloud storage bucket with # the symbols. In this case, just bail out. return not environment.get_value('LOCAL_DEVELOPMENT') -def get_kernel_prefix_and_full_hash(build_id): +def get_kernel_prefix_and_full_hash( + build_id: str) -> tuple[str | None, str | None]: """Download repo.prop and return the full hash and prefix.""" android_kernel_repo_data = _get_repo_prop_data(build_id, constants.LKL_BUILD_TARGET) @@ -42,7 +43,7 @@ def get_kernel_prefix_and_full_hash(build_id): return None, None -def _get_repo_prop_data(build_id, fuzz_target): +def _get_repo_prop_data(build_id: str, fuzz_target: str) -> str | None: """Downloads repo.prop and returuns the data based on build_id and target.""" symbols_directory = os.path.join( environment.get_value('SYMBOLS_DIR'), fuzz_target) @@ -58,8 +59,8 @@ def _get_repo_prop_data(build_id, fuzz_target): return None -def _download_kernel_repo_prop_if_needed(symbols_directory, build_id, - fuzz_target): +def _download_kernel_repo_prop_if_needed(symbols_directory: str, build_id: str, + fuzz_target: str) -> None: """Downloads the repo.prop for an LKL fuzzer""" if not _should_download_symbols(): return diff --git a/src/clusterfuzz/_internal/platforms/windows/gestures.py b/src/clusterfuzz/_internal/platforms/windows/gestures.py index be490d30dd3..36cad5954fd 100644 --- a/src/clusterfuzz/_internal/platforms/windows/gestures.py +++ b/src/clusterfuzz/_internal/platforms/windows/gestures.py @@ -16,18 +16,19 @@ import ast import random import time +from typing import Any from clusterfuzz._internal.base import utils from clusterfuzz._internal.metrics import logs try: - from pywinauto import application + from pywinauto import application # type: ignore except ImportError: # This can be imported from appengine, so make sure we don't exception out. - pass + application: Any = None -def find_windows_for_process(process_id): +def find_windows_for_process(process_id: int) -> list[Any]: """Return visible windows belonging to a process.""" pids = utils.get_process_ids(process_id) if not pids: @@ -59,7 +60,7 @@ def find_windows_for_process(process_id): return visible_windows -def get_random_gestures(gesture_count): +def get_random_gestures(gesture_count: int) -> list[str]: """Return list of random gesture command strings.""" gestures_types = [ 'key,Letters', 'key,Letters', 'key,Letters', 'key,Letters', 'mouse,MA', @@ -118,8 +119,8 @@ def get_random_gestures(gesture_count): return gestures -def run_gestures(gestures, process_id, process_status, start_time, timeout, - windows): +def run_gestures(gestures: list[str], process_id: int, process_status: Any, + start_time: float, timeout: float, windows: list[Any]) -> None: """Run the provided interaction gestures.""" if not windows: windows += find_windows_for_process(process_id) diff --git a/src/clusterfuzz/_internal/remote_task/remote_task_adapters.py b/src/clusterfuzz/_internal/remote_task/remote_task_adapters.py index b79740977bd..42dc213b110 100644 --- a/src/clusterfuzz/_internal/remote_task/remote_task_adapters.py +++ b/src/clusterfuzz/_internal/remote_task/remote_task_adapters.py @@ -18,6 +18,7 @@ from clusterfuzz._internal.base import feature_flags from clusterfuzz._internal.batch import service as batch_service from clusterfuzz._internal.k8s import service as k8s_service +from clusterfuzz._internal.remote_task import remote_task_types from clusterfuzz._internal.swarming.service import SwarmingService @@ -41,8 +42,11 @@ class RemoteTaskAdapters(Enum): SWARMING = ('swarming', SwarmingService, feature_flags.FeatureFlags.SWARMING_REMOTE_EXECUTION, 0.0) - def __init__(self, adapter_id, service, feature_flag, default_weight): - self.id = adapter_id - self.feature_flag = feature_flag - self.default_weight = default_weight - self.service = service + def __init__(self, adapter_id: str, + service: type[remote_task_types.RemoteTaskInterface], + feature_flag: feature_flags.FeatureFlags, + default_weight: float) -> None: + self.id: str = adapter_id + self.feature_flag: feature_flags.FeatureFlags = feature_flag + self.default_weight: float = default_weight + self.service: type[remote_task_types.RemoteTaskInterface] = service diff --git a/src/clusterfuzz/_internal/remote_task/remote_task_gate.py b/src/clusterfuzz/_internal/remote_task/remote_task_gate.py index 0425f29c348..17997bbdbb0 100644 --- a/src/clusterfuzz/_internal/remote_task/remote_task_gate.py +++ b/src/clusterfuzz/_internal/remote_task/remote_task_gate.py @@ -21,6 +21,7 @@ import collections import random +from typing import Optional from clusterfuzz._internal import swarming from clusterfuzz._internal.base import feature_flags @@ -38,14 +39,15 @@ class RemoteTaskGate(remote_task_types.RemoteTaskInterface): allowing for flexible distribution and A/B testing. """ - def __init__(self): + def __init__(self) -> None: # Instantiate and cache the service clients for each defined adapter. - self._service_map = { + self._service_map: dict[str, remote_task_types.RemoteTaskInterface] = { adapter.id: adapter.service() for adapter in remote_task_adapters.RemoteTaskAdapters if adapter.feature_flag.enabled } - self._adapters = remote_task_adapters.RemoteTaskAdapters + self._adapters: type[remote_task_adapters.RemoteTaskAdapters] = ( + remote_task_adapters.RemoteTaskAdapters) def _get_adapter(self) -> str: """Performs a weighted random choice to select a remote backend. @@ -58,21 +60,25 @@ def _get_adapter(self) -> str: weights = list(frequencies.values()) return random.choices(population, weights)[0] - def _is_swarming_applicable(self): + def _is_swarming_applicable(self) -> bool: return feature_flags.FeatureFlags.SWARMING_REMOTE_EXECUTION.enabled - def _is_swarming_task(self, job_type): + def _is_swarming_task(self, job_type: str) -> bool: return swarming.is_swarming_task(job_type) - def _handle_swarming_job(self, module, job_type, input_download_url): + def _handle_swarming_job( + self, module: str, job_type: str, input_download_url: str + ) -> Optional[remote_task_types.RemoteTask] | list[remote_task_types. + RemoteTask]: return self._service_map['swarming'].create_utask_main_job( module, job_type, input_download_url) def _handle_swarming_jobs(self, - remote_tasks: list[remote_task_types.RemoteTask]): + remote_tasks: list[remote_task_types.RemoteTask] + ) -> list[remote_task_types.RemoteTask]: return self._service_map['swarming'].create_utask_main_jobs(remote_tasks) - def get_job_frequency(self): + def get_job_frequency(self) -> dict[str, float]: """Returns the frequency distribution for all remote task adapters. This function calculates the proportion of tasks that should be sent to each @@ -88,7 +94,9 @@ def get_job_frequency(self): A dictionary mapping each adapter's ID (e.g., 'gcp_batch') to its calculated frequency (a float between 0.0 and 1.0). """ - frequencies = {adapter.id: 0.0 for adapter in self._adapters} + frequencies: dict[str, float] = { + adapter.id: 0.0 for adapter in self._adapters + } total_weight = 0.0 for adapter in self._adapters: @@ -121,7 +129,10 @@ def get_job_frequency(self): logs.info('Job frequencies', frequencies=frequencies) return frequencies - def create_utask_main_job(self, module, job_type, input_download_url): + def create_utask_main_job( + self, module: str, job_type: str, input_download_url: str + ) -> Optional[remote_task_types.RemoteTask] | list[remote_task_types. + RemoteTask]: """Creates a single remote task, selecting a backend dynamically.""" if self._is_swarming_applicable() and self._is_swarming_task(job_type): return self._handle_swarming_job(module, job_type, input_download_url) @@ -131,7 +142,8 @@ def create_utask_main_job(self, module, job_type, input_download_url): return service.create_utask_main_job(module, job_type, input_download_url) def create_utask_main_jobs(self, - remote_tasks: list[remote_task_types.RemoteTask]): + remote_tasks: list[remote_task_types.RemoteTask] + ) -> list[remote_task_types.RemoteTask]: """Creates a batch of remote tasks, distributing them across backends. This method manages the distribution of tasks in two stages: @@ -150,8 +162,9 @@ def create_utask_main_jobs(self, according to their frequencies. This ensures precise adherence to the distribution ratios (e.g., exactly 70/30 split for 100 tasks). """ - tasks_by_adapter = collections.defaultdict(list) - unscheduled_tasks = [] + tasks_by_adapter: dict[str, list[remote_task_types.RemoteTask]] = ( + collections.defaultdict(list)) + unscheduled_tasks: list[remote_task_types.RemoteTask] = [] if self._is_swarming_applicable(): logs.info(f'[Swarming] enabled, pushing {len(remote_tasks)} tasks.') diff --git a/src/clusterfuzz/_internal/remote_task/remote_task_types.py b/src/clusterfuzz/_internal/remote_task/remote_task_types.py index 91e508eabba..e96060a7f7f 100644 --- a/src/clusterfuzz/_internal/remote_task/remote_task_types.py +++ b/src/clusterfuzz/_internal/remote_task/remote_task_types.py @@ -14,19 +14,29 @@ """Remote task types.""" import abc +from typing import Optional from clusterfuzz._internal.base import tasks class RemoteTask(tasks.Task): """Represents a single ClusterFuzz task to be executed on a remote worker. - + This class holds the necessary information to execute a ClusterFuzz command, such as 'fuzz' or 'progression', in a remote environment like GCP Batch. It is used to enqueue tasks and track their state. """ - def __init__(self, command, job_type, input_download_url, pubsub_task=None): + command: str + job_type: str + input_download_url: Optional[str] + pubsub_task: Optional[tasks.Task] + + def __init__(self, + command: str, + job_type: str, + input_download_url: Optional[str], + pubsub_task: Optional[tasks.Task] = None) -> None: super().__init__(command, input_download_url, job_type) self.command = command self.job_type = job_type @@ -36,15 +46,16 @@ def __init__(self, command, job_type, input_download_url, pubsub_task=None): class RemoteTaskInterface(abc.ABC): """Interface for a remote task execution client. - + This interface defines the contract for a client that can create and manage remote jobs. Each client is responsible for translating a ClusterFuzz task specification into a job that can be executed in its target environment. """ @abc.abstractmethod - def create_utask_main_job(self, module: str, job_type: str, - input_download_url: str): + def create_utask_main_job( + self, module: str, job_type: str, + input_download_url: str) -> Optional[RemoteTask] | list[RemoteTask]: """Creates a single remote task for a uworker main task. Returns the task that couldn't be created. """ diff --git a/src/clusterfuzz/_internal/scripts/copy_corpus.py b/src/clusterfuzz/_internal/scripts/copy_corpus.py index cef884c5aa7..db1b7203b47 100644 --- a/src/clusterfuzz/_internal/scripts/copy_corpus.py +++ b/src/clusterfuzz/_internal/scripts/copy_corpus.py @@ -20,17 +20,17 @@ import sys import time -GCLOUD_STORAGE_CMD = ['gcloud', 'storage'] -RETRY_COUNT = 5 -SLEEP_WAIT = 60 +GCLOUD_STORAGE_CMD: list[str] = ['gcloud', 'storage'] +RETRY_COUNT: int = 5 +SLEEP_WAIT: int = 60 -USAGE_REMINDER_MESSAGE = """If you set up two or more experimental jobs, +USAGE_REMINDER_MESSAGE: str = """If you set up two or more experimental jobs, remember to copy production buckets only once. For creating two or more buckets with the same content, copy the first experimental bucket over to the others to make sure that contents of the buckets will be exactly the same.""" -def _run_command(command): +def _run_command(command: list[str]) -> str: """Runs a command and prints it.""" print( 'Running command [{time}]:'.format( @@ -39,7 +39,8 @@ def _run_command(command): for _ in range(RETRY_COUNT): try: - return subprocess.check_output(command, stderr=subprocess.STDOUT) + return subprocess.check_output( + command, stderr=subprocess.STDOUT, text=True) except subprocess.CalledProcessError as e: print('Command failed with non-zero exit code. Output:\n%s' % e.output) @@ -50,9 +51,12 @@ def _run_command(command): sys.exit(-1) -def _copy_corpus(source_bucket, source_project, target_bucket, target_project): +def _copy_corpus(source_bucket: str, source_project: str | None, + target_bucket: str, target_project: str | None) -> None: """Copy corpus from a source bucket to target bucket, keeping their project - names into account.""" + + names into account. + """ # Ensure that gcloud is installed. subprocess.check_call([GCLOUD_STORAGE_CMD[0], '-v']) @@ -88,7 +92,7 @@ def _copy_corpus(source_bucket, source_project, target_bucket, target_project): print('Copy corpus finished successfully.') -def main(): +def main() -> None: arg_parser = argparse.ArgumentParser(description='Corpus copier. %s' % USAGE_REMINDER_MESSAGE) arg_parser.add_argument( diff --git a/src/clusterfuzz/_internal/swarming/__init__.py b/src/clusterfuzz/_internal/swarming/__init__.py index 2c0755e9c91..45f3ca137cd 100644 --- a/src/clusterfuzz/_internal/swarming/__init__.py +++ b/src/clusterfuzz/_internal/swarming/__init__.py @@ -15,6 +15,7 @@ import base64 import json +from typing import Any import uuid from clusterfuzz._internal.base import utils @@ -29,7 +30,7 @@ from clusterfuzz._internal.system import environment -def has_swarming_env_vars(job_environment: dict) -> bool: +def has_swarming_env_vars(job_environment: dict[str, Any]) -> bool: """Returns True if the job environment contains swarming env vars.""" return bool( utils.string_is_true(job_environment.get('IS_SWARMING_JOB')) or @@ -63,11 +64,11 @@ def is_swarming_task(job_name: str, job: data_types.Job | None = None) -> bool: def _get_instance_spec(swarming_config: local_config.SwarmingConfig, - job: data_types.Job) -> dict | None: + job: data_types.Job) -> dict[str, Any] | None: return swarming_config.get('mapping').get(job.platform, None) -def _get_task_name(job_name: str): +def _get_task_name(job_name: str) -> str: return f't-{str(uuid.uuid4()).lower()}-{job_name}' @@ -80,7 +81,8 @@ def get_swarming_config() -> local_config.SwarmingConfig | None: return None -def _get_task_dimensions(job: data_types.Job, platform_specific_dimensions: list +def _get_task_dimensions(job: data_types.Job, + platform_specific_dimensions: list[dict[str, Any]] ) -> list[swarming_pb2.StringPair]: # pylint: disable=no-member """ Gets all swarming dimensions for a task. Job dimensions have more precedence than static dimensions""" @@ -90,7 +92,7 @@ def _get_task_dimensions(job: data_types.Job, platform_specific_dimensions: list '[Swarming] No dimensions set. Reason: failed to retrieve config') return [] - unique_dimensions = {} + unique_dimensions: dict[str, str] = {} if job.platform == 'ANDROID_EMULATOR': unique_dimensions['os'] = 'Linux' else: @@ -139,8 +141,8 @@ def _append_metadata_env_var( logs.warning(f'{env_var_name} is not set or cannot be fetched.') -def _get_env_vars(logs_project_id: str, - instance_spec: dict) -> list[swarming_pb2.StringPair]: # pylint: disable=no-member +def _get_env_vars(logs_project_id: str | None, instance_spec: dict[str, Any] + ) -> list[swarming_pb2.StringPair]: # pylint: disable=no-member """Retrieve required environment variables from metadata and config.""" default_task_environment = [ swarming_pb2.StringPair(key='UWORKER', value='True'), # pylint: disable=no-member diff --git a/src/clusterfuzz/_internal/swarming/api.py b/src/clusterfuzz/_internal/swarming/api.py index 50a9f8091d5..09b248e5617 100644 --- a/src/clusterfuzz/_internal/swarming/api.py +++ b/src/clusterfuzz/_internal/swarming/api.py @@ -13,12 +13,13 @@ # limitations under the License. """Swarming pRPC API client.""" +from typing import cast from typing import Optional from google.auth import exceptions as auth_exceptions from google.protobuf import json_format from google.protobuf.timestamp_pb2 import \ - Timestamp # pylint: disable=no-name-in-module + Timestamp # pylint: disable=no-name-in-module # pyright: ignore[reportAttributeAccessIssue] from requests.exceptions import HTTPError from clusterfuzz._internal.base import utils @@ -29,17 +30,17 @@ from clusterfuzz._internal.swarming import get_swarming_config # TODO(b/516627559): Move scopes to config file -_SWARMING_SCOPES = [ +_SWARMING_SCOPES: list[str] = [ 'https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/userinfo.email' ] -_COUNT_TASKS_ENDPOINT = 'swarming.v2.Tasks/CountTasks' -_NEW_TASK_ENDPOINT = 'swarming.v2.Tasks/NewTask' +_COUNT_TASKS_ENDPOINT: str = 'swarming.v2.Tasks/CountTasks' +_NEW_TASK_ENDPOINT: str = 'swarming.v2.Tasks/NewTask' -_MIN_TASK_START_TIME = "2026-06-01T00:00:00Z" -_MIN_TASK_START_TIME_PROTO = json_format.Parse(f'"{_MIN_TASK_START_TIME}"', - Timestamp()) +_MIN_TASK_START_TIME: str = "2026-06-01T00:00:00Z" +_MIN_TASK_START_TIME_PROTO: Timestamp = json_format.Parse( + f'"{_MIN_TASK_START_TIME}"', Timestamp()) class SwarmingApiError(Exception): @@ -52,7 +53,7 @@ class SwarmingApi: _config: SwarmingConfig _base_url: str = "" - def __init__(self, config: SwarmingConfig): + def __init__(self, config: SwarmingConfig) -> None: self._config = config self._base_url = f"https://{self._config.get('swarming_server')}/prpc/" @@ -78,7 +79,7 @@ def _get_token(self) -> str: logs.error('[Swarming] Failed to get credentials. None found.') return "" - return creds.token + return cast(str, creds.token) except (auth_exceptions.DefaultCredentialsError, auth_exceptions.RefreshError, auth_exceptions.TransportError) as e: logs.error(f'[Swarming] Failed to get token with: {e}.') diff --git a/src/clusterfuzz/_internal/swarming/service.py b/src/clusterfuzz/_internal/swarming/service.py index 6fd101fe812..8c649792acc 100644 --- a/src/clusterfuzz/_internal/swarming/service.py +++ b/src/clusterfuzz/_internal/swarming/service.py @@ -24,7 +24,7 @@ from clusterfuzz._internal.swarming.api import SwarmingApi from clusterfuzz._internal.swarming.api import SwarmingApiError -SWARMING_MAIN_QUEUE_LIMIT_DEFAULT = 25 +SWARMING_MAIN_QUEUE_LIMIT_DEFAULT: int = 25 class SwarmingService(remote_task_types.RemoteTaskInterface): @@ -32,11 +32,12 @@ class SwarmingService(remote_task_types.RemoteTaskInterface): _api: SwarmingApi - def __init__(self): - self._api = SwarmingApi.create() - if self._api is None: + def __init__(self) -> None: + api = SwarmingApi.create() + if api is None: raise ValueError( 'Failed to instantiate SwarmingApi. Swarming config not available.') + self._api = api # pylint: disable=no-member def _get_dimension(self, request: swarming_pb2.NewTaskRequest, @@ -77,8 +78,9 @@ def _is_queue_full(self, return False - def create_utask_main_job(self, module: str, job_type: str, - input_download_url: str): + def create_utask_main_job( + self, module: str, job_type: str, + input_download_url: str) -> remote_task_types.RemoteTask | None: """Creates a single swarming task for a uworker main task.""" command = task_utils.get_command_from_module(module) swarming_task = remote_task_types.RemoteTask(command, job_type, @@ -97,7 +99,7 @@ def create_utask_main_jobs(self, """Creates many remote tasks for uworker main tasks. Returns the tasks that couldn't be created. """ - unscheduled_tasks = [] + unscheduled_tasks: list[remote_task_types.RemoteTask] = [] logs.info(f'[Swarming] Pushing {len(remote_tasks)} tasks trough service.') for i, task in enumerate(remote_tasks): if not swarming.is_swarming_task(task.job_type): @@ -133,7 +135,7 @@ def create_utask_main_jobs(self, unscheduled_tasks.extend(remote_tasks[i:]) break - response: swarming_pb2.NewTaskResponse + response: swarming_pb2.TaskRequestResponse try: response = self._api.push_task(task_req) except SwarmingApiError as api_failure: diff --git a/src/clusterfuzz/_internal/system/archive.py b/src/clusterfuzz/_internal/system/archive.py index bb3a67e10c7..b36b9f75a1b 100644 --- a/src/clusterfuzz/_internal/system/archive.py +++ b/src/clusterfuzz/_internal/system/archive.py @@ -14,46 +14,55 @@ """Functions for handling archives.""" import abc +from collections.abc import Callable import dataclasses +import enum import io import os import tarfile -from typing import BinaryIO -from typing import Callable -from typing import List -from typing import Optional -from typing import Union +import types +from typing import Any +from typing import IO +from typing import Self import zipfile import requests from clusterfuzz._internal.metrics import logs -FILE_ATTRIBUTE = '0o10' -SYMLINK_ATTRIBUTE = '0o12' +FILE_ATTRIBUTE: str = '0o10' +SYMLINK_ATTRIBUTE: str = '0o12' # File extensions for archive files. -ZIP_FILE_EXTENSIONS = ['.zip'] -TAR_FILE_EXTENSIONS = [ +ZIP_FILE_EXTENSIONS: list[str] = ['.zip'] +TAR_FILE_EXTENSIONS: list[str] = [ '.tar', '.tar.bz2', '.tar.gz', '.tb2', '.tbz', '.tbz2', '.tgz' ] -LZMA_FILE_EXTENSIONS = ['.tar.lzma', '.tar.xz'] +LZMA_FILE_EXTENSIONS: list[str] = ['.tar.lzma', '.tar.xz'] -ARCHIVE_FILE_EXTENSIONS = ( +ARCHIVE_FILE_EXTENSIONS: list[str] = ( ZIP_FILE_EXTENSIONS + TAR_FILE_EXTENSIONS + LZMA_FILE_EXTENSIONS) # This is the size of the internal buffer that we're using to cache HTTP # range bytes. -HTTP_BUFFER_SIZE = 50 * 1024 * 1024 # 50 MB -HTTP_REQUEST_TIMEOUT = 5 # 5 seconds +HTTP_BUFFER_SIZE: int = 50 * 1024 * 1024 # 50 MB +HTTP_REQUEST_TIMEOUT: int = 5 # 5 seconds -StrBytesPathLike = Union[str, bytes, os.PathLike] +StrBytesPathLike = str | bytes | os.PathLike[Any] MatchCallback = Callable[[str], bool] -def _is_attempting_path_traversal(archive_name: StrBytesPathLike, - output_dir: StrBytesPathLike, - filename: StrBytesPathLike) -> bool: +class ArchiveType(enum.IntEnum): + """Type of the archive.""" + UNKNOWN = 0 + ZIP = 1 + TAR = 2 + TAR_LZMA = 3 + + +def _is_attempting_path_traversal(archive_name: str | os.PathLike[str] | None, + output_dir: str | os.PathLike[str], + filename: str) -> bool: """Detects whether there is a path traversal attempt. Args: @@ -64,19 +73,19 @@ def _is_attempting_path_traversal(archive_name: StrBytesPathLike, Returns: Whether there is a path traversal attempt """ - output_dir = os.path.realpath(output_dir) + output_dir_path = os.path.realpath(output_dir) if os.path.isabs(filename): logs.error('Directory traversal attempted while unpacking archive %s ' '(absolute file path=%s). Aborting.' % (archive_name, filename)) return True - real_file_path = os.path.realpath(os.path.join(output_dir, filename)) - if (not real_file_path.startswith(output_dir + os.sep) and - real_file_path != output_dir): + real_file_path = os.path.realpath(os.path.join(output_dir_path, filename)) + if (not real_file_path.startswith(output_dir_path + os.sep) and + real_file_path != output_dir_path): logs.error( 'Directory traversal attempted while unpacking archive %s ' '(file path=%s, real file path=%s). Aborting.' % - (archive_name, os.path.join(output_dir, filename), real_file_path)) + (archive_name, os.path.join(output_dir_path, filename), real_file_path)) return True return False @@ -102,14 +111,16 @@ class ArchiveReader(abc.ABC): """Abstract class for representing an archive reader. """ - def __enter__(self): + def __enter__(self) -> Self: return self - def __exit__(self, *args): + def __exit__(self, exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None) -> None: self.close() @abc.abstractmethod - def list_members(self) -> List[ArchiveMemberInfo]: + def list_members(self) -> list[ArchiveMemberInfo]: """Lists all members contained in the archives. Returns: @@ -120,8 +131,8 @@ def list_members(self) -> List[ArchiveMemberInfo]: @abc.abstractmethod def extract(self, member: str, - path: Union[str, os.PathLike], - trusted: bool = False) -> str: + path: str | os.PathLike[str], + trusted: bool = False) -> str | None: """Extracts `member` out of the archive to the provided path. If `member` is a directory in the archive, only the directory itself will be extracted, not its content. @@ -137,7 +148,7 @@ def extract(self, raise NotImplementedError @abc.abstractmethod - def open(self, member: str) -> BinaryIO: + def open(self, member: str) -> IO[bytes] | None: """Opens `member`. Args: @@ -155,8 +166,8 @@ def close(self) -> None: raise NotImplementedError def extract_all(self, - path: Union[str, os.PathLike], - members: Optional[List[str]] = None, + path: str | os.PathLike[str], + members: list[str] | None = None, trusted: bool = False) -> bool: """Extract the whole archive content or the members listed in `members`. @@ -187,7 +198,7 @@ def extract_all(self, return not error_occurred - def try_open(self, member: str) -> Optional[BinaryIO]: + def try_open(self, member: str) -> IO[bytes] | None: """Tries to open the archive. Does not throw. Args: @@ -198,10 +209,10 @@ def try_open(self, member: str) -> Optional[BinaryIO]: """ try: return self.open(member) - except: + except Exception: return None - def get_first_file_matching(self, search_string: str) -> str: + def get_first_file_matching(self, search_string: str) -> str | None: """Gets the name of the first matching member. Args: @@ -219,7 +230,8 @@ def get_first_file_matching(self, search_string: str) -> str: return file.name return None - def extracted_size(self, file_match_callback: MatchCallback = None) -> int: + def extracted_size(self, + file_match_callback: MatchCallback | None = None) -> int: """Gets the total extracted size of the files matched by file_match_callback. If file_match_callback is None, gets the extracted size of the whole archive. @@ -261,10 +273,12 @@ def file_exists(self, path: str) -> bool: class TarArchiveReader(ArchiveReader): """A tar archive reader. Currently supports TAR and TAR_LZMA archives. """ + _archive: tarfile.TarFile + _archive_path: str | os.PathLike[str] def __init__(self, - archive_path: Union[str, os.PathLike], - file_obj: BinaryIO = None) -> None: + archive_path: str | os.PathLike[str], + file_obj: IO[bytes] | None = None) -> None: if file_obj: self._archive = tarfile.open(fileobj=file_obj) else: @@ -273,14 +287,14 @@ def __init__(self, self._archive = tarfile.open(archive_path, mode=mode) self._archive_path = archive_path - def list_members(self) -> List[ArchiveMemberInfo]: + def list_members(self) -> list[ArchiveMemberInfo]: return [ ArchiveMemberInfo( name=f.name, is_dir=f.isdir(), size_bytes=f.size, mode=f.mode) for f in self._archive.getmembers() ] - def open(self, member: str): + def open(self, member: str) -> IO[bytes] | None: return self._archive.extractfile(member) def close(self) -> None: @@ -288,8 +302,8 @@ def close(self) -> None: def extract(self, member: str, - path: Union[str, os.PathLike], - trusted: bool = False): + path: str | os.PathLike[str], + trusted: bool = False) -> str | None: # If the output directory is a symlink, get its actual path since we will be # doing directory traversal checks later when unpacking the archive. output_directory = os.path.realpath(path) @@ -306,11 +320,12 @@ def extract(self, class ZipArchiveReader(ArchiveReader): """A zip archive reader. """ + _zip_archive: zipfile.ZipFile - def __init__(self, file) -> None: + def __init__(self, file: str | os.PathLike[str] | IO[bytes]) -> None: self._zip_archive = zipfile.ZipFile(file, mode='r') - def list_members(self) -> List[ArchiveMemberInfo]: + def list_members(self) -> list[ArchiveMemberInfo]: return [ ArchiveMemberInfo( name=f.filename, @@ -320,7 +335,7 @@ def list_members(self) -> List[ArchiveMemberInfo]: for f in self._zip_archive.infolist() ] - def open(self, member): + def open(self, member: str) -> IO[bytes]: return self._zip_archive.open(name=member) def close(self) -> None: @@ -328,8 +343,8 @@ def close(self) -> None: def extract(self, member: str, - path: Union[str, os.PathLike], - trusted: bool = False): + path: str | os.PathLike[str], + trusted: bool = False) -> str | None: # If the output directory is a symlink, get its actual path since we will be # doing directory traversal checks later when unpacking the archive. output_directory = os.path.realpath(path) @@ -385,8 +400,8 @@ class ArchiveError(Exception): def open( # pylint: disable=redefined-builtin - archive_path: str, - file_obj: Optional[BinaryIO] = None) -> ArchiveReader: + archive_path: str | os.PathLike[str], + file_obj: IO[bytes] | None = None) -> ArchiveReader: """Opens the archive and gets the appropriate archive reader based on the `archive_path`. If `file_obj` is not none, the binary file-like object will be used to read the archive instead of opening `archive_path`. @@ -410,14 +425,6 @@ def open( # pylint: disable=redefined-builtin raise ArchiveError('Unhandled archive type.') -class ArchiveType: - """Type of the archive.""" - UNKNOWN = 0 - ZIP = 1 - TAR = 2 - TAR_LZMA = 3 - - @dataclasses.dataclass class CacheBlock: """Represents a cache entry for the HttpZipFile. @@ -439,6 +446,12 @@ class HttpZipFile(io.IOBase): See https://docs.python.org/3/glossary.html#term-file-like-object for the definition of a file-like object. """ + uri: str + session: requests.Session + file_size: int + _current_block: CacheBlock + _pos: int + _read_bytes: int @staticmethod def is_uri_compatible(uri: str) -> bool: @@ -449,7 +462,7 @@ def is_uri_compatible(uri: str) -> bool: logs.warning(f'Request to {uri} failed: {e}') return False - def __init__(self, uri: str): + def __init__(self, uri: str) -> None: self.uri = uri self.session = requests.Session() # This slows down the overall performances, because compressing something @@ -457,13 +470,13 @@ def __init__(self, uri: str): # waste time decoding the data twice. self.session.headers.pop('Accept-Encoding') resp = self.session.head(self.uri) - self.file_size = int(resp.headers.get('Content-Length', default=0)) + self.file_size = int(resp.headers.get('Content-Length', 0)) self._current_block = CacheBlock(0, 0, b'') self._pos = 0 self._read_bytes = 0 assert resp.headers.get('Accept-Ranges') is not None - def close(self): + def close(self) -> None: logs.info('HttpZipFile: read %d / %d (bytes) from %s' % (self._read_bytes, self.file_size, self.uri)) super().close() @@ -534,7 +547,7 @@ def _fetch_from_cache(self, start: int, end: int) -> bytes: inner_end = end - self._current_block.start return self._current_block.content[inner_start:inner_end + 1] - def read(self, size: Optional[int] = -1) -> bytes: + def read(self, size: int | None = -1) -> bytes: """Read into this file-object. Args: @@ -560,7 +573,7 @@ def read(self, size: Optional[int] = -1) -> bytes: self._pos += read_size return content - def read1(self, size: Optional[int] = -1) -> bytes: + def read1(self, size: int | None = -1) -> bytes: """Read into the file-object in at most on system call. This is exactly similar to the read implementation in our case. @@ -573,7 +586,7 @@ def read1(self, size: Optional[int] = -1) -> bytes: return self.read(size) -def get_archive_type(archive_path: str) -> ArchiveType: +def get_archive_type(archive_path: str | os.PathLike[str]) -> ArchiveType: """Get the type of the archive. Args: @@ -582,11 +595,13 @@ def get_archive_type(archive_path: str) -> ArchiveType: Returns: the type of the archive, or ArchiveType.UNKNOWN if unknown. """ + path_str = os.fspath(archive_path) if isinstance( + archive_path, os.PathLike) else archive_path - def has_extension(extensions): + def has_extension(extensions: list[str]) -> bool: """Returns True if |archive_path| endswith an extension in |extensions|.""" for extension in extensions: - if archive_path.endswith(extension): + if path_str.endswith(extension): return True return False @@ -602,7 +617,7 @@ def has_extension(extensions): return ArchiveType.UNKNOWN -def is_archive(filename: str) -> bool: +def is_archive(filename: str | os.PathLike[str]) -> bool: """Return true if the file is an archive. Args: diff --git a/src/clusterfuzz/_internal/system/environment.py b/src/clusterfuzz/_internal/system/environment.py index cea2241979c..feb7e327115 100644 --- a/src/clusterfuzz/_internal/system/environment.py +++ b/src/clusterfuzz/_internal/system/environment.py @@ -14,6 +14,9 @@ """Environment functions.""" import ast +from collections.abc import Callable +from collections.abc import Mapping +from collections.abc import MutableMapping import enum import functools import os @@ -21,6 +24,10 @@ import socket import subprocess import sys +from typing import Any +from typing import overload +from typing import ParamSpec +from typing import TypeVar import uuid import yaml @@ -30,12 +37,12 @@ # Tools supporting customization of options via ADDITIONAL_{TOOL_NAME}_OPTIONS. # FIXME: Support ADDITIONAL_UBSAN_OPTIONS and ADDITIONAL_LSAN_OPTIONS in an # ASAN instrumented build. -SUPPORTED_MEMORY_TOOLS_FOR_OPTIONS = [ +SUPPORTED_MEMORY_TOOLS_FOR_OPTIONS: list[str] = [ 'HWASAN', 'ASAN', 'KASAN', 'CFI', 'MSAN', 'TSAN', 'UBSAN', 'NOSANITIZER', 'MTE' ] -SANITIZER_NAME_MAP = { +SANITIZER_NAME_MAP: dict[str, str] = { 'ASAN': 'address', 'CFI': 'cfi', 'MSAN': 'memory', @@ -44,7 +51,7 @@ 'NOSANITIZER': 'nosanitizer', } -COMMON_SANITIZER_OPTIONS = { +COMMON_SANITIZER_OPTIONS: dict[str, int] = { 'handle_abort': 1, 'handle_segv': 1, 'handle_sigbus': 1, @@ -54,6 +61,10 @@ 'use_sigaltstack': 1, } +_P = ParamSpec('_P') +_R = TypeVar('_R') +_T = TypeVar('_T') + class UtaskMainRuntime(enum.Enum): @@ -63,7 +74,7 @@ class UtaskMainRuntime(enum.Enum): SWARMING = 'swarming' -def _eval_value(value_string): +def _eval_value(value_string: str) -> Any: """Returns evaluated value.""" try: return ast.literal_eval(value_string) @@ -72,13 +83,13 @@ def _eval_value(value_string): return value_string -def join_memory_tool_options(options): +def join_memory_tool_options(options: Mapping[str, Any]) -> str: """Joins a dict holding memory tool options into a string that can be set in the environment.""" return ':'.join(f'{key}={str(val)}' for key, val in sorted(options.items())) -def _maybe_convert_to_int(value): +def _maybe_convert_to_int(value: str) -> int | str: """Returns the int representation contained by string |value| if it contains one. Otherwise returns |value|.""" try: @@ -89,12 +100,12 @@ def _maybe_convert_to_int(value): # Matches anything that isn't an unquoted (ie: not between two single or two # double quotes) colon. -UNQUOTED_COLON_REGEX = re.compile('((?:[^\'":]|\'[^\']*\'|"[^"]*")+)') +UNQUOTED_COLON_REGEX = re.compile(r'((?:[^\'":]|\'[^\']*\'|"[^"]*")+)') -def _parse_memory_tool_options(options_str): +def _parse_memory_tool_options(options_str: str) -> dict[str, Any]: """Parses memory tool options into a dict.""" - parsed = {} + parsed: dict[str, Any] = {} for item in UNQUOTED_COLON_REGEX.split(options_str): # Regex split can give us empty strings at the beginning and the end. Skip @@ -117,7 +128,7 @@ def _parse_memory_tool_options(options_str): return parsed -def _quote_value_if_needed(value): +def _quote_value_if_needed(value: str) -> str: """Quote environment value as needed for certain platforms like Windows.""" result = value if ' ' in result or ':' in result: @@ -126,13 +137,13 @@ def _quote_value_if_needed(value): return result -def copy(): +def copy() -> dict[str, str]: """Return a safe copy of the environment.""" environment_copy = os.environ.copy() return environment_copy -def disable_lsan(): +def disable_lsan() -> None: """Disable leak detection (if enabled).""" if get_current_memory_tool_var() != 'ASAN_OPTIONS': return @@ -142,10 +153,12 @@ def disable_lsan(): set_memory_tool_options('ASAN_OPTIONS', sanitizer_options) -def get_asan_options(redzone_size, malloc_context_size, quarantine_size_mb, - bot_platform, leaks, disable_ubsan): +def get_asan_options( + redzone_size: int | float | None, malloc_context_size: int | None, + quarantine_size_mb: int | None, bot_platform: str | None, + leaks: bool | None, disable_ubsan: bool | None) -> dict[str, Any]: """Generates default ASAN options.""" - asan_options = {} + asan_options: dict[str, Any] = {} # Default options needed for all cases. asan_options['alloc_dealloc_mismatch'] = 0 @@ -220,7 +233,7 @@ def get_asan_options(redzone_size, malloc_context_size, quarantine_size_mb, return asan_options -def get_cpu_arch(): +def get_cpu_arch() -> str | None: """Return cpu architecture.""" if is_android(): # FIXME: Handle this import in a cleaner way. @@ -231,7 +244,7 @@ def get_cpu_arch(): return None -def get_current_memory_tool_var(): +def get_current_memory_tool_var() -> str | None: """Get the environment variable name for the current job type's sanitizer.""" memory_tool_name = get_memory_tool_name(get_value('JOB_NAME')) if not memory_tool_name: @@ -240,7 +253,19 @@ def get_current_memory_tool_var(): return memory_tool_name + '_OPTIONS' -def get_memory_tool_options(env_var, default_value=None): +@overload +def get_memory_tool_options( + env_var: str, default_value: None = None) -> dict[str, Any] | None: + ... + + +@overload +def get_memory_tool_options(env_var: str, + default_value: _T) -> dict[str, Any] | _T: + ... + + +def get_memory_tool_options(env_var: str, default_value: Any = None) -> Any: """Get the current memory tool options as a dict. Returns |default_value| if |env_var| isn't set. Otherwise returns a dictionary containing the memory tool options and their values.""" @@ -251,7 +276,7 @@ def get_memory_tool_options(env_var, default_value=None): return default_value -def get_instrumented_libraries_paths(): +def get_instrumented_libraries_paths() -> list[str] | None: """Get the instrumented libraries path for the current sanitizer.""" memory_tool_name = get_memory_tool_name(get_value('JOB_NAME')) if not memory_tool_name: @@ -270,7 +295,7 @@ def get_instrumented_libraries_paths(): return paths.split(':') -def get_default_tool_path(tool_name): +def get_default_tool_path(tool_name: str) -> str: """Get the default tool for this platform (from scripts/ dir).""" if is_android(): # For android devices, we do symbolization on the host machine, which is @@ -286,7 +311,7 @@ def get_default_tool_path(tool_name): return tool_path -def get_environment_settings_as_string(): +def get_environment_settings_as_string() -> str: """Return environment settings as a string. Includes settings for memory debugging tools (e.g. ASAN_OPTIONS for ASAN), application binary revision, application command line, etc.""" @@ -337,9 +362,9 @@ def get_environment_settings_as_string(): return environment_string -def get_sanitizer_options_for_display(): +def get_sanitizer_options_for_display() -> list[str]: """Return a list of sanitizer options with quoted values.""" - result = [] + result: list[str] = [] for tool in SUPPORTED_MEMORY_TOOLS_FOR_OPTIONS: options_variable = tool + '_OPTIONS' options_value = os.getenv(options_variable) @@ -351,7 +376,7 @@ def get_sanitizer_options_for_display(): return result -def get_llvm_symbolizer_path(): +def get_llvm_symbolizer_path() -> str | None: """Get the path of the llvm-symbolizer binary.""" llvm_symbolizer_path = get_value('LLVM_SYMBOLIZER_PATH') @@ -382,17 +407,17 @@ def get_llvm_symbolizer_path(): return llvm_symbolizer_path -def get_root_directory(): +def get_root_directory() -> str: """Return root directory.""" return get_value('ROOT_DIR') -def get_startup_scripts_directory(): +def get_startup_scripts_directory() -> str: """Return path to startup scripts.""" return os.path.join(get_value('ROOT_DIR'), 'src', 'python', 'bot', 'startup') -def get_config_directory(): +def get_config_directory() -> str: """Return the path to the configs directory.""" config_dir = get_value('CONFIG_DIR_OVERRIDE') if config_dir: @@ -406,22 +431,23 @@ def get_config_directory(): return os.path.join(get_root_directory(), 'src', 'appengine', 'config') -def get_gae_config_directory(): +def get_gae_config_directory() -> str: """Return the path to the google appengine configs directory.""" return os.path.join(get_config_directory(), 'gae') -def get_gce_config_directory(): +def get_gce_config_directory() -> str: """Return the path to the google compute engine configs directory.""" return os.path.join(get_config_directory(), 'gce') -def get_resources_directory(): +def get_resources_directory() -> str: """Return the path to the resources directory.""" return os.path.join(get_root_directory(), 'resources') -def get_platform_resources_directory(platform_override=None): +def get_platform_resources_directory( + platform_override: str | None = None) -> str: """Return the path to platform-specific resources directory.""" plt = platform_override or platform() @@ -432,12 +458,13 @@ def get_platform_resources_directory(platform_override=None): return os.path.join(get_resources_directory(), 'platform', plt.lower()) -def get_suppressions_directory(): +def get_suppressions_directory() -> str: """Return the path to the suppressions directory.""" return os.path.join(get_config_directory(), 'suppressions') -def get_suppressions_file(sanitizer, suffix='suppressions'): +def get_suppressions_file(sanitizer: str, + suffix: str = 'suppressions') -> str | None: """Return the path to sanitizer suppressions file, if exists.""" sanitizer_suppressions_filename = '{sanitizer}_{suffix}.txt'.format( sanitizer=sanitizer, suffix=suffix) @@ -453,10 +480,10 @@ def get_suppressions_file(sanitizer, suffix='suppressions'): return sanitizer_suppressions_file_path -def get_lsan_options(): +def get_lsan_options() -> dict[str, Any]: """Generates default LSAN options.""" lsan_suppressions_path = get_suppressions_file('lsan') - lsan_options = { + lsan_options: dict[str, Any] = { 'print_suppressions': 0, } @@ -469,9 +496,9 @@ def get_lsan_options(): return lsan_options -def get_kasan_options(): +def get_kasan_options() -> dict[str, Any]: """Generates default KASAN options.""" - kasan_options = {'symbolize': 0} + kasan_options: dict[str, Any] = {'symbolize': 0} # Add common sanitizer options. kasan_options.update(COMMON_SANITIZER_OPTIONS) @@ -479,9 +506,9 @@ def get_kasan_options(): return kasan_options -def get_msan_options(): +def get_msan_options() -> dict[str, Any]: """Generates default MSAN options.""" - msan_options = {'symbolize': 0} + msan_options: dict[str, Any] = {'symbolize': 0} # Add common sanitizer options. msan_options.update(COMMON_SANITIZER_OPTIONS) @@ -489,7 +516,7 @@ def get_msan_options(): return msan_options -def get_platform_id(): +def get_platform_id() -> str: """Return a platform id as a lowercase string.""" bot_platform = platform() if is_android_cuttlefish(): @@ -504,7 +531,7 @@ def get_platform_id(): return bot_platform.lower() -def get_platform_group(): +def get_platform_group() -> str: """Return the platform group (specified via QUEUE_OVERRIDE) if it exists, otherwise platform().""" platform_group = get_value('QUEUE_OVERRIDE') @@ -514,7 +541,7 @@ def get_platform_group(): return platform() -def get_memory_tool_name(job_name): +def get_memory_tool_name(job_name: str | None) -> str: """Figures out name of memory debugging tool.""" for tool in SUPPORTED_MEMORY_TOOLS_FOR_OPTIONS: if tool_matches(tool, job_name): @@ -524,7 +551,7 @@ def get_memory_tool_name(job_name): return 'ASAN' -def get_memory_tool_display_string(job_name): +def get_memory_tool_display_string(job_name: str | None) -> str: """Return memory tool string for a testcase.""" memory_tool_name = get_memory_tool_name(job_name) sanitizer_name = SANITIZER_NAME_MAP.get(memory_tool_name) @@ -534,7 +561,7 @@ def get_memory_tool_display_string(job_name): return 'Sanitizer: %s (%s)' % (sanitizer_name, memory_tool_name) -def get_executable_filename(executable_name): +def get_executable_filename(executable_name: str) -> str: """Return the filename for the given executable.""" if platform() != 'WINDOWS': return executable_name @@ -546,11 +573,11 @@ def get_executable_filename(executable_name): return executable_name + extension -def get_tsan_options(): +def get_tsan_options() -> dict[str, Any]: """Generates default TSAN options.""" tsan_suppressions_path = get_suppressions_file('tsan') - tsan_options = { + tsan_options: dict[str, Any] = { 'atexit_sleep_ms': 200, 'flush_memory_ms': 2000, 'history_size': 3, @@ -570,12 +597,12 @@ def get_tsan_options(): return tsan_options -def get_ubsan_options(): +def get_ubsan_options() -> dict[str, Any]: """Generates default UBSAN options.""" # Note that UBSAN can work together with ASAN as well. ubsan_suppressions_path = get_suppressions_file('ubsan') - ubsan_options = { + ubsan_options: dict[str, Any] = { 'halt_on_error': 1, 'print_stacktrace': 1, 'print_suppressions': 0, @@ -597,7 +624,7 @@ def get_ubsan_options(): return ubsan_options -def get_ubsan_disabled_options(): +def get_ubsan_disabled_options() -> dict[str, Any]: """Generates ubsan options """ return { 'halt_on_error': 0, @@ -606,12 +633,31 @@ def get_ubsan_disabled_options(): } -def get_value_string(environment_variable, default_value=None): +@overload +def get_value_string(environment_variable: str, + default_value: None = None) -> str | None: + ... + + +@overload +def get_value_string(environment_variable: str, default_value: str) -> str: + ... + + +@overload +def get_value_string(environment_variable: str, default_value: _T) -> str | _T: + ... + + +def get_value_string(environment_variable: str, + default_value: Any = None) -> Any: """Get environment variable (as a string).""" return os.getenv(environment_variable, default_value) -def get_value(environment_variable, default_value=None, env=None): +def get_value(environment_variable: str, + default_value: Any = None, + env: Mapping[str, str] | None = None) -> Any: """Return an environment variable value.""" if env is None: env = os.environ @@ -630,7 +676,7 @@ def get_value(environment_variable, default_value=None, env=None): return _eval_value(value_string) -def _job_substring_match(search_string, job_name): +def _job_substring_match(search_string: str, job_name: str | None) -> bool: """Return a bool on whether a string exists in a provided job name or use from environment if available (case insensitive).""" job_name = job_name or get_value('JOB_NAME') @@ -640,57 +686,60 @@ def _job_substring_match(search_string, job_name): return search_string in job_name.lower() -def is_afl_job(job_name=None): +def is_afl_job(job_name: str | None = None) -> bool: """Return True if the current job uses AFL.""" return get_engine_for_job(job_name) == 'afl' -def is_ios_job(job_name=None): +def is_ios_job(job_name: str | None = None) -> bool: """Return True if the current job is for iOS.""" return _job_substring_match('ios_', job_name) -def is_chromeos_job(job_name=None): +def is_chromeos_job(job_name: str | None = None) -> bool: """Return True if the current job is for ChromeOS.""" return _job_substring_match('chromeos', job_name) -def is_lkl_job(job_name=None): +def is_lkl_job(job_name: str | None = None) -> bool: """Return True if the current job is for ChromeOS.""" return _job_substring_match('lkl', job_name) -def is_chromeos_system_job(job_name=None): +def is_chromeos_system_job(job_name: str | None = None) -> bool: """Return True if the current job is for ChromeOS system (i.e. not libFuzzer or entire Chrome browser for Chrome on ChromeOS).""" return is_chromeos_job(job_name) and get_value('CHROMEOS_SYSTEM') -def is_libfuzzer_job(job_name=None): +def is_libfuzzer_job(job_name: str | None = None) -> bool: """Return True if the current job uses libFuzzer.""" return get_engine_for_job(job_name) == 'libFuzzer' -def is_honggfuzz_job(job_name=None): +def is_honggfuzz_job(job_name: str | None = None) -> bool: """Return True if the current job uses honggfuzz.""" return get_engine_for_job(job_name) == 'honggfuzz' -def is_centipede_fuzzer_job(job_name=None): +def is_centipede_fuzzer_job(job_name: str | None = None) -> bool: """Return True if the current job uses Centipede.""" return get_engine_for_job(job_name) == 'centipede' -def is_engine_fuzzer_job(job_name=None): +def is_engine_fuzzer_job(job_name: str | None = None) -> bool: """Return True if this is an engine fuzzer.""" return bool(get_engine_for_job(job_name)) -def get_engine_for_job(job_name=None): +def get_engine_for_job(job_name: str | None = None) -> str | None: """Get the engine for the given job.""" if not job_name: job_name = get_value('JOB_NAME') + if not job_name: + return None + for engine in fuzzing.ENGINES: if engine.lower() in job_name: return engine @@ -698,7 +747,7 @@ def get_engine_for_job(job_name=None): return None -def is_minimization_supported(): +def is_minimization_supported() -> bool: """Return True if the current job supports minimization. Currently blackbox-fuzzer jobs or libfuzzer support minimization, unless @@ -708,18 +757,18 @@ def is_minimization_supported(): return not skipped and (not is_engine_fuzzer_job() or is_libfuzzer_job()) -def is_posix(): +def is_posix() -> bool: """Return True if we are on a posix platform (linux/unix and mac os).""" return os.name == 'posix' -def is_trusted_host(ensure_connected=True): +def is_trusted_host(ensure_connected: bool = True) -> bool: """Return whether or not the current bot is a trusted host.""" return get_value('TRUSTED_HOST') and (not ensure_connected or get_value('WORKER_BOT_NAME')) -def is_untrusted_worker(): +def is_untrusted_worker() -> bool: """Return whether or not the current bot is an untrusted worker.""" return get_value('UNTRUSTED_WORKER') @@ -756,31 +805,33 @@ def is_running_on_swarming() -> bool: return get_value('SWARMING_BOT') is True -def is_running_on_app_engine(): +def is_running_on_app_engine() -> bool: """Return True if we are running on appengine (local or production).""" - return (os.getenv('GAE_ENV') or is_running_on_app_engine_development() or - os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/')) + return bool( + os.getenv('GAE_ENV') or is_running_on_app_engine_development() or + os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/')) -def is_running_on_app_engine_development(): +def is_running_on_app_engine_development() -> bool: """Return True if running on the local development appengine server.""" return (os.getenv('GAE_ENV') == 'dev' or os.getenv('SERVER_SOFTWARE', '').startswith('Development/')) -def parse_environment_definition(environment_string): +def parse_environment_definition( + environment_string: str | None) -> dict[str, str]: """Parses a job's environment definition.""" if not environment_string: return {} definitions = [environment_string.splitlines()] - values = {} + values: dict[str, str] = {} for definition in definitions: for line in definition: if line.startswith('#') or not line.strip(): continue - m = re.match('([^ =]+)[ ]*=[ ]*(.*)', line) + m = re.match(r'([^ =]+)[ ]*=[ ]*(.*)', line) if m: key = m.group(1).strip() value = m.group(2).strip() @@ -797,16 +848,16 @@ def is_running_on_k8s() -> bool: return bool(env_value) -def base_platform(override): +def base_platform(override: str) -> str: """Return the base platform when an override is provided.""" return override.split(':')[0] -def platform(): +def platform() -> str: """Return the operating system type, unless an override is provided.""" environment_override = get_value('OS_OVERRIDE') if environment_override: - return environment_override.upper() + return str(environment_override).upper() if sys.platform.startswith('win'): return 'WINDOWS' @@ -818,7 +869,7 @@ def platform(): raise ValueError('Unsupported platform "%s".' % sys.platform) -def remove_key(key_name): +def remove_key(key_name: str | None) -> None: """Remove environment |key| and its associated value.""" if not key_name: return @@ -830,10 +881,10 @@ def remove_key(key_name): # Used by reset_environment to store the initial environment. -_initial_environment = None +_initial_environment: dict[str, str] | None = None -def reset_environment(): +def reset_environment() -> None: """Resets environment variables to their initial state. Saves the initial state on first call.""" global _initial_environment @@ -853,7 +904,7 @@ def reset_environment(): untrusted_env.reset_environment() -def set_common_environment_variables(): +def set_common_environment_variables() -> None: """Sets environment variables common for different memory debugging tools.""" # G_SLICE = always-malloc: make glib use system malloc. # NSS_DISABLE_UNLOAD = 1: make nss skip dlclosing dynamically loaded modules, @@ -865,12 +916,14 @@ def set_common_environment_variables(): set_value('NACL_DANGEROUS_SKIP_QUALIFICATION_TEST', 1) -def set_memory_tool_options(env_var, options_dict): +def set_memory_tool_options(env_var: str, + options_dict: Mapping[str, Any]) -> None: """Set current memory tool options.""" set_value(env_var, join_memory_tool_options(options_dict)) -def update_symbolizer_options(tool_options, symbolize_inline_frames=False): +def update_symbolizer_options(tool_options: dict[str, Any], + symbolize_inline_frames: bool = False) -> None: """Checks and updates the necessary symbolizer options such as `external_symbolizer_path` and `symbolize_inline_frames`.""" if 'external_symbolizer_path' not in tool_options: @@ -886,12 +939,13 @@ def update_symbolizer_options(tool_options, symbolize_inline_frames=False): }) -def reset_current_memory_tool_options(redzone_size=0, - malloc_context_size=0, - leaks=True, - symbolize_inline_frames=False, - quarantine_size_mb=None, - disable_ubsan=False): +def reset_current_memory_tool_options( + redzone_size: int | float | None = 0, + malloc_context_size: int | None = 0, + leaks: bool | None = True, + symbolize_inline_frames: bool = False, + quarantine_size_mb: int | None = None, + disable_ubsan: bool | None = False) -> None: """Resets environment variables for memory debugging tool to default values.""" # FIXME: Handle these imports in a cleaner way. @@ -906,7 +960,7 @@ def reset_current_memory_tool_options(redzone_size=0, set_value('MEMORY_TOOL', tool_name) bot_platform = platform() - tool_options = {} + tool_options: dict[str, Any] = {} # Default options for memory debuggin tool used. if tool_name in ['ASAN', 'HWASAN']: @@ -948,19 +1002,22 @@ def reset_current_memory_tool_options(redzone_size=0, android.sanitizer.set_options(tool_name, joined_tool_options) -def set_default_vars(): +def set_default_vars() -> None: """Set default environment vars and values.""" env_file_path = os.path.join(get_value('ROOT_DIR'), 'bot', 'env.yaml') with open(env_file_path) as file_handle: env_file_contents = file_handle.read() env_vars_and_values = yaml.safe_load(env_file_contents) + if not isinstance(env_vars_and_values, dict): + return + for variable, value in env_vars_and_values.items(): # We cannot call set_value here. - os.environ[variable] = str(value) + os.environ[str(variable)] = str(value) -def set_bot_environment(): +def set_bot_environment() -> bool: """Set environment for the bots.""" root_dir = get_value('ROOT_DIR') @@ -1036,7 +1093,7 @@ def set_bot_environment(): return True -def set_local_log_only(): +def set_local_log_only() -> None: """Force logs to be local-only.""" # We set this to an empty string because currently the logs @@ -1046,7 +1103,7 @@ def set_local_log_only(): set_value('LOG_TO_CONSOLE', 'True') -def set_tsan_max_history_size(): +def set_tsan_max_history_size() -> None: """Sets maximum history size for TSAN tool.""" tsan_options = get_value('TSAN_OPTIONS') if not tsan_options: @@ -1061,7 +1118,8 @@ def set_tsan_max_history_size(): set_value('TSAN_OPTIONS', tsan_options) -def set_task_id_vars(task_name, task_id=None): +def set_task_id_vars(task_name: str, + task_id: uuid.UUID | str | None = None) -> None: """Sets environment vars for task name and ID.""" if not task_id: task_id = uuid.uuid4() @@ -1069,7 +1127,9 @@ def set_task_id_vars(task_name, task_id=None): set_value('CF_TASK_NAME', task_name) -def set_value(environment_variable, value, env=None): +def set_value(environment_variable: Any, + value: Any, + env: MutableMapping[str, str] | None = None) -> None: """Set an environment variable.""" if env is None: env = os.environ @@ -1085,7 +1145,7 @@ def set_value(environment_variable, value, env=None): value_str) -def get_initial_task_name(): +def get_initial_task_name() -> str | None: """Returns the name of the task that this task (postprocess or utask_main) is part of.""" initial_task_payload = get_value('INITIAL_TASK_PAYLOAD') @@ -1094,18 +1154,20 @@ def get_initial_task_name(): return initial_task_payload.split(' ')[0] -def tool_matches(tool_name, job_name): +def tool_matches(tool_name: str, job_name: str | None) -> bool: """Return if the memory debugging tool is used in this job.""" + if not job_name: + return False match_prefix = '(.*[^a-zA-Z]|^)%s' matches_tool = re.match(match_prefix % tool_name.lower(), job_name.lower()) return bool(matches_tool) -def appengine_noop(func): +def appengine_noop(func: Callable[_P, _R]) -> Callable[_P, _R | None]: """Wrap a function into no-op and return None if running on App Engine.""" @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R | None: if is_running_on_app_engine(): return None @@ -1114,11 +1176,11 @@ def wrapper(*args, **kwargs): return wrapper -def bot_noop(func): +def bot_noop(func: Callable[_P, _R]) -> Callable[_P, _R | None]: """Wrap a function into no-op and return None if running on bot.""" @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R | None: is_bot = not is_running_on_app_engine() if is_bot: return None @@ -1128,25 +1190,25 @@ def wrapper(*args, **kwargs): return wrapper -def is_local_development(): +def is_local_development() -> bool: """Return True if running in local development environment (e.g. running a bot locally, excludes tests).""" return bool(get_value('LOCAL_DEVELOPMENT') and not get_value('PY_UNITTESTS')) -def is_production(): +def is_production() -> bool: """Returns True if there are no environmental indicators of local development ocurring.""" return not (is_local_development() or get_value('UNTRUSTED_RUNNER_TESTS') or get_value('LOCAL_DEVELOPMENT') or get_value('UTASK_TESTS')) -def local_noop(func): +def local_noop(func: Callable[_P, _R]) -> Callable[_P, _R | None]: """Wrap a function into no-op and return None if running in local development environment.""" @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R | None: if (is_local_development() or is_running_on_app_engine_development()): return None @@ -1155,50 +1217,50 @@ def wrapper(*args, **kwargs): return wrapper -def is_ephemeral(): +def is_ephemeral() -> bool: """Return whether or not we are an ephemeral bot.""" return get_value('EPHEMERAL') -def is_android(plt=None): +def is_android(plt: str | None = None) -> bool: """Return True if we are on android platform.""" return 'ANDROID' in (plt or platform()) -def is_android_auto(plt=None): +def is_android_auto(plt: str | None = None) -> bool: """Return True if we are on automotive platform.""" return 'ANDROID_AUTO' in (plt or platform()) -def is_android_cuttlefish(plt=None): +def is_android_cuttlefish(plt: str | None = None) -> bool: """Return True if we are on android cuttlefish platform.""" return 'ANDROID_X86' in (plt or platform()) -def is_android_emulator(): +def is_android_emulator() -> bool: """Return True if we are on android emulator platform.""" return 'ANDROID_EMULATOR' in get_platform_group() -def is_android_real_device(): +def is_android_real_device() -> bool: """Return True if we are on a real android device.""" return base_platform(platform()) == 'ANDROID' -def is_lib(): +def is_lib() -> bool: """Whether or not we're in libClusterFuzz.""" return get_value('LIB_CF') -def is_i386(job_type): +def is_i386(job_type: str) -> bool: return '_i386' in job_type -def if_redis_available(func): +def if_redis_available(func: Callable[_P, _R]) -> Callable[_P, _R | None]: """Wrap a function if redis is available and return None if not.""" @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R | None: if get_value('REDIS_HOST'): return func(*args, **kwargs) @@ -1207,8 +1269,10 @@ def wrapper(*args, **kwargs): return wrapper -def is_testcase_deprecated(platform_id=None): +def is_testcase_deprecated(platform_id: str | None = None) -> bool: """Whether or not the device or branch is deprecated.""" + if not platform_id: + return False if is_android( platform_id.upper()) and not is_android_cuttlefish(platform_id.upper()): @@ -1220,7 +1284,8 @@ def is_testcase_deprecated(platform_id=None): return False -def can_testcase_run_on_platform(testcase_platform_id, current_platform_id): +def can_testcase_run_on_platform(testcase_platform_id: str, + current_platform_id: str) -> bool: """Whether or not the testcase can run on the current platform.""" if not is_android(testcase_platform_id.upper()) or not is_android( current_platform_id.upper()): @@ -1241,7 +1306,7 @@ def can_testcase_run_on_platform(testcase_platform_id, current_platform_id): return False -def is_tworker(): +def is_tworker() -> bool: return get_value('TWORKER', False) diff --git a/src/clusterfuzz/_internal/system/fast_http.py b/src/clusterfuzz/_internal/system/fast_http.py index 1a6d91f17c2..d54c81b22c9 100644 --- a/src/clusterfuzz/_internal/system/fast_http.py +++ b/src/clusterfuzz/_internal/system/fast_http.py @@ -13,10 +13,10 @@ # limitations under the License. """Tools for fast HTTP operations.""" import asyncio +from collections.abc import AsyncIterator +from collections.abc import Sequence import itertools -from typing import List -from typing import Sequence -from typing import Tuple +from typing import Any import urllib.parse import aiohttp @@ -29,7 +29,7 @@ _HTTP_TIMEOUT_SECONDS = aiohttp.ClientTimeout(total=300) -def download_urls(urls_and_filepaths: List[Tuple[str, str]]) -> List[bool]: +def download_urls(urls_and_filepaths: Sequence[tuple[str, str]]) -> list[bool]: """Downloads multiple urls to filepaths in parallel and asynchronously. Tolerates errors. Returns a list of whether each download was successful.""" utils.python_gc() @@ -50,22 +50,22 @@ def download_urls(urls_and_filepaths: List[Tuple[str, str]]) -> List[bool]: return list(itertools.chain(*download_results)) -def _download_files(urls_and_paths: Sequence[Tuple[str, str]]) -> List[bool]: +def _download_files(urls_and_paths: Sequence[tuple[str, str]]) -> list[bool]: """Non-async function that starts an event loop to asynchronously download URLs.""" urls, paths = list(zip(*urls_and_paths)) return asyncio.run(_async_download_files(list(urls), list(paths))) -async def _async_download_files(urls: List[str], - paths: List[str]) -> List[bool]: +async def _async_download_files(urls: Sequence[str], + paths: Sequence[str]) -> list[bool]: """Asynchronously downloads multiple files.""" async with aiohttp.ClientSession(timeout=_HTTP_TIMEOUT_SECONDS) as session: tasks = [ asyncio.create_task(_error_tolerant_download_file(session, url, path)) for url, path in zip(urls, paths) ] - return await asyncio.gather(*tasks) + return list(await asyncio.gather(*tasks)) async def _error_tolerant_download_file(session: aiohttp.ClientSession, @@ -75,13 +75,13 @@ async def _error_tolerant_download_file(session: aiohttp.ClientSession, # TODO(metzman): Catch specific exceptions. await _async_download_file(session, url, path) return True - except: + except Exception: logs.warning(f'Failed to download {url}.') return False async def _async_download_file(session: aiohttp.ClientSession, url: str, - path: str): + path: str) -> None: """Asynchronously downloads |url| and writes it to |path|.""" async with session.get(url) as response: response.raise_for_status() @@ -92,15 +92,16 @@ async def _async_download_file(session: aiohttp.ClientSession, url: str, file_handle.write(chunk) -def _get_blob_url(bucket_name, blob_name): +def _get_blob_url(bucket_name: str, blob_name: str) -> str: blob_name = urllib.parse.quote(blob_name, safe='') return ( f'https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{blob_name}' ) -async def delete_blob_async(bucket_name: str, blob_name: str, session, - auth_token): +async def delete_blob_async(bucket_name: str, blob_name: str, + session: aiohttp.ClientSession, + auth_token: str) -> None: """Asynchronously deletes a GCS blob.""" url = _get_blob_url(bucket_name, blob_name) headers = { @@ -114,12 +115,13 @@ async def delete_blob_async(bucket_name: str, blob_name: str, session, logs.error(f'Failed to delete blob {blob_name}. Status code + resp: ' f'{response.status} {response_text}.') except google.api_core.exceptions.NotFound: - logs.info(f'Not found: {blob_name} {response_text}.') + logs.info(f'Not found: {blob_name}.') except Exception as e: logs.error(f'Couldn\'t delete {blob_name}: {e}.') -async def list_blobs_async(bucket_name, path, auth_token): +async def list_blobs_async(bucket_name: str, path: str, + auth_token: str) -> AsyncIterator[dict[str, Any]]: """Asynchronously lists blobs, yielding dicts containing their size, updated time and name.""" async with aiohttp.ClientSession(timeout=_HTTP_TIMEOUT_SECONDS) as session: diff --git a/src/clusterfuzz/_internal/system/minijail.py b/src/clusterfuzz/_internal/system/minijail.py index d9cd88bb541..15a1dc26b9e 100644 --- a/src/clusterfuzz/_internal/system/minijail.py +++ b/src/clusterfuzz/_internal/system/minijail.py @@ -13,20 +13,27 @@ # limitations under the License. """Running processes with minijail.""" -from collections import namedtuple +from collections.abc import Sequence import os import shutil import signal import subprocess import tempfile - +import types +from typing import Any +from typing import IO +from typing import NamedTuple +from typing import Self + +from clusterfuzz._internal.bot.untrusted_runner import \ + environment as untrusted_environment from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import new_process from clusterfuzz._internal.system import shell -def _get_minijail_path(): +def _get_minijail_path() -> str: """Get the minijail path. Returns: @@ -36,7 +43,7 @@ def _get_minijail_path(): 'minijail0') -def _get_minijail_user_namespace_args(): +def _get_minijail_user_namespace_args() -> list[str]: """Get user namespace arguments for minijail. Returns: @@ -61,25 +68,27 @@ def _get_minijail_user_namespace_args(): return arguments -def _create_chroot_dir(base_dir): +def _create_chroot_dir(base_dir: str | None) -> str: """Create dir for chroot.""" return tempfile.mkdtemp(dir=base_dir) -def _create_tmp_mount(base_dir): +def _create_tmp_mount(base_dir: str | None) -> str: """Create a tmp mount in base_dir.""" return tempfile.mkdtemp(dir=base_dir) -ChrootBinding = namedtuple('ChrootBinding', - ['src_path', 'dest_path', 'writeable']) +class ChrootBinding(NamedTuple): + src_path: str + dest_path: str + writeable: bool class MinijailChroot: """Minijail environment.""" # Default directories to bind from host to chroot. Mostly library directories. - DEFAULT_BINDINGS = [ + DEFAULT_BINDINGS: list[str] = [ '/lib', '/lib32', '/lib64', @@ -87,7 +96,16 @@ class MinijailChroot: '/usr/lib32', ] - def __init__(self, base_dir=None, bindings=None, use_existing_base=False): + _chroot_dir: str + _tmp_mount: str + _bindings: list[ChrootBinding] + + def __init__( + self, + base_dir: str | None = None, + bindings: Sequence[ChrootBinding] | None = None, + use_existing_base: bool = False, + ) -> None: """Inits the MinijailChroot. Args: @@ -98,6 +116,7 @@ def __init__(self, base_dir=None, bindings=None, use_existing_base=False): if not use_existing_base: self._chroot_dir = _create_chroot_dir(base_dir=base_dir) else: + assert base_dir is not None self._chroot_dir = base_dir # Create /tmp, /proc directories. @@ -128,7 +147,7 @@ def __init__(self, base_dir=None, bindings=None, use_existing_base=False): for binding in bindings: self.add_binding(binding) - def _mknod(self, path, file_type, major, minor): + def _mknod(self, path: str, file_type: str, major: int, minor: int) -> None: """Creates a special file.""" try: with open(os.devnull) as devnull: @@ -143,7 +162,7 @@ def _mknod(self, path, file_type, major, minor): except subprocess.CalledProcessError as e: logs.error('Failed to call mknod.', output=e.output) - def _create_devices(self): + def _create_devices(self) -> None: """Create /dev/null, /dev/random, /dev/urandom, and /dev/shm.""" dev_dir = os.path.join(self._chroot_dir, 'dev') os.mkdir(dev_dir) @@ -152,7 +171,7 @@ def _create_devices(self): self._mknod(os.path.join(dev_dir, 'urandom'), 'c', 1, 9) os.mkdir(os.path.join(dev_dir, 'shm')) - def _makedirs(self, directory): + def _makedirs(self, directory: str) -> None: """Create directories for binding in chroot. Args: @@ -165,18 +184,18 @@ def _makedirs(self, directory): os.path.join(self._chroot_dir, directory), create_intermediates=True) @property - def bindings(self): + def bindings(self) -> list[ChrootBinding]: return self._bindings @property - def directory(self): + def directory(self) -> str: return self._chroot_dir @property - def tmp_directory(self): + def tmp_directory(self) -> str: return self._tmp_mount - def add_binding(self, binding): + def add_binding(self, binding: ChrootBinding) -> None: """Adds a directory to be bound to the chroot. Args: @@ -188,7 +207,7 @@ def add_binding(self, binding): self._makedirs(binding.dest_path) self._bindings.append(binding) - def get_binding(self, src_path): + def get_binding(self, src_path: str) -> ChrootBinding | None: """Returns binding for src_path. Args: @@ -201,12 +220,12 @@ def get_binding(self, src_path): if os.path.abspath(x.src_path) == os.path.abspath(src_path)), None) - def close(self): + def close(self) -> None: """Cleanup the chroot environment.""" shutil.rmtree(self._chroot_dir, ignore_errors=True) shutil.rmtree(self._tmp_mount, ignore_errors=True) - def remove_binding(self, binding): + def remove_binding(self, binding: ChrootBinding) -> None: """Remove a directory bound to the chroot. This function does not delete the given directory. @@ -215,11 +234,16 @@ def remove_binding(self, binding): """ self._bindings.remove(binding) - def __enter__(self): + def __enter__(self) -> Self: """Context manager override.""" return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: types.TracebackType | None, + ) -> None: """Context manager override.""" self.close() @@ -227,21 +251,29 @@ def __exit__(self, exc_type, exc_value, traceback): class ChromeOSChroot(MinijailChroot): """Minijail environment for ChromeOS fuzzers.""" - DEFAULT_BINDINGS = [] + DEFAULT_BINDINGS: list[str] = [] - def __init__(self, chroot_dir, bindings=None): + def __init__( + self, + chroot_dir: str, + bindings: Sequence[ChrootBinding] | None = None, + ) -> None: # Do clean up in case close() was not called. self.remove_created_dirs(chroot_dir) super().__init__(chroot_dir, bindings, use_existing_base=True) - def remove_created_dirs(self, chroot_dir, minijail_created_dirs=None): + def remove_created_dirs( + self, + chroot_dir: str, + minijail_created_dirs: Sequence[str] | None = None, + ) -> None: if minijail_created_dirs is None: minijail_created_dirs = ['tmp', 'proc', 'dev'] for directory in minijail_created_dirs: shell.remove_directory(os.path.join(chroot_dir, directory)) - def remove_binding(self, binding): + def remove_binding(self, binding: ChrootBinding) -> None: """Overriden version of remove_binding that ensures the bound directory is removed. This is necessary because unlike in regular minijails, we do not delete the entire chroot directory.""" @@ -249,7 +281,7 @@ def remove_binding(self, binding): shell.remove_directory(abs_path) super().remove_binding(binding) - def close(self): + def close(self) -> None: """Overrides MinijailChroot.close(). Closes the chroot environment. Does not delete the chroot directory.""" for binding in self._bindings: @@ -260,18 +292,26 @@ def close(self): class MinijailChildProcess(new_process.ChildProcess): """Minijail child process.""" - def __init__(self, popen, command, max_stdout_len, stdout_file, - jailed_pid_file): + _jailed_pid_file: IO[Any] + + def __init__( + self, + popen: subprocess.Popen, + command: list[str], + max_stdout_len: int | None, + stdout_file: Any, + jailed_pid_file: IO[Any], + ) -> None: super().__init__(popen, command, max_stdout_len, stdout_file) self._jailed_pid_file = jailed_pid_file - def terminate(self): + def terminate(self) -> None: """Send SIGTERM to the jailed process.""" self._jailed_pid_file.seek(0) jailed_pid = int(self._jailed_pid_file.read()) os.kill(jailed_pid, signal.SIGTERM) - def kill(self): + def kill(self) -> None: """Kill minijail and all child processes.""" os.killpg(self.popen.pid, signal.SIGKILL) @@ -279,7 +319,7 @@ def kill(self): class MinijailProcessRunner(new_process.ProcessRunner): """ProcessRunner wrapper for minijail.""" - MINIJAIL_ARGS = [ + MINIJAIL_ARGS: list[str] = [ '-T', 'static', # don't use preload. '-c', @@ -293,17 +333,27 @@ class MinijailProcessRunner(new_process.ProcessRunner): 'proc,/proc,proc,1' # Mount procfs RO (1 == MS_RDONLY). ] - PATH_ENVIRONMENT_VALUE = '/bin:/usr/bin' + PATH_ENVIRONMENT_VALUE: str = '/bin:/usr/bin' + + _chroot: MinijailChroot - def __init__(self, chroot, executable_path, default_args=None): + def __init__( + self, + chroot: MinijailChroot, + executable_path: str, + default_args: Sequence[str] | None = None, + ) -> None: super().__init__(executable_path, default_args=default_args) self._chroot = chroot @property - def chroot(self): + def chroot(self) -> MinijailChroot: return self._chroot - def get_command(self, additional_args=None): + def get_command( + self, + additional_args: Sequence[str] | None = None, + ) -> list[str]: """ProcessRunner.get_command override to prepend minijail.""" base_command = super().get_command(additional_args) command = [_get_minijail_path()] @@ -320,14 +370,16 @@ def get_command(self, additional_args=None): command.extend(base_command) return command - def run(self, - additional_args=None, - max_stdout_len=None, - extra_env=None, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - **popen_args): + def run( + self, + additional_args: Sequence[str] | None = None, + max_stdout_len: int | None = None, + extra_env: dict[str, str] | None = None, + stdin: Any = subprocess.PIPE, + stdout: Any = subprocess.PIPE, + stderr: Any = subprocess.STDOUT, + **popen_args: Any, + ) -> MinijailChildProcess: """ProcessRunner.run override.""" pid_file = tempfile.NamedTemporaryFile() @@ -341,8 +393,6 @@ def run(self, command.insert(2, pid_file.name) passed_env = popen_args.pop('env', None) - from clusterfuzz._internal.bot.untrusted_runner import \ - environment as untrusted_environment env = untrusted_environment.get_env_for_untrusted_process(passed_env) if extra_env is not None: env.update(extra_env) diff --git a/src/clusterfuzz/_internal/system/new_process.py b/src/clusterfuzz/_internal/system/new_process.py index 403c0375db6..4f419521577 100644 --- a/src/clusterfuzz/_internal/system/new_process.py +++ b/src/clusterfuzz/_internal/system/new_process.py @@ -13,43 +13,47 @@ # limitations under the License. """Process handling utilities.""" +from collections.abc import Callable +from collections.abc import Sequence import os import subprocess import sys import tempfile import threading import time +from typing import Any +from typing import cast import urllib.request try: import psutil except ImportError: - psutil = None + psutil = None # type: ignore from clusterfuzz._internal.base import utils from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -TOOL_ARGS = { +TOOL_ARGS: dict[str, list[str]] = { 'unshare': [ '-c', # Map current user to same user in user namespace. '-n', # Enter network namespace. ], } -TOOL_URLS = { +TOOL_URLS: dict[str, str] = { 'extra_sanitizers': 'https://storage.googleapis.com/oss-fuzz-sanitizers/latest' } -def _end_process(terminate_function, process_result): +def _end_process(terminate_function: Callable[[], Any], + process_result: 'ProcessResult') -> None: """Ends a running process. Ignores exceptions. Args: - process: A subprocess.Popen object. terminate_function: The function to terminate the process. process_result: A ProcessResult object where timeout information will be written to. @@ -62,15 +66,17 @@ def _end_process(terminate_function, process_result): process_result.timed_out = True -def wait_process(process, - timeout, - input_data=None, - terminate_before_kill=False, - terminate_wait_time=None): +def wait_process( + process: 'ChildProcess', + timeout: float, + input_data: str | bytes | None = None, + terminate_before_kill: bool = False, + terminate_wait_time: float | None = None, +) -> 'ProcessResult': """Waits until either the process exits or times out. Args: - process: A subprocess.Popen object. + process: A ChildProcess object. timeout: Maximum number of seconds to wait for before sending a signal. input_data: Input to be sent to the process. terminate_before_kill: A bool indicating that SIGTERM should be sent to @@ -84,7 +90,11 @@ def wait_process(process, result = ProcessResult() is_windows = environment.platform() == 'WINDOWS' - def create_timeout_timer(interval, function, args): + def create_timeout_timer( + interval: float, + function: Callable[..., Any], + args: Sequence[Any], + ) -> threading.Timer: """Helper to create a timer that logs timeout info. This function takes the same arguments as threading.Timer. @@ -98,7 +108,7 @@ def create_timeout_timer(interval, function, args): A threading.Timer object. """ - def target(): + def target() -> None: binary_name = os.path.basename(process.command[0]) action_fn = args[0] logs.info(f"TIMEOUT: Terminating {binary_name} after {interval} seconds " @@ -113,8 +123,10 @@ def target(): first_timeout_function = process.terminate # Use a second timer to send the process kill. + terminate_wait = ( + terminate_wait_time if terminate_wait_time is not None else 0.0) second_timer = create_timeout_timer( - timeout + terminate_wait_time, + timeout + terminate_wait, _end_process, [process.kill, result], ) @@ -149,8 +161,12 @@ def target(): return result -def kill_process_tree(root_pid): +def kill_process_tree(root_pid: int) -> None: """Kill process tree.""" + if psutil is None: + logs.warning('psutil is not available.') + return + try: parent = psutil.Process(root_pid) children = parent.children(recursive=True) @@ -173,12 +189,14 @@ def kill_process_tree(root_pid): class ChildProcess: """A class representing a process that's running.""" - def __init__(self, - popen, - command, - max_stdout_len=None, - stdout_file=None, - interactive=False): + def __init__( + self, + popen: subprocess.Popen, + command: list[str], + max_stdout_len: int | None = None, + stdout_file: Any = None, + interactive: bool = False, + ) -> None: self._popen = popen self._command = command self._max_stdout_len = max_stdout_len @@ -186,23 +204,28 @@ def __init__(self, self._interactive = interactive @property - def command(self): + def command(self) -> list[str]: return self._command @property - def popen(self): + def popen(self) -> subprocess.Popen: return self._popen - def communicate(self, input=None): # pylint: disable=redefined-builtin + def communicate( + self, + input: str | bytes | None = None, # pylint: disable=redefined-builtin + ) -> tuple[Any, Any]: """subprocess.Popen.communicate.""" stdout = b'' stderr = b'' if self._interactive: - if input: + if input and self._popen.stdin: self._popen.stdin.write(input) while True: + if not self._popen.stdout: + break line = self._popen.stdout.readline() if not line: break @@ -218,22 +241,22 @@ def communicate(self, input=None): # pylint: disable=redefined-builtin else: stdout, stderr = self._popen.communicate(input) - if not self._max_stdout_len: + if not self._max_stdout_len or self._stdout_file is None: return stdout, stderr with self._stdout_file: return utils.read_from_handle_truncated(self._stdout_file, self._max_stdout_len), stderr - def poll(self): + def poll(self) -> int | None: """subprocess.Popen.poll.""" return self._popen.poll() - def kill(self): + def kill(self) -> None: """Kills running process and all of its associated children.""" kill_process_tree(self._popen.pid) - def terminate(self): + def terminate(self) -> None: """subprocess.Popen.terminate.""" try: self._popen.terminate() @@ -254,12 +277,20 @@ class ProcessResult: timed_out: Whether or not the process timed out. """ - def __init__(self, - command=None, - return_code=None, - output=None, - time_executed=None, - timed_out=False): + command: list[str] | None + return_code: int | None + output: Any + time_executed: float | None + timed_out: bool + + def __init__( + self, + command: list[str] | None = None, + return_code: int | None = None, + output: Any = None, + time_executed: float | None = None, + timed_out: bool = False, + ) -> None: """Inits the ProcessResult.""" self.command = command self.return_code = return_code @@ -278,24 +309,32 @@ class ProcessRunner: cwd: Optional current working directory for the process. """ - def __init__(self, executable_path, default_args=None, cwd=None): + def __init__( + self, + executable_path: str, + default_args: Sequence[str] | None = None, + cwd: str | None = None, + ) -> None: """Inits ProcessRunner.""" self._executable_path = executable_path - self._default_args = [] + self._default_args: list[str] = [] self.cwd = cwd if default_args: self.default_args.extend(default_args) @property - def executable_path(self): + def executable_path(self) -> str: return self._executable_path @property - def default_args(self): + def default_args(self) -> list[str]: return self._default_args - def get_command(self, additional_args=None): + def get_command( + self, + additional_args: Sequence[str] | None = None, + ) -> list[str]: """Returns the command line for running the executable. Args: @@ -313,14 +352,16 @@ def get_command(self, additional_args=None): return command - def run(self, - additional_args=None, - max_stdout_len=None, - extra_env=None, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - **popen_args): + def run( + self, + additional_args: Sequence[str] | None = None, + max_stdout_len: int | None = None, + extra_env: dict[str, str] | None = None, + stdin: Any = subprocess.PIPE, + stdout: Any = subprocess.PIPE, + stderr: Any = subprocess.STDOUT, + **popen_args: Any, + ) -> ChildProcess: """Runs the executable. Does not block the caller. @@ -386,18 +427,20 @@ def run(self, # Note: changes to this function may require changes to # untrusted_runner.proto. - def run_and_wait(self, - additional_args=None, - timeout=None, - terminate_before_kill=False, - terminate_wait_time=None, - input_data=None, - max_stdout_len=None, - extra_env=None, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - **popen_args) -> ProcessResult: + def run_and_wait( + self, + additional_args: Sequence[str] | None = None, + timeout: float | None = None, + terminate_before_kill: bool = False, + terminate_wait_time: float | None = None, + input_data: str | bytes | None = None, + max_stdout_len: int | None = None, + extra_env: dict[str, str] | None = None, + stdin: Any = subprocess.PIPE, + stdout: Any = subprocess.PIPE, + stderr: Any = subprocess.STDOUT, + **popen_args: Any, + ) -> ProcessResult: """Runs the executable. Blocks the caller until the process exits. @@ -453,9 +496,10 @@ def run_and_wait(self, class UnicodeProcessRunnerMixin: """Mixin for process runner subclasses to output unicode output.""" - def run_and_wait(self, *args, **kwargs) -> ProcessResult: # pylint: disable=arguments-differ + def run_and_wait(self, *args: Any, **kwargs: Any) -> ProcessResult: # pylint: disable=arguments-differ """Overridden run_and_wait which always decodes the output.""" - result = ProcessRunner.run_and_wait(self, *args, **kwargs) + result = ProcessRunner.run_and_wait( + cast(ProcessRunner, self), *args, **kwargs) if result.output is not None: result.output = utils.decode_to_unicode(result.output) @@ -469,7 +513,10 @@ class UnicodeProcessRunner(UnicodeProcessRunnerMixin, ProcessRunner): class ModifierProcessRunnerMixin: """ProcessRunner mixin with modifiers.""" - def tool_prefix(self, tool): + _executable_path: str + _default_args: list[str] + + def tool_prefix(self, tool: str) -> list[str]: """Prefix the command with a tool and its args""" if not environment.get_value(f'USE_{tool.upper()}'): return [] @@ -479,7 +526,7 @@ def tool_prefix(self, tool): tool_path = environment.get_default_tool_path(tool) if not os.path.exists(tool_path) and tool in TOOL_URLS: - urllib.request.urlretrieve(TOOL_URLS.get(tool), tool_path) + urllib.request.urlretrieve(TOOL_URLS[tool], tool_path) if os.path.exists(tool_path): os.chmod(tool_path, 0o755) if not os.path.exists(tool_path): @@ -487,7 +534,10 @@ def tool_prefix(self, tool): return [tool_path] + TOOL_ARGS.get(tool, []) - def get_command(self, additional_args=None): + def get_command( + self, + additional_args: Sequence[str] | None = None, + ) -> list[str]: """Overridden get_command.""" command = [self._executable_path] command.extend(self._default_args) diff --git a/src/clusterfuzz/_internal/system/process_handler.py b/src/clusterfuzz/_internal/system/process_handler.py index 27f697e6fbd..34abaa54303 100644 --- a/src/clusterfuzz/_internal/system/process_handler.py +++ b/src/clusterfuzz/_internal/system/process_handler.py @@ -13,6 +13,7 @@ # limitations under the License. """Functions for process management.""" +from collections.abc import Sequence import copy import datetime import os @@ -21,6 +22,8 @@ import sys import threading import time +from typing import Any +from typing import TYPE_CHECKING from clusterfuzz._internal.base import utils from clusterfuzz._internal.crash_analysis import crash_analyzer @@ -31,15 +34,21 @@ from clusterfuzz._internal.system import environment from clusterfuzz._internal.system import shell -# FIXME: Find a better way to handle this case. These imports -# will fail and are not needed from App Engine. -try: +if TYPE_CHECKING: import multiprocessing import mozprocess import psutil -except ImportError: - pass +else: + # FIXME: Find a better way to handle this case. These imports + # will fail and are not needed from App Engine. + try: + import multiprocessing + + import mozprocess + import psutil + except ImportError: + pass # On Android, we need to wait a little after a crash occurred to get the full # logcat output. This makes sure we get all the stack frames since there is no @@ -51,7 +60,7 @@ CRASH_ANALYSIS_TIME = 1.5 # Test timeout if not specified. -DEFAULT_TEST_TIMEOUT = 10 +DEFAULT_TEST_TIMEOUT: float = 10 # Time to wait for cleanup after process if finished. PROCESS_CLEANUP_WAIT_TIME = 5 @@ -67,14 +76,14 @@ class ProcessStatus: """Process exited notification.""" - def __init__(self): - self.finished = False + def __init__(self) -> None: + self.finished: bool = False - def __call__(self): + def __call__(self) -> None: self.finished = True -def start_process(process_handle): +def start_process(process_handle: Any) -> None: """Start the process using process handle and override list2cmdline for Windows.""" is_win = environment.platform() == 'WINDOWS' @@ -82,16 +91,17 @@ def start_process(process_handle): # Override list2cmdline on Windows to return first index of list as string. # This is to workaround a mozprocess bug since it passes command as list # and not as string. - subprocess.list2cmdline_orig = subprocess.list2cmdline - subprocess.list2cmdline = lambda s: s[0] + setattr(subprocess, 'list2cmdline_orig', subprocess.list2cmdline) + setattr(subprocess, 'list2cmdline', lambda s: s[0]) try: process_handle.run() finally: if is_win: - subprocess.list2cmdline = subprocess.list2cmdline_orig + setattr(subprocess, 'list2cmdline', + getattr(subprocess, 'list2cmdline_orig')) -def cleanup_defunct_processes(): +def cleanup_defunct_processes() -> None: """Cleans up defunct processes.""" # Defunct processes happen only on unix platforms. if environment.platform() != 'WINDOWS': @@ -110,14 +120,15 @@ def cleanup_defunct_processes(): # Note: changes to this function may require changes to untrusted_runner.proto. # This should only be used for running target black box applications which # return text output. -def run_process(cmdline, - current_working_directory=None, - timeout=DEFAULT_TEST_TIMEOUT, - need_shell=False, - gestures=None, - env_copy=None, - testcase_run=True, - ignore_children=True): +def run_process( + cmdline: str, + current_working_directory: str | None = None, + timeout: float = DEFAULT_TEST_TIMEOUT, + need_shell: bool = False, + gestures: list[str] | None = None, + env_copy: dict[str, str] | None = None, + testcase_run: bool = True, + ignore_children: bool = True) -> tuple[int | None, float | None, str]: """Executes a process with a given command line and other parameters.""" if environment.is_trusted_host() and testcase_run: from clusterfuzz._internal.bot.untrusted_runner import remote_process_host @@ -154,14 +165,15 @@ def run_process(cmdline, # Initialize variables. adb_output = None - process_output = '' + process_output: Any = '' process_status = None - return_code = 0 + process_handle = None + return_code: int | None = 0 process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5) start_time = time.time() watch_for_process_exit = ( environment.get_value('WATCH_FOR_PROCESS_EXIT') if is_android else True) - window_list = [] + window_list: list[str] = [] # Get gesture start time from last element in gesture list. gestures = copy.deepcopy(gestures) @@ -208,11 +220,11 @@ def run_process(cmdline, # so that the next check is after 1 second. gesture_start_time += 1 - if plt == 'LINUX': + if plt == 'LINUX' and process_handle: linux.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) - elif plt == 'WINDOWS': + elif plt == 'WINDOWS' and process_handle: windows.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) @@ -244,7 +256,8 @@ def run_process(cmdline, # On desktop, we bail out as soon as the process finishes. if process_status and process_status.finished: # Wait for process shutdown and set return code. - process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME) + if process_handle: + process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME) break # Process output based on platform. @@ -298,10 +311,10 @@ def run_process(cmdline, # Get the return code in case the process has finished already. # If the process hasn't finished, return_code will be None which is what # callers expect unless the output indicates a crash. - return_code = process_handle.poll() + return_code = process_handle.poll() if process_handle else None # If the process is still running, then terminate it. - if not process_status.finished: + if process_status and not process_status.finished: launcher_with_interpreter = shell.get_execute_command( launcher) if launcher else None if (launcher_with_interpreter and @@ -310,11 +323,13 @@ def run_process(cmdline, # except for APP_NAME. # It is expected that, if the launcher script terminated normally, it # cleans up all the child processes it created itself. - terminate_root_and_child_processes(process_handle.pid) + if process_handle: + terminate_root_and_child_processes(process_handle.pid) else: try: # kill() here actually sends SIGTERM on posix. - process_handle.kill() + if process_handle: + process_handle.kill() except: pass @@ -351,14 +366,14 @@ def run_process(cmdline, return return_code, round(time.time() - start_time, 1), output -def cleanup_stale_processes(): +def cleanup_stale_processes() -> None: """Kill stale processes left behind by a job.""" terminate_multiprocessing_children() terminate_stale_application_instances() cleanup_defunct_processes() -def close_queue(queue_to_close): +def close_queue(queue_to_close: Any) -> None: """Close the queue.""" if environment.is_trusted_host(): # We don't use multiprocessing.Queue on trusted hosts. @@ -370,7 +385,7 @@ def close_queue(queue_to_close): logs.error('Unable to close queue.') -def get_process(): +def get_process() -> Any: """Return a multiprocessing process object (with bug fixes).""" if environment.is_trusted_host(): # forking/multiprocessing is unsupported because of the RPC connection. @@ -383,7 +398,7 @@ def get_process(): return multiprocessing.Process -def get_runtime_snapshot(): +def get_runtime_snapshot() -> str: """Return a list of current processes and their command lines as string.""" process_strings = [] for process in psutil.process_iter(): @@ -404,7 +419,7 @@ def get_runtime_snapshot(): return '\n'.join(sorted(process_strings)) -def get_queue(): +def get_queue() -> Any: """Return a multiprocessing queue object.""" if environment.is_trusted_host(): # We don't use multiprocessing.Process on trusted hosts. No need to use @@ -422,7 +437,7 @@ def get_queue(): return result_queue -def terminate_hung_threads(threads): +def terminate_hung_threads(threads: Sequence[Any]) -> None: """Terminate hung threads.""" start_time = time.time() while time.time() - start_time < THREAD_FINISH_WAIT_TIME: @@ -446,7 +461,7 @@ def terminate_hung_threads(threads): pass -def terminate_root_and_child_processes(root_pid): +def terminate_root_and_child_processes(root_pid: int) -> None: """Terminate the root process along with any children it spawned.""" app_name = environment.get_value('APP_NAME') direct_children = utils.get_process_ids(root_pid, recursive=False) @@ -479,7 +494,7 @@ def terminate_root_and_child_processes(root_pid): terminate_process(root_pid, kill=True) -def terminate_multiprocessing_children(): +def terminate_multiprocessing_children() -> None: """Terminate all children created with multiprocessing module.""" child_list = multiprocessing.active_children() for child in child_list: @@ -490,7 +505,7 @@ def terminate_multiprocessing_children(): pass -def terminate_stale_application_instances(): +def terminate_stale_application_instances() -> None: """Kill stale instances of the application running for this command.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import remote_process_host @@ -589,7 +604,7 @@ def terminate_stale_application_instances(): datetime.timedelta(seconds=duration))) -def terminate_process(process_id, kill=False): +def terminate_process(process_id: int, kill: bool = False) -> None: """Terminates a process by its process id.""" try: process = psutil.Process(process_id) @@ -603,7 +618,8 @@ def terminate_process(process_id, kill=False): logs.warning('Failed to terminate process.') -def terminate_processes_matching_names(match_strings, kill=False): +def terminate_processes_matching_names(match_strings: str | Sequence[str], + kill: bool = False) -> None: """Terminates processes matching particular names (case sensitive).""" if isinstance(match_strings, str): match_strings = [match_strings] @@ -619,10 +635,14 @@ def terminate_processes_matching_names(match_strings, kill=False): terminate_process(process_info['pid'], kill) -def terminate_processes_matching_cmd_line(match_strings, - kill=False, - exclude_strings=None): +def terminate_processes_matching_cmd_line( + match_strings: str | Sequence[str | None] | None, + kill: bool = False, + exclude_strings: Sequence[str] | None = None) -> None: """Terminates processes matching particular command line (case sensitive).""" + if not match_strings: + return + if exclude_strings is None: exclude_strings = [] @@ -644,7 +664,7 @@ def terminate_processes_matching_cmd_line(match_strings, terminate_process(process_info['pid'], kill) -def scripts_are_running(expected_scripts): +def scripts_are_running(expected_scripts: list[str] | set[str]) -> bool: """Check if all target scripts are running as expected.""" scripts_left = expected_scripts.copy() for process in psutil.process_iter(): diff --git a/src/clusterfuzz/_internal/system/shell.py b/src/clusterfuzz/_internal/system/shell.py index d98a54caaf0..20ffa28043e 100644 --- a/src/clusterfuzz/_internal/system/shell.py +++ b/src/clusterfuzz/_internal/system/shell.py @@ -13,6 +13,9 @@ # limitations under the License. """Shell related functions.""" +from collections.abc import Callable +from collections.abc import Iterator +from collections.abc import Sequence import contextlib import os import re @@ -21,6 +24,7 @@ import subprocess import sys import tempfile +from typing import Any import uuid from clusterfuzz._internal.base import persistent_cache @@ -30,7 +34,7 @@ try: import psutil except ImportError: - psutil = None + psutil = None # type: ignore _DEFAULT_LOW_DISK_SPACE_THRESHOLD = 5 * 1024 * 1024 * 1024 # 5 GB. _TRUSTED_HOST_LOW_DISK_SPACE_THRESHOLD = 2 * 1024 * 1024 # 2 GB. @@ -38,10 +42,10 @@ HANDLE_OUTPUT_FILE_TYPE_REGEX = re.compile( br'.*pid:\s*(\d+)\s*type:\s*File\s*([a-fA-F0-9]+):\s*(.*)') -_system_temp_dir = None +_system_temp_dir: str | None = None -def _low_disk_space_threshold(): +def _low_disk_space_threshold() -> int: """Get the low disk space threshold.""" if environment.is_trusted_host(ensure_connected=False): # Trusted hosts can run with less free space as they do not store builds or @@ -51,7 +55,7 @@ def _low_disk_space_threshold(): return _DEFAULT_LOW_DISK_SPACE_THRESHOLD -def copy_file(source_file_path, destination_file_path): +def copy_file(source_file_path: str, destination_file_path: str) -> bool: """Faster version of shutil.copy with buffer size.""" if not os.path.exists(source_file_path): logs.error('Source file %s for copy not found.' % source_file_path) @@ -77,12 +81,12 @@ def copy_file(source_file_path, destination_file_path): return True -def clear_build_directory(): +def clear_build_directory() -> None: """Clears the build directory.""" remove_directory(environment.get_value('BUILDS_DIR'), recreate=True) -def clear_build_urls_directory(): +def clear_build_urls_directory() -> None: """Clears the build url directory.""" remove_directory(environment.get_value('BUILD_URLS_DIR'), recreate=True) @@ -91,23 +95,23 @@ def clear_build_urls_directory(): file_host.clear_build_urls_directory() -def clear_crash_stacktraces_directory(): +def clear_crash_stacktraces_directory() -> None: """Clears the crash stacktraces directory.""" remove_directory( environment.get_value('CRASH_STACKTRACES_DIR'), recreate=True) -def clear_common_data_bundles_directory(): +def clear_common_data_bundles_directory() -> None: """Clear the common data bundle directory.""" remove_directory(environment.get_value('FUZZ_DATA'), recreate=True) -def clear_data_bundles_directory(): +def clear_data_bundles_directory() -> None: """Clears the data bundles directory.""" remove_directory(environment.get_value('DATA_BUNDLES_DIR'), recreate=True) -def clear_data_directories(): +def clear_data_directories() -> None: """Clear all data directories.""" clear_build_directory() clear_build_urls_directory() @@ -121,7 +125,7 @@ def clear_data_directories(): persistent_cache.clear_values(clear_all=True) -def clear_data_directories_on_low_disk_space(): +def clear_data_directories_on_low_disk_space() -> None: """Clear all data directories on low disk space. This should ideally never happen, but when it does, we do this to keep the bot working in sane state.""" free_disk_space = get_free_disk_space() @@ -138,19 +142,19 @@ def clear_data_directories_on_low_disk_space(): clear_data_directories() -def clear_device_temp_directories(): +def clear_device_temp_directories() -> None: """Clear device specific temp directories.""" if environment.is_android(): from clusterfuzz._internal.platforms import android android.device.clear_temp_directories() -def clear_fuzzers_directories(): +def clear_fuzzers_directories() -> None: """Clears the fuzzers directory.""" remove_directory(environment.get_value('FUZZERS_DIR'), recreate=True) -def clear_temp_directory(clear_user_profile_directories=True): +def clear_temp_directory(clear_user_profile_directories: bool = True) -> None: """Clear the temporary directories.""" temp_directory = environment.get_value('BOT_TMPDIR') remove_directory(temp_directory, recreate=True) @@ -174,12 +178,12 @@ def clear_temp_directory(clear_user_profile_directories=True): @environment.local_noop -def clear_system_temp_directory(): +def clear_system_temp_directory() -> None: """Clear system specific temp directory.""" if environment.get_value('DEBUG_TASK'): return - def _delete_object(path, delete_func): + def _delete_object(path: str, delete_func: Callable[[str], Any]) -> None: """Delete a object with its delete function, ignoring any error.""" try: delete_func(path) @@ -209,7 +213,7 @@ def _delete_object(path, delete_func): logs.info('Cleared system temp directory: %s' % _system_temp_dir) -def clear_testcase_directories(): +def clear_testcase_directories() -> None: """Clears the testcase directories.""" remove_directory(environment.get_value('FUZZ_INPUTS'), recreate=True) remove_directory(environment.get_value('FUZZ_INPUTS_DISK'), recreate=True) @@ -222,7 +226,7 @@ def clear_testcase_directories(): file_host.clear_testcase_directories() -def close_open_file_handles_if_needed(path): +def close_open_file_handles_if_needed(path: str) -> None: """Try to close all open file handle for a specific path.""" if environment.platform() != 'WINDOWS': # Handle closing is only applicable on Windows platform. @@ -247,7 +251,9 @@ def close_open_file_handles_if_needed(path): (handle_executable_path, file_handle_id, process_id)) -def create_directory(directory, create_intermediates=False, recreate=False): +def create_directory(directory: str, + create_intermediates: bool = False, + recreate: bool = False) -> bool: """Creates |directory|. Create intermediate directories if |create_intermediates|. Ignore if it already exists and |recreate| is False.""" @@ -269,7 +275,7 @@ def create_directory(directory, create_intermediates=False, recreate=False): return True -def execute_command(shell_command): +def execute_command(shell_command: str) -> bytes: """Run a command, returning its output.""" try: process_handle = subprocess.Popen( @@ -281,12 +287,12 @@ def execute_command(shell_command): output, _ = process_handle.communicate() except: logs.error('Error while executing command %s.' % shell_command) - return '' + return b'' return output -def get_command(command_line): +def get_command(command_line: str) -> str | list[str]: """Get the command to pass to subprocess.""" if environment.platform() == 'WINDOWS': return command_line @@ -294,12 +300,12 @@ def get_command(command_line): return shlex.split(command_line, posix=True) -def get_command_line_from_argument_list(argument_list): +def get_command_line_from_argument_list(argument_list: Sequence[str]) -> str: """Convert a list of arguments to a string.""" return subprocess.list2cmdline(argument_list) -def get_directory_file_count(directory_path): +def get_directory_file_count(directory_path: str) -> int: """Returns number of files within a directory (recursively).""" file_count = 0 for (root, _, files) in walk(directory_path): @@ -312,7 +318,7 @@ def get_directory_file_count(directory_path): return file_count -def get_directory_size(directory_path): +def get_directory_size(directory_path: str) -> int: """Returns size of a directory (in bytes).""" directory_size = 0 for (root, _, files) in walk(directory_path): @@ -323,7 +329,7 @@ def get_directory_size(directory_path): return directory_size -def get_files_list(directory_path): +def get_files_list(directory_path: str) -> list[str]: """Returns a list of files in a directory (recursively).""" files_list = [] for (root, _, files) in walk(directory_path): @@ -336,7 +342,7 @@ def get_files_list(directory_path): return files_list -def get_free_disk_space(path='/'): +def get_free_disk_space(path: str = '/') -> int | None: """Return free disk space.""" if not os.path.exists(path): return None @@ -348,7 +354,7 @@ def get_free_disk_space(path='/'): return psutil.disk_usage(path).free -def get_interpreter(file_to_execute): +def get_interpreter(file_to_execute: str) -> str | None: """Gives the interpreter needed to execute |file_to_execute|.""" interpreters = { '.bash': 'bash', @@ -368,7 +374,7 @@ def get_interpreter(file_to_execute): return interpreter -def get_execute_command(file_to_execute): +def get_execute_command(file_to_execute: str) -> str: """Return command to execute |file_to_execute|.""" interpreter_path = get_interpreter(file_to_execute) @@ -383,7 +389,7 @@ def get_execute_command(file_to_execute): return command -def move(src, dst): +def move(src: str, dst: str) -> bool: """Wrapper around shutil.move(src, dst). If shutil.move throws an shutil.Error the exception is caught, an error is logged, and False is returned.""" try: @@ -394,7 +400,7 @@ def move(src, dst): return False -def remove_empty_files(root_path): +def remove_empty_files(root_path: str) -> None: """Removes empty files in a path recursively""" for directory, _, filenames in walk(root_path): for filename in filenames: @@ -409,7 +415,7 @@ def remove_empty_files(root_path): (path, sys.exc_info()[0])) -def remove_empty_directories(path): +def remove_empty_directories(path: str) -> None: """Removes empty folder in a path recursively.""" if not os.path.isdir(path): return @@ -430,7 +436,7 @@ def remove_empty_directories(path): logs.error('Unable to remove empty folder %s.' % path) -def remove_file(file_path): +def remove_file(file_path: str) -> None: """Removes a file, ignoring any error if it occurs.""" try: if os.path.exists(file_path): @@ -439,12 +445,12 @@ def remove_file(file_path): pass -def _get_random_filename(): +def _get_random_filename() -> str: return str(uuid.uuid4()).lower() @contextlib.contextmanager -def get_tempfile(prefix='', suffix=''): +def get_tempfile(prefix: str = '', suffix: str = '') -> Iterator[str]: """Returns path to a temporary file.""" tempdir = environment.get_value('BOT_TMPDIR', '/tmp') os.makedirs(tempdir, exist_ok=True) @@ -456,12 +462,14 @@ def get_tempfile(prefix='', suffix=''): os.remove(filepath) -def remove_directory(directory, recreate=False, ignore_errors=False): +def remove_directory(directory: str, + recreate: bool = False, + ignore_errors: bool = False) -> bool: """Removes a directory tree.""" # Log errors as warnings if |ignore_errors| is set. log_error_func = logs.warning if ignore_errors else logs.error - def clear_read_only(func, path, _): + def clear_read_only(func: Callable[[str], Any], path: str, _: Any) -> None: """Clear the read-only bit and reattempt the removal again. This is needed on Windows.""" @@ -521,6 +529,7 @@ def clear_read_only(func, path, _): return True -def walk(directory, **kwargs): +def walk(directory: str, + **kwargs: Any) -> Iterator[tuple[str, list[str], list[str]]]: """Wrapper around walk to resolve compatibility issues.""" return os.walk(directory, **kwargs) diff --git a/src/clusterfuzz/_internal/tests/core/base/dates_test.py b/src/clusterfuzz/_internal/tests/core/base/dates_test.py new file mode 100644 index 00000000000..8d39bb09c15 --- /dev/null +++ b/src/clusterfuzz/_internal/tests/core/base/dates_test.py @@ -0,0 +1,41 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for dates module.""" + +import datetime +import unittest + +from clusterfuzz._internal.base import dates + + +class DatesTest(unittest.TestCase): + """Tests for date utility functions.""" + + def test_time_has_expired_true(self): + """Test that time_has_expired returns True when timestamp is older.""" + now = datetime.datetime(2026, 1, 1, 12, 0, 0) + past = datetime.datetime(2026, 1, 1, 11, 0, 0) + self.assertTrue(dates.time_has_expired(past, compare_to=now, seconds=1800)) + self.assertTrue(dates.time_has_expired(past, compare_to=now, minutes=30)) + + def test_time_has_expired_false(self): + """Test that time_has_expired returns False when timestamp is recent.""" + now = datetime.datetime(2026, 1, 1, 12, 0, 0) + past = datetime.datetime(2026, 1, 1, 11, 30, 0) + self.assertFalse(dates.time_has_expired(past, compare_to=now, hours=1)) + self.assertFalse(dates.time_has_expired(past, compare_to=now, days=1)) + + def test_initialize_timezone_from_environment(self): + """Test initialize_timezone_from_environment execution.""" + dates.initialize_timezone_from_environment() diff --git a/src/clusterfuzz/_internal/tests/core/base/feature_flags_test.py b/src/clusterfuzz/_internal/tests/core/base/feature_flags_test.py new file mode 100644 index 00000000000..b9b3de9aa5b --- /dev/null +++ b/src/clusterfuzz/_internal/tests/core/base/feature_flags_test.py @@ -0,0 +1,63 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""FeatureFlag tests.""" +import unittest + +from clusterfuzz._internal.base import feature_flags +from clusterfuzz._internal.datastore import data_types +from clusterfuzz._internal.tests.test_libs import test_utils + + +@test_utils.with_cloud_emulators('datastore') +class FeatureFlagsTest(unittest.TestCase): + """Test FeatureFlags.""" + + def test_get_and_set(self) -> None: + """Test getting and setting feature flags.""" + # Create a boolean feature flag. + flag = data_types.FeatureFlag(id=feature_flags.FeatureFlags.TEST_FLAG.value) + flag.description = 'A test boolean flag' + flag.enabled = True + flag.put() + + # Create a float feature flag. + flag_float = data_types.FeatureFlag( + id=feature_flags.FeatureFlags.TEST_FLOAT_FLAG.value) + flag_float.description = 'A test float flag' + flag_float.enabled = True + flag_float.value = 1.23 + flag_float.string_value = 'checker' + flag_float.put() + + # Retrieve and verify. + self.assertIsNotNone(feature_flags.FeatureFlags.TEST_FLAG.flag) + self.assertTrue(feature_flags.FeatureFlags.TEST_FLAG.enabled) + self.assertEqual(feature_flags.FeatureFlags.TEST_FLAG.description, + 'A test boolean flag') + self.assertIsNone(feature_flags.FeatureFlags.TEST_FLAG.content) + + retrieved_flag_float = feature_flags.FeatureFlags.TEST_FLOAT_FLAG.flag + self.assertIsNotNone(retrieved_flag_float) + self.assertTrue(feature_flags.FeatureFlags.TEST_FLOAT_FLAG.enabled) + self.assertEqual(feature_flags.FeatureFlags.TEST_FLOAT_FLAG.content, 1.23) + self.assertEqual(feature_flags.FeatureFlags.TEST_FLOAT_FLAG.string_value, + 'checker') + + def test_not_found_in_db(self) -> None: + """Test that a feature flag returns empty or False values when not found in the DB.""" + self.assertFalse(feature_flags.FeatureFlags.TEST_FLAG.enabled) + self.assertIsNone(feature_flags.FeatureFlags.TEST_FLAG.flag) + self.assertIsNone(feature_flags.FeatureFlags.TEST_FLAG.content) + self.assertEqual(feature_flags.FeatureFlags.TEST_FLAG.description, '') + self.assertEqual(feature_flags.FeatureFlags.TEST_FLAG.string_value, '') diff --git a/src/clusterfuzz/_internal/tests/core/base/modules_test.py b/src/clusterfuzz/_internal/tests/core/base/modules_test.py new file mode 100644 index 00000000000..34c89f583ff --- /dev/null +++ b/src/clusterfuzz/_internal/tests/core/base/modules_test.py @@ -0,0 +1,103 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""modules tests.""" +# pylint: disable=protected-access +import os +import sys +import unittest +from unittest import mock + +from clusterfuzz._internal.base import modules + + +class ConfigModulesDirectoryTest(unittest.TestCase): + """Test _config_modules_directory.""" + + def test_default_config_dir(self): + """Test default config directory when override is not set.""" + with mock.patch.dict(os.environ, {}, clear=True): + res = modules._config_modules_directory('/root') + self.assertEqual( + res, os.path.join('/root', 'src', 'appengine', 'config', 'modules')) + + def test_override_config_dir(self): + """Test config directory when CONFIG_DIR_OVERRIDE is set.""" + with mock.patch.dict(os.environ, {'CONFIG_DIR_OVERRIDE': '/custom_config'}): + res = modules._config_modules_directory('/root') + self.assertEqual(res, os.path.join('/custom_config', 'modules')) + + +class PatchAppEngineModulesForBotsTest(unittest.TestCase): + """Test _patch_appengine_modules_for_bots.""" + + def test_appengine_environment(self): + """Test that it does nothing on App Engine (SERVER_SOFTWARE set).""" + with mock.patch.dict(os.environ, {'SERVER_SOFTWARE': 'Development/2.0'}): + mock_auth = mock.MagicMock() + mock_auth.app_identity = mock.MagicMock() + with mock.patch.dict( + sys.modules, { + 'google': mock.MagicMock(), + 'google.auth': mock.MagicMock(app_engine=mock_auth), + 'google.auth.app_engine': mock_auth + }): + modules._patch_appengine_modules_for_bots() + self.assertIsNotNone(mock_auth.app_identity) + + def test_bot_environment(self): + """Test patching on bot environment.""" + with mock.patch.dict(os.environ, {}, clear=True): + mock_auth = mock.MagicMock() + mock_auth.app_identity = mock.MagicMock() + with mock.patch.dict( + sys.modules, { + 'google': mock.MagicMock(), + 'google.auth': mock.MagicMock(app_engine=mock_auth), + 'google.auth.app_engine': mock_auth + }): + modules._patch_appengine_modules_for_bots() + self.assertIsNone(mock_auth.app_identity) + + +class FixModuleSearchPathsTest(unittest.TestCase): + """Test fix_module_search_paths.""" + + @mock.patch( + 'clusterfuzz._internal.base.modules._patch_appengine_modules_for_bots') + @mock.patch('site.addsitedir') + @mock.patch('os.path.exists') + def test_fix_module_search_paths(self, mock_exists, mock_addsitedir, + mock_patch): + """Test fix_module_search_paths adds paths correctly.""" + mock_exists.return_value = True + root = '/fake_root' + with mock.patch.dict( + os.environ, { + 'ROOT_DIR': root, + 'PYTHONPATH': '' + }, clear=True): + with mock.patch.object(sys, 'path', ['/existing_path']): + modules.fix_module_search_paths() + + expected_config_modules = os.path.join(root, 'src', 'appengine', + 'config', 'modules') + expected_third_party = os.path.join(root, 'src', 'third_party') + expected_src = os.path.join(root, 'src') + + self.assertIn(expected_config_modules, sys.path) + self.assertIn(expected_third_party, sys.path) + self.assertIn(expected_src, sys.path) + + mock_addsitedir.assert_called_once_with(expected_third_party) + mock_patch.assert_called_once() diff --git a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py index fd92f73173b..da23811da61 100644 --- a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py +++ b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py @@ -387,8 +387,8 @@ def setUp(self): func='test_func', sinfo=None, ) - self.log_record.created = datetime.datetime(2023, 10, 26, 12, 0, - 0).timestamp() + self.log_record.created = datetime.datetime( + 2023, 10, 26, 12, 0, 0, tzinfo=datetime.timezone.utc).timestamp() self.original_env = dict(os.environ) diff --git a/src/clusterfuzz/fuzz/engine.py b/src/clusterfuzz/fuzz/engine.py index 84768902bd0..3d303971195 100644 --- a/src/clusterfuzz/fuzz/engine.py +++ b/src/clusterfuzz/fuzz/engine.py @@ -13,7 +13,9 @@ # limitations under the License. """Fuzzing engine interface.""" -_ENGINES = {} +from typing import Any + +_ENGINES: dict[str, type['Engine']] = {} class Error(Exception): @@ -24,7 +26,12 @@ class FuzzOptions: """Represents options passed to the engine. Can be overridden to provide more options.""" - def __init__(self, corpus_dir, arguments, strategies): + corpus_dir: str + arguments: list[str] + strategies: Any + + def __init__(self, corpus_dir: str, arguments: list[str], + strategies: Any) -> None: self.corpus_dir = corpus_dir self.arguments = arguments self.strategies = strategies @@ -33,7 +40,13 @@ def __init__(self, corpus_dir, arguments, strategies): class Crash: """Represents a crash found by the fuzzing engine.""" - def __init__(self, input_path, stacktrace, reproduce_args, crash_time): + input_path: str + stacktrace: str + reproduce_args: list[str] + crash_time: float + + def __init__(self, input_path: str, stacktrace: str, + reproduce_args: list[str], crash_time: float) -> None: self.input_path = input_path self.stacktrace = stacktrace self.reproduce_args = reproduce_args @@ -44,13 +57,20 @@ class FuzzResult: """Represents a result of a fuzzing session: a list of crashes found and the stats generated.""" + logs: str + command: Any + crashes: list[Crash] + stats: Any + time_executed: Any + timed_out: bool | None + def __init__(self, - logs, - command, - crashes, - stats, - time_executed, - timed_out=None): + logs: str, + command: Any, + crashes: list[Crash], + stats: Any, + time_executed: Any, + timed_out: bool | None = None) -> None: self.logs = logs self.command = command self.crashes = crashes @@ -62,7 +82,13 @@ def __init__(self, class ReproduceResult: """Results from running a testcase against a target.""" - def __init__(self, command, return_code, time_executed, output): + command: Any + return_code: Any + time_executed: Any + output: str + + def __init__(self, command: Any, return_code: Any, time_executed: Any, + output: str) -> None: self.command = command self.return_code = return_code self.time_executed = time_executed @@ -72,15 +98,17 @@ def __init__(self, command, return_code, time_executed, output): class Engine: """Base interface for a grey box fuzzing engine.""" - def __init__(self): + do_strategies: bool + + def __init__(self) -> None: self.do_strategies = True @property - def name(self): + def name(self) -> str: """Get the name of the engine.""" raise NotImplementedError - def fuzz_additional_processing_timeout(self, options): + def fuzz_additional_processing_timeout(self, options: Any) -> int: """Return the maximum additional timeout in seconds for additional operations in fuzz() (e.g. merging back new items). @@ -93,7 +121,8 @@ def fuzz_additional_processing_timeout(self, options): del options return 0 - def prepare(self, corpus_dir, target_path, build_dir): + def prepare(self, corpus_dir: str, target_path: str, + build_dir: str) -> FuzzOptions: """Prepare for a fuzzing session, by generating options. Returns a FuzzOptions object. @@ -107,7 +136,8 @@ def prepare(self, corpus_dir, target_path, build_dir): """ raise NotImplementedError - def fuzz(self, target_path, options, reproducers_dir, max_time): + def fuzz(self, target_path: str, options: Any, reproducers_dir: str, + max_time: Any) -> FuzzResult: """Run a fuzz session. Args: @@ -122,8 +152,8 @@ def fuzz(self, target_path, options, reproducers_dir, max_time): """ raise NotImplementedError - def reproduce(self, target_path, input_path, arguments, - max_time) -> ReproduceResult: + def reproduce(self, target_path: str, input_path: str, arguments: list[str], + max_time: Any) -> ReproduceResult: """Reproduce a crash given an input. Args: @@ -140,8 +170,9 @@ def reproduce(self, target_path, input_path, arguments, """ raise NotImplementedError - def minimize_corpus(self, target_path, arguments, input_dirs, output_dir, - reproducers_dir, max_time): + def minimize_corpus(self, target_path: str, arguments: list[str], + input_dirs: list[str], output_dir: str, + reproducers_dir: str, max_time: Any) -> FuzzResult: """Optional (but recommended): run corpus minimization. Args: @@ -162,8 +193,9 @@ def minimize_corpus(self, target_path, arguments, input_dirs, output_dir, """ raise NotImplementedError - def minimize_testcase(self, target_path, arguments, input_path, output_path, - max_time): + def minimize_testcase(self, target_path: str, arguments: list[str], + input_path: str, output_path: str, + max_time: Any) -> ReproduceResult: """Optional (but recommended): Minimize a testcase. Args: @@ -181,7 +213,8 @@ def minimize_testcase(self, target_path, arguments, input_path, output_path, """ raise NotImplementedError - def cleanse(self, target_path, arguments, input_path, output_path, max_time): + def cleanse(self, target_path: str, arguments: list[str], input_path: str, + output_path: str, max_time: Any) -> ReproduceResult: """Optional (but recommended): Cleanse a testcase. Args: @@ -200,7 +233,7 @@ def cleanse(self, target_path, arguments, input_path, output_path, max_time): raise NotImplementedError -def register(name, engine_class): +def register(name: str, engine_class: type[Engine]) -> None: """Register a fuzzing engine.""" if name in _ENGINES: raise ValueError('Engine {name} is already registered'.format(name=name)) @@ -208,9 +241,11 @@ def register(name, engine_class): _ENGINES[name] = engine_class -def get(name): +def get(name: str | None) -> Engine | None: """Gets an implementation of a fuzzing engine, or None if one does not exist.""" + if not name: + return None engine_class = _ENGINES.get(name) if engine_class: return engine_class() diff --git a/src/clusterfuzz/stacktraces/__init__.py b/src/clusterfuzz/stacktraces/__init__.py index 672c17a8c19..cc32ef04933 100644 --- a/src/clusterfuzz/stacktraces/__init__.py +++ b/src/clusterfuzz/stacktraces/__init__.py @@ -12,13 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. """Stack parsing module.""" +from collections.abc import Callable import os import re import string import subprocess +from typing import cast from clusterfuzz._internal.base import utils from clusterfuzz._internal.crash_analysis import crash_analyzer +from clusterfuzz._internal.crash_analysis.stack_parsing import stack_parser from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.platforms.linux.lkl import constants as lkl_constants from clusterfuzz._internal.system import environment @@ -29,52 +32,95 @@ class CrashInfo: """Parsed crash information.""" - def __init__(self): - self.crash_type = '' - self.crash_address = '' - self.crash_state = '' - self.crash_stacktrace = '' - self.crash_categories = set() - self.frame_count = 0 - self.process_name = None - self.process_died = False + crash_type: str + crash_address: str + crash_state: str + crash_stacktrace: str + crash_categories: set[str] + frame_count: int + process_name: str | None + process_died: bool + + # Following fields are for internal use only and subject to change. Do not + # rely on these. + frames: list[list[stack_parser.StackFrame | None]] + last_frame_id: int + raw_frames: list[str] + + # Additional tracking for Java bugs. + found_java_exception: bool + + # Additional tracking for bad casts. + found_bad_cast_crash_end_marker: bool + + # Additional tracking for check failures. + check_failure_source_file: str + + # Additional tracking for fatal errors. + fatal_error_occurred: bool + + # Additional tracking for lkl bugs. + lkl_kernel_build_id: str | None + + # Additional tracking for fuzzing directory frames, + fuzzer_dir_frames: int + + is_kasan: bool + is_lkl: bool + is_golang: bool + is_python: bool + is_js: bool + found_python_crash: bool + found_golang_crash: bool + found_android_kernel_crash: bool + is_trusty: bool + + def __init__(self) -> None: + self.crash_type: str = '' + self.crash_address: str = '' + self.crash_state: str = '' + self.crash_stacktrace: str = '' + self.crash_categories: set[str] = set() + self.frame_count: int = 0 + self.process_name: str | None = None + self.process_died: bool = False # Following fields are for internal use only and subject to change. Do not # rely on these. - self.frames = [] - self.last_frame_id = -1 - self.raw_frames = [] + self.frames: list[list[stack_parser.StackFrame | None]] = [] + self.last_frame_id: int = -1 + self.raw_frames: list[str] = [] # Additional tracking for Java bugs. - self.found_java_exception = False + self.found_java_exception: bool = False # Additional tracking for bad casts. - self.found_bad_cast_crash_end_marker = False + self.found_bad_cast_crash_end_marker: bool = False # Additional tracking for check failures. - self.check_failure_source_file = '' + self.check_failure_source_file: str = '' # Additional tracking for fatal errors. - self.fatal_error_occurred = False + self.fatal_error_occurred: bool = False # Additional tracking for lkl bugs. - self.lkl_kernel_build_id = None + self.lkl_kernel_build_id: str | None = None # Additional tracking for fuzzing directory frames, - self.fuzzer_dir_frames = 0 + self.fuzzer_dir_frames: int = 0 - self.is_kasan = False - self.is_lkl = False - self.is_golang = False - self.is_python = False - self.is_js = False - self.found_python_crash = False - self.found_golang_crash = False - self.found_android_kernel_crash = False - self.is_trusty = False + self.is_kasan: bool = False + self.is_lkl: bool = False + self.is_golang: bool = False + self.is_python: bool = False + self.is_js: bool = False + self.found_python_crash: bool = False + self.found_golang_crash: bool = False + self.found_android_kernel_crash: bool = False + self.is_trusty: bool = False -def _filter_stack_frame(stack_frame): +def _filter_stack_frame(stack_frame: str) -> str: """Filter stack frame.""" # Filter out anonymous namespaces. anonymous_namespaces = [ @@ -102,13 +148,23 @@ def _filter_stack_frame(stack_frame): class StackParser: """Stack parser.""" - def __init__(self, - symbolized=True, - detect_ooms_and_hangs=True, - detect_v8_runtime_errors=False, - custom_stack_frame_ignore_regexes=None, - fuzz_target=None, - include_ubsan=True): + stack_frame_ignore_regex: re.Pattern[str] + stack_frame_ignore_regex_if_symbolized: re.Pattern[str] + detect_ooms_and_hangs: bool + detect_v8_runtime_errors: bool + symbolized: bool + fuzz_target: str | None + include_ubsan: bool + + def __init__( + self, + symbolized: bool = True, + detect_ooms_and_hangs: bool = True, + detect_v8_runtime_errors: bool = False, + custom_stack_frame_ignore_regexes: list[str] | None = None, + fuzz_target: str | None = None, + include_ubsan: bool = True, + ) -> None: if not custom_stack_frame_ignore_regexes: custom_stack_frame_ignore_regexes = [] @@ -126,7 +182,7 @@ def __init__(self, self.fuzz_target = fuzz_target self.include_ubsan = include_ubsan - def ignore_stack_frame(self, stack_frame): + def ignore_stack_frame(self, stack_frame: str) -> bool: """Return true if stack frame should not used in determining the crash state.""" # No data, should ignore. @@ -151,34 +207,36 @@ def ignore_stack_frame(self, stack_frame): return False - def get_rank(self, crash_type): + def get_rank(self, crash_type: str) -> int: """Return the assignment rank of a given crash type.""" # Higher ranked types will not be overwritten with lower ranked types. # Types missing in this list have a default rank of 0. high_rank_types = {'V8 sandbox violation': 2, 'Timeout': 1} return high_rank_types.get(crash_type, 0) - def update_crash_type(self, state: CrashInfo, new_type): + def update_crash_type(self, state: CrashInfo, new_type: str | None) -> None: # Prioritize the crash type associated with the latest stack frame, unless # the previous type has an explicitly assigned higher priority. if new_type is not None and self.get_rank(new_type) >= self.get_rank( state.crash_type): state.crash_type = new_type - def update_state_on_match(self, - compiled_regex: re.Pattern, - line: str, - state: CrashInfo, - new_type=None, - new_state=None, - new_frame_count=None, - new_address=None, - address_from_group=None, - type_from_group=None, - state_from_group=None, - address_filter=lambda s: s, - type_filter=lambda s: s, - reset=False) -> re.Match | None: + def update_state_on_match( + self, + compiled_regex: re.Pattern[str], + line: str, + state: CrashInfo, + new_type: str | None = None, + new_state: str | None = None, + new_frame_count: int | None = None, + new_address: str | None = None, + address_from_group: int | str | None = None, + type_from_group: int | str | None = None, + state_from_group: int | str | None = None, + address_filter: Callable[[str], str] = lambda s: s, + type_filter: Callable[[str], str] = lambda s: s, + reset: bool = False, + ) -> re.Match[str] | None: """Update the specified parts of the state if we have a match.""" match = compiled_regex.match(line) @@ -217,16 +275,19 @@ def update_state_on_match(self, return match - def add_frame_on_match(self, - compiled_regex: re.Pattern, - line: str, - state: CrashInfo, - group=0, - frame_filter=_filter_stack_frame, - demangle=False, - can_ignore=True, - frame_spec=None, - frame_override_func=None): + def add_frame_on_match( + self, + compiled_regex: re.Pattern[str], + line: str, + state: CrashInfo, + group: int | str = 0, + frame_filter: Callable[[str], str] = _filter_stack_frame, + demangle: bool = False, + can_ignore: bool = True, + frame_spec: stack_parser.StackFrameSpec | None = None, + frame_override_func: Callable[[str, stack_parser.StackFrame | None], str] + | None = None, + ) -> re.Match[str] | None: """Add a frame to the crash state if we have a match on this line.""" match = compiled_regex.match(line) if not match: @@ -250,8 +311,8 @@ def add_frame_on_match(self, ['c++filt', '-n', frame], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - frame, _ = pipe.communicate() - frame = frame.decode('utf-8') + frame_bytes, _ = pipe.communicate() + frame = frame_bytes.decode('utf-8') # Try to parse the frame with the various stackframes. frame_struct = None @@ -263,7 +324,7 @@ def add_frame_on_match(self, if (frame_struct and frame_struct.module_offset and not frame_struct.function_name): - def new_frame_filter(s): + def new_frame_filter(s: str) -> str: return s frame_filter = new_frame_filter @@ -307,7 +368,13 @@ def new_frame_filter(s): return match - def update_state_on_check_failure(self, state, line, regex, crash_type): + def update_state_on_check_failure( + self, + state: CrashInfo, + line: str, + regex: re.Pattern[str], + crash_type: str, + ) -> None: """Update the state if the crash is a CHECK failure.""" check_match = self.update_state_on_match( regex, line, state, new_type=crash_type, reset=True, new_frame_count=1) @@ -316,7 +383,13 @@ def update_state_on_check_failure(self, state, line, regex, crash_type): source_file = fix_filename_string(check_match.group(1)) state.crash_state = '%s in %s\n' % (failure_string, source_file) - def match_assert(self, line, state, regex, group=1): + def match_assert( + self, + line: str, + state: CrashInfo, + regex: re.Pattern[str], + group: int | str = 1, + ) -> None: """Match an assert.""" assert_match = self.update_state_on_match( regex, line, state, new_type='ASSERT', new_frame_count=1) @@ -324,7 +397,7 @@ def match_assert(self, line, state, regex, group=1): # For asserts, we want to actually use the match as the crash state. state.crash_state = assert_match.group(group) + '\n' - def filter_crash_parameters(self, state): + def filter_crash_parameters(self, state: CrashInfo) -> CrashInfo: """Normalize crash parameters into generic format regardless of the tool used.""" # Filter crash state represented in |state|. @@ -374,7 +447,7 @@ def filter_crash_parameters(self, state): # Normalize access size parameter if greater than 16 bytes. m = re.match('([^0-9]+)([0-9]+)', state.crash_type, re.DOTALL) if m: - num = int(m.group(2)) + num: int | str = int(m.group(2)) if num > 16: num = '{*}' @@ -396,7 +469,11 @@ def filter_crash_parameters(self, state): return state - def remove_lkl_kernel_times_and_set_params(self, state, crash_data): + def remove_lkl_kernel_times_and_set_params( + self, + state: CrashInfo, + crash_data: str, + ) -> str: """Filter kernel time from crash_data if LKL.""" result = '' for line in crash_data.splitlines(): @@ -409,7 +486,7 @@ def remove_lkl_kernel_times_and_set_params(self, state, crash_data): return result @staticmethod - def split_stacktrace(stacktrace: str): + def split_stacktrace(stacktrace: str) -> list[str]: """Split stacktrace by line, and handle special cases.""" # Fix a known malformed traceback pattern: # A newline character is missing between the important crash info line and @@ -785,7 +862,9 @@ def parse(self, stacktrace: str) -> CrashInfo: # with the access type (defaulting to empty if absent). if 'SEGV_MTESERR' in line: mte_match = ANDROID_SEGV_REGEX.search(line) - state.crash_type = f"Tag-mismatch{(mte_match.group(2) or '')}" + state.crash_type = ( + f"Tag-mismatch{(cast(re.Match[str], mte_match).group(2) or '')}" + ) # Set the crash address for SEGVs. if 'SIGSEGV' in line: @@ -1391,7 +1470,7 @@ def parse(self, stacktrace: str) -> CrashInfo: return state -def filter_addresses_and_numbers(stack_frame): +def filter_addresses_and_numbers(stack_frame: str) -> str: """Return a normalized string without unique addresses and numbers.""" # Remove offset part from end of every line. result = re.sub(r'\+0x[0-9a-fA-F]+\n', '\n', stack_frame, re.DOTALL) @@ -1415,7 +1494,10 @@ def filter_addresses_and_numbers(stack_frame): return re.sub(number_expression, 'NUMBER', result, flags=re.X) -def should_ignore_line_for_crash_processing(line, state): +def should_ignore_line_for_crash_processing( + line: str, + state: CrashInfo, +) -> bool: """Check to see if a line should be displayed in a report, but ignored when processing crashes.""" # If we detected that the process had died, we won't use any further stack @@ -1461,7 +1543,7 @@ def should_ignore_line_for_crash_processing(line, state): return False -def fix_sanitizer_crash_type(crash_type): +def fix_sanitizer_crash_type(crash_type: str) -> str: """Ensure that Sanitizer crashes use generic formats.""" # General normalization. crash_type = crash_type.lower().replace('_', '-').capitalize() @@ -1472,7 +1554,7 @@ def fix_sanitizer_crash_type(crash_type): return crash_type -def fix_win_cdb_crash_type(crash_type): +def fix_win_cdb_crash_type(crash_type: str) -> str: """Convert a Windows CDB crash type into ASAN like format.""" # Strip application verifier string from crash type suffix. crash_type = utils.strip_from_right(crash_type, '_AVRF') @@ -1487,7 +1569,7 @@ def fix_win_cdb_crash_type(crash_type): return crash_type -def fix_check_failure_string(failure_string): +def fix_check_failure_string(failure_string: str) -> str: """Cleanup values that should not be included in CHECK failure strings.""" # Remove |CHECK_FAILURE_PATTERN| from start of failure string. failure_string = utils.strip_from_left(failure_string, CHECK_FAILURE_PATTERN) @@ -1507,7 +1589,7 @@ def fix_check_failure_string(failure_string): return failure_string.strip(' .\'"[]') -def fix_filename_string(filename_string): +def fix_filename_string(filename_string: str) -> str: """Fix filename string to remove line number, path and other invalid chars.""" # Remove invalid chars at ends first. filename_string = filename_string.strip(' .\'"[]') @@ -1524,43 +1606,43 @@ def fix_filename_string(filename_string): return filename_string -def get_fault_description_for_android_kernel(code): +def get_fault_description_for_android_kernel(code: str) -> str: """Return human readable fault description based on numeric FSR value.""" # Convert code from string to number. try: - code = int(code, 16) + code_int = int(code, 16) except: return 'BUG' # Figure out where is out-of-bounds read or write. - if code & 0x800 == 0: + if code_int & 0x800 == 0: fault = 'READ' else: fault = 'WRITE' fault += ' ' # The full status code is bits 12, 10, and 0-3, but we're ignoring 12 and 10. - status = code & 0b1111 + status = code_int & 0b1111 try: fault += ANDROID_KERNEL_STATUS_TO_STRING[status] except KeyError: fault += 'Unknown' - fault += ' (%s)' % str(code) + fault += ' (%s)' % str(code_int) return 'Kernel failure\n' + fault -def filter_kasan_crash_type(crash_type): +def filter_kasan_crash_type(crash_type: str) -> str: """Filter a KASan crash type.""" return 'Kernel failure\n%s' % crash_type.replace(' ', '-').capitalize() -def filter_kernel_panic_crash_type(crash_type): +def filter_kernel_panic_crash_type(crash_type: str) -> str: """Filter a kernel panic crash type.""" return 'Kernel failure\n%s' % crash_type.replace(' ', '-') -def update_crash_state_for_stack_overflow_if_needed(state): +def update_crash_state_for_stack_overflow_if_needed(state: CrashInfo) -> None: """For stack-overflow bugs, updates crash state based on cycle detected.""" if state.crash_type != 'Stack-overflow': return @@ -1570,7 +1652,7 @@ def update_crash_state_for_stack_overflow_if_needed(state): for cycle_length in range(1, MAX_CYCLE_LENGTH + 1): # Create frame potential cycles of a given length starting from # |frame_index|. - frame_potential_cycles = [] + frame_potential_cycles: list[list[str]] = [] end_reached = False for i in range(0, REPEATED_CYCLE_COUNT): start_index = frame_index + i * cycle_length @@ -1597,7 +1679,10 @@ def update_crash_state_for_stack_overflow_if_needed(state): return -def llvm_test_one_input_override(frame, frame_struct): +def llvm_test_one_input_override( + frame: str, + frame_struct: stack_parser.StackFrame | None, +) -> str: """Override frame matching for LLVMFuzzerTestOneInput frames.""" if not frame.startswith('LLVMFuzzerTestOneInput'): return frame @@ -1611,12 +1696,12 @@ def llvm_test_one_input_override(frame, frame_struct): return frame -def reverse_python_stacktrace(stacktrace): +def reverse_python_stacktrace(stacktrace: list[str]) -> list[str]: """Extract a Python stacktrace. Python stacktraces are a bit special: they are reversed, and followed by a sanitizer one, so we need to extract them, reverse them, and put their "title" back on top.""" - python_stacktrace_split = [] + python_stacktrace_split: list[str] = [] in_python_stacktrace = False for line in stacktrace: @@ -1636,7 +1721,7 @@ def reverse_python_stacktrace(stacktrace): return python_stacktrace_split -def update_kasan_crash_details(state, line): +def update_kasan_crash_details(state: CrashInfo, line: str) -> None: """For KASan crashes, additional information about a bad access may exist.""" if state.crash_type.startswith('Kernel failure'): kasan_access_match = KASAN_ACCESS_TYPE_ADDRESS_REGEX.match(line) diff --git a/src/clusterfuzz/stacktraces/constants.py b/src/clusterfuzz/stacktraces/constants.py index 3f129e9f478..e691381ed03 100644 --- a/src/clusterfuzz/stacktraces/constants.py +++ b/src/clusterfuzz/stacktraces/constants.py @@ -17,162 +17,183 @@ from clusterfuzz._internal.crash_analysis.stack_parsing import stack_parser -C_CPP_EXTENSIONS = ['c', 'cc', 'cpp', 'cxx', 'h', 'hh', 'hpp', 'hxx'] +C_CPP_EXTENSIONS: list[str] = ['c', 'cc', 'cpp', 'cxx', 'h', 'hh', 'hpp', 'hxx'] # Patterns which cannot be compiled directly, or which are used for direct # comparison. -CHECK_FAILURE_PATTERN = r'Check failed: ' -JNI_ERROR_STRING = r'JNI DETECTED ERROR IN APPLICATION:' +CHECK_FAILURE_PATTERN: str = r'Check failed: ' +JNI_ERROR_STRING: str = r'JNI DETECTED ERROR IN APPLICATION:' # Common log prefix format for Google fatal logs. -GOOGLE_LOG_FATAL_PREFIX = r'^F\d{4}\s+\d{2}:\d{2}:\d{2}\.\d+\s+\d+\s+(.*):\d+\]' +GOOGLE_LOG_FATAL_PREFIX: str = ( + r'^F\d{4}\s+\d{2}:\d{2}:\d{2}\.\d+\s+\d+\s+(.*):\d+\]') # Compiled regular expressions. -ANDROID_ABORT_REGEX = re.compile(r'^Abort message: (.*)') -ANDROID_FATAL_EXCEPTION_REGEX = re.compile(r'.*FATAL EXCEPTION.*:') -ANDROID_KERNEL_ERROR_REGEX = re.compile( +ANDROID_ABORT_REGEX: re.Pattern[str] = re.compile(r'^Abort message: (.*)') +ANDROID_FATAL_EXCEPTION_REGEX: re.Pattern[str] = re.compile( + r'.*FATAL EXCEPTION.*:') +ANDROID_KERNEL_ERROR_REGEX: re.Pattern[str] = re.compile( r'.*Internal error: (Oops)?( -|:) (BUG|[0-9a-fA-F]+)') -ANDROID_KERNEL_STACK_FRAME_REGEX = re.compile( +ANDROID_KERNEL_STACK_FRAME_REGEX: re.Pattern[str] = re.compile( # e.g. "[ 1998.156940] [] " r'[^(]*\[\<([x0-9a-fA-F]+)\>\]\s+' # e.g. "(msm_vidc_prepare_buf+0xa0/0x124)"; function (3), offset (4) r'\(?(([\w]+)\+([\w]+)/[\w]+)\)?') -ANDROID_KERNEL_STACK_FRAME_NO_ADDRESS_REGEX = re.compile( +ANDROID_KERNEL_STACK_FRAME_NO_ADDRESS_REGEX: re.Pattern[str] = re.compile( # e.g. "(msm_vidc_prepare_buf+0xa0/0x124)"; function (2), offset (3) r'\(?(([\w]+)\+([\w]+)/[\w]+)\)?') -ANDROID_KERNEL_TIME_REGEX = re.compile(r'^\[\s*\d+\.\d+\]\s') +ANDROID_KERNEL_TIME_REGEX: re.Pattern[str] = re.compile(r'^\[\s*\d+\.\d+\]\s') # Parentheses are optional. -ANDROID_PROCESS_NAME_REGEX = re.compile(r'.*[(](.*)[)]$') -ANDROID_SEGV_REGEX = re.compile(r'.*signal.*\(SIG.*fault addr ([^ ]*)(.*)') -ANDROID_SIGABRT_REGEX = re.compile(r'.*signal.*\(SIGABRT.*fault addr --------') -ANDROID_SIGTRAP_REGEX = re.compile( +ANDROID_PROCESS_NAME_REGEX: re.Pattern[str] = re.compile(r'.*[(](.*)[)]$') +ANDROID_SEGV_REGEX: re.Pattern[str] = re.compile( + r'.*signal.*\(SIG.*fault addr ([^ ]*)(.*)') +ANDROID_SIGABRT_REGEX: re.Pattern[str] = re.compile( + r'.*signal.*\(SIGABRT.*fault addr --------') +ANDROID_SIGTRAP_REGEX: re.Pattern[str] = re.compile( r'.*signal.*\(SIGTRAP.*fault addr ([^ ]*)(.*)') -ASAN_INVALID_FREE_REGEX = re.compile( +ASAN_INVALID_FREE_REGEX: re.Pattern[str] = re.compile( r'.*AddressSanitizer: ' r'attempting free on address which was not malloc\(\)-ed: ' r'([xX0-9a-fA-F]+)') -ASAN_DOUBLE_FREE_REGEX = re.compile( +ASAN_DOUBLE_FREE_REGEX: re.Pattern[str] = re.compile( r'.*(AddressSanitizer).*double-free' r' on (unknown address |address |)([xX0-9a-fA-F]+)') -ASAN_MEMCPY_OVERLAP_REGEX = re.compile( +ASAN_MEMCPY_OVERLAP_REGEX: re.Pattern[str] = re.compile( r'.*(AddressSanitizer).*memcpy-param-overlap' r'[^\[]*([\[].*[)])') -ASAN_REGEX = re.compile( +ASAN_REGEX: re.Pattern[str] = re.compile( r'.*ERROR: (HWAddressSanitizer|AddressSanitizer)[: ]*[ ]*([^(:;]+)') -ASSERT_REGEX = re.compile( +ASSERT_REGEX: re.Pattern[str] = re.compile( r'(?:\[.*?\]|.*\.(?:%s):.*)?' % ('|'.join(C_CPP_EXTENSIONS)) + r'\s*(?:ASSERT(?:ION)? FAIL(?:URE|ED)|panic): (.*)', re.IGNORECASE) -ASSERT_REGEX_GOOGLE = re.compile(GOOGLE_LOG_FATAL_PREFIX + - r'.*assertion failed at\s.*\sin\s*.*: (.*)') -ASSERT_REGEX_GLIBC = re.compile( +ASSERT_REGEX_GOOGLE: re.Pattern[str] = re.compile( + GOOGLE_LOG_FATAL_PREFIX + r'.*assertion failed at\s.*\sin\s*.*: (.*)') +ASSERT_REGEX_GLIBC: re.Pattern[str] = re.compile( r'.*:\s*assertion [`\'"]?(.*?)[`\'"]? failed\.?$', re.IGNORECASE) # The 'assertion .* failed' is suffixed with the failure reason. E.g., # 'assertion __dest < len() failed: sample_array[] index out of bounds', # 'assertion !full() failed: sample_function() called on an empty vector' # Add the lookahead group to prevent DoS, see # https://github.com/google/clusterfuzz/issues/3978. -ASSERT_REGEX_GLIBC_SUFFIXED = re.compile( +ASSERT_REGEX_GLIBC_SUFFIXED: re.Pattern[str] = re.compile( r'(?=.*assertion .* failed:.).*.*\S.*\/.*:\d+:\s*assertion .* failed:\s*(\S.*)' # pylint: disable=line-too-long ) -ASSERT_NOT_REACHED_REGEX = re.compile(r'^\s*SHOULD NEVER BE REACHED\s*$') -CENTIPEDE_TIMEOUT_REGEX = re.compile(r'(?:%s)' % '|'.join([ +ASSERT_NOT_REACHED_REGEX: re.Pattern[str] = re.compile( + r'^\s*SHOULD NEVER BE REACHED\s*$') +CENTIPEDE_TIMEOUT_REGEX: re.Pattern[str] = re.compile(r'(?:%s)' % '|'.join([ r'========= Timeout of \d+ seconds exceeded; exiting', r'========= Per-input timeout exceeded:' ])) -CENTIPEDE_STACK_LIMIT_REGEX = re.compile( +CENTIPEDE_STACK_LIMIT_REGEX: re.Pattern[str] = re.compile( r'^========= Stack limit exceeded: \d+ > \d+ \(byte\); aborting$') -CFI_ERROR_REGEX = re.compile( +CFI_ERROR_REGEX: re.Pattern[str] = re.compile( r'(.*): runtime error: control flow integrity check for type (.*) ' r'failed during (.*vtable address ([xX0-9a-fA-F]+)|.*)') -CFI_INVALID_DOWNCAST_REGEX = re.compile(r'.*note: vtable is of type (.*)') -CFI_INVALID_VPTR_REGEX = re.compile(r'.*note: invalid vtable$') -CFI_FUNC_DEFINED_HERE_REGEX = re.compile(r'.*note: .* defined here$') -CFI_NODEBUG_ERROR_MARKER_REGEX = re.compile( +CFI_INVALID_DOWNCAST_REGEX: re.Pattern[str] = re.compile( + r'.*note: vtable is of type (.*)') +CFI_INVALID_VPTR_REGEX: re.Pattern[str] = re.compile(r'.*note: invalid vtable$') +CFI_FUNC_DEFINED_HERE_REGEX: re.Pattern[str] = re.compile( + r'.*note: .* defined here$') +CFI_NODEBUG_ERROR_MARKER_REGEX: re.Pattern[str] = re.compile( r'CFI: Most likely a control flow integrity violation;.*') -CHROME_CHECK_FAILURE_REGEX = re.compile( +CHROME_CHECK_FAILURE_REGEX: re.Pattern[str] = re.compile( r'\s*\[[^\]]*[:]([^\](]*\([0-9]+\)|[^\]:]*[:][0-9]+).*\].*(?:Check failed:|DCHECK failed:|NOTREACHED hit.)\s*([^;]*)' # pylint: disable=line-too-long ) -CHROME_STACK_FRAME_REGEX = re.compile( +CHROME_STACK_FRAME_REGEX: re.Pattern[str] = re.compile( r'[ ]*(#(?P[0-9]+)[ ]' # frame id (2) r'([xX0-9a-fA-F]+)[ ])' # addr (3) r'([^/\\]+)$') # rest, usually fun (4); may have off -CHROME_WIN_STACK_FRAME_REGEX = re.compile( +CHROME_WIN_STACK_FRAME_REGEX: re.Pattern[str] = re.compile( r'[ ]*([^/\\]+) ' # fun (1) r'\[([xX0-9a-fA-F]+)\+' # fun_base (2) r'(\d+)\]' # off[dec] (3) r'( \((.*):(\d+)\))?') # if available, file (5) and line (6) -CHROME_MAC_STACK_FRAME_REGEX = re.compile( +CHROME_MAC_STACK_FRAME_REGEX: re.Pattern[str] = re.compile( r'(?P\d+)\s+' # frame id (1) r'(([\w ]+)|(\?\?\?))\s+' # image (2) r'([xX0-9a-fA-F]+)\s+' # addr[hex] (5) r'([^/\\]+)\s*\+\s*' # fun (6) r'(\d+)') # off[dec] (7) -MSAN_TSAN_REGEX = re.compile( +MSAN_TSAN_REGEX: re.Pattern[str] = re.compile( r'.*(ThreadSanitizer|MemorySanitizer):\s+(?!ABRT)(?!ILL)([^(:]+)') -EXTRA_SANITIZERS_COMMAND_INJECTION_REGEX = re.compile( +EXTRA_SANITIZERS_COMMAND_INJECTION_REGEX: re.Pattern[str] = re.compile( r'===BUG DETECTED: Shell (corruption|injection)===') -EXTRA_SANITIZERS_ARBITRARY_FILE_OPEN_REGEX = re.compile( +EXTRA_SANITIZERS_ARBITRARY_FILE_OPEN_REGEX: re.Pattern[str] = re.compile( r'===BUG DETECTED: Arbitrary file open===') -EXTRA_SANITIZERS_ARBITRARY_DNS = re.compile( +EXTRA_SANITIZERS_ARBITRARY_DNS: re.Pattern[str] = re.compile( r'===BUG DETECTED: Arbitrary domain name resolution===') -EXTRA_SANITIZERS_PYSECSAN = re.compile(r'===BUG DETECTED: PySecSan:') -FATAL_ERROR_GENERIC_FAILURE = re.compile(r'#\s+()(.*)') -FATAL_ERROR_CHECK_FAILURE = re.compile( +EXTRA_SANITIZERS_PYSECSAN: re.Pattern[str] = re.compile( + r'===BUG DETECTED: PySecSan:') +FATAL_ERROR_GENERIC_FAILURE: re.Pattern[str] = re.compile(r'#\s+()(.*)') +FATAL_ERROR_CHECK_FAILURE: re.Pattern[str] = re.compile( r'#\s+(Check failed: |RepresentationChangerError: node #\d+:)(.*)') -FATAL_ERROR_DCHECK_FAILURE = re.compile(r'#\s+(Debug check failed: )(.*)') -FATAL_ERROR_REGEX = re.compile(r'#\s*Fatal error(?: in (.*))?$') -FATAL_ERROR_LINE_REGEX = re.compile(r'#\s*Fatal error in (.*), line [0-9]+') -FATAL_ERROR_UNREACHABLE = re.compile(r'# un(reachable|implemented) code') -FUZZER_DIR_REGEX = re.compile(r'^\s*#\d 0x.*(?:fuzzer|fuzz/|/fuzz)', - re.IGNORECASE) -FUZZER_EXIT_REGEX = re.compile(r'^\s*(?:#0|#1) 0x.*(?:fuzzer|fuzz/|/fuzz)', - re.IGNORECASE) -GENERIC_SEGV_HANDLER_REGEX = re.compile( +FATAL_ERROR_DCHECK_FAILURE: re.Pattern[str] = re.compile( + r'#\s+(Debug check failed: )(.*)') +FATAL_ERROR_REGEX: re.Pattern[str] = re.compile( + r'#\s*Fatal error(?: in (.*))?$') +FATAL_ERROR_LINE_REGEX: re.Pattern[str] = re.compile( + r'#\s*Fatal error in (.*), line [0-9]+') +FATAL_ERROR_UNREACHABLE: re.Pattern[str] = re.compile( + r'# un(reachable|implemented) code') +FUZZER_DIR_REGEX: re.Pattern[str] = re.compile( + r'^\s*#\d 0x.*(?:fuzzer|fuzz/|/fuzz)', re.IGNORECASE) +FUZZER_EXIT_REGEX: re.Pattern[str] = re.compile( + r'^\s*(?:#0|#1) 0x.*(?:fuzzer|fuzz/|/fuzz)', re.IGNORECASE) +GENERIC_SEGV_HANDLER_REGEX: re.Pattern[str] = re.compile( 'Received signal 11 (?:SEGV_[A-Z]+|) ([0-9a-f]*)') -GOOGLE_CHECK_FAILURE_REGEX = re.compile(GOOGLE_LOG_FATAL_PREFIX + - r'\s*Check failed[:]\s*(.*)') -GOOGLE_LOG_FATAL_REGEX = re.compile(GOOGLE_LOG_FATAL_PREFIX + r'\s*(.*)') -GPU_PROCESS_FAILURE = re.compile(r'.*GPU process exited unexpectedly.*') -HWASAN_ALLOCATION_TAIL_OVERWRITTEN_ADDRESS_REGEX = re.compile( +GOOGLE_CHECK_FAILURE_REGEX: re.Pattern[str] = re.compile( + GOOGLE_LOG_FATAL_PREFIX + r'\s*Check failed[:]\s*(.*)') +GOOGLE_LOG_FATAL_REGEX: re.Pattern[str] = re.compile(GOOGLE_LOG_FATAL_PREFIX + + r'\s*(.*)') +GPU_PROCESS_FAILURE: re.Pattern[str] = re.compile( + r'.*GPU process exited unexpectedly.*') +HWASAN_ALLOCATION_TAIL_OVERWRITTEN_ADDRESS_REGEX: re.Pattern[str] = re.compile( r'.*ERROR: HWAddressSanitizer: allocation-tail-overwritten; ' r'heap object \[([xX0-9a-fA-F]+),.*of size') -JAZZER_JAVA_SECURITY_EXCEPTION_REGEX = re.compile( +JAZZER_JAVA_SECURITY_EXCEPTION_REGEX: re.Pattern[str] = re.compile( '== Java Exception: .*FuzzerSecurityIssue') -JAZZER_JAVA_EXCEPTION_REGEX = re.compile('== Java Exception: .*') -JAVA_EXCEPTION_CRASH_STATE_REGEX = re.compile(r'\s*at (.*)\(.*\)') -KERNEL_BUG = re.compile(r'kernel BUG at (.*)') -KASAN_ACCESS_TYPE_REGEX = re.compile(r'(Read|Write) of size ([0-9]+)') -KASAN_ACCESS_TYPE_ADDRESS_REGEX = re.compile( +JAZZER_JAVA_EXCEPTION_REGEX: re.Pattern[str] = re.compile( + '== Java Exception: .*') +JAVA_EXCEPTION_CRASH_STATE_REGEX: re.Pattern[str] = re.compile( + r'\s*at (.*)\(.*\)') +KERNEL_BUG: re.Pattern[str] = re.compile(r'kernel BUG at (.*)') +KASAN_ACCESS_TYPE_REGEX: re.Pattern[str] = re.compile( + r'(Read|Write) of size ([0-9]+)') +KASAN_ACCESS_TYPE_ADDRESS_REGEX: re.Pattern[str] = re.compile( r'(Read|Write) of size ([0-9]+) at (addr|address) ([a-f0-9]+)') -KASAN_CRASH_TYPE_ADDRESS_REGEX = re.compile( +KASAN_CRASH_TYPE_ADDRESS_REGEX: re.Pattern[str] = re.compile( r'BUG: KASAN: (.*) (in|on).*(addr|address) ([a-f0-9]+)') -KASAN_CRASH_TYPE_ADDRESS_RANGE_REGEX = re.compile( +KASAN_CRASH_TYPE_ADDRESS_RANGE_REGEX: re.Pattern[str] = re.compile( r'KASAN: (.*?) (in|on) range \[([a-z0-9]+)-([a-z0-9]+)\]') -KASAN_CRASH_TYPE_FUNCTION_REGEX = re.compile( +KASAN_CRASH_TYPE_FUNCTION_REGEX: re.Pattern[str] = re.compile( r'BUG: KASAN: (.*) (in|on).* ([\w]+)\+([\w]+)\/([\w]+)') -KASAN_GPF_REGEX = re.compile(r'general protection fault:.*KASAN') -KERNEL_PANIC = re.compile(r'Kernel panic - not syncing: (.*)') -LIBFUZZER_DEADLY_SIGNAL_REGEX = re.compile( +KASAN_GPF_REGEX: re.Pattern[str] = re.compile( + r'general protection fault:.*KASAN') +KERNEL_PANIC: re.Pattern[str] = re.compile(r'Kernel panic - not syncing: (.*)') +LIBFUZZER_DEADLY_SIGNAL_REGEX: re.Pattern[str] = re.compile( r'.*ERROR:\s*libFuzzer:\s*deadly signal') -LIBFUZZER_FUZZ_TARGET_EXITED_REGEX = re.compile( +LIBFUZZER_FUZZ_TARGET_EXITED_REGEX: re.Pattern[str] = re.compile( r'.*ERROR:\s*libFuzzer:\s*fuzz target exited') -LIBFUZZER_OVERWRITES_CONST_INPUT_REGEX = re.compile( +LIBFUZZER_OVERWRITES_CONST_INPUT_REGEX: re.Pattern[str] = re.compile( r'.*ERROR:\s*libFuzzer:\s*fuzz target overwrites its const input') -LIBFUZZER_TIMEOUT_REGEX = re.compile(r'.*ERROR:\s*libFuzzer:\s*timeout') -LIBRARY_NOT_FOUND_ANDROID_REGEX = re.compile( +LIBFUZZER_TIMEOUT_REGEX: re.Pattern[str] = re.compile( + r'.*ERROR:\s*libFuzzer:\s*timeout') +LIBRARY_NOT_FOUND_ANDROID_REGEX: re.Pattern[str] = re.compile( r'.*: library ([`\'"])(.*)\1 not found') -LIBRARY_NOT_FOUND_LINUX_REGEX = re.compile( +LIBRARY_NOT_FOUND_LINUX_REGEX: re.Pattern[str] = re.compile( r'.*error while loading shared libraries: ([^:]*): ' r'cannot open shared object file') -LINUX_GDB_CRASH_TYPE_REGEX = re.compile(r'Program received signal ([a-zA-Z]+),') -LINUX_GDB_CRASH_ADDRESS_REGEX = re.compile(r'rip[ ]+([xX0-9a-fA-F]+)') -LINUX_GDB_CRASH_ADDRESS_NO_REGISTERS_REGEX = re.compile( +LINUX_GDB_CRASH_TYPE_REGEX: re.Pattern[str] = re.compile( + r'Program received signal ([a-zA-Z]+),') +LINUX_GDB_CRASH_ADDRESS_REGEX: re.Pattern[str] = re.compile( + r'rip[ ]+([xX0-9a-fA-F]+)') +LINUX_GDB_CRASH_ADDRESS_NO_REGISTERS_REGEX: re.Pattern[str] = re.compile( r'^(0[xX][0-9a-fA-F]+)\s+in\s+') -LSAN_DIRECT_LEAK_REGEX = re.compile(r'Direct leak of ') -LSAN_INDIRECT_LEAK_REGEX = re.compile(r'Indirect leak of ') -MAC_GDB_CRASH_ADDRESS_REGEX = re.compile( +LSAN_DIRECT_LEAK_REGEX: re.Pattern[str] = re.compile(r'Direct leak of ') +LSAN_INDIRECT_LEAK_REGEX: re.Pattern[str] = re.compile(r'Indirect leak of ') +MAC_GDB_CRASH_ADDRESS_REGEX: re.Pattern[str] = re.compile( r'Reason:.*at address[^0-9]*([0-9a-zA-Z]+)') -OUT_OF_MEMORY_REGEX = re.compile(r'.*(?:%s).*' % '|'.join([ +OUT_OF_MEMORY_REGEX: re.Pattern[str] = re.compile(r'.*(?:%s).*' % '|'.join([ r'# Allocation failed.*out of memory', r'::OnNoMemory', r'ERROR.*Sanitizer failed to allocate', @@ -199,32 +220,35 @@ r'A device memory allocation has failed\.', # To detect gpu OOM errors. r'__rust_alloc_error_handler', # Rust allocation failure on OOM. ])) -RUNTIME_ERROR_REGEX = re.compile(r'#\s*Runtime error in (.*)') -RUNTIME_ERROR_LINE_REGEX = re.compile(r'#\s*Runtime error in (.*), line [0-9]+') -RUST_ASSERT_REGEX = re.compile(r'thread\s.*\spanicked at \'([^\']*)', - re.IGNORECASE) -SAN_ABRT_REGEX = re.compile(r'.*[a-zA-Z]+Sanitizer: ABRT ') -SAN_BREAKPOINT_REGEX = re.compile(r'.*[a-zA-Z]+Sanitizer: breakpoint ') -SAN_CHECK_FAILURE_REGEX = re.compile( +RUNTIME_ERROR_REGEX: re.Pattern[str] = re.compile(r'#\s*Runtime error in (.*)') +RUNTIME_ERROR_LINE_REGEX: re.Pattern[str] = re.compile( + r'#\s*Runtime error in (.*), line [0-9]+') +RUST_ASSERT_REGEX: re.Pattern[str] = re.compile( + r'thread\s.*\spanicked at \'([^\']*)', re.IGNORECASE) +SAN_ABRT_REGEX: re.Pattern[str] = re.compile(r'.*[a-zA-Z]+Sanitizer: ABRT ') +SAN_BREAKPOINT_REGEX: re.Pattern[str] = re.compile( + r'.*[a-zA-Z]+Sanitizer: breakpoint ') +SAN_CHECK_FAILURE_REGEX: re.Pattern[str] = re.compile( r'.*Sanitizer CHECK failed[:]\s*[^ ]*\s*(.*)') -SAN_CRASH_TYPE_ADDRESS_REGEX = re.compile( +SAN_CRASH_TYPE_ADDRESS_REGEX: re.Pattern[str] = re.compile( r'[ ]*([^ ]*|Atomic [^ ]*) of size ([^ ]*) at ([^ ]*)') -SAN_DEADLYSIGNAL_REGEX = re.compile( +SAN_DEADLYSIGNAL_REGEX: re.Pattern[str] = re.compile( r'(Address|Leak|Memory|UndefinedBehavior|Thread)Sanitizer:DEADLYSIGNAL') # Use the lookahead group to prevent DoS, see # https://github.com/google/clusterfuzz/issues/3978. -CONCATENATED_SAN_DEADLYSIGNAL_REGEX = re.compile( - r'\n(?=.*Sanitizer\:DEADLYSIGNAL.*)([^\n]*\S[^\n]*)(' + - SAN_DEADLYSIGNAL_REGEX.pattern + r')\n') -SPLIT_CONCATENATED_SAN_DEADLYSIGNAL_REGEX = r'\n\1\n\2\n' -SAN_FPE_REGEX = re.compile(r'.*[a-zA-Z]+Sanitizer: FPE ') -SAN_ILL_REGEX = re.compile(r'.*[a-zA-Z]+Sanitizer: ILL ') -SAN_TRAP_REGEX = re.compile(r'.*[a-zA-Z]+Sanitizer: TRAP ') -SAN_SEGV_CRASH_TYPE_REGEX = re.compile( +CONCATENATED_SAN_DEADLYSIGNAL_REGEX: re.Pattern[ + str] = re.compile(r'\n(?=.*Sanitizer\:DEADLYSIGNAL.*)([^\n]*\S[^\n]*)(' + + SAN_DEADLYSIGNAL_REGEX.pattern + r')\n') +SPLIT_CONCATENATED_SAN_DEADLYSIGNAL_REGEX: str = r'\n\1\n\2\n' +SAN_FPE_REGEX: re.Pattern[str] = re.compile(r'.*[a-zA-Z]+Sanitizer: FPE ') +SAN_ILL_REGEX: re.Pattern[str] = re.compile(r'.*[a-zA-Z]+Sanitizer: ILL ') +SAN_TRAP_REGEX: re.Pattern[str] = re.compile(r'.*[a-zA-Z]+Sanitizer: TRAP ') +SAN_SEGV_CRASH_TYPE_REGEX: re.Pattern[str] = re.compile( r'.*The signal is caused by a ([A-Z]+) memory access.') # FIXME: Replace when better ways to check signal crashes are available. -SAN_SIGNAL_REGEX = re.compile(r'.*SCARINESS: (\d+) \(signal\)', re.DOTALL) -SAN_STACK_FRAME_REGEX = re.compile( +SAN_SIGNAL_REGEX: re.Pattern[str] = re.compile(r'.*SCARINESS: (\d+) \(signal\)', + re.DOTALL) +SAN_STACK_FRAME_REGEX: re.Pattern[str] = re.compile( # frame id (1) r'\s*#(?P\d+)\s+' # addr (2) @@ -247,116 +271,133 @@ r'|' r'(\((((.*)\+([xX0-9a-fA-F]+))|(.*))\))' r')') -SAN_ADDR_REGEX = re.compile(r'.*(ERROR: [a-zA-Z]+Sanitizer)[: ]*(.*) on ' - r'(unknown address |address |)([xX0-9a-fA-F]+)') -SAN_SEGV_REGEX = re.compile(r'.*([a-zA-Z]+Sanitizer).*(SEGV|access-violation) ' - r'on unknown address ([xX0-9a-fA-F]+)') -SECURITY_CHECK_FAILURE_REGEX = re.compile( +SAN_ADDR_REGEX: re.Pattern[str] = re.compile( + r'.*(ERROR: [a-zA-Z]+Sanitizer)[: ]*(.*) on ' + r'(unknown address |address |)([xX0-9a-fA-F]+)') +SAN_SEGV_REGEX: re.Pattern[str] = re.compile( + r'.*([a-zA-Z]+Sanitizer).*(SEGV|access-violation) ' + r'on unknown address ([xX0-9a-fA-F]+)') +SECURITY_CHECK_FAILURE_REGEX: re.Pattern[str] = re.compile( r'.*\[[^\]]*[:]([^\](]*).*\].*Security CHECK failed[:]\s*(.*)\.\s*') -SECURITY_DCHECK_FAILURE_REGEX = re.compile( +SECURITY_DCHECK_FAILURE_REGEX: re.Pattern[str] = re.compile( r'.*\[[^\]]*[:]([^\](]*).*\].*Security DCHECK failed[:]\s*(.*)\.\s*') -SYMBOL_NOT_FOUND_REGEX = re.compile( +SYMBOL_NOT_FOUND_REGEX: re.Pattern[str] = re.compile( r'.*: cannot locate symbol ([`\'"])(.*)\1 referenced by') -TRUSTY_STACK_FRAME_REGEX = re.compile( +TRUSTY_STACK_FRAME_REGEX: re.Pattern[str] = re.compile( r'(uSP)\+([a-zA-Z0-9]{6}): (0x[a-fA-F0-9]{16}) in (\w+)') -UBSAN_ASSUMPTION_VIOLATION = re.compile( +UBSAN_ASSUMPTION_VIOLATION: re.Pattern[str] = re.compile( r'.*assumption is violated during execution.*') -UBSAN_DIVISION_BY_ZERO_REGEX = re.compile(r'.*division by zero.*') -UBSAN_FLOAT_CAST_OVERFLOW_REGEX = re.compile(r'.*outside the range of ' - r'representable values.*') -UBSAN_IMPLICIT_CONVERSION_REGEX = re.compile( +UBSAN_DIVISION_BY_ZERO_REGEX: re.Pattern[str] = re.compile( + r'.*division by zero.*') +UBSAN_FLOAT_CAST_OVERFLOW_REGEX: re.Pattern[str] = re.compile( + r'.*outside the range of ' + r'representable values.*') +UBSAN_IMPLICIT_CONVERSION_REGEX: re.Pattern[str] = re.compile( r'.*implicit conversion from type.*') -UBSAN_INCORRECT_FUNCTION_POINTER_REGEX = re.compile( +UBSAN_INCORRECT_FUNCTION_POINTER_REGEX: re.Pattern[str] = re.compile( r'.*call to function [^\s]+ through pointer to incorrect function type.*') -UBSAN_INDEX_OOB_REGEX = re.compile(r'.*out of bounds for type.*') -UBSAN_UNSIGNED_INTEGER_OVERFLOW_REGEX = re.compile( +UBSAN_INDEX_OOB_REGEX: re.Pattern[str] = re.compile( + r'.*out of bounds for type.*') +UBSAN_UNSIGNED_INTEGER_OVERFLOW_REGEX: re.Pattern[str] = re.compile( r'.*unsigned integer overflow.*') -UBSAN_INTEGER_OVERFLOW_REGEX = re.compile( +UBSAN_INTEGER_OVERFLOW_REGEX: re.Pattern[str] = re.compile( r'.*(integer overflow|' r'(negation|division) of.*cannot be represented in type).*') -UBSAN_INVALID_BOOL_VALUE_REGEX = re.compile( +UBSAN_INVALID_BOOL_VALUE_REGEX: re.Pattern[str] = re.compile( r'.*not a valid value for type \'(bool|BOOL)\'.*') -UBSAN_INVALID_BUILTIN_REGEX = re.compile(r'.*, which is not a valid argument.*') -UBSAN_INVALID_ENUM_VALUE_REGEX = re.compile(r'.*not a valid value for type.*') -UBSAN_MISALIGNED_ADDRESS_REGEX = re.compile(r'.*misaligned address.*') -UBSAN_NO_RETURN_VALUE_REGEX = re.compile( +UBSAN_INVALID_BUILTIN_REGEX: re.Pattern[str] = re.compile( + r'.*, which is not a valid argument.*') +UBSAN_INVALID_ENUM_VALUE_REGEX: re.Pattern[str] = re.compile( + r'.*not a valid value for type.*') +UBSAN_MISALIGNED_ADDRESS_REGEX: re.Pattern[str] = re.compile( + r'.*misaligned address.*') +UBSAN_NO_RETURN_VALUE_REGEX: re.Pattern[str] = re.compile( r'.*reached the end of a value-returning function.*') -UBSAN_NULL_ARGUMENT_REGEX = re.compile( +UBSAN_NULL_ARGUMENT_REGEX: re.Pattern[str] = re.compile( r'.*null pointer passed as .*, which is declared to never be null.*') -UBSAN_NULL_POINTER_READ_REGEX = re.compile(r'.*load of null pointer.*') -UBSAN_NULL_POINTER_REFERENCE_REGEX = re.compile( +UBSAN_NULL_POINTER_READ_REGEX: re.Pattern[str] = re.compile( + r'.*load of null pointer.*') +UBSAN_NULL_POINTER_REFERENCE_REGEX: re.Pattern[str] = re.compile( r'.*(binding to|access within|call on) null pointer.*') -UBSAN_NULL_POINTER_WRITE_REGEX = re.compile(r'.*store to null pointer.*') -UBSAN_OBJECT_SIZE_REGEX = re.compile( +UBSAN_NULL_POINTER_WRITE_REGEX: re.Pattern[str] = re.compile( + r'.*store to null pointer.*') +UBSAN_OBJECT_SIZE_REGEX: re.Pattern[str] = re.compile( r'.*address .* with insufficient space for an object of type.*') -UBSAN_POINTER_OVERFLOW_REGEX = re.compile( +UBSAN_POINTER_OVERFLOW_REGEX: re.Pattern[str] = re.compile( r'.*((addition|subtraction) of unsigned offset |' r'pointer index expression with base |' r'applying non-zero offset [0-9]+ to null pointer|' r'applying zero offset to null pointer).*') -UBSAN_RETURNS_NONNULL_ATTRIBUTE_REGEX = re.compile( +UBSAN_RETURNS_NONNULL_ATTRIBUTE_REGEX: re.Pattern[str] = re.compile( r'.*null pointer returned from function declared to never return null.*') -UBSAN_RUNTIME_ERROR_REGEX = re.compile(r'(.*): runtime error: (.*)') -UBSAN_SHIFT_ERROR_REGEX = re.compile(r'.*shift.*') -UBSAN_UNREACHABLE_REGEX = re.compile( +UBSAN_RUNTIME_ERROR_REGEX: re.Pattern[str] = re.compile( + r'(.*): runtime error: (.*)') +UBSAN_SHIFT_ERROR_REGEX: re.Pattern[str] = re.compile(r'.*shift.*') +UBSAN_UNREACHABLE_REGEX: re.Pattern[str] = re.compile( r'.*execution reached an unreachable program point.*') -UBSAN_UPCAST_OF_NULL_POINTER = re.compile(r'.*upcast of null pointer of type.*') -UBSAN_VLA_BOUND_REGEX = re.compile( +UBSAN_UPCAST_OF_NULL_POINTER: re.Pattern[str] = re.compile( + r'.*upcast of null pointer of type.*') +UBSAN_VLA_BOUND_REGEX: re.Pattern[str] = re.compile( r'.*variable length array bound evaluates to non-positive value.*') -UBSAN_VPTR_REGEX = re.compile( +UBSAN_VPTR_REGEX: re.Pattern[str] = re.compile( r'(.*): runtime error: ' r'(member access within|member call on|downcast of)' r' address ([xX0-9a-fA-F]+) .* of type (.*)') -UBSAN_VPTR_INVALID_DOWNCAST_REGEX = re.compile( +UBSAN_VPTR_INVALID_DOWNCAST_REGEX: re.Pattern[str] = re.compile( r'.*note: object is of type (.*)') -UBSAN_VPTR_INVALID_OFFSET_REGEX = re.compile( +UBSAN_VPTR_INVALID_OFFSET_REGEX: re.Pattern[str] = re.compile( r'.*at offset (\d+) within object of type (.*)') -UBSAN_VPTR_INVALID_VPTR_REGEX = re.compile(r'.*note: object has invalid vptr') -V8_ABORT_FAILURE_REGEX = re.compile( +UBSAN_VPTR_INVALID_VPTR_REGEX: re.Pattern[str] = re.compile( + r'.*note: object has invalid vptr') +V8_ABORT_FAILURE_REGEX: re.Pattern[str] = re.compile( r'^abort: (CSA_(?:ASSERT|DCHECK) failed: .*)') -V8_ABORT_METADATA_REGEX = re.compile(r'(.*?) \[(.*):\d+\]$') -V8_CORRECTNESS_FAILURE_REGEX = re.compile(r'#\s*V8 correctness failure') -V8_CORRECTNESS_METADATA_REGEX = re.compile( +V8_ABORT_METADATA_REGEX: re.Pattern[str] = re.compile(r'(.*?) \[(.*):\d+\]$') +V8_CORRECTNESS_FAILURE_REGEX: re.Pattern[str] = re.compile( + r'#\s*V8 correctness failure') +V8_CORRECTNESS_METADATA_REGEX: re.Pattern[str] = re.compile( r'#\s*V8 correctness ((configs|sources|suppression): .*)') -V8_ERROR_REGEX = re.compile(r'\s*\[[^\]]*\] V8 error: (.+)\.$') -V8_SANDBOX_VIOLATION_REGEX = re.compile(r'## V8 sandbox violation detected!$') -WINDOWS_CDB_STACK_FRAME_REGEX = re.compile( +V8_ERROR_REGEX: re.Pattern[str] = re.compile(r'\s*\[[^\]]*\] V8 error: (.+)\.$') +V8_SANDBOX_VIOLATION_REGEX: re.Pattern[str] = re.compile( + r'## V8 sandbox violation detected!$') +WINDOWS_CDB_STACK_FRAME_REGEX: re.Pattern[str] = re.compile( r'([0-9a-zA-Z`]+) ' # Child EBP or SP; remove ` if needed (1) r'([0-9a-zA-Z`]+) ' # RetAddr; remove ` if needed (2) r'([0-9a-zA-Z_]+)' # mod (3) r'!(.*)\+' # fun (4) r'([xX0-9a-fA-F]+)') # off (5) -WINDOWS_CDB_STACK_START_REGEX = re.compile(r'ChildEBP RetAddr') -WINDOWS_CDB_CRASH_TYPE_ADDRESS_REGEX = re.compile( +WINDOWS_CDB_STACK_START_REGEX: re.Pattern[str] = re.compile(r'ChildEBP RetAddr') +WINDOWS_CDB_CRASH_TYPE_ADDRESS_REGEX: re.Pattern[str] = re.compile( r'Attempt to (.*) [^ ]* address (.*)') -WINDOWS_CDB_CRASH_TYPE_REGEX = re.compile( +WINDOWS_CDB_CRASH_TYPE_REGEX: re.Pattern[str] = re.compile( r'.*DEFAULT_BUCKET_ID[ ]*[:][ ]*([a-zA-Z_]+)') -WINDOWS_CDB_STACK_OVERFLOW_REGEX = re.compile( +WINDOWS_CDB_STACK_OVERFLOW_REGEX: re.Pattern[str] = re.compile( r'.*ExceptionCode: .*\(Stack overflow\).*') -WINDOWS_SAN_ILL_REGEX = re.compile(r'.*[a-zA-Z]+Sanitizer: illegal-instruction') +WINDOWS_SAN_ILL_REGEX: re.Pattern[str] = re.compile( + r'.*[a-zA-Z]+Sanitizer: illegal-instruction') -WYCHEPROOF_JAVA_EXCEPTION = re.compile( +WYCHEPROOF_JAVA_EXCEPTION: re.Pattern[str] = re.compile( r'.*\) (.*\(com\.google\.security\.wycheproof\.[a-zA-Z0-9]*\))') # Golang specific regular expressions. -GOLANG_DIVISION_BY_ZERO_REGEX = re.compile( +GOLANG_DIVISION_BY_ZERO_REGEX: re.Pattern[str] = re.compile( r'^(?:.* )?panic( in .*)?: runtime error: integer divide by zero.*', re.IGNORECASE) -GOLANG_INDEX_OUT_OF_RANGE_REGEX = re.compile( +GOLANG_INDEX_OUT_OF_RANGE_REGEX: re.Pattern[str] = re.compile( r'^(?:.* )?panic( in .*)?: runtime error: index out of range.*', re.IGNORECASE) -GOLANG_INVALID_MEMORY_ADDRESS_REGEX = re.compile( +GOLANG_INVALID_MEMORY_ADDRESS_REGEX: re.Pattern[str] = re.compile( r'^(?:.* )?panic( in .*)?: runtime error: invalid memory address.*', re.IGNORECASE) -GOLANG_MAKESLICE_LEN_OUT_OF_RANGE_REGEX = re.compile( +GOLANG_MAKESLICE_LEN_OUT_OF_RANGE_REGEX: re.Pattern[str] = re.compile( r'^(?:.* )?panic( in .*)?: runtime error: makeslice: len out of range.*', re.IGNORECASE) -GOLANG_SLICE_BOUNDS_OUT_OF_RANGE_REGEX = re.compile( +GOLANG_SLICE_BOUNDS_OUT_OF_RANGE_REGEX: re.Pattern[str] = re.compile( r'^(?:.* )?panic( in .*)?: runtime error: slice bounds out of range.*', re.IGNORECASE) -GOLANG_STACK_OVERFLOW_REGEX = re.compile(r'^fatal error: stack overflow.*') +GOLANG_STACK_OVERFLOW_REGEX: re.Pattern[str] = re.compile( + r'^fatal error: stack overflow.*') -GOLANG_CRASH_TYPES_MAP = [ +GOLANG_CRASH_TYPES_MAP: list[tuple[re.Pattern[str], str]] = [ (GOLANG_DIVISION_BY_ZERO_REGEX, 'Integer divide by zero'), (GOLANG_INDEX_OUT_OF_RANGE_REGEX, 'Index out of range'), (GOLANG_INVALID_MEMORY_ADDRESS_REGEX, 'Invalid memory address'), @@ -365,33 +406,33 @@ (GOLANG_STACK_OVERFLOW_REGEX, 'Stack overflow'), ] -GOLANG_FATAL_ERROR_REGEX = re.compile(r'^fatal error: (.*)') +GOLANG_FATAL_ERROR_REGEX: re.Pattern[str] = re.compile(r'^fatal error: (.*)') -GOLANG_STACK_FRAME_FUNCTION_REGEX = re.compile( +GOLANG_STACK_FRAME_FUNCTION_REGEX: re.Pattern[str] = re.compile( r'^([0-9a-zA-Z\.\-\_\\\/\(\)\*]+)\([x0-9a-f\s,\.{}]*\)$') # Python specific regular expressions. -PYTHON_UNHANDLED_EXCEPTION = re.compile( +PYTHON_UNHANDLED_EXCEPTION: re.Pattern[str] = re.compile( r'^\s*=== Uncaught Python exception: ===$') -PYTHON_CRASH_TYPES_MAP = [ +PYTHON_CRASH_TYPES_MAP: list[tuple[re.Pattern[str], str]] = [ (PYTHON_UNHANDLED_EXCEPTION, 'Uncaught exception'), (EXTRA_SANITIZERS_PYSECSAN, 'PySecSan'), ] -PYTHON_STACK_FRAME_FUNCTION_REGEX = re.compile( +PYTHON_STACK_FRAME_FUNCTION_REGEX: re.Pattern[str] = re.compile( # File "/gzip.py", line 421, in _read_gzip_header r'^\s*File "([^"]+)", line (\d+), in (.+)$') -JAZZER_JS_UNCAUGHT_EXCEPTION = re.compile( +JAZZER_JS_UNCAUGHT_EXCEPTION: re.Pattern[str] = re.compile( r'^\s*==\d+== Uncaught Exception: Jazzer.js: .*') -JAZZER_JS_STACK_FRAME_FUNCTION_REGEX = re.compile( +JAZZER_JS_STACK_FRAME_FUNCTION_REGEX: re.Pattern[str] = re.compile( # at exploreMe (/out/example/target.ts:6:15) r'^\s* at ([^\s]+) \(.*:\d+:\d+\)') # Mappings of Android kernel error status codes to strings. -ANDROID_KERNEL_STATUS_TO_STRING = { +ANDROID_KERNEL_STATUS_TO_STRING: dict[int, str] = { 0b0001: 'Alignment Fault', 0b0100: 'Instruction Cache Maintenance Fault', 0b1100: 'L1 Translation', @@ -407,7 +448,7 @@ } # Ignore lists. -STACK_FRAME_IGNORE_REGEXES = [ +STACK_FRAME_IGNORE_REGEXES: list[str] = [ # Function names (exact match). r'^abort$', r'^exit$', @@ -652,7 +693,7 @@ r'^SkMutex::~SkMutex', # sometimes released atexit giving a secondary crash ] -STACK_FRAME_IGNORE_REGEXES_IF_SYMBOLIZED = [ +STACK_FRAME_IGNORE_REGEXES_IF_SYMBOLIZED: list[str] = [ r'.*libc\.so', r'.*libc\+\+\.so', r'.*libc\+\+_shared\.so', @@ -660,7 +701,7 @@ r'.*libc-.*\.so', ] -IGNORE_CRASH_TYPES_FOR_ABRT_BREAKPOINT_AND_ILLS = [ +IGNORE_CRASH_TYPES_FOR_ABRT_BREAKPOINT_AND_ILLS: list[str] = [ 'Arbitrary file open', 'ASSERT', 'CHECK failure', @@ -677,7 +718,7 @@ 'V8 sandbox violation', ] -STATE_STOP_MARKERS = [ +STATE_STOP_MARKERS: list[str] = [ 'Direct leak of', 'Uninitialized value was stored to memory at', 'allocated by thread', @@ -686,7 +727,7 @@ 'previously allocated by', ] -UBSAN_CRASH_TYPES_MAP = [ +UBSAN_CRASH_TYPES_MAP: list[tuple[re.Pattern[str], str]] = [ (UBSAN_DIVISION_BY_ZERO_REGEX, 'Divide-by-zero'), (UBSAN_FLOAT_CAST_OVERFLOW_REGEX, 'Float-cast-overflow'), (UBSAN_IMPLICIT_CONVERSION_REGEX, 'Implicit-conversion'), @@ -717,30 +758,32 @@ ] # Additional regexes for cleaning up format. -STRIP_STRUCTURE_REGEXES = [ +STRIP_STRUCTURE_REGEXES: list[re.Pattern[str]] = [ re.compile(r'^in (.*)'), # sanitizers have prefix for function if present re.compile(r'^\((.*)\)$'), # sanitizers wrap module if no function ] # Other constants. -LINE_LENGTH_CAP = 80 -MAX_CRASH_STATE_FRAMES = 3 -MAX_CYCLE_LENGTH = 10 -REPEATED_CYCLE_COUNT = 3 +LINE_LENGTH_CAP: int = 80 +MAX_CRASH_STATE_FRAMES: int = 3 +MAX_CYCLE_LENGTH: int = 10 +REPEATED_CYCLE_COUNT: int = 3 # Stackframe format specifications. -CHROME_STACK_FRAME_SPEC = stack_parser.StackFrameSpec( - address=3, function_name=4) -CHROME_WIN_STACK_FRAME_SPEC = stack_parser.StackFrameSpec( - function_name=1, - function_base=2, - function_offset=3, - filename=5, - fileline=6, - base=10) -CHROME_MAC_STACK_FRAME_SPEC = stack_parser.StackFrameSpec( - address=5, function_name=6, function_offset=7, module_name=2, base=10) -SAN_STACK_FRAME_SPEC = stack_parser.StackFrameSpec( +CHROME_STACK_FRAME_SPEC: stack_parser.StackFrameSpec = ( + stack_parser.StackFrameSpec(address=3, function_name=4)) +CHROME_WIN_STACK_FRAME_SPEC: stack_parser.StackFrameSpec = ( + stack_parser.StackFrameSpec( + function_name=1, + function_base=2, + function_offset=3, + filename=5, + fileline=6, + base=10)) +CHROME_MAC_STACK_FRAME_SPEC: stack_parser.StackFrameSpec = ( + stack_parser.StackFrameSpec( + address=5, function_name=6, function_offset=7, module_name=2, base=10)) +SAN_STACK_FRAME_SPEC: stack_parser.StackFrameSpec = stack_parser.StackFrameSpec( address=2, function_name=[7, 5, 23], function_offset=8, @@ -748,5 +791,6 @@ fileline=[13, 17], module_name=[19, 31], module_offset=[21, 32]) -WINDOWS_CDB_STACK_FRAME_SPEC = stack_parser.StackFrameSpec( - address=1, function_name=4, function_offset=5, module_name=3) +WINDOWS_CDB_STACK_FRAME_SPEC: stack_parser.StackFrameSpec = ( + stack_parser.StackFrameSpec( + address=1, function_name=4, function_offset=5, module_name=3)) diff --git a/src/clusterfuzz/stacktraces/crash_comparer.py b/src/clusterfuzz/stacktraces/crash_comparer.py index 8e440c643e1..eeca0055da5 100644 --- a/src/clusterfuzz/stacktraces/crash_comparer.py +++ b/src/clusterfuzz/stacktraces/crash_comparer.py @@ -12,5 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Expose crash comparer to external users.""" -# pylint: disable=unused-import from clusterfuzz._internal.crash_analysis.crash_comparer import CrashComparer + +__all__ = [ + 'CrashComparer', +] diff --git a/src/local/butler/common.py b/src/local/butler/common.py index 50844c419b6..b44051848d6 100644 --- a/src/local/butler/common.py +++ b/src/local/butler/common.py @@ -242,48 +242,20 @@ def _pip(): return 'pip3' -def _pipfile_to_requirements(pipfile_dir, requirements_path, dev=False): - """Output a requirements.txt given a locked Pipfile.""" - dev_arg = '' - if dev: - dev_arg = '--dev' - - return_code, output = execute( - f'python3.11 -m pipenv requirements {dev_arg}', - exit_on_error=False, - cwd=pipfile_dir, - extra_environments={'PIPENV_IGNORE_VIRTUALENVS': '1'}, - stderr=subprocess.DEVNULL) - if return_code != 0: - # Older pipenv version. - return_code, output = execute( - f'python3.11 -m pipenv lock -r --no-header {dev_arg}', - exit_on_error=False, - cwd=pipfile_dir, - extra_environments={'PIPENV_IGNORE_VIRTUALENVS': '1'}, - stderr=subprocess.DEVNULL) - - if return_code != 0: - raise RuntimeError('Failed to generate requirements from Pipfile.') - - with open(requirements_path, 'wb') as f: - f.write(output) - - -def _install_pip(requirements_path, target_path): - """Perform pip install using requirements_path onto target_path.""" +def _install_third_party(target_path, extra): + """Directly install packages for an extra into target_path.""" if os.path.exists(target_path): shutil.rmtree(target_path) + root_dir = os.environ.get('ROOT_DIR', '.') execute( - '{pip} install -r {requirements_path} --upgrade --target {target_path}'. - format( - pip=_pip(), - requirements_path=requirements_path, - target_path=target_path)) + f'uv export --extra {extra} --no-dev --no-hashes | ' + f'uv pip install --target {target_path} -r /dev/stdin ' + '--python 3.11 --upgrade', + cwd=root_dir) -def _install_platform_pip(requirements_path, target_path, platform_name): +def _install_platform_pip(target_path, platform_name): """Install platform specific pip packages.""" pip_platform = constants.PLATFORMS.get(platform_name) if not pip_platform: @@ -298,13 +270,17 @@ def _install_platform_pip(requirements_path, target_path, platform_name): pip_platforms = pip_platform pip_abi = constants.ABIS[platform_name] + root_dir = os.environ.get('ROOT_DIR', '.') for pip_platform in pip_platforms: temp_dir = tempfile.mkdtemp() return_code, _ = execute( + 'uv export --extra platform-specific --no-dev --no-hashes ' + '--no-header | ' f'{_pip()} download --no-deps --only-binary=:all: ' - f'--platform={pip_platform} --abi={pip_abi} -r {requirements_path} -d ' - f'{temp_dir}', + f'--platform={pip_platform} --abi={pip_abi} -r /dev/stdin ' + f'-d {temp_dir}', + cwd=root_dir, exit_on_error=False) if return_code != 0: @@ -329,18 +305,13 @@ def _remove_invalid_files(): def install_dependencies(platform_name=None): """Install dependencies for bots.""" - _pipfile_to_requirements('src', 'src/requirements.txt') - # Hack: Use "dev-packages" to specify App Engine only packages. - _pipfile_to_requirements('src', 'src/appengine/requirements.txt', dev=True) - - _install_pip('src/requirements.txt', 'src/third_party') + _install_third_party(target_path='src/third_party', extra='core') if platform_name: _install_platform_pip( - 'src/platform_requirements.txt', - 'src/third_party', - platform_name=platform_name) + target_path='src/third_party', platform_name=platform_name) - _install_pip('src/appengine/requirements.txt', 'src/appengine/third_party') + _install_third_party( + target_path='src/appengine/third_party', extra='appengine') _remove_invalid_files() execute('bower install --allow-root') diff --git a/src/local/butler/guard.py b/src/local/butler/guard.py index 516cb78488a..580b565f6b1 100644 --- a/src/local/butler/guard.py +++ b/src/local/butler/guard.py @@ -22,12 +22,12 @@ def check_virtualenv(): raise RuntimeError('Python 2 is no longer supported!') is_in_virtualenv = bool(os.getenv('VIRTUAL_ENV')) - if not is_in_virtualenv: raise RuntimeError( - 'You are not in a virtual env environment. Please install it with' - ' `./local/install_deps.bash` or load it with' - ' `pipenv shell`. Then, you can re-run this command.') + 'You are not in a virtual env environment. Please install it with ' + '`./local/install_deps.bash` or load it with ' + '`source .venv/bin/activate` (or run with `uv run butler.py`). ' + 'Then, you can re-run this command.') def check(): diff --git a/src/local/butler/py_unittest.py b/src/local/butler/py_unittest.py index d401f42764b..6f50a4a2c1e 100644 --- a/src/local/butler/py_unittest.py +++ b/src/local/butler/py_unittest.py @@ -243,11 +243,9 @@ def execute(args): test_directory = APPENGINE_TEST_DIRECTORY sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine'))) - for i, path in enumerate(sys.path): - if 'third_party' in path: - # Replace third_party with App Engine third_party/. - sys.path[i] = os.path.abspath( - os.path.join('src', 'appengine', 'third_party')) + appengine_third_party = os.path.abspath( + os.path.join('src', 'appengine', 'third_party')) + sys.path.insert(0, appengine_third_party) elif args.target == 'core': test_directory = CORE_TEST_DIRECTORY diff --git a/src/platform_requirements.txt b/src/platform_requirements.txt deleted file mode 100644 index eeb19228808..00000000000 --- a/src/platform_requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -grpcio==1.62.2 -kubernetes==34.1.0 -protobuf==4.23.4 -psutil==5.9.4 \ No newline at end of file diff --git a/typings/google/cloud/ndb/__init__.pyi b/typings/google/cloud/ndb/__init__.pyi new file mode 100644 index 00000000000..bd30f861838 --- /dev/null +++ b/typings/google/cloud/ndb/__init__.pyi @@ -0,0 +1,96 @@ +from ._datastore_api import EVENTUAL as EVENTUAL +from ._datastore_api import EVENTUAL_CONSISTENCY as EVENTUAL_CONSISTENCY +from ._datastore_api import STRONG as STRONG +from ._datastore_query import Cursor as Cursor +from ._datastore_query import QueryIterator as QueryIterator +from ._transaction import in_transaction as in_transaction +from ._transaction import transaction as transaction +from ._transaction import transaction_async as transaction_async +from .client import Client as Client +from .context import AutoBatcher as AutoBatcher +from .context import Context as Context +from .context import ContextOptions as ContextOptions +from .context import TransactionOptions as TransactionOptions +from .context import get_context as get_context +from .context import get_toplevel_context as get_toplevel_context +from .context import non_transactional as non_transactional +from .context import transactional as transactional +from .context import transactional_async as transactional_async +from .context import transactional_tasklet as transactional_tasklet +from .exceptions import BadArgumentError as BadArgumentError +from .exceptions import BadFilterError as BadFilterError +from .exceptions import BadQueryError as BadQueryError +from .exceptions import ContextError as ContextError +from .exceptions import Error as Error +from .exceptions import InternalError as InternalError +from .exceptions import Timeout as Timeout +from .global_cache import GlobalCache as GlobalCache +from .global_cache import MemcacheCache as MemcacheCache +from .global_cache import RedisCache as RedisCache +from .key import Key as Key +from .model import BadProjectionError as BadProjectionError +from .model import BlobKey as BlobKey +from .model import BlobKeyProperty as BlobKeyProperty +from .model import BlobProperty as BlobProperty +from .model import BooleanProperty as BooleanProperty +from .model import ComputedProperty as ComputedProperty +from .model import ComputedPropertyError as ComputedPropertyError +from .model import DateProperty as DateProperty +from .model import DateTimeProperty as DateTimeProperty +from .model import Expando as Expando +from .model import FloatProperty as FloatProperty +from .model import GenericProperty as GenericProperty +from .model import GeoPt as GeoPt +from .model import GeoPtProperty as GeoPtProperty +from .model import Index as Index +from .model import IndexProperty as IndexProperty +from .model import IndexState as IndexState +from .model import IntegerProperty as IntegerProperty +from .model import InvalidPropertyError as InvalidPropertyError +from .model import JsonProperty as JsonProperty +from .model import KeyProperty as KeyProperty +from .model import KindError as KindError +from .model import LocalStructuredProperty as LocalStructuredProperty +from .model import MetaModel as MetaModel +from .model import Model as Model +from .model import ModelAdapter as ModelAdapter +from .model import ModelAttribute as ModelAttribute +from .model import ModelKey as ModelKey +from .model import PickleProperty as PickleProperty +from .model import Property as Property +from .model import ReadonlyPropertyError as ReadonlyPropertyError +from .model import Rollback as Rollback +from .model import StringProperty as StringProperty +from .model import StructuredProperty as StructuredProperty +from .model import TextProperty as TextProperty +from .model import TimeProperty as TimeProperty +from .model import UnprojectedPropertyError as UnprojectedPropertyError +from .model import User as User +from .model import UserNotFoundError as UserNotFoundError +from .model import UserProperty as UserProperty +from .model import delete_multi as delete_multi +from .model import delete_multi_async as delete_multi_async +from .model import get_indexes as get_indexes +from .model import get_indexes_async as get_indexes_async +from .model import get_multi as get_multi +from .model import get_multi_async as get_multi_async +from .model import make_connection as make_connection +from .model import put_multi as put_multi +from .model import put_multi_async as put_multi_async +from .polymodel import PolyModel as PolyModel +from .query import AND as AND +from .query import ConjunctionNode as ConjunctionNode +from .query import DisjunctionNode as DisjunctionNode +from .query import FalseNode as FalseNode +from .query import FilterNode as FilterNode +from .query import Node as Node +from .query import OR as OR +from .query import ParameterNode as ParameterNode +from .query import PropertyOrder as PropertyOrder +from .query import Query as Query +from .query import gql as gql +from .tasklets import Future as Future +from .tasklets import Return as Return +from .tasklets import add_flow_variable as add_flow_variable +from .tasklets import sleep as sleep +from .tasklets import tasklet as tasklet diff --git a/typings/google/cloud/ndb/_datastore_api.pyi b/typings/google/cloud/ndb/_datastore_api.pyi new file mode 100644 index 00000000000..55a04b05171 --- /dev/null +++ b/typings/google/cloud/ndb/_datastore_api.pyi @@ -0,0 +1,3 @@ +EVENTUAL: str +EVENTUAL_CONSISTENCY: str +STRONG: str diff --git a/typings/google/cloud/ndb/_datastore_query.pyi b/typings/google/cloud/ndb/_datastore_query.pyi new file mode 100644 index 00000000000..8c937f5cc9e --- /dev/null +++ b/typings/google/cloud/ndb/_datastore_query.pyi @@ -0,0 +1,2 @@ +from typing import Any, Generic, Iterator, Optional, TypeVar, Union +from .query import Cursor, QueryIterator diff --git a/typings/google/cloud/ndb/_transaction.pyi b/typings/google/cloud/ndb/_transaction.pyi new file mode 100644 index 00000000000..53b7edf74fb --- /dev/null +++ b/typings/google/cloud/ndb/_transaction.pyi @@ -0,0 +1,52 @@ +from typing import Any, Callable, Optional, TypeVar +from .tasklets import Future + +_T = TypeVar('_T') + +def in_transaction() -> bool: ... + +def transaction( + callback: Callable[..., _T], + retries: int = ..., + read_only: bool = ..., + join: bool = ..., + xg: bool = ..., + propagation: Any = ..., +) -> _T: ... + +def transaction_async( + callback: Callable[..., _T], + retries: int = ..., + read_only: bool = ..., + join: bool = ..., + xg: bool = ..., + propagation: Any = ..., +) -> Future[_T]: ... + +def transactional( + retries: int = ..., + read_only: bool = ..., + xg: bool = ..., + propagation: Any = ..., + join: bool = ..., +) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... + +def transactional_async( + retries: int = ..., + read_only: bool = ..., + xg: bool = ..., + propagation: Any = ..., + join: bool = ..., +) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... + +def transactional_tasklet( + retries: int = ..., + read_only: bool = ..., + xg: bool = ..., + propagation: Any = ..., + join: bool = ..., +) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... + +def non_transactional( + callback: Optional[Callable[..., _T]] = None, +) -> Any: ... diff --git a/typings/google/cloud/ndb/client.pyi b/typings/google/cloud/ndb/client.pyi new file mode 100644 index 00000000000..fc39cac057b --- /dev/null +++ b/typings/google/cloud/ndb/client.pyi @@ -0,0 +1,20 @@ +from typing import Any, Optional +from .context import Context + +class Client: + def __init__( + self, + project: Optional[str] = None, + namespace: Optional[str] = None, + database: Optional[str] = None, + credentials: Optional[Any] = None, + client_info: Optional[Any] = None, + client_options: Optional[Any] = None, + ) -> None: ... + def context( + self, + namespace: Optional[str] = None, + database: Optional[str] = None, + global_cache: Optional[Any] = None, + legacy_data: bool = True, + ) -> Context: ... diff --git a/typings/google/cloud/ndb/context.pyi b/typings/google/cloud/ndb/context.pyi new file mode 100644 index 00000000000..a3e0e3dc97b --- /dev/null +++ b/typings/google/cloud/ndb/context.pyi @@ -0,0 +1,54 @@ +from typing import Any, Callable, ContextManager, Dict, Optional, TypeVar + +_T = TypeVar('_T') + +class ContextOptions: + def __init__(self, **kwargs: Any) -> None: ... + +class TransactionOptions: + NESTED: int + MANDATORY: int + ALLOWED: int + INDEPENDENT: int + def __init__(self, **kwargs: Any) -> None: ... + +class AutoBatcher: ... + +class Context: + def __init__(self, client: Any, **kwargs: Any) -> None: ... + def use(self) -> ContextManager['Context']: ... + def __enter__(self) -> 'Context': ... + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ... + def clear_cache(self) -> None: ... + def flush(self) -> None: ... + def set_memcache_policy(self, policy: Any) -> None: ... + def set_cache_policy(self, policy: Any) -> None: ... + +class _State: + context: Optional[Context] + +_state: _State + +def get_context(raise_context_error: bool = True) -> Optional[Context]: ... +def get_toplevel_context() -> Context: ... +def transactional( + retries: int = 3, + read_only: bool = False, + propagation: Optional[int] = None, + xg: bool = False, +) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... +def transactional_async( + retries: int = 3, + read_only: bool = False, + propagation: Optional[int] = None, + xg: bool = False, +) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... +def transactional_tasklet( + retries: int = 3, + read_only: bool = False, + propagation: Optional[int] = None, + xg: bool = False, +) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... +def non_transactional( + allow_existing: bool = True, +) -> Callable[[Callable[..., _T]], Callable[..., _T]]: ... diff --git a/typings/google/cloud/ndb/exceptions.pyi b/typings/google/cloud/ndb/exceptions.pyi new file mode 100644 index 00000000000..c553119d1a9 --- /dev/null +++ b/typings/google/cloud/ndb/exceptions.pyi @@ -0,0 +1,9 @@ +class Error(Exception): ... +class ContextError(Error): ... +class BadFilterError(Error): ... +class BadArgumentError(Error): ... +class Rollback(Error): ... +class BadQueryError(Error): ... +class Timeout(Error): ... +class BadRequestError(Error): ... +class InternalError(Error): ... diff --git a/typings/google/cloud/ndb/global_cache.pyi b/typings/google/cloud/ndb/global_cache.pyi new file mode 100644 index 00000000000..215989714ce --- /dev/null +++ b/typings/google/cloud/ndb/global_cache.pyi @@ -0,0 +1,5 @@ +from typing import Any + +class GlobalCache: ... +class MemcacheCache(GlobalCache): ... +class RedisCache(GlobalCache): ... diff --git a/typings/google/cloud/ndb/key.pyi b/typings/google/cloud/ndb/key.pyi new file mode 100644 index 00000000000..88c56ce1c7c --- /dev/null +++ b/typings/google/cloud/ndb/key.pyi @@ -0,0 +1,26 @@ +from typing import Any, Iterator, List, Optional, Sequence, Tuple, Union, overload + +class Key: + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + def id(self) -> Any: ... + def string_id(self) -> Optional[str]: ... + def integer_id(self) -> Optional[int]: ... + def kind(self) -> str: ... + def parent(self) -> Optional['Key']: ... + def root(self) -> 'Key': ... + def pairs(self) -> Tuple[Tuple[str, Union[int, str]], ...]: ... + def flat(self) -> Tuple[Union[str, int], ...]: ... + def app(self) -> str: ... + def namespace(self) -> Optional[str]: ... + def urlsafe(self) -> bytes: ... + def get(self, **kwargs: Any) -> Optional[Any]: ... + def get_async(self, **kwargs: Any) -> Any: ... + def delete(self, **kwargs: Any) -> None: ... + def delete_async(self, **kwargs: Any) -> Any: ... + @classmethod + def from_old_key(cls, old_key: Any) -> 'Key': ... + def to_old_key(self) -> Any: ... + def __eq__(self, other: Any) -> bool: ... + def __ne__(self, other: Any) -> bool: ... + def __hash__(self) -> int: ... + def __repr__(self) -> str: ... diff --git a/typings/google/cloud/ndb/model.pyi b/typings/google/cloud/ndb/model.pyi new file mode 100644 index 00000000000..0172d601340 --- /dev/null +++ b/typings/google/cloud/ndb/model.pyi @@ -0,0 +1,654 @@ +import datetime +from typing import ( + Any, + Callable, + Dict, + Generic, + Iterable, + Iterator, + List, + Literal, + Optional, + Sequence, + Set, + Tuple, + Type, + TypeVar, + Union, + dataclass_transform, + overload, +) + +from .key import Key +from .query import FilterNode, PropertyOrder, Query + +_T = TypeVar('_T') +_M = TypeVar('_M', bound='Model') + +class Property(Generic[_T]): + _name: str + _code_name: str + _indexed: bool + _repeated: bool + _required: bool + _default: Any + def __init__( + self, + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> None: ... + @overload + def __get__(self: Property[_T], instance: None, owner: Any) -> Property[_T]: ... + @overload + def __get__(self: Property[_T], instance: Any, owner: Any) -> _T: ... + def __set__(self, instance: Any, value: _T) -> None: ... + def __delete__(self, instance: Any) -> None: ... + def __eq__(self, other: Any) -> FilterNode: ... # type: ignore[override] + def __ne__(self, other: Any) -> FilterNode: ... # type: ignore[override] + def __lt__(self, other: Any) -> FilterNode: ... + def __le__(self, other: Any) -> FilterNode: ... + def __gt__(self, other: Any) -> FilterNode: ... + def __ge__(self, other: Any) -> FilterNode: ... + def IN(self, values: Iterable[Any]) -> FilterNode: ... + def __neg__(self) -> PropertyOrder: ... + def __pos__(self) -> PropertyOrder: ... + +class StringProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[List[str]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[Optional[str]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class TextProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + compressed: bool = False, + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[List[str]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + compressed: bool = False, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[Optional[str]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class IntegerProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[List[int]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[Optional[int]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class FloatProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[List[float]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[Optional[float]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class BooleanProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[List[bool]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + write_empty_list: Optional[bool] = None, + ) -> Property[Optional[bool]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class DateTimeProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + auto_now: bool = False, + auto_now_add: bool = False, + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + tzinfo: Optional[datetime.tzinfo] = None, + ) -> Property[List[datetime.datetime]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + auto_now: bool = False, + auto_now_add: bool = False, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + tzinfo: Optional[datetime.tzinfo] = None, + ) -> Property[Optional[datetime.datetime]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class DateProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + auto_now: bool = False, + auto_now_add: bool = False, + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[List[datetime.date]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + auto_now: bool = False, + auto_now_add: bool = False, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[Optional[datetime.date]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class TimeProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + auto_now: bool = False, + auto_now_add: bool = False, + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[List[datetime.time]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + auto_now: bool = False, + auto_now_add: bool = False, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[Optional[datetime.time]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class JsonProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + compressed: bool = False, + json_type: Optional[Type[Any]] = None, + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[List[Any]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + compressed: bool = False, + json_type: Optional[Type[Any]] = None, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[Any]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class PickleProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + compressed: bool = False, + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[List[Any]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + compressed: bool = False, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[Any]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class BlobProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + compressed: bool = False, + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[List[bytes]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + compressed: bool = False, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[Optional[bytes]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class BlobKeyProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[List[str]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[Optional[str]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class GenericProperty(Property[Any]): + def __init__( + self, + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> None: ... + +class GeoPtProperty(Property[Any]): + def __init__( + self, + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> None: ... + +class UserProperty(Property[Any]): + def __init__( + self, + name: Optional[str] = None, + auto_current_user: bool = False, + auto_current_user_add: bool = False, + indexed: Optional[bool] = None, + repeated: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> None: ... + +class KeyProperty(Property[Any]): + @overload + def __new__( + cls, + name: Optional[str] = None, + *, + repeated: Literal[True], + kind: Optional[Union[str, Type['Model']]] = None, + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[List[Key]]: ... + @overload + def __new__( + cls, + name: Optional[str] = None, + kind: Optional[Union[str, Type['Model']]] = None, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + choices: Optional[Iterable[Any]] = None, + validator: Optional[Callable[[Any, Any], Any]] = None, + verbose_name: Optional[str] = None, + ) -> Property[Optional[Key]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class ComputedProperty(Property[_T]): + def __init__( + self, + func: Callable[[Any], _T], + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[bool] = None, + verbose_name: Optional[str] = None, + ) -> None: ... + +class StructuredProperty(Property[Any]): + @overload + def __new__( + cls, + modelclass: Type[_M], + name: Optional[str] = None, + *, + repeated: Literal[True], + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + ) -> Property[List[_M]]: ... + @overload + def __new__( + cls, + modelclass: Type[_M], + name: Optional[str] = None, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + ) -> Property[Optional[_M]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class LocalStructuredProperty(StructuredProperty): + @overload + def __new__( + cls, + modelclass: Type[_M], + name: Optional[str] = None, + *, + repeated: Literal[True], + compressed: bool = False, + indexed: Optional[bool] = None, + required: Optional[bool] = None, + default: Any = None, + ) -> Property[List[_M]]: ... + @overload + def __new__( + cls, + modelclass: Type[_M], + name: Optional[str] = None, + compressed: bool = False, + indexed: Optional[bool] = None, + repeated: Optional[Literal[False]] = None, + required: Optional[bool] = None, + default: Any = None, + ) -> Property[Optional[_M]]: ... + @overload + def __new__(cls, *args: Any, **kwargs: Any) -> Property[Any]: ... + +class ModelAdapter: ... +class ModelAttribute: ... +class ModelKey(Key): ... + +class MetaModel(type): ... + +class Model(metaclass=MetaModel): + key: Key + _kind_map: Dict[str, Type['Model']] + _properties: Dict[str, Property[Any]] + _use_cache: bool + _use_memcache: bool + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + @classmethod + def query(cls: Type[_M], *args: Any, **kwargs: Any) -> Query[_M]: ... + @classmethod + def gql(cls: Type[_M], query_string: str, *args: Any, **kwds: Any) -> Query[_M]: ... + @classmethod + def get_by_id(cls: Type[_M], id: Union[int, str], parent: Optional[Key] = None, **kwargs: Any) -> Optional[_M]: ... + @classmethod + def get_by_id_async(cls: Type[_M], id: Union[int, str], parent: Optional[Key] = None, **kwargs: Any) -> Any: ... + @classmethod + def get_or_insert(cls: Type[_M], name: str, parent: Optional[Key] = None, **kwargs: Any) -> _M: ... + @classmethod + def get_or_insert_async(cls: Type[_M], name: str, parent: Optional[Key] = None, **kwargs: Any) -> Any: ... + @classmethod + def _get_kind(cls) -> str: ... + def put(self, **kwargs: Any) -> Key: ... + def put_async(self, **kwargs: Any) -> Any: ... + def to_dict(self, include: Optional[Sequence[str]] = None, exclude: Optional[Sequence[str]] = None) -> Dict[str, Any]: ... + def populate(self, **kwargs: Any) -> None: ... + def _pre_put_hook(self) -> None: ... + def _post_put_hook(self, future: Any) -> None: ... + @classmethod + def _pre_delete_hook(cls, key: Key) -> None: ... + @classmethod + def _post_delete_hook(cls, key: Key, future: Any) -> None: ... + +class Expando(Model): ... + +class BlobKey: + def __init__(self, blob_key: str) -> None: ... + +class GeoPt: + lat: float + lon: float + def __init__(self, lat: float, lon: float) -> None: ... + +class User: + def __init__(self, email: Optional[str] = None, _auth_domain: Optional[str] = None, _user_id: Optional[str] = None) -> None: ... + def email(self) -> str: ... + def nickname(self) -> str: ... + def user_id(self) -> Optional[str]: ... + +class Index: ... +class IndexProperty: ... +class IndexState: ... + +def get_multi(keys: Sequence[Key], **kwargs: Any) -> List[Optional[Model]]: ... +def get_multi_async(keys: Sequence[Key], **kwargs: Any) -> Any: ... +def put_multi(entities: Sequence[Model], **kwargs: Any) -> List[Key]: ... +def put_multi_async(entities: Sequence[Model], **kwargs: Any) -> Any: ... +def delete_multi(keys: Sequence[Key], **kwargs: Any) -> List[None]: ... +def delete_multi_async(keys: Sequence[Key], **kwargs: Any) -> Any: ... +def get_indexes(**kwargs: Any) -> Any: ... +def get_indexes_async(**kwargs: Any) -> Any: ... +def make_connection(**kwargs: Any) -> Any: ... + +class BadProjectionError(Exception): ... +class ComputedPropertyError(Exception): ... +class InvalidPropertyError(Exception): ... +class KindError(Exception): ... +class ReadonlyPropertyError(Exception): ... +class Rollback(Exception): ... +class UnprojectedPropertyError(Exception): ... +class UserNotFoundError(Exception): ... diff --git a/typings/google/cloud/ndb/polymodel.pyi b/typings/google/cloud/ndb/polymodel.pyi new file mode 100644 index 00000000000..7b4eae04815 --- /dev/null +++ b/typings/google/cloud/ndb/polymodel.pyi @@ -0,0 +1,5 @@ +from typing import Any +from .model import Model + +class PolyModel(Model): + _class_key_: Any diff --git a/typings/google/cloud/ndb/query.pyi b/typings/google/cloud/ndb/query.pyi new file mode 100644 index 00000000000..58bf2f240e2 --- /dev/null +++ b/typings/google/cloud/ndb/query.pyi @@ -0,0 +1,66 @@ +from typing import Any, Generic, Iterable, Iterator, List, Optional, Sequence, Tuple, Type, TypeVar, Union + +_M = TypeVar('_M') + +class Node: + def __and__(self, other: 'Node') -> 'ConjunctionNode': ... + def __or__(self, other: 'Node') -> 'DisjunctionNode': ... + def __invert__(self) -> 'Node': ... + +class FalseNode(Node): ... +class ParameterNode(Node): ... + +class FilterNode(Node): + name: str + opsymbol: str + value: Any + def __init__(self, name: str, opsymbol: str, value: Any) -> None: ... + +class ConjunctionNode(Node): + nodes: List[Node] + def __init__(self, *nodes: Node) -> None: ... + +class DisjunctionNode(Node): + nodes: List[Node] + def __init__(self, *nodes: Node) -> None: ... + +def AND(*nodes: Node) -> ConjunctionNode: ... +def OR(*nodes: Node) -> DisjunctionNode: ... + +class PropertyOrder: + name: str + reverse: bool + def __init__(self, name: str, reverse: bool = False) -> None: ... + def __neg__(self) -> 'PropertyOrder': ... + +class Cursor: + def __init__(self, urlsafe: Optional[Union[bytes, str]] = None) -> None: ... + def urlsafe(self) -> bytes: ... + +class QueryIterator(Iterator[_M], Generic[_M]): + def cursor_after(self) -> Cursor: ... + def cursor_before(self) -> Cursor: ... + def __iter__(self) -> 'QueryIterator[_M]': ... + def __next__(self) -> _M: ... + +class Query(Generic[_M]): + kind: Optional[str] + ancestor: Optional[Any] + filters: Optional[Node] + orders: Optional[List[PropertyOrder]] + namespace: Optional[str] + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + def filter(self, *filters: Node) -> 'Query[_M]': ... + def order(self, *orders: Union[PropertyOrder, Any]) -> 'Query[_M]': ... + def fetch(self, limit: Optional[int] = None, **kwargs: Any) -> List[_M]: ... + def fetch_async(self, limit: Optional[int] = None, **kwargs: Any) -> Any: ... + def fetch_page(self, page_size: int, **kwargs: Any) -> Tuple[List[_M], Optional[Cursor], bool]: ... + def fetch_page_async(self, page_size: int, **kwargs: Any) -> Any: ... + def get(self, **kwargs: Any) -> Optional[_M]: ... + def get_async(self, **kwargs: Any) -> Any: ... + def count(self, limit: Optional[int] = None, **kwargs: Any) -> int: ... + def count_async(self, limit: Optional[int] = None, **kwargs: Any) -> Any: ... + def iter(self, **kwargs: Any) -> QueryIterator[_M]: ... + def __iter__(self) -> QueryIterator[_M]: ... + +def gql(query_string: str, *args: Any, **kwds: Any) -> Query[Any]: ... diff --git a/typings/google/cloud/ndb/tasklets.pyi b/typings/google/cloud/ndb/tasklets.pyi new file mode 100644 index 00000000000..ff677963180 --- /dev/null +++ b/typings/google/cloud/ndb/tasklets.pyi @@ -0,0 +1,27 @@ +from typing import Any, Callable, Generator, Generic, Optional, TypeVar, Union + +_T = TypeVar('_T') + +class Future(Generic[_T]): + def __init__(self, info: Optional[str] = None) -> None: ... + def done(self) -> bool: ... + def running(self) -> bool: ... + def wait(self) -> None: ... + def get_result(self) -> _T: ... + def check_success(self) -> None: ... + def set_result(self, result: _T) -> None: ... + def set_exception(self, exception: Any) -> None: ... + def add_done_callback(self, callback: Callable[['Future[_T]'], Any]) -> None: ... + def add_immediate_callback(self, callback: Callable[['Future[_T]'], Any]) -> None: ... + @classmethod + def wait_any(cls, futures: Any) -> 'Future[Any]': ... + @classmethod + def wait_all(cls, futures: Any) -> None: ... + +def tasklet( + func: Callable[..., Union[Generator[Any, Any, _T], _T]] +) -> Callable[..., Future[_T]]: ... + +def Return(result: _T) -> None: ... +def sleep(seconds: float) -> Future[None]: ... +def add_flow_variable(name: str, initial_value: Any = None) -> Any: ... diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000000..01212b5af3d --- /dev/null +++ b/uv.lock @@ -0,0 +1,2110 @@ +version = 1 +revision = 3 +requires-python = "==3.11.*" + +[[package]] +name = "aiohappyeyeballs" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.10.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/28/ca549838018140b92a19001a8628578b0f2a3b38c16826212cc6f706e6d4/aiohttp-3.10.5.tar.gz", hash = "sha256:f071854b47d39591ce9a17981c46790acb30518e2f83dfca8db2dfa091178691", size = 7524360, upload-time = "2024-08-19T20:10:48.406Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/90/54ccb1e4eadfb6c95deff695582453f6208584431d69bf572782e9ae542b/aiohttp-3.10.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8c6a4e5e40156d72a40241a25cc226051c0a8d816610097a8e8f517aeacd59a2", size = 586455, upload-time = "2024-08-19T20:08:31.409Z" }, + { url = "https://files.pythonhosted.org/packages/c3/7a/95e88c02756e7e718f054e1bb3ec6ad5d0ee4a2ca2bb1768c5844b3de30a/aiohttp-3.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c634a3207a5445be65536d38c13791904fda0748b9eabf908d3fe86a52941cf", size = 397255, upload-time = "2024-08-19T20:08:32.659Z" }, + { url = "https://files.pythonhosted.org/packages/07/4f/767387b39990e1ee9aba8ce642abcc286d84d06e068dc167dab983898f18/aiohttp-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4aff049b5e629ef9b3e9e617fa6e2dfeda1bf87e01bcfecaf3949af9e210105e", size = 388973, upload-time = "2024-08-19T20:08:33.9Z" }, + { url = "https://files.pythonhosted.org/packages/61/46/0df41170a4d228c07b661b1ba9d87101d99a79339dc93b8b1183d8b20545/aiohttp-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1942244f00baaacaa8155eca94dbd9e8cc7017deb69b75ef67c78e89fdad3c77", size = 1326126, upload-time = "2024-08-19T20:08:35.606Z" }, + { url = "https://files.pythonhosted.org/packages/af/20/da0d65e07ce49d79173fed41598f487a0a722e87cfbaa8bb7e078a7c1d39/aiohttp-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e04a1f2a65ad2f93aa20f9ff9f1b672bf912413e5547f60749fa2ef8a644e061", size = 1364538, upload-time = "2024-08-19T20:08:37.754Z" }, + { url = "https://files.pythonhosted.org/packages/aa/20/b59728405114e57541ba9d5b96033e69d004e811ded299537f74237629ca/aiohttp-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f2bfc0032a00405d4af2ba27f3c429e851d04fad1e5ceee4080a1c570476697", size = 1399896, upload-time = "2024-08-19T20:08:39.635Z" }, + { url = "https://files.pythonhosted.org/packages/2a/92/006690c31b830acbae09d2618e41308fe4c81c0679b3b33a3af859e0b7bf/aiohttp-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424ae21498790e12eb759040bbb504e5e280cab64693d14775c54269fd1d2bb7", size = 1312914, upload-time = "2024-08-19T20:08:41.417Z" }, + { url = "https://files.pythonhosted.org/packages/d4/71/1a253ca215b6c867adbd503f1e142117527ea8775e65962bc09b2fad1d2c/aiohttp-3.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:975218eee0e6d24eb336d0328c768ebc5d617609affaca5dbbd6dd1984f16ed0", size = 1271301, upload-time = "2024-08-19T20:08:43.39Z" }, + { url = "https://files.pythonhosted.org/packages/0a/ab/5d1d9ff9ce6cce8fa54774d0364e64a0f3cd50e512ff09082ced8e5217a1/aiohttp-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4120d7fefa1e2d8fb6f650b11489710091788de554e2b6f8347c7a20ceb003f5", size = 1291652, upload-time = "2024-08-19T20:08:45.335Z" }, + { url = "https://files.pythonhosted.org/packages/75/5f/f90510ea954b9ae6e7a53d2995b97a3e5c181110fdcf469bc9238445871d/aiohttp-3.10.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b90078989ef3fc45cf9221d3859acd1108af7560c52397ff4ace8ad7052a132e", size = 1286289, upload-time = "2024-08-19T20:08:47.546Z" }, + { url = "https://files.pythonhosted.org/packages/be/9e/1f523414237798660921817c82b9225a363af436458caf584d2fa6a2eb4a/aiohttp-3.10.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ba5a8b74c2a8af7d862399cdedce1533642fa727def0b8c3e3e02fcb52dca1b1", size = 1341848, upload-time = "2024-08-19T20:08:49.698Z" }, + { url = "https://files.pythonhosted.org/packages/f6/36/443472ddaa85d7d80321fda541d9535b23ecefe0bf5792cc3955ea635190/aiohttp-3.10.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:02594361128f780eecc2a29939d9dfc870e17b45178a867bf61a11b2a4367277", size = 1361619, upload-time = "2024-08-19T20:08:51.942Z" }, + { url = "https://files.pythonhosted.org/packages/19/f6/3ecbac0bc4359c7d7ba9e85c6b10f57e20edaf1f97751ad2f892db231ad0/aiohttp-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4fc029e135859f533025bc82047334e24b0d489e75513144f25408ecaf058", size = 1320869, upload-time = "2024-08-19T20:08:53.708Z" }, + { url = "https://files.pythonhosted.org/packages/34/7e/ed74ffb36e3a0cdec1b05d8fbaa29cb532371d5a20058b3a8052fc90fe7c/aiohttp-3.10.5-cp311-cp311-win32.whl", hash = "sha256:e1ca1ef5ba129718a8fc827b0867f6aa4e893c56eb00003b7367f8a733a9b072", size = 359271, upload-time = "2024-08-19T20:08:55.186Z" }, + { url = "https://files.pythonhosted.org/packages/98/1b/718901f04bc8c886a742be9e83babb7b93facabf7c475cc95e2b3ab80b4d/aiohttp-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:349ef8a73a7c5665cca65c88ab24abe75447e28aa3bc4c93ea5093474dfdf0ff", size = 379143, upload-time = "2024-08-19T20:08:56.604Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "antlr4-python3-runtime" +version = "4.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/14/8ac135ec7cc9db3f768e2d032776718c6b23f74e63543f0974b4873500b2/antlr4-python3-runtime-4.7.2.tar.gz", hash = "sha256:168cdcec8fb9152e84a87ca6fd261b3d54c8f6358f42ab3b813b14a7193bb50b", size = 112278, upload-time = "2018-12-18T18:50:52.313Z" } + +[[package]] +name = "astroid" +version = "2.15.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lazy-object-proxy" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/3d/c18b0854d0d2eb3aca20c149cff5c90e6b84a5366066768d98636f5045ed/astroid-2.15.8.tar.gz", hash = "sha256:6c107453dffee9055899705de3c9ead36e74119cee151e5a9aaf7f0b0e020a6a", size = 344362, upload-time = "2023-09-26T12:40:28.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/b6/c0b5394ec6149e0129421f1a762b805e0e583974bc3cd65e3c7ce7c95444/astroid-2.15.8-py3-none-any.whl", hash = "sha256:1aa149fc5c6589e3d0ece885b4491acd80af4f087baafa3fb5203b113e68cd3c", size = 278329, upload-time = "2023-09-26T12:40:25.988Z" }, +] + +[[package]] +name = "async-timeout" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "backports-tarfile" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991", size = 86406, upload-time = "2024-05-28T17:01:54.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181, upload-time = "2024-05-28T17:01:53.112Z" }, +] + +[[package]] +name = "bcrypt" +version = "4.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/07/6a6f2047a9dc9d012b7b977e4041d37d078b76b44b7ee4daf331c1e6fb35/bcrypt-4.1.2.tar.gz", hash = "sha256:33313a1200a3ae90b75587ceac502b048b840fc69e7f7a0905b5f87fac7a1258", size = 26356, upload-time = "2023-12-15T14:53:25.981Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/cc/5a73c2ecfa9f255423530e8aeaceb0590da12e4c83c99fdac17093f5ce42/bcrypt-4.1.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:ac621c093edb28200728a9cca214d7e838529e557027ef0581685909acd28b5e", size = 528438, upload-time = "2023-12-15T14:52:41.282Z" }, + { url = "https://files.pythonhosted.org/packages/22/2e/32c1810b8470aca98c33892fc8c559c1be95eba711cb1bb82fbbf2a4752a/bcrypt-4.1.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea505c97a5c465ab8c3ba75c0805a102ce526695cd6818c6de3b1a38f6f60da1", size = 695692, upload-time = "2023-12-15T14:52:43.585Z" }, + { url = "https://files.pythonhosted.org/packages/41/ed/e446078ebe94d8ccac7170ff4bab83d8c86458c6fcfc7c5a4b449974fdd6/bcrypt-4.1.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57fa9442758da926ed33a91644649d3e340a71e2d0a5a8de064fb621fd5a3326", size = 699072, upload-time = "2023-12-15T14:52:45.688Z" }, + { url = "https://files.pythonhosted.org/packages/6d/7c/761ab4586beb7aa14b3fa2f382794746a218fffe1d22d9e10926200c8ccd/bcrypt-4.1.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb3bd3321517916696233b5e0c67fd7d6281f0ef48e66812db35fc963a422a1c", size = 695934, upload-time = "2023-12-15T14:52:47.098Z" }, + { url = "https://files.pythonhosted.org/packages/91/21/6350647549656138a067788d67bdb5ee89ffc2f025618ebf60d3806274c4/bcrypt-4.1.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6cad43d8c63f34b26aef462b6f5e44fdcf9860b723d2453b5d391258c4c8e966", size = 699120, upload-time = "2023-12-15T14:52:48.557Z" }, + { url = "https://files.pythonhosted.org/packages/54/fc/fd9a299d4dfd7da38b4570e487ea2465fb92021ab31a08bd66b3caba0baa/bcrypt-4.1.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:44290ccc827d3a24604f2c8bcd00d0da349e336e6503656cb8192133e27335e2", size = 725857, upload-time = "2023-12-15T14:52:49.942Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5b/dfcd8b7422a8f3b4ce3d28d64307e2f3502e3b5c540dde35eccda2d6c763/bcrypt-4.1.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:732b3920a08eacf12f93e6b04ea276c489f1c8fb49344f564cca2adb663b3e4c", size = 730079, upload-time = "2023-12-15T14:52:51.902Z" }, + { url = "https://files.pythonhosted.org/packages/21/d9/7924b194b3aa9bcc39f4592470995841efe71015cb8a79abae9bb043ec28/bcrypt-4.1.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1c28973decf4e0e69cee78c68e30a523be441972c826703bb93099868a8ff5b5", size = 732432, upload-time = "2023-12-15T14:52:53.478Z" }, + { url = "https://files.pythonhosted.org/packages/bf/26/ec53ccf5cadc81891d53cf0c117cff0f973d98cab6e9d6979578ca5aceeb/bcrypt-4.1.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b8df79979c5bae07f1db22dcc49cc5bccf08a0380ca5c6f391cbb5790355c0b0", size = 750035, upload-time = "2023-12-15T14:52:54.829Z" }, + { url = "https://files.pythonhosted.org/packages/b0/df/a1ac4188ee865236aba0a747773985a0f39211037f75a2d881a3be206a4e/bcrypt-4.1.2-cp37-abi3-win32.whl", hash = "sha256:fbe188b878313d01b7718390f31528be4010fed1faa798c5a1d0469c9c48c369", size = 171222, upload-time = "2023-12-15T14:52:56.796Z" }, + { url = "https://files.pythonhosted.org/packages/a1/c8/09eb0bd262b8b64f5ce99cb7f99984769fd1dbf35bdcd63d41a7b713c09f/bcrypt-4.1.2-cp37-abi3-win_amd64.whl", hash = "sha256:9800ae5bd5077b13725e2e3934aa3c9c37e49d3ea3d06318010aa40f54c63551", size = 158340, upload-time = "2023-12-15T14:52:58.7Z" }, + { url = "https://files.pythonhosted.org/packages/a4/72/a1276d2fbf5d1af0e29ff9fb5220ce1d49a5f94ccbfb4f9141c963ff9d0e/bcrypt-4.1.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:71b8be82bc46cedd61a9f4ccb6c1a493211d031415a34adde3669ee1b0afbb63", size = 528460, upload-time = "2023-12-15T14:53:00.723Z" }, + { url = "https://files.pythonhosted.org/packages/42/c4/13c4bba7e25633b2e94724c642aa93ce376c476d80ecd50d73f0fe2eb38f/bcrypt-4.1.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e3c6642077b0c8092580c819c1684161262b2e30c4f45deb000c38947bf483", size = 695634, upload-time = "2023-12-15T14:53:02.761Z" }, + { url = "https://files.pythonhosted.org/packages/72/3d/925adb5f5bef7616b504227a431fcaadd9630044802b5c81a31a560b4369/bcrypt-4.1.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:387e7e1af9a4dd636b9505a465032f2f5cb8e61ba1120e79a0e1cd0b512f3dfc", size = 698763, upload-time = "2023-12-15T14:53:04.781Z" }, + { url = "https://files.pythonhosted.org/packages/b6/1b/1c1cf4efe142dfe6fab912c16766d3eab65b87f33f1d13a08238afce5fdf/bcrypt-4.1.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f70d9c61f9c4ca7d57f3bfe88a5ccf62546ffbadf3681bb1e268d9d2e41c91a7", size = 695781, upload-time = "2023-12-15T14:53:06.858Z" }, + { url = "https://files.pythonhosted.org/packages/42/9d/a88027b5a8752f4b1831d957470f48e23cebc112aaf762880f3adbfba9cf/bcrypt-4.1.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2a298db2a8ab20056120b45e86c00a0a5eb50ec4075b6142db35f593b97cb3fb", size = 698875, upload-time = "2023-12-15T14:53:09.013Z" }, + { url = "https://files.pythonhosted.org/packages/05/76/6232380b99d85a2154ae06966b4bf6ce805878a7a92c3211295063b0b6be/bcrypt-4.1.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ba55e40de38a24e2d78d34c2d36d6e864f93e0d79d0b6ce915e4335aa81d01b1", size = 725674, upload-time = "2023-12-15T14:53:10.436Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c5/243674ec98288af9da31f5b137686746986d5d298dc520e243032160fd1b/bcrypt-4.1.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3566a88234e8de2ccae31968127b0ecccbb4cddb629da744165db72b58d88ca4", size = 729832, upload-time = "2023-12-15T14:53:12.391Z" }, + { url = "https://files.pythonhosted.org/packages/88/fd/6025f5530e6ac2513404aa2ab3fb935b9d992dbf24f255f03b5972dace74/bcrypt-4.1.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b90e216dc36864ae7132cb151ffe95155a37a14e0de3a8f64b49655dd959ff9c", size = 732222, upload-time = "2023-12-15T14:53:14.133Z" }, + { url = "https://files.pythonhosted.org/packages/85/23/756228cbc426049c264c86d163ec1b4fb1b06114f26b25fb63132af56126/bcrypt-4.1.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:69057b9fc5093ea1ab00dd24ede891f3e5e65bee040395fb1e66ee196f9c9b4a", size = 749705, upload-time = "2023-12-15T14:53:15.563Z" }, + { url = "https://files.pythonhosted.org/packages/ca/9e/abc56ba85897eeca1f3755343a7b6b55f63c048516ebc5790145a7cdfddb/bcrypt-4.1.2-cp39-abi3-win32.whl", hash = "sha256:02d9ef8915f72dd6daaef40e0baeef8a017ce624369f09754baf32bb32dba25f", size = 170970, upload-time = "2023-12-15T14:53:17.084Z" }, + { url = "https://files.pythonhosted.org/packages/53/5b/73803e5bf877e07739deaeecb2e356f4cc9ae3b766558959a898f7a993e0/bcrypt-4.1.2-cp39-abi3-win_amd64.whl", hash = "sha256:be3ab1071662f6065899fe08428e45c16aa36e28bc42921c4901a191fda6ee42", size = 158307, upload-time = "2023-12-15T14:53:18.422Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + +[[package]] +name = "cachecontrol" +version = "0.14.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/f6/c972b32d80760fb79d6b9eeb0b3010a46b89c0b23cf6329417ff7886cd22/cachecontrol-0.14.4.tar.gz", hash = "sha256:e6220afafa4c22a47dd0badb319f84475d79108100d04e26e8542ef7d3ab05a1", size = 16150, upload-time = "2025-11-14T04:32:13.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/79/c45f2d53efe6ada1110cf6f9fca095e4ff47a0454444aefdde6ac4789179/cachecontrol-0.14.4-py3-none-any.whl", hash = "sha256:b7ac014ff72ee199b5f8af1de29d60239954f223e948196fa3d84adaffc71d2b", size = 22247, upload-time = "2025-11-14T04:32:11.733Z" }, +] + +[[package]] +name = "cachetools" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380, upload-time = "2025-02-20T21:01:19.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, +] + +[[package]] +name = "certifi" +version = "2024.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/71/da/e94e26401b62acd6d91df2b52954aceb7f561743aa5ccc32152886c76c96/certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", size = 164886, upload-time = "2024-02-02T01:22:17.364Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1", size = 163774, upload-time = "2024-02-02T01:22:14.86Z" }, +] + +[[package]] +name = "cffi" +version = "1.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/a8/050ab4f0c3d4c1b8aaa805f70e26e84d0e27004907c5b8ecc1d31815f92a/cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", size = 508501, upload-time = "2022-06-30T18:18:32.799Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/8b/2e8c2469eaf89f7273ac685164949a7e644cdfe5daf1c036564208c3d26b/cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", size = 179198, upload-time = "2022-06-30T18:16:06.056Z" }, + { url = "https://files.pythonhosted.org/packages/f9/96/fc9e118c47b7adc45a0676f413b4a47554e5f3b6c99b8607ec9726466ef1/cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", size = 174221, upload-time = "2022-06-30T18:16:09.137Z" }, + { url = "https://files.pythonhosted.org/packages/10/72/617ee266192223a38b67149c830bd9376b69cf3551e1477abc72ff23ef8e/cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", size = 441694, upload-time = "2022-06-30T18:16:12.057Z" }, + { url = "https://files.pythonhosted.org/packages/91/bc/b7723c2fe7a22eee71d7edf2102cd43423d5f95ff3932ebaa2f82c7ec8d0/cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", size = 470613, upload-time = "2022-06-30T18:16:15.3Z" }, + { url = "https://files.pythonhosted.org/packages/5d/4e/4e0bb5579b01fdbfd4388bd1eb9394a989e1336203a4b7f700d887b233c1/cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", size = 472199, upload-time = "2022-06-30T18:16:18.027Z" }, + { url = "https://files.pythonhosted.org/packages/37/5a/c37631a86be838bdd84cc0259130942bf7e6e32f70f4cab95f479847fb91/cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", size = 462588, upload-time = "2022-06-30T18:16:21.148Z" }, + { url = "https://files.pythonhosted.org/packages/71/d7/0fe0d91b0bbf610fb7254bb164fa8931596e660d62e90fb6289b7ee27b09/cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", size = 450543, upload-time = "2022-06-30T18:16:24.497Z" }, + { url = "https://files.pythonhosted.org/packages/d3/56/3e94aa719ae96eeda8b68b3ec6e347e0a23168c6841dc276ccdcdadc9f32/cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", size = 474253, upload-time = "2022-06-30T18:16:27.156Z" }, + { url = "https://files.pythonhosted.org/packages/87/ee/ddc23981fc0f5e7b5356e98884226bcb899f95ebaefc3e8e8b8742dd7e22/cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", size = 170313, upload-time = "2022-06-30T18:16:29.939Z" }, + { url = "https://files.pythonhosted.org/packages/43/a0/cc7370ef72b6ee586369bacd3961089ab3d94ae712febf07a244f1448ffd/cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", size = 179001, upload-time = "2022-06-30T18:16:34.521Z" }, +] + +[[package]] +name = "chardet" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", size = 1868453, upload-time = "2017-06-08T14:34:35.581Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", size = 133356, upload-time = "2017-06-08T14:34:33.552Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/63/09/c1bc53dab74b1816a00d8d030de5bf98f724c52c1635e07681d312f20be8/charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", size = 104809, upload-time = "2023-11-01T04:04:59.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/77/02839016f6fbbf808e8b38601df6e0e66c17bbab76dff4613f7511413597/charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", size = 191647, upload-time = "2023-11-01T04:02:55.329Z" }, + { url = "https://files.pythonhosted.org/packages/3e/33/21a875a61057165e92227466e54ee076b73af1e21fe1b31f1e292251aa1e/charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", size = 121434, upload-time = "2023-11-01T04:02:57.173Z" }, + { url = "https://files.pythonhosted.org/packages/dd/51/68b61b90b24ca35495956b718f35a9756ef7d3dd4b3c1508056fa98d1a1b/charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", size = 118979, upload-time = "2023-11-01T04:02:58.442Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a6/7ee57823d46331ddc37dd00749c95b0edec2c79b15fc0d6e6efb532e89ac/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", size = 136582, upload-time = "2023-11-01T04:02:59.776Z" }, + { url = "https://files.pythonhosted.org/packages/74/f1/0d9fe69ac441467b737ba7f48c68241487df2f4522dd7246d9426e7c690e/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", size = 146645, upload-time = "2023-11-01T04:03:02.186Z" }, + { url = "https://files.pythonhosted.org/packages/05/31/e1f51c76db7be1d4aef220d29fbfa5dbb4a99165d9833dcbf166753b6dc0/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", size = 139398, upload-time = "2023-11-01T04:03:04.255Z" }, + { url = "https://files.pythonhosted.org/packages/40/26/f35951c45070edc957ba40a5b1db3cf60a9dbb1b350c2d5bef03e01e61de/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", size = 140273, upload-time = "2023-11-01T04:03:05.983Z" }, + { url = "https://files.pythonhosted.org/packages/07/07/7e554f2bbce3295e191f7e653ff15d55309a9ca40d0362fcdab36f01063c/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", size = 142577, upload-time = "2023-11-01T04:03:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/d8/b5/eb705c313100defa57da79277d9207dc8d8e45931035862fa64b625bfead/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", size = 137747, upload-time = "2023-11-01T04:03:08.886Z" }, + { url = "https://files.pythonhosted.org/packages/19/28/573147271fd041d351b438a5665be8223f1dd92f273713cb882ddafe214c/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", size = 143375, upload-time = "2023-11-01T04:03:10.613Z" }, + { url = "https://files.pythonhosted.org/packages/cf/7c/f3b682fa053cc21373c9a839e6beba7705857075686a05c72e0f8c4980ca/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", size = 148474, upload-time = "2023-11-01T04:03:11.973Z" }, + { url = "https://files.pythonhosted.org/packages/1e/49/7ab74d4ac537ece3bc3334ee08645e231f39f7d6df6347b29a74b0537103/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", size = 140232, upload-time = "2023-11-01T04:03:13.505Z" }, + { url = "https://files.pythonhosted.org/packages/2d/dc/9dacba68c9ac0ae781d40e1a0c0058e26302ea0660e574ddf6797a0347f7/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", size = 140859, upload-time = "2023-11-01T04:03:17.362Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c2/4a583f800c0708dd22096298e49f887b49d9746d0e78bfc1d7e29816614c/charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", size = 92509, upload-time = "2023-11-01T04:03:21.453Z" }, + { url = "https://files.pythonhosted.org/packages/57/ec/80c8d48ac8b1741d5b963797b7c0c869335619e13d4744ca2f67fc11c6fc/charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", size = 99870, upload-time = "2023-11-01T04:03:22.723Z" }, + { url = "https://files.pythonhosted.org/packages/28/76/e6222113b83e3622caa4bb41032d0b1bf785250607392e1b778aca0b8a7d/charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", size = 48543, upload-time = "2023-11-01T04:04:58.622Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "clusterfuzz" +version = "0.1.0" +source = { virtual = "." } + +[package.optional-dependencies] +appengine = [ + { name = "charset-normalizer" }, + { name = "firebase-admin" }, + { name = "flask" }, + { name = "itsdangerous" }, + { name = "parameterized" }, + { name = "pyjwt" }, + { name = "requests-toolbelt" }, + { name = "werkzeug" }, +] +core = [ + { name = "aiohttp" }, + { name = "antlr4-python3-runtime" }, + { name = "bcrypt" }, + { name = "certifi" }, + { name = "cffi" }, + { name = "crcmod" }, + { name = "cryptography" }, + { name = "future" }, + { name = "google-api-core" }, + { name = "google-api-python-client" }, + { name = "google-auth" }, + { name = "google-auth-oauthlib" }, + { name = "google-cloud-batch" }, + { name = "google-cloud-core" }, + { name = "google-cloud-datastore" }, + { name = "google-cloud-logging" }, + { name = "google-cloud-monitoring" }, + { name = "google-cloud-ndb" }, + { name = "google-cloud-profiler" }, + { name = "google-cloud-secret-manager" }, + { name = "google-cloud-storage" }, + { name = "google-crc32c" }, + { name = "grpcio" }, + { name = "httplib2" }, + { name = "jinja2" }, + { name = "jira" }, + { name = "kubernetes" }, + { name = "mozprocess" }, + { name = "oauth2client" }, + { name = "protobuf" }, + { name = "psutil" }, + { name = "pygithub" }, + { name = "pyopenssl" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "pyyaml" }, + { name = "redis" }, + { name = "requests" }, + { name = "sendgrid" }, + { name = "wrapt" }, +] +platform-specific = [ + { name = "grpcio" }, + { name = "kubernetes" }, + { name = "protobuf" }, + { name = "psutil" }, +] + +[package.dev-dependencies] +dev = [ + { name = "fabric" }, + { name = "grpcio-tools" }, + { name = "gunicorn" }, + { name = "isort" }, + { name = "mypy-protobuf" }, + { name = "nodeenv" }, + { name = "parameterized" }, + { name = "paramiko" }, + { name = "pyfakefs" }, + { name = "pylint" }, + { name = "pylint-protobuf" }, + { name = "pyright" }, + { name = "pywinauto", marker = "sys_platform == 'win32'" }, + { name = "pyyaml" }, + { name = "selenium" }, + { name = "setuptools" }, + { name = "twine" }, + { name = "ty" }, + { name = "webtest" }, + { name = "yapf" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiohttp", marker = "extra == 'core'", specifier = "==3.10.5" }, + { name = "antlr4-python3-runtime", marker = "extra == 'core'", specifier = "==4.7.2" }, + { name = "bcrypt", marker = "extra == 'core'", specifier = "==4.1.2" }, + { name = "certifi", marker = "extra == 'core'", specifier = "==2024.2.2" }, + { name = "cffi", marker = "extra == 'core'", specifier = "==1.15.1" }, + { name = "charset-normalizer", marker = "extra == 'appengine'", specifier = "==3.3.2" }, + { name = "crcmod", marker = "extra == 'core'", specifier = "==1.7" }, + { name = "cryptography", marker = "extra == 'core'", specifier = "==37.0.4" }, + { name = "firebase-admin", marker = "extra == 'appengine'", specifier = "==6.2.0" }, + { name = "flask", marker = "extra == 'appengine'", specifier = "==2.2.2" }, + { name = "future", marker = "extra == 'core'", specifier = "==0.17.1" }, + { name = "google-api-core", marker = "extra == 'core'", specifier = "==2.11.1" }, + { name = "google-api-python-client", marker = "extra == 'core'", specifier = "==2.93.0" }, + { name = "google-auth", marker = "extra == 'core'", specifier = "==2.22.0" }, + { name = "google-auth-oauthlib", marker = "extra == 'core'", specifier = "==0.4.1" }, + { name = "google-cloud-batch", marker = "extra == 'core'", specifier = "==0.11.0" }, + { name = "google-cloud-core", marker = "extra == 'core'", specifier = "==2.3.3" }, + { name = "google-cloud-datastore", marker = "extra == 'core'", specifier = "==2.16.1" }, + { name = "google-cloud-logging", marker = "extra == 'core'", specifier = "==3.6.0" }, + { name = "google-cloud-monitoring", marker = "extra == 'core'", specifier = "==2.15.1" }, + { name = "google-cloud-ndb", marker = "extra == 'core'", specifier = "==2.2.1" }, + { name = "google-cloud-profiler", marker = "extra == 'core'", specifier = "==4.1.0" }, + { name = "google-cloud-secret-manager", marker = "extra == 'core'", specifier = "==2.17.0" }, + { name = "google-cloud-storage", marker = "extra == 'core'", specifier = "==2.10.0" }, + { name = "google-crc32c", marker = "extra == 'core'", specifier = "==1.5.0" }, + { name = "grpcio", marker = "extra == 'core'", specifier = "==1.62.2" }, + { name = "grpcio", marker = "extra == 'platform-specific'", specifier = "==1.62.2" }, + { name = "httplib2", marker = "extra == 'core'", specifier = "==0.19.0" }, + { name = "itsdangerous", marker = "extra == 'appengine'", specifier = "==2.0.1" }, + { name = "jinja2", marker = "extra == 'core'", specifier = "==3.1.4" }, + { name = "jira", marker = "extra == 'core'", specifier = "==2.0.0" }, + { name = "kubernetes", marker = "extra == 'core'", specifier = "==34.1.0" }, + { name = "kubernetes", marker = "extra == 'platform-specific'", specifier = "==34.1.0" }, + { name = "mozprocess", marker = "extra == 'core'", specifier = "==1.3.1" }, + { name = "oauth2client", marker = "extra == 'core'", specifier = "==4.1.3" }, + { name = "parameterized", marker = "extra == 'appengine'", specifier = "==0.9.0" }, + { name = "protobuf", marker = "extra == 'core'", specifier = "==4.23.4" }, + { name = "protobuf", marker = "extra == 'platform-specific'", specifier = "==4.23.4" }, + { name = "psutil", marker = "extra == 'core'", specifier = "==5.9.4" }, + { name = "psutil", marker = "extra == 'platform-specific'", specifier = "==5.9.4" }, + { name = "pygithub", marker = "extra == 'core'", specifier = "==1.55" }, + { name = "pyjwt", marker = "extra == 'appengine'", specifier = "==2.7.0" }, + { name = "pyopenssl", marker = "extra == 'core'", specifier = "==22.0.0" }, + { name = "python-dateutil", marker = "extra == 'core'", specifier = "==2.8.1" }, + { name = "pytz", marker = "extra == 'core'", specifier = "==2023.3" }, + { name = "pyyaml", marker = "extra == 'core'", specifier = "==6.0.1" }, + { name = "redis", marker = "extra == 'core'", specifier = "==4.6.0" }, + { name = "requests", marker = "extra == 'core'", specifier = "==2.21.0" }, + { name = "requests-toolbelt", marker = "extra == 'appengine'", specifier = "==0.9.1" }, + { name = "sendgrid", marker = "extra == 'core'", specifier = "==6.0.4" }, + { name = "werkzeug", marker = "extra == 'appengine'", specifier = "==2.2.2" }, + { name = "wrapt", marker = "extra == 'core'", specifier = "==1.16.0" }, +] +provides-extras = ["core", "appengine", "platform-specific"] + +[package.metadata.requires-dev] +dev = [ + { name = "fabric", specifier = "==1.14.1" }, + { name = "grpcio-tools", specifier = "==1.62.2" }, + { name = "gunicorn" }, + { name = "isort" }, + { name = "mypy-protobuf", specifier = "==3.4.0" }, + { name = "nodeenv", specifier = "==1.9.1" }, + { name = "parameterized", specifier = "==0.9.0" }, + { name = "paramiko", specifier = "==2.6.0" }, + { name = "pyfakefs", specifier = "==4.4.0" }, + { name = "pylint", specifier = "~=2.4" }, + { name = "pylint-protobuf", specifier = "==0.21.0" }, + { name = "pyright", specifier = "==1.1.411" }, + { name = "pywinauto", marker = "sys_platform == 'win32'", specifier = "==0.6.8" }, + { name = "pyyaml", specifier = "==6.0.1" }, + { name = "selenium", specifier = "==3.141.0" }, + { name = "setuptools", specifier = "==80.9.0" }, + { name = "twine" }, + { name = "ty", specifier = "==0.0.69" }, + { name = "webtest", specifier = "==3.0.0" }, + { name = "yapf", specifier = "==0.22.0" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "comtypes" +version = "1.4.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/2a/65274c13327f637ec13af8d39f2cf579d9ebe7a0e683696b5f05236d2805/comtypes-1.4.16.tar.gz", hash = "sha256:cd66d1add01265cface4df51ba1e31cd1657e04463c281c802e737e79e1ba93c", size = 260252, upload-time = "2026-03-02T23:11:42.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/7c/0eb685107290b6221c03c46d39214a4e42a124189691cb83ae3228257f46/comtypes-1.4.16-py3-none-any.whl", hash = "sha256:e18d85179ff12955524c5a8c3bc09cb3c0d890f1da4d7123d14244c7b78f84c8", size = 296230, upload-time = "2026-03-02T23:11:41.049Z" }, +] + +[[package]] +name = "crcmod" +version = "1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/b0/e595ce2a2527e169c3bcd6c33d2473c1918e0b7f6826a043ca1245dd4e5b/crcmod-1.7.tar.gz", hash = "sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e", size = 89670, upload-time = "2010-06-27T14:35:29.538Z" } + +[[package]] +name = "cryptography" +version = "37.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/d9/5fcd312d5cce0b4d7ee8b551a0ea99e4ea9db0fdbf6dd455a19042e3370b/cryptography-37.0.4.tar.gz", hash = "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", size = 585913, upload-time = "2022-07-05T13:33:04.789Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/f0/8bc2246a422eb5cd1fe7cfc2ed522e4e3e0fd6f1c828193c0860c7030ca6/cryptography-37.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", size = 5389724, upload-time = "2022-07-05T13:33:24.754Z" }, + { url = "https://files.pythonhosted.org/packages/c5/93/23f1cc4a39cee6ca0dc75550dc204e5af71e8bf3012d23feb1bd5b06edea/cryptography-37.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", size = 2847016, upload-time = "2022-07-05T13:33:31.369Z" }, + { url = "https://files.pythonhosted.org/packages/ca/44/2384260ffa2fa974894ec5f70896b328cd55a19dc367cf5c7ef32d5b3ba8/cryptography-37.0.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", size = 4079427, upload-time = "2022-07-05T13:34:10.373Z" }, + { url = "https://files.pythonhosted.org/packages/55/68/17d21988cec2dec825ce7fb965cc44d5f64e9f48f414084510f5836c5cb3/cryptography-37.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", size = 3686786, upload-time = "2022-07-05T13:35:11.684Z" }, + { url = "https://files.pythonhosted.org/packages/fd/5a/f47456f062b0c5bd828198992fca1f78bcc7aeadd216d9ce6c3348188b92/cryptography-37.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", size = 4011894, upload-time = "2022-07-05T13:35:18.582Z" }, + { url = "https://files.pythonhosted.org/packages/20/8b/66600f5851ec7893ace9b74445d7eaf3499571b347e339d18c76c876b0f9/cryptography-37.0.4-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", size = 4151466, upload-time = "2022-07-05T13:34:16.445Z" }, + { url = "https://files.pythonhosted.org/packages/86/82/5e81dbf8a94c011e5240595149626d92e78a110f01311face1ab08431566/cryptography-37.0.4-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", size = 4052936, upload-time = "2022-07-05T13:33:47.347Z" }, + { url = "https://files.pythonhosted.org/packages/d4/d7/fa8688ca6ba6dbe44a8ecab9b34cbba0a5ab42c5a3609371968ba3e7f44a/cryptography-37.0.4-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", size = 4109191, upload-time = "2022-07-05T13:35:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/6b/c0/4cfdc2fa58f86ccb5dcd017c9aa2125a5132e9b52868ccc5d46d0542d29c/cryptography-37.0.4-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", size = 4234788, upload-time = "2022-07-05T13:34:38.672Z" }, + { url = "https://files.pythonhosted.org/packages/1d/07/9c6779f9aac823c760465e59dd6a2ff3fffdd8c8174ef2fbe05f1b77650e/cryptography-37.0.4-cp36-abi3-win32.whl", hash = "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", size = 2054314, upload-time = "2022-07-05T13:34:42.512Z" }, + { url = "https://files.pythonhosted.org/packages/e5/4b/45759a0238628b48113ce899ef499d39433d3ac384a132796cd9fb4977a1/cryptography-37.0.4-cp36-abi3-win_amd64.whl", hash = "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", size = 2422362, upload-time = "2022-07-05T13:34:46.363Z" }, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, +] + +[[package]] +name = "dill" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, +] + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "docutils" +version = "0.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/a4/5180d9afc57e8fca05601dd652bdff19604c218814037fe90ffc7625a50a/docutils-0.23.tar.gz", hash = "sha256:746f5060322511280a1e50eb76846ed6bf2342984b2ac04dc42caa1a8d78799e", size = 2303823, upload-time = "2026-05-27T17:41:06.934Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/91/30151a39f7570f448ed84529390628a651d7f27c87d73c9b887f8189695e/docutils-0.23-py3-none-any.whl", hash = "sha256:25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea", size = 634701, upload-time = "2026-05-27T17:40:58.442Z" }, +] + +[[package]] +name = "durationpy" +version = "0.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/a4/e44218c2b394e31a6dd0d6b095c4e1f32d0be54c2a4b250032d717647bab/durationpy-0.10.tar.gz", hash = "sha256:1fa6893409a6e739c9c72334fc65cca1f355dbdd93405d30f726deb5bde42fba", size = 3335, upload-time = "2025-05-17T13:52:37.26Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/0d/9feae160378a3553fa9a339b0e9c1a048e147a4127210e286ef18b730f03/durationpy-0.10-py3-none-any.whl", hash = "sha256:3b41e1b601234296b4fb368338fdcd3e13e0b4fb5b67345948f4f2bf9868b286", size = 3922, upload-time = "2025-05-17T13:52:36.463Z" }, +] + +[[package]] +name = "fabric" +version = "1.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "paramiko" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/5e/08caf338095d35fde7e374979d53c8fd025eb7e827bf5d56489d13d29545/Fabric-1.14.1.tar.gz", hash = "sha256:66097883bb3e5beecacae92b82b2bd489d10a8fd4f06ce1cb27019de2e6d76a8", size = 220463, upload-time = "2018-11-28T00:05:31.999Z" } + +[[package]] +name = "firebase-admin" +version = "6.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachecontrol" }, + { name = "google-api-core", extra = ["grpc"], marker = "platform_python_implementation != 'PyPy'" }, + { name = "google-api-python-client" }, + { name = "google-cloud-firestore", marker = "platform_python_implementation != 'PyPy'" }, + { name = "google-cloud-storage" }, + { name = "pyjwt", extra = ["crypto"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/0b/9c2a1827bbba8f21fb9bc6c08231b6891071f805aea0727489f3d4c9492f/firebase_admin-6.2.0.tar.gz", hash = "sha256:e3b334d18bbea039f2f3e8a792ad6870d2a7cc79a13ed10659dedd63f5b475e4", size = 94523, upload-time = "2023-06-22T18:51:06.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/f0/e259b1345aa79e73a594cb0df82793965614fccb7516a930192d8690fcd7/firebase_admin-6.2.0-py3-none-any.whl", hash = "sha256:e3c42351fb6194d7279a6fd9209a947005fb4ee7e9037d19762e6cb3da4a82e1", size = 119891, upload-time = "2023-06-22T18:51:05.088Z" }, +] + +[[package]] +name = "flask" +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/b6/53cfa30eed5aa7343daff36622843688ba8c6fe9829bb2b92e193ab1163f/Flask-2.2.2.tar.gz", hash = "sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b", size = 677389, upload-time = "2022-08-08T23:26:33.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/43/15f4f9ab225b0b25352412e8daa3d0e3d135fcf5e127070c74c3632c8b4c/Flask-2.2.2-py3-none-any.whl", hash = "sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526", size = 101477, upload-time = "2022-08-08T23:26:30.228Z" }, +] + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/03/077f869d540370db12165c0aa51640a873fb661d8b315d1d4d67b284d7ac/frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84", size = 86912, upload-time = "2025-10-06T05:35:45.98Z" }, + { url = "https://files.pythonhosted.org/packages/df/b5/7610b6bd13e4ae77b96ba85abea1c8cb249683217ef09ac9e0ae93f25a91/frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9", size = 50046, upload-time = "2025-10-06T05:35:47.009Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ef/0e8f1fe32f8a53dd26bdd1f9347efe0778b0fddf62789ea683f4cc7d787d/frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93", size = 50119, upload-time = "2025-10-06T05:35:48.38Z" }, + { url = "https://files.pythonhosted.org/packages/11/b1/71a477adc7c36e5fb628245dfbdea2166feae310757dea848d02bd0689fd/frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f", size = 231067, upload-time = "2025-10-06T05:35:49.97Z" }, + { url = "https://files.pythonhosted.org/packages/45/7e/afe40eca3a2dc19b9904c0f5d7edfe82b5304cb831391edec0ac04af94c2/frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695", size = 233160, upload-time = "2025-10-06T05:35:51.729Z" }, + { url = "https://files.pythonhosted.org/packages/a6/aa/7416eac95603ce428679d273255ffc7c998d4132cfae200103f164b108aa/frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52", size = 228544, upload-time = "2025-10-06T05:35:53.246Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3d/2a2d1f683d55ac7e3875e4263d28410063e738384d3adc294f5ff3d7105e/frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581", size = 243797, upload-time = "2025-10-06T05:35:54.497Z" }, + { url = "https://files.pythonhosted.org/packages/78/1e/2d5565b589e580c296d3bb54da08d206e797d941a83a6fdea42af23be79c/frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567", size = 247923, upload-time = "2025-10-06T05:35:55.861Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c3/65872fcf1d326a7f101ad4d86285c403c87be7d832b7470b77f6d2ed5ddc/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b", size = 230886, upload-time = "2025-10-06T05:35:57.399Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/ac9ced601d62f6956f03cc794f9e04c81719509f85255abf96e2510f4265/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92", size = 245731, upload-time = "2025-10-06T05:35:58.563Z" }, + { url = "https://files.pythonhosted.org/packages/b9/49/ecccb5f2598daf0b4a1415497eba4c33c1e8ce07495eb07d2860c731b8d5/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d", size = 241544, upload-time = "2025-10-06T05:35:59.719Z" }, + { url = "https://files.pythonhosted.org/packages/53/4b/ddf24113323c0bbcc54cb38c8b8916f1da7165e07b8e24a717b4a12cbf10/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd", size = 241806, upload-time = "2025-10-06T05:36:00.959Z" }, + { url = "https://files.pythonhosted.org/packages/a7/fb/9b9a084d73c67175484ba2789a59f8eebebd0827d186a8102005ce41e1ba/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967", size = 229382, upload-time = "2025-10-06T05:36:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/95/a3/c8fb25aac55bf5e12dae5c5aa6a98f85d436c1dc658f21c3ac73f9fa95e5/frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25", size = 39647, upload-time = "2025-10-06T05:36:03.409Z" }, + { url = "https://files.pythonhosted.org/packages/0a/f5/603d0d6a02cfd4c8f2a095a54672b3cf967ad688a60fb9faf04fc4887f65/frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b", size = 44064, upload-time = "2025-10-06T05:36:04.368Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/c2c9ab44e181f043a86f9a8f84d5124b62dbcb3a02c0977ec72b9ac1d3e0/frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a", size = 39937, upload-time = "2025-10-06T05:36:05.669Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "future" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/90/52/e20466b85000a181e1e144fd8305caf2cf475e2f9674e797b222f8105f5f/future-0.17.1.tar.gz", hash = "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8", size = 829119, upload-time = "2018-10-31T00:20:25.785Z" } + +[[package]] +name = "google-api-core" +version = "2.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/b8/f727ada5b63aba53848e3791dd57be7481d5c9bf86978600ca9cca4ab03e/google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a", size = 129114, upload-time = "2023-06-14T18:47:12.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/c4/c3cd048b6cbeba8d9ae50dd7643ac065b85237338aa7501b0efae91eb4d9/google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a", size = 120538, upload-time = "2023-06-14T18:47:05.785Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, + { name = "grpcio-status" }, +] + +[[package]] +name = "google-api-python-client" +version = "2.93.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-auth-httplib2" }, + { name = "httplib2" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/dd/3b0de4b9633d7a2410b5b19c42c6c41410d4a87907fff5e077e2117ffb01/google-api-python-client-2.93.0.tar.gz", hash = "sha256:62ee28e96031a10a1c341f226a75ac6a4f16bdb1d888dc8222b2cdca133d0031", size = 10929939, upload-time = "2023-07-11T15:46:24.065Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/84/08219af4747b7dd5b98becfdb592c7f556fd41298e50aa63a64631729a4f/google_api_python_client-2.93.0-py2.py3-none-any.whl", hash = "sha256:f34abb671afd488bd19d30721ea20fb30d3796ddd825d6f91f26d8c718a9f07d", size = 11420014, upload-time = "2023-07-11T15:46:18.249Z" }, +] + +[[package]] +name = "google-auth" +version = "2.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "pyasn1-modules" }, + { name = "rsa" }, + { name = "six" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a4/3a/b6ab1073d2ac98c1b4f9036a4d37d5720f783bd4dc6e2c0ae516d3b13326/google-auth-2.22.0.tar.gz", hash = "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce", size = 231433, upload-time = "2023-07-11T17:24:49.539Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/8d/bff87fc722553a5691d8514da5523c23547f3894189ba03b57592e37bdc2/google_auth-2.22.0-py2.py3-none-any.whl", hash = "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873", size = 181773, upload-time = "2023-07-11T17:24:46.963Z" }, +] + +[[package]] +name = "google-auth-httplib2" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "httplib2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/46/79983cb738f0eb14e6ab4f43457aa9652f8d46bc4376b178f676b68c5c37/google_auth_httplib2-0.4.1.tar.gz", hash = "sha256:125b1bb4fcfdd2d97f19b673c1f46f831603d0acaffe415c8a35dadb312552a1", size = 11158, upload-time = "2026-08-06T06:24:00.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/8a/96410d0c3e02b584d1e7f5d4000b2e1710855650b226cd0601f2b154940a/google_auth_httplib2-0.4.1-py3-none-any.whl", hash = "sha256:67088a8387dc3f66016a690634f1cb9b7a48a31406f8998e088efc64203e7a12", size = 9529, upload-time = "2026-08-06T06:22:50.53Z" }, +] + +[[package]] +name = "google-auth-oauthlib" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "requests-oauthlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/5a/2b5a4c1294a4e8903bdba122083bd505dc51688a95d4670cde89dc45e6ed/google-auth-oauthlib-0.4.1.tar.gz", hash = "sha256:88d2cd115e3391eb85e1243ac6902e76e77c5fe438b7276b297fbe68015458dd", size = 18687, upload-time = "2019-10-04T16:27:15.165Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/b8/88def36e74bee9fce511c9519571f4e485e890093ab7442284f4ffaef60b/google_auth_oauthlib-0.4.1-py2.py3-none-any.whl", hash = "sha256:a92a0f6f41a0fb6138454fbc02674e64f89d82a244ea32f98471733c8ef0e0e1", size = 18012, upload-time = "2019-10-04T16:27:13.698Z" }, +] + +[[package]] +name = "google-cloud-appengine-logging" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "grpcio" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/38/89317773c64b5a7e9b56b9aecb2e39ac02d8d6d09fb5b276710c6892e690/google_cloud_appengine_logging-1.8.0.tar.gz", hash = "sha256:84b705a69e4109fc2f68dfe36ce3df6a34d5c3d989eee6d0ac1b024dda0ba6f5", size = 18071, upload-time = "2026-01-15T13:14:40.024Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/66/4a9be8afb1d0bf49472478cec20fefe4f4cb3a6e67be2231f097041e7339/google_cloud_appengine_logging-1.8.0-py3-none-any.whl", hash = "sha256:a4ce9ce94a9fd8c89ed07fa0b06fcf9ea3642f9532a1be1a8c7b5f82c0a70ec6", size = 18380, upload-time = "2026-01-09T14:52:58.154Z" }, +] + +[[package]] +name = "google-cloud-audit-log" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/d2/ad96950410f8a05e921a6da2e1a6ba4aeca674bbb5dda8200c3c7296d7ad/google_cloud_audit_log-0.4.0.tar.gz", hash = "sha256:8467d4dcca9f3e6160520c24d71592e49e874838f174762272ec10e7950b6feb", size = 44682, upload-time = "2025-10-17T02:33:44.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/25/532886995f11102ad6de290496de5db227bd3a73827702445928ad32edcb/google_cloud_audit_log-0.4.0-py3-none-any.whl", hash = "sha256:6b88e2349df45f8f4cc0993b687109b1388da1571c502dc1417efa4b66ec55e0", size = 44890, upload-time = "2025-10-17T02:30:55.11Z" }, +] + +[[package]] +name = "google-cloud-batch" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/37/a7f06a716e24fdd179e61ebe9dae8bf23d11e497521d2d5d6d274e3f96bd/google-cloud-batch-0.11.0.tar.gz", hash = "sha256:f102a7230dd201236e1e2804d4ca025995bd2141887df8a95a64cc419b33b476", size = 129783, upload-time = "2023-05-31T18:45:59.65Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/4a/a2f612dcf2bfd9a6f04e5982c93e8b2e259d5400340a26f4490b21e3ae43/google_cloud_batch-0.11.0-py2.py3-none-any.whl", hash = "sha256:d2daf19987e72369e9d33f1b634f0963ec90eb08f9d3c5d0a1ab940c0bd74d51", size = 128174, upload-time = "2023-05-31T18:45:57.664Z" }, +] + +[[package]] +name = "google-cloud-core" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/60/dcc26e42d3754ac57c51a524f53c988f2aa755faec4cc00a232bb0077637/google-cloud-core-2.3.3.tar.gz", hash = "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb", size = 35242, upload-time = "2023-07-05T19:44:34.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/40/02045f776fdb6e44194f34b6375a26ce8a61bd9bd03cd8930ed91cf51a62/google_cloud_core-2.3.3-py2.py3-none-any.whl", hash = "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863", size = 29948, upload-time = "2023-07-05T19:44:32.2Z" }, +] + +[[package]] +name = "google-cloud-datastore" +version = "2.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-cloud-core" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/62/82b0748c97cc54190a49901249bcc2b7a21d67d0a27d586c5297d78e8a70/google-cloud-datastore-2.16.1.tar.gz", hash = "sha256:710ef27ebdfb50340f4671c532ad4c1525665985e9421984ffcd6b96b57e34bb", size = 205802, upload-time = "2023-07-05T15:14:37.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/10/1281beff5567742559bfd447719fa61037b1b4e52ac644497aa16e8196ce/google_cloud_datastore-2.16.1-py2.py3-none-any.whl", hash = "sha256:153d8dac05f0248d7f7d6f0986a31f8fb22d3cdfb764718eac0f28e3cadc8144", size = 176407, upload-time = "2023-07-05T15:14:35.232Z" }, +] + +[[package]] +name = "google-cloud-firestore" +version = "2.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "google-cloud-core" }, + { name = "grpcio" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2c/a2/ec932f4cccccc6c0e527e2244efde3c8f0c8f537b4d387655328f95fc5d9/google_cloud_firestore-2.26.0.tar.gz", hash = "sha256:103fc2bdedcb801a31590fef9a0d50754ee63b803946f9afc904817c85bd935d", size = 622368, upload-time = "2026-03-23T22:43:56.504Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/cb/3fe572f724863e2d41d71da82856d3449809e9e30017f0ae46bcf0af1446/google_cloud_firestore-2.26.0-py3-none-any.whl", hash = "sha256:307e6d3aed1a15e7083d9bac1871cb439be17620e035de960225bb00acc771de", size = 416926, upload-time = "2026-03-23T22:43:31.345Z" }, +] + +[[package]] +name = "google-cloud-logging" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-cloud-appengine-logging" }, + { name = "google-cloud-audit-log" }, + { name = "google-cloud-core" }, + { name = "grpc-google-iam-v1" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/c4/be34f3f4f14b0b885da7354193c7183d65af21f05a095c6340266061b7ac/google-cloud-logging-3.6.0.tar.gz", hash = "sha256:42134223956850ddd64877c88042d31f78658e7b067a5a8e3dd28236b71f3c32", size = 229229, upload-time = "2023-07-14T17:27:24.298Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/3d/17e8c0bd49af17a70d2aeca29900e7a88f8d1bf02fad86520ee05ce9ad79/google_cloud_logging-3.6.0-py2.py3-none-any.whl", hash = "sha256:6beb843cb88a9bc2f6df94ce69048243ebb360ee06979f5311ea7a0ec09bd097", size = 195892, upload-time = "2023-07-14T17:27:21.851Z" }, +] + +[[package]] +name = "google-cloud-monitoring" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/51/49dad24016ab5fbb816330da31eb7f49dfee675450077d9b3a9082cc0c92/google-cloud-monitoring-2.15.1.tar.gz", hash = "sha256:086a845a34a6e105ab8a3a4809392ea6367a65b28b7561c26101534a8705ed37", size = 301070, upload-time = "2023-07-05T15:47:12.758Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/ba/6a2d59a8f9e0bc3d80c3f5d7b42517e5025f48eeed81a8dc989c72f1ea40/google_cloud_monitoring-2.15.1-py2.py3-none-any.whl", hash = "sha256:4394e5e031f30d622a24739678ec48a45400ead94af9a6032cbff7f66194dc12", size = 323419, upload-time = "2023-07-05T15:47:10.39Z" }, +] + +[[package]] +name = "google-cloud-ndb" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-cloud-datastore" }, + { name = "protobuf" }, + { name = "pymemcache" }, + { name = "pytz" }, + { name = "redis" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/7d/30510aa0c5253e073a681cd287367f0247977fc278f61bad5ce4ec53d900/google-cloud-ndb-2.2.1.tar.gz", hash = "sha256:39f5a81afa426b3acb81bcc696c84253bba81908cef40dd9fb730b6708d0ffaa", size = 159260, upload-time = "2023-09-15T20:39:50.545Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/df/704c100b318830a88cc4213df50d90e0481a05208b420f1be5c59d45267c/google_cloud_ndb-2.2.1-py3-none-any.whl", hash = "sha256:e849b48c984f31616e99e1fcc797d1deb1d883a6f38f58f77768b814fb84b57e", size = 178998, upload-time = "2023-09-15T20:39:48.002Z" }, +] + +[[package]] +name = "google-cloud-profiler" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-python-client" }, + { name = "google-auth" }, + { name = "google-auth-httplib2" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/be/fa9b4915315c764a38b42aac1a6355bb65deb5cb8a97c1a01c0de425780d/google-cloud-profiler-4.1.0.tar.gz", hash = "sha256:2d90f9c6d4c075ad6d43752ae39424c3d0bad63e6549a2c761881f9a235067ae", size = 32305, upload-time = "2023-08-16T16:57:56.332Z" } + +[[package]] +name = "google-cloud-secret-manager" +version = "2.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "grpc-google-iam-v1" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/63/093608cf497a744a3a86a9d53258fbf553d84865397c7ed8dc50809aaa3e/google-cloud-secret-manager-2.17.0.tar.gz", hash = "sha256:8254e2960c06a8dc91aeac3c894afba0893a674582f0e0ecfc22f894e6173c2f", size = 133174, upload-time = "2023-12-07T20:15:05.534Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/41/3cdf92fb2599f8b4e14b418b72f662b1b0697a942df45a4c4084b9b19be0/google_cloud_secret_manager-2.17.0-py2.py3-none-any.whl", hash = "sha256:2bf58be569710773f0757d7a788e2860767aa1c148a5255e53b228da3fbcf446", size = 115804, upload-time = "2023-12-07T20:15:02.958Z" }, +] + +[[package]] +name = "google-cloud-storage" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-cloud-core" }, + { name = "google-resumable-media" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/20/51e9676cc112ec7344c0a8690361175dc1f41ed6edf618eae8af87d92a49/google-cloud-storage-2.10.0.tar.gz", hash = "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7", size = 5504936, upload-time = "2023-06-26T20:34:54.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/14/c9d4faae7ea4bff4405152cbc762b61100aa6949273b4eb3203d23308670/google_cloud_storage-2.10.0-py2.py3-none-any.whl", hash = "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7", size = 114565, upload-time = "2023-06-26T20:34:51.797Z" }, +] + +[[package]] +name = "google-crc32c" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/a5/4bb58448fffd36ede39684044df93a396c13d1ea3516f585767f9f960352/google-crc32c-1.5.0.tar.gz", hash = "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7", size = 12689, upload-time = "2022-09-01T14:33:40.108Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/0f/7f89ae2b22c55273110a44a7ed55a2948bc213fb58983093fbefcdfd2d13/google_crc32c-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273", size = 32093, upload-time = "2022-09-01T14:32:29.306Z" }, + { url = "https://files.pythonhosted.org/packages/41/3f/8141b03ad127fc569c3efda2bfe31d64665e02e2b8b7fbf7b25ea914c27a/google_crc32c-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298", size = 30071, upload-time = "2022-09-01T14:32:30.523Z" }, + { url = "https://files.pythonhosted.org/packages/fc/76/3ef124b893aa280e45e95d2346160f1d1d5c0ffc89d3f6e446c83116fb91/google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57", size = 32702, upload-time = "2022-09-01T14:32:31.59Z" }, + { url = "https://files.pythonhosted.org/packages/fd/71/299a368347aeab3c89896cdfb67703161becbf5afbc1748a1850094828dc/google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438", size = 30041, upload-time = "2022-09-01T14:32:34.078Z" }, + { url = "https://files.pythonhosted.org/packages/72/92/2a2fa23db7d0b0382accbdf09768c28f7c07fc8c354cdcf2f44a47f4314e/google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906", size = 32317, upload-time = "2022-09-01T14:32:35.553Z" }, + { url = "https://files.pythonhosted.org/packages/0f/99/e7e288f1b50baf4964ff39fa79d9259d004ae44db35c8280ff4ffea362d5/google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183", size = 570024, upload-time = "2022-09-01T14:32:36.715Z" }, + { url = "https://files.pythonhosted.org/packages/88/ea/e53fbafcd0be2349d9c2a6912646cdfc47cfc5c22be9a8a5156552e33821/google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd", size = 36047, upload-time = "2022-09-01T14:32:37.867Z" }, + { url = "https://files.pythonhosted.org/packages/02/94/d2ea867760d5a27b3e9eb40ff31faf7f03f949e51d4e3b3ae24f759b5963/google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c", size = 582541, upload-time = "2022-09-01T14:32:39.077Z" }, + { url = "https://files.pythonhosted.org/packages/b7/09/768d2ca0c10a0765f83c6d06a5e40f3083cb75b8e7718ac22edff997aefc/google_crc32c-1.5.0-cp311-cp311-win32.whl", hash = "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709", size = 23928, upload-time = "2022-09-01T14:32:40.947Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8b/02bf4765c487901c8660290ade9929d65a6151c367ba32e75d136ef2d0eb/google_crc32c-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968", size = 27318, upload-time = "2022-09-01T14:32:42.116Z" }, +] + +[[package]] +name = "google-resumable-media" +version = "2.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/f5/f35505e6091614e285056a495488cb0a9c1a9dcc88a4a3c91bbc5fd4835b/google_resumable_media-2.10.1.tar.gz", hash = "sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0", size = 2164548, upload-time = "2026-08-06T06:24:50.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/ba/77ef49baf338c03a11deadac984e3161b3f2b4fa4bb5aab160e7ca0fd522/google_resumable_media-2.10.1-py3-none-any.whl", hash = "sha256:4e2cbc704207ddc09f23b1f18e8ef4a4ccbfe0f1768b370e5c969704adbd0a1c", size = 81533, upload-time = "2026-08-06T06:23:45.464Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.73.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/96/a0205167fa0154f4a542fd6925bdc63d039d88dab3588b875078107e6f06/googleapis_common_protos-1.73.0.tar.gz", hash = "sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a", size = 147323, upload-time = "2026-03-06T21:53:09.727Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/28/23eea8acd65972bbfe295ce3666b28ac510dfcb115fac089d3edb0feb00a/googleapis_common_protos-1.73.0-py3-none-any.whl", hash = "sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8", size = 297578, upload-time = "2026-03-06T21:52:33.933Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, +] + +[[package]] +name = "grpc-google-iam-v1" +version = "0.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos", extra = ["grpc"] }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/1e/1011451679a983f2f5c6771a1682542ecb027776762ad031fd0d7129164b/grpc_google_iam_v1-0.14.3.tar.gz", hash = "sha256:879ac4ef33136c5491a6300e27575a9ec760f6cdf9a2518798c1b8977a5dc389", size = 23745, upload-time = "2025-10-15T21:14:53.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/bd/330a1bbdb1afe0b96311249e699b6dc9cfc17916394fd4503ac5aca2514b/grpc_google_iam_v1-0.14.3-py3-none-any.whl", hash = "sha256:7a7f697e017a067206a3dfef44e4c634a34d3dee135fe7d7a4613fe3e59217e6", size = 32690, upload-time = "2025-10-15T21:14:51.72Z" }, +] + +[[package]] +name = "grpcio" +version = "1.62.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/7a/cb869ec6d7f634c0153b4a38ae25bb33699b9307ea17f8cb22b609a46605/grpcio-1.62.2.tar.gz", hash = "sha256:c77618071d96b7a8be2c10701a98537823b9c65ba256c0b9067e0594cdbd954d", size = 26316089, upload-time = "2024-04-18T17:59:08.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/b5/599940bd37c807d7c3f72d0a99bd35158ca3c972345d2c53b4a3de5506d7/grpcio-1.62.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:c1624aa686d4b36790ed1c2e2306cc3498778dffaf7b8dd47066cf819028c3ad", size = 4775120, upload-time = "2024-04-18T17:53:28.405Z" }, + { url = "https://files.pythonhosted.org/packages/cf/fe/6c99e20e4a7deee2590524b0922f5093f5121afb425c5bd4a78828ffafb1/grpcio-1.62.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:1c1bb80299bdef33309dff03932264636450c8fdb142ea39f47e06a7153d3063", size = 10020281, upload-time = "2024-04-18T17:53:32.267Z" }, + { url = "https://files.pythonhosted.org/packages/73/ee/9af6205181a259b31e8223c01d837d31b228d389d121bf3293c544a5476a/grpcio-1.62.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:db068bbc9b1fa16479a82e1ecf172a93874540cb84be69f0b9cb9b7ac3c82670", size = 5294556, upload-time = "2024-04-18T17:53:35.985Z" }, + { url = "https://files.pythonhosted.org/packages/1f/cd/3a1117df4dc6a0637b8939185636257d629671847a8795c64850fe154c27/grpcio-1.62.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2cc8a308780edbe2c4913d6a49dbdb5befacdf72d489a368566be44cadaef1a", size = 5830518, upload-time = "2024-04-18T17:53:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/26/d6/fb04ae7cde69fbf0c2a7fbfa2fb93b338094ec57fe34555e9cb9befa0459/grpcio-1.62.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0695ae31a89f1a8fc8256050329a91a9995b549a88619263a594ca31b76d756", size = 5546336, upload-time = "2024-04-18T17:53:42.72Z" }, + { url = "https://files.pythonhosted.org/packages/dd/0a/c1b3aacc6d60b7b7c0f3e2101f6e839107884919c79b41cb940020361ae5/grpcio-1.62.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88b4f9ee77191dcdd8810241e89340a12cbe050be3e0d5f2f091c15571cd3930", size = 6116967, upload-time = "2024-04-18T17:53:46.389Z" }, + { url = "https://files.pythonhosted.org/packages/9c/11/cc50e796652aa94e073a065ecf80680a9ca48258bc1ce57cf72744397f7a/grpcio-1.62.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a0204532aa2f1afd467024b02b4069246320405bc18abec7babab03e2644e75", size = 5778771, upload-time = "2024-04-18T17:53:49.369Z" }, + { url = "https://files.pythonhosted.org/packages/6c/90/2f9b54e8c6dcc094937af4ff75cc045ce1cfbe805c4c148418528cf0b966/grpcio-1.62.2-cp311-cp311-win32.whl", hash = "sha256:6e784f60e575a0de554ef9251cbc2ceb8790914fe324f11e28450047f264ee6f", size = 3170797, upload-time = "2024-04-18T17:53:52.725Z" }, + { url = "https://files.pythonhosted.org/packages/3c/9c/fe8b37679adf57c46323810b4d8a87b3416ca7b9721fcf816a30b35c23df/grpcio-1.62.2-cp311-cp311-win_amd64.whl", hash = "sha256:112eaa7865dd9e6d7c0556c8b04ae3c3a2dc35d62ad3373ab7f6a562d8199200", size = 3769992, upload-time = "2024-04-18T17:53:56.106Z" }, +] + +[[package]] +name = "grpcio-status" +version = "1.62.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/14/3bda4a77fe3da7cb309a136628042c9179424e941f1d41cbb0aca27a5c39/grpcio-status-1.62.2.tar.gz", hash = "sha256:62e1bfcb02025a1cd73732a2d33672d3e9d0df4d21c12c51e0bbcaf09bab742a", size = 13064, upload-time = "2024-04-18T17:59:22.157Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/63/56a8c67a77947d0ebc31a03c5dea8d2c933ab3ad30019b0bafa3e50a5ef6/grpcio_status-1.62.2-py3-none-any.whl", hash = "sha256:206ddf0eb36bc99b033f03b2c8e95d319f0044defae9b41ae21408e7e0cda48f", size = 14451, upload-time = "2024-04-18T17:56:13.731Z" }, +] + +[[package]] +name = "grpcio-tools" +version = "1.62.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/09/dfb87373a34bf6ce3261d65f8fb102a163c850c72a3e84902777ed44aa1d/grpcio-tools-1.62.2.tar.gz", hash = "sha256:5fd5e1582b678e6b941ee5f5809340be5e0724691df5299aae8226640f94e18f", size = 4538350, upload-time = "2024-04-18T17:59:25.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/43/7e062d7c2302366f2cf130f414bbc9a684782831f5c5c6d559b47e569957/grpcio_tools-1.62.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:472505d030135d73afe4143b0873efe0dcb385bd6d847553b4f3afe07679af00", size = 2700593, upload-time = "2024-04-18T17:56:44.015Z" }, + { url = "https://files.pythonhosted.org/packages/8b/fe/99e75396f2893b7dc408c1ab18f45a3ea0dd59b13bf98c6f1892c5901c66/grpcio_tools-1.62.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:ec674b4440ef4311ac1245a709e87b36aca493ddc6850eebe0b278d1f2b6e7d1", size = 5147591, upload-time = "2024-04-18T17:56:47.027Z" }, + { url = "https://files.pythonhosted.org/packages/24/34/c673496464df8e5eab5afeb1cda68006293581cbe503914511aeeeee5eed/grpcio_tools-1.62.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:184b4174d4bd82089d706e8223e46c42390a6ebac191073b9772abc77308f9fa", size = 2719541, upload-time = "2024-04-18T17:56:49.448Z" }, + { url = "https://files.pythonhosted.org/packages/aa/f2/daeceb79d6483cd99429701fa547234e29c1ea5815e5ed68032dd8d53575/grpcio_tools-1.62.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c195d74fe98541178ece7a50dad2197d43991e0f77372b9a88da438be2486f12", size = 3070969, upload-time = "2024-04-18T17:56:51.975Z" }, + { url = "https://files.pythonhosted.org/packages/45/f5/42461cacfaeede95f6177dded6765c7262b29294826d2037adaad3fb2a57/grpcio_tools-1.62.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a34d97c62e61bfe9e6cff0410fe144ac8cca2fc979ad0be46b7edf026339d161", size = 2805005, upload-time = "2024-04-18T17:56:55.78Z" }, + { url = "https://files.pythonhosted.org/packages/12/a7/abd8df333f2b207ac1ffd7ad81fb5e12c86334e02663ae44124d70759d4c/grpcio_tools-1.62.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cbb8453ae83a1db2452b7fe0f4b78e4a8dd32be0f2b2b73591ae620d4d784d3d", size = 3685148, upload-time = "2024-04-18T17:56:59.068Z" }, + { url = "https://files.pythonhosted.org/packages/be/4f/a5e84de94c8b9c00a602e4dc40049ca486402e82a42936427991f337f27d/grpcio_tools-1.62.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4f989e5cebead3ae92c6abf6bf7b19949e1563a776aea896ac5933f143f0c45d", size = 3297941, upload-time = "2024-04-18T17:57:01.822Z" }, + { url = "https://files.pythonhosted.org/packages/ce/9a/9dbace60d39fc03bd44c788eda01c6b9d46153d7caffae4a6a6b81b8a2c6/grpcio_tools-1.62.2-cp311-cp311-win32.whl", hash = "sha256:c48fabe40b9170f4e3d7dd2c252e4f1ff395dc24e49ac15fc724b1b6f11724da", size = 910242, upload-time = "2024-04-18T17:57:05.531Z" }, + { url = "https://files.pythonhosted.org/packages/6e/23/0569f9e21fb58c7d38250179ac2c821e08467afcd62919309a47f1fac303/grpcio_tools-1.62.2-cp311-cp311-win_amd64.whl", hash = "sha256:8c616d0ad872e3780693fce6a3ac8ef00fc0963e6d7815ce9dcfae68ba0fc287", size = 1057042, upload-time = "2024-04-18T17:57:08.965Z" }, +] + +[[package]] +name = "gunicorn" +version = "26.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/b7/a4a3f632f823e432ce6bc65f62961b7980c898c77f075a2f7118cb3846fe/gunicorn-26.0.0.tar.gz", hash = "sha256:ca9346f85e3a4aeeb64d491045c16b9a35647abd37ea15efe53080eb8b090baf", size = 727286, upload-time = "2026-05-05T06:38:25.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/40/9c2384fc2be4ad25dd4a49decd5ad9ea5a3639814c11bd40ab77cb9f0a14/gunicorn-26.0.0-py3-none-any.whl", hash = "sha256:40233d26a5f0d1872916188c276e21641155111c2853f0c2cd55260aec0d24fc", size = 212009, upload-time = "2026-05-05T06:38:23.007Z" }, +] + +[[package]] +name = "httplib2" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/ef/f0e05d5886a9c25dea4b18be06cd7bcaddbae0168cc576f3568f9bd6a35a/httplib2-0.19.0.tar.gz", hash = "sha256:e0d428dad43c72dbce7d163b7753ffc7a39c097e6788ef10f4198db69b92f08e", size = 263522, upload-time = "2021-02-07T20:07:47.61Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/7e/51e5bd333c0afa1c7bdbf98eb3b0ccf5167e2b1ecc8b4d13e9cc29291f81/httplib2-0.19.0-py3-none-any.whl", hash = "sha256:749c32603f9bf16c1277f59531d502e8f1c2ca19901ae653b49c4ed698f0820e", size = 95399, upload-time = "2021-02-07T20:07:46.329Z" }, +] + +[[package]] +name = "idna" +version = "2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7/idna-2.8.tar.gz", hash = "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", size = 174481, upload-time = "2018-12-04T17:11:30.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2c/cd551d81dbe15200be1cf41cd03869a46fe7226e7450af7a6545bfc474c9/idna-2.8-py2.py3-none-any.whl", hash = "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c", size = 58594, upload-time = "2018-12-04T17:11:33.338Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, +] + +[[package]] +name = "isort" +version = "5.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/87/f9/c1eb8635a24e87ade2efce21e3ce8cd6b8630bb685ddc9cdaca1349b2eb5/isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109", size = 175303, upload-time = "2023-12-13T20:37:26.124Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/b3/8def84f539e7d2289a02f0524b944b15d7c75dab7628bedf1c4f0992029c/isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6", size = 92310, upload-time = "2023-12-13T20:37:23.244Z" }, +] + +[[package]] +name = "itsdangerous" +version = "2.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/66/d6c5859dcac92b442626427a8c7a42322068c5cd5d4a463ce78b93f730b7/itsdangerous-2.0.1.tar.gz", hash = "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0", size = 59336, upload-time = "2021-05-18T15:09:44.532Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/96/26f935afba9cd6140216da5add223a0c465b99d0f112b68a4ca426441019/itsdangerous-2.0.1-py3-none-any.whl", hash = "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c", size = 18326, upload-time = "2021-05-18T15:09:42.542Z" }, +] + +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, +] + +[[package]] +name = "jaraco-context" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-tarfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, +] + +[[package]] +name = "jaraco-functools" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/1f/c23395957d41ccf27c4e535c3d334c4051e5395b3752057ba4cbaec35c56/jaraco_functools-4.6.0.tar.gz", hash = "sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280", size = 20837, upload-time = "2026-07-14T01:28:02.544Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/36/ecc85bc96c273dc8a11273ed4782272975e6338d4a3e9228621175edf0e3/jaraco_functools-4.6.0-py3-none-any.whl", hash = "sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30", size = 11677, upload-time = "2026-07-14T01:28:01.59Z" }, +] + +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/55/39036716d19cab0747a5020fc7e907f362fbf48c984b14e62127f7e68e5d/jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369", size = 240245, upload-time = "2024-05-05T23:42:02.455Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/80/3a54838c3fb461f6fec263ebf3a3a41771bd05190238de3486aae8540c36/jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d", size = 133271, upload-time = "2024-05-05T23:41:59.928Z" }, +] + +[[package]] +name = "jira" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "defusedxml" }, + { name = "oauthlib", extra = ["signedtoken"] }, + { name = "pbr" }, + { name = "requests" }, + { name = "requests-oauthlib" }, + { name = "requests-toolbelt" }, + { name = "setuptools" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/25/22de2a31d679e23b0865d6c39c6fd8e28da0bcfea3a1da1eeee0e4fdd286/jira-2.0.0.tar.gz", hash = "sha256:e2a94adff98e45b29ded030adc76103eab34fa7d4d57303f211f572bedba0e93", size = 95561, upload-time = "2018-07-12T17:43:51.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/c2/4203b6cad72d115b0ad62ae7028567b538c8933745f665d0eee5082baf67/jira-2.0.0-py2.py3-none-any.whl", hash = "sha256:9adeead4d5f5a6aff74c630787f8bd2d4b0e154f3a3036641298064e91b2d25d", size = 57560, upload-time = "2018-07-12T17:43:50.114Z" }, +] + +[[package]] +name = "keyring" +version = "25.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "secretstorage", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, +] + +[[package]] +name = "kubernetes" +version = "34.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "durationpy" }, + { name = "google-auth" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "requests-oauthlib" }, + { name = "six" }, + { name = "urllib3" }, + { name = "websocket-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/55/3f880ef65f559cbed44a9aa20d3bdbc219a2c3a3bac4a30a513029b03ee9/kubernetes-34.1.0.tar.gz", hash = "sha256:8fe8edb0b5d290a2f3ac06596b23f87c658977d46b5f8df9d0f4ea83d0003912", size = 1083771, upload-time = "2025-09-29T20:23:49.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/ec/65f7d563aa4a62dd58777e8f6aa882f15db53b14eb29aba0c28a20f7eb26/kubernetes-34.1.0-py2.py3-none-any.whl", hash = "sha256:bffba2272534e224e6a7a74d582deb0b545b7c9879d2cd9e4aae9481d1f2cc2a", size = 2008380, upload-time = "2025-09-29T20:23:47.684Z" }, +] + +[[package]] +name = "lazy-object-proxy" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/a2/69df9c6ba6d316cfd81fe2381e464db3e6de5db45f8c43c6a23504abf8cb/lazy_object_proxy-1.12.0.tar.gz", hash = "sha256:1f5a462d92fd0cfb82f1fab28b51bfb209fabbe6aabf7f0d51472c0c124c0c61", size = 43681, upload-time = "2025-08-22T13:50:06.783Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/b3/4684b1e128a87821e485f5a901b179790e6b5bc02f89b7ee19c23be36ef3/lazy_object_proxy-1.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1cf69cd1a6c7fe2dbcc3edaa017cf010f4192e53796538cc7d5e1fedbfa4bcff", size = 26656, upload-time = "2025-08-22T13:42:30.605Z" }, + { url = "https://files.pythonhosted.org/packages/3a/03/1bdc21d9a6df9ff72d70b2ff17d8609321bea4b0d3cffd2cea92fb2ef738/lazy_object_proxy-1.12.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:efff4375a8c52f55a145dc8487a2108c2140f0bec4151ab4e1843e52eb9987ad", size = 68832, upload-time = "2025-08-22T13:42:31.675Z" }, + { url = "https://files.pythonhosted.org/packages/3d/4b/5788e5e8bd01d19af71e50077ab020bc5cce67e935066cd65e1215a09ff9/lazy_object_proxy-1.12.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1192e8c2f1031a6ff453ee40213afa01ba765b3dc861302cd91dbdb2e2660b00", size = 69148, upload-time = "2025-08-22T13:42:32.876Z" }, + { url = "https://files.pythonhosted.org/packages/79/0e/090bf070f7a0de44c61659cb7f74c2fe02309a77ca8c4b43adfe0b695f66/lazy_object_proxy-1.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3605b632e82a1cbc32a1e5034278a64db555b3496e0795723ee697006b980508", size = 67800, upload-time = "2025-08-22T13:42:34.054Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d2/b320325adbb2d119156f7c506a5fbfa37fcab15c26d13cf789a90a6de04e/lazy_object_proxy-1.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a61095f5d9d1a743e1e20ec6d6db6c2ca511961777257ebd9b288951b23b44fa", size = 68085, upload-time = "2025-08-22T13:42:35.197Z" }, + { url = "https://files.pythonhosted.org/packages/6a/48/4b718c937004bf71cd82af3713874656bcb8d0cc78600bf33bb9619adc6c/lazy_object_proxy-1.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:997b1d6e10ecc6fb6fe0f2c959791ae59599f41da61d652f6c903d1ee58b7370", size = 26535, upload-time = "2025-08-22T13:42:36.521Z" }, + { url = "https://files.pythonhosted.org/packages/41/a0/b91504515c1f9a299fc157967ffbd2f0321bce0516a3d5b89f6f4cad0355/lazy_object_proxy-1.12.0-pp39.pp310.pp311.graalpy311-none-any.whl", hash = "sha256:c3b2e0af1f7f77c4263759c4824316ce458fabe0fceadcd24ef8ca08b2d1e402", size = 15072, upload-time = "2025-08-22T13:50:05.498Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "more-itertools" +version = "11.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/1d/f4da6f02cdffe04d6362210b807146a26044c88d839208aec273bb0d9184/more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d", size = 145772, upload-time = "2026-05-22T14:14:29.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, +] + +[[package]] +name = "mozfile" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/f8/a1f0076490d50dbe8bdcf15df97856a4734f459aaf0a4d42c64a11ab7231/mozfile-3.0.0.tar.gz", hash = "sha256:92ca1a786abbdf5e6a7aada62d3a4e28f441ef069c7623223add45268e53c789", size = 7699, upload-time = "2022-10-13T13:09:09.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/cd/fe6b0afa57fbf026631de1435682735dae0aa0ffbe3855f62806da31c55b/mozfile-3.0.0-py2.py3-none-any.whl", hash = "sha256:3b0afcda2fa8b802ef657df80a56f21619008f61fcc14b756124028d7b7adf5c", size = 8227, upload-time = "2022-10-13T13:09:08.03Z" }, +] + +[[package]] +name = "mozinfo" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distro" }, + { name = "mozfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/35/96cccb2244a08247f5c1b5e810d6117d35a30e4a3e29679ed0c7dd2406c6/mozinfo-1.2.3.tar.gz", hash = "sha256:5d2b8a5f1b362692f221e33eb3ff47454a580db1a1384614cdc637b31131b438", size = 6358, upload-time = "2023-07-28T13:50:55.068Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/b2/0efcb9aa6d1362aa00b567c8f355f028332a5a533f80c45e5dacd12a5466/mozinfo-1.2.3-py2.py3-none-any.whl", hash = "sha256:90e0cfb377fc2cc3fad023d38c1f6d60a9135400ff5684a04abf79ca5cc3c521", size = 7454, upload-time = "2023-07-28T13:50:52.402Z" }, +] + +[[package]] +name = "mozprocess" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mozinfo" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/a0/72e23715343b34f702f95399d1ce49c1cd12868048548a009d538edbb842/mozprocess-1.3.1.tar.gz", hash = "sha256:7dc38ec3c11693e9944ade1558392c04f37fd8df68d3ec7a20372dfe96b2e5bb", size = 20571, upload-time = "2023-07-13T17:21:24.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/41/32683c60bbdd97643c9b08d8719c178d674e35e8d5d2ec4b18a874bfcfb7/mozprocess-1.3.1-py2.py3-none-any.whl", hash = "sha256:ae343fabb72840195278b73ad426fb04d35d66b15a664f5c47221f4e4f0842e5", size = 22113, upload-time = "2023-07-13T17:21:21.239Z" }, +] + +[[package]] +name = "msgpack" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/6b/e9b1cdc042c4458801d2545ed782a95f3d6ba8e270cce8745b8603c7f748/msgpack-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:29a3f6e9667868429d8240dfd063ea5ffdc1321c13d783aa23827a38de0dcb22", size = 82812, upload-time = "2026-06-18T16:12:45.022Z" }, + { url = "https://files.pythonhosted.org/packages/0c/3a/dd518a1bf78ed1e9ad8afe57307c079a00eafe4b3068932a27ca1ea56b4f/msgpack-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aded5bdf32609dc7987a49bbbd15a8ef096193f96dd8bbeb791de729e650acf5", size = 82739, upload-time = "2026-06-18T16:12:46.025Z" }, + { url = "https://files.pythonhosted.org/packages/70/e0/7ba9e1542bf0771a27b8b37c1316e3f95ae9d748fd765284655c476ad4ef/msgpack-1.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:146ee4e9ce80b365c6d4c47073da9da7bcec473e58194ceee5dd7620ace77e06", size = 414233, upload-time = "2026-06-18T16:12:47.029Z" }, + { url = "https://files.pythonhosted.org/packages/03/8d/671d81534ea0e2b0e8a121be100020da09eb78861fe3aa8f3ef7dcd3bed1/msgpack-1.2.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a28d076ca7c82b9c8728ad90b7147489449557038bed50e4241eb832395169b4", size = 423843, upload-time = "2026-06-18T16:12:48.19Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b6/e5c737515ed1f166664b87601b532f58cbb73d8aa6a90b99f7c2c5037e8e/msgpack-1.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d31c0ac0c640f877804c67cb2bc9f4e23dc2db97e96c2e67fa27d38283b41f8", size = 390772, upload-time = "2026-06-18T16:12:49.624Z" }, + { url = "https://files.pythonhosted.org/packages/a8/46/62ed8c2e87d7021eab19921594d961ef3aa3794eec76c716dc30f3bfd433/msgpack-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ff92d7feeaf5bc26c51495b69e2f99ed97ab79346fb6555f44be7dd2ac6503b", size = 409559, upload-time = "2026-06-18T16:12:50.936Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/59aa3887b860bbf43532835e192b1c388a17590d6068ae4f8b2bc74c906e/msgpack-1.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:779197a6513bab3c3632265e3d0f7cb3227e62510841a6f34f1eaa37efbb345e", size = 387838, upload-time = "2026-06-18T16:12:52.161Z" }, + { url = "https://files.pythonhosted.org/packages/09/11/f8563e471093420cf6478cb3271a0175d8402b82d879783d4035d2d03360/msgpack-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:67f6dd22fa72a93752643f07889796d62739a13415ee630169a8ce764f86cf9f", size = 421732, upload-time = "2026-06-18T16:12:53.556Z" }, + { url = "https://files.pythonhosted.org/packages/57/cf/e673683c4c6c90c1022b24c65af4b03eda72b182a1176ef6449069d66acc/msgpack-1.2.1-cp311-cp311-win32.whl", hash = "sha256:91054a783328e0ea7954b8771095705c8d2243b814743fbaadf14552c9c52c5d", size = 64091, upload-time = "2026-06-18T16:12:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/ca212739d179f9083bff2c7c08c24101c3555a334fadc2b876b18768a3ae/msgpack-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2eda0b7ebb1283a98d3e4492ac933c8af6aff59fd3df1c3ed024f536af4b1dc8", size = 70462, upload-time = "2026-06-18T16:12:55.898Z" }, + { url = "https://files.pythonhosted.org/packages/6d/be/6798347b425e26f35db82e69dd83c09716c856a3714e7bffc4c0860fd830/msgpack-1.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:6ee967f7c7e1df2890c671ff2ee51a28ded0efc95da3e507176dee881ce36c66", size = 65059, upload-time = "2026-06-18T16:12:57.053Z" }, +] + +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/f1/a90635c4f88fb913fbf4ce660b83b7445b7a02615bda034b2f8eb38fd597/multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d", size = 76626, upload-time = "2026-01-26T02:43:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/267e64eaf6fc637a15b35f5de31a566634a2740f97d8d094a69d34f524a4/multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e", size = 44706, upload-time = "2026-01-26T02:43:27.607Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a4/d45caf2b97b035c57267791ecfaafbd59c68212004b3842830954bb4b02e/multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855", size = 44356, upload-time = "2026-01-26T02:43:28.661Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d2/0a36c8473f0cbaeadd5db6c8b72d15bbceeec275807772bfcd059bef487d/multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3", size = 244355, upload-time = "2026-01-26T02:43:31.165Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/8c65be997fd7dd311b7d39c7b6e71a0cb449bad093761481eccbbe4b42a2/multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e", size = 246433, upload-time = "2026-01-26T02:43:32.581Z" }, + { url = "https://files.pythonhosted.org/packages/01/fb/4dbd7e848d2799c6a026ec88ad39cf2b8416aa167fcc903baa55ecaa045c/multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a", size = 225376, upload-time = "2026-01-26T02:43:34.417Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/4a3a6341eac3830f6053062f8fbc9a9e54407c80755b3f05bc427295c2d0/multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8", size = 257365, upload-time = "2026-01-26T02:43:35.741Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a2/dd575a69c1aa206e12d27d0770cdf9b92434b48a9ef0cd0d1afdecaa93c4/multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0", size = 254747, upload-time = "2026-01-26T02:43:36.976Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/21b27c560c13822ed93133f08aa6372c53a8e067f11fbed37b4adcdac922/multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144", size = 246293, upload-time = "2026-01-26T02:43:38.258Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a4/23466059dc3854763423d0ad6c0f3683a379d97673b1b89ec33826e46728/multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49", size = 242962, upload-time = "2026-01-26T02:43:40.034Z" }, + { url = "https://files.pythonhosted.org/packages/1f/67/51dd754a3524d685958001e8fa20a0f5f90a6a856e0a9dcabff69be3dbb7/multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71", size = 237360, upload-time = "2026-01-26T02:43:41.752Z" }, + { url = "https://files.pythonhosted.org/packages/64/3f/036dfc8c174934d4b55d86ff4f978e558b0e585cef70cfc1ad01adc6bf18/multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3", size = 245940, upload-time = "2026-01-26T02:43:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/3d/20/6214d3c105928ebc353a1c644a6ef1408bc5794fcb4f170bb524a3c16311/multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c", size = 253502, upload-time = "2026-01-26T02:43:44.371Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/c653bc4ae1be70a0f836b82172d643fcf1dade042ba2676ab08ec08bff0f/multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0", size = 247065, upload-time = "2026-01-26T02:43:45.745Z" }, + { url = "https://files.pythonhosted.org/packages/c8/11/a854b4154cd3bd8b1fd375e8a8ca9d73be37610c361543d56f764109509b/multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa", size = 241870, upload-time = "2026-01-26T02:43:47.054Z" }, + { url = "https://files.pythonhosted.org/packages/13/bf/9676c0392309b5fdae322333d22a829715b570edb9baa8016a517b55b558/multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a", size = 41302, upload-time = "2026-01-26T02:43:48.753Z" }, + { url = "https://files.pythonhosted.org/packages/c9/68/f16a3a8ba6f7b6dc92a1f19669c0810bd2c43fc5a02da13b1cbf8e253845/multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b", size = 45981, upload-time = "2026-01-26T02:43:49.921Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/9dd5305253fa00cd3c7555dbef69d5bf4133debc53b87ab8d6a44d411665/multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6", size = 43159, upload-time = "2026-01-26T02:43:51.635Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, +] + +[[package]] +name = "mypy-protobuf" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, + { name = "types-protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/3a/189e1a3f2d378c9e23e26fe76649ad0c5530e7f3e825a71af88e1b73a365/mypy-protobuf-3.4.0.tar.gz", hash = "sha256:7d75a079651b105076776a35a5405e3fa773b8a167118f1b712e443e9a6c18a2", size = 23540, upload-time = "2022-10-20T06:18:07.864Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/47/5bad06a9546d9b8115b4f8f1e216a6033055ad06bec0bbcbfe3918a27b70/mypy_protobuf-3.4.0-py3-none-any.whl", hash = "sha256:da33dfde7547ff57e5ba5564126cbfa114f14413b2fa50759b1fa5de1e4ab511", size = 15885, upload-time = "2022-10-20T06:18:05.549Z" }, +] + +[[package]] +name = "nh3" +version = "0.3.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/1b/ef84624f14954d270f74060a19fc550dd4f06656399447569afb584d8c06/nh3-0.3.6.tar.gz", hash = "sha256:f3736c9dd3d1856f80cd031715b84ca75cda2bbb1ac802c3da26bfce590838d7", size = 24684, upload-time = "2026-06-22T00:47:02.008Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/ab/a7653bce9a3b204be6a6931767a9e23595807bb84790ce6685e4d7e5bd08/nh3-0.3.6-cp38-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:a43ebd7543555c3ac1bc353023d0794e75cb76f6f18f19c32e95441496c0cc25", size = 1443564, upload-time = "2026-06-22T00:46:36.66Z" }, + { url = "https://files.pythonhosted.org/packages/41/21/e1084ab18eb589506335c7c7576f2d4643e9a0c0e33983ef0e549a256b96/nh3-0.3.6-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1b160831c9cdb06a6c79c2f9cdb11386602938f9af260d1c457a85add4f6f69", size = 838002, upload-time = "2026-06-22T00:46:38.101Z" }, + { url = "https://files.pythonhosted.org/packages/b0/94/f48d08e6f72a406300fa11d8acd929fea1a80d4bf750fa292cb10785f126/nh3-0.3.6-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d14bf7982e7a77c0c775634c29c07ce08b38a046df73e1c1f139b3e82f18a38e", size = 823045, upload-time = "2026-06-22T00:46:39.495Z" }, + { url = "https://files.pythonhosted.org/packages/25/bb/431615ba1d1d3eb63cde0f974f2114edf863a8a3f6049a12fed23fc241d3/nh3-0.3.6-cp38-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:44673b27010051ab5a5e438a86ec31bbda61d4a77d7e900af6b7be3037c1abae", size = 1093171, upload-time = "2026-06-22T00:46:41.21Z" }, + { url = "https://files.pythonhosted.org/packages/0e/24/a0d80182a18919665fefd19c1c06f1d1df1c9a6455d0252de40c034a0bc3/nh3-0.3.6-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6b7beece07525dc6e6b0fc2f104442de2ba328360ad00e50cbe2e1fd620447d", size = 1049217, upload-time = "2026-06-22T00:46:42.804Z" }, + { url = "https://files.pythonhosted.org/packages/0a/13/6f1e302ca674ac74362e150848ad56a1be5145391204f74facdb8e94df12/nh3-0.3.6-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:455469a29951edc92bc48b47ac2281c3f2609e6c4f6a047056449f8c2c23facf", size = 917372, upload-time = "2026-06-22T00:46:44.495Z" }, + { url = "https://files.pythonhosted.org/packages/5b/67/314f6151bad77a93d751978a344033e1fc890822f05f0416079338e34231/nh3-0.3.6-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:905f877dc66dd7aea4a76e54bcb26acb5ff8216f720c0017ccf63e0e6035698e", size = 806699, upload-time = "2026-06-22T00:46:45.99Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a6/bfaa00046e58603507dcfc266c4778e3ab7adf68a5dedd73b6274b8d9314/nh3-0.3.6-cp38-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:25c733bee928530556b1db0ea46c52cf5aa686146e38e60a6fc7cb801ef91cec", size = 835165, upload-time = "2026-06-22T00:46:47.617Z" }, + { url = "https://files.pythonhosted.org/packages/30/a8/fb2c38845efb703a9173bffdfc745fc64d2b0e55cfc73a3647d2f028250c/nh3-0.3.6-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2f90d9a0cfdbee218994fdaaeeb5a0fde62d08f35e4eef0378ec1e2200172fd0", size = 858282, upload-time = "2026-06-22T00:46:49.276Z" }, + { url = "https://files.pythonhosted.org/packages/68/17/06e72a18ee9b572914447338237ca7eb164c0df901f141bc10d1282247a2/nh3-0.3.6-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:82ca5bf427ad1b216b65ede1a2e2d87dc49bec417ceba0f297213107d3cd9d78", size = 1014328, upload-time = "2026-06-22T00:46:51.026Z" }, + { url = "https://files.pythonhosted.org/packages/11/f9/3966c61455668c08853bf5e33b4bed93c421f3194ce4de896dc248d6f6ce/nh3-0.3.6-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f5ed5fe84aee7f39db95c214a7421bf0499fbf500fec6d86a4e29bfc37971438", size = 1098207, upload-time = "2026-06-22T00:46:52.674Z" }, + { url = "https://files.pythonhosted.org/packages/19/d3/479cb4ae440424825735d60525b53e3c77fd60fd6e6afc0e984f00eb0178/nh3-0.3.6-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:082675ff87b9385ec430ffe6d5847ba7456cc39b73720cd4add472f9f4cffd56", size = 1056961, upload-time = "2026-06-22T00:46:54.335Z" }, + { url = "https://files.pythonhosted.org/packages/17/0c/6cdb5ee1e127be50dc8391e54bddc1f64e87bf4bfad0c55633320e2e02db/nh3-0.3.6-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:36d06341bd501240d320f5942481ed5e6846136b666e1ba4faf802b78ebc875f", size = 1033829, upload-time = "2026-06-22T00:46:56.258Z" }, + { url = "https://files.pythonhosted.org/packages/e9/55/9de666ad975d6ccd77d799ea0add55ee2347aa81286ce21b2a97c070746b/nh3-0.3.6-cp38-abi3-win32.whl", hash = "sha256:5276ef17bdba9ad8040575c74072008b13aae429436e9d0429e718bb5f90f4da", size = 609081, upload-time = "2026-06-22T00:46:57.665Z" }, + { url = "https://files.pythonhosted.org/packages/82/fa/2b5d684e3edf1e81bfd02d298c78c3e3da77ca1d8a2be3183a79544a7548/nh3-0.3.6-cp38-abi3-win_amd64.whl", hash = "sha256:f338ac7d594c067679f1e99b4f5ec3906842979560f9d8f15d6bdfa39a353b10", size = 624461, upload-time = "2026-06-22T00:46:59.163Z" }, + { url = "https://files.pythonhosted.org/packages/7b/e5/7cafee2f0413ca4cb0ef3bd111e94d408a48810008b283ad8aee00dd1809/nh3-0.3.6-cp38-abi3-win_arm64.whl", hash = "sha256:69f365963f63a1e9bff53bdbb3c542c7c2efed3e163c9d5d83a772a2ac468c21", size = 603060, upload-time = "2026-06-22T00:47:00.596Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437, upload-time = "2024-06-04T18:44:11.171Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, +] + +[[package]] +name = "oauth2client" +version = "4.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httplib2" }, + { name = "pyasn1" }, + { name = "pyasn1-modules" }, + { name = "rsa" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/7b/17244b1083e8e604bf154cf9b716aecd6388acd656dd01893d0d244c94d9/oauth2client-4.1.3.tar.gz", hash = "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6", size = 155910, upload-time = "2018-09-07T21:38:18.036Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/a9/4f25a14d23f0786b64875b91784607c2277eff25d48f915e39ff0cff505a/oauth2client-4.1.3-py2.py3-none-any.whl", hash = "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac", size = 98206, upload-time = "2018-09-07T21:38:16.742Z" }, +] + +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + +[package.optional-dependencies] +signedtoken = [ + { name = "cryptography" }, + { name = "pyjwt" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "parameterized" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/49/00c0c0cc24ff4266025a53e41336b79adaa5a4ebfad214f433d623f9865e/parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1", size = 24351, upload-time = "2023-03-27T02:01:11.592Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/2f/804f58f0b856ab3bf21617cccf5b39206e6c4c94c2cd227bde125ea6105f/parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b", size = 20475, upload-time = "2023-03-27T02:01:09.31Z" }, +] + +[[package]] +name = "paramiko" +version = "2.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bcrypt" }, + { name = "cryptography" }, + { name = "pynacl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/68/dde7919279d4ecdd1607a7eb425a2874ccd49a73a5a71f8aa4f0102d3eb8/paramiko-2.6.0.tar.gz", hash = "sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041", size = 1018679, upload-time = "2019-06-23T22:49:01.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/80/74dace9e48b0ef923633dfb5e48798f58a168e4734bca8ecfaf839ba051a/paramiko-2.6.0-py2.py3-none-any.whl", hash = "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", size = 199863, upload-time = "2019-06-23T22:48:59.057Z" }, +] + +[[package]] +name = "pbr" +version = "7.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/ab/1de9a4f730edde1bdbbc2b8d19f8fa326f036b4f18b2f72cfbea7dc53c26/pbr-7.0.3.tar.gz", hash = "sha256:b46004ec30a5324672683ec848aed9e8fc500b0d261d40a3229c2d2bbfcedc29", size = 135625, upload-time = "2025-11-03T17:04:56.274Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/db/61efa0d08a99f897ef98256b03e563092d36cc38dc4ebe4a85020fe40b31/pbr-7.0.3-py2.py3-none-any.whl", hash = "sha256:ff223894eb1cd271a98076b13d3badff3bb36c424074d26334cd25aebeecea6b", size = 131898, upload-time = "2025-11-03T17:04:54.875Z" }, +] + +[[package]] +name = "pkginfo" +version = "1.12.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/03/e26bf3d6453b7fda5bd2b84029a426553bb373d6277ef6b5ac8863421f87/pkginfo-1.12.1.2.tar.gz", hash = "sha256:5cd957824ac36f140260964eba3c6be6442a8359b8c48f4adf90210f33a04b7b", size = 451828, upload-time = "2025-02-19T15:27:37.188Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/3d/f4f2ba829efb54b6cd2d91349c7463316a9cc55a43fc980447416c88540f/pkginfo-1.12.1.2-py3-none-any.whl", hash = "sha256:c783ac885519cab2c34927ccfa6bf64b5a704d7c69afaea583dd9b7afe969343", size = 32717, upload-time = "2025-02-19T15:27:33.071Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.11.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/98/0bf930c4f97d0266b58a89e36c015f56232c52b5d2f207215d48cca9e8f7/platformdirs-4.11.2.tar.gz", hash = "sha256:3a2ae5fca3520a01ab1be8b45613537f52ddf5b5f6f53d88233892dfbf0cd82d", size = 32716, upload-time = "2026-08-10T15:48:06.092Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/e2/4e6eee633809c376c024821b91ade709cbfd040ec53939ffbcc292aa7eee/platformdirs-4.11.2-py3-none-any.whl", hash = "sha256:7f89089b6ea71bda7962953edcf784b2e2d9d285b40ad88be2bb75c6e9d82ab4", size = 23361, upload-time = "2026-08-10T15:48:04.855Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/f1/8a8cc1c2c7e7934ab77e0163414f736fadbc0f5e8dd9673b952355ac175b/propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78", size = 90744, upload-time = "2026-05-08T20:59:45.799Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f4/651b1225e976bd1a2ba5cfba0c29d096581c2636b437e3a9a7ab6276270a/propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959", size = 52033, upload-time = "2026-05-08T20:59:47.408Z" }, + { url = "https://files.pythonhosted.org/packages/15/a8/8ede85d6aa1f79fc7dc2f8fd2c8d65920b8272c3892903c8a1affde48cfb/propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7", size = 52754, upload-time = "2026-05-08T20:59:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/7d/fe/b3551b41bbc2f5b5bb088fc6920567cd43101253e68fbaa261339eb96fe1/propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511", size = 57573, upload-time = "2026-05-08T20:59:50.778Z" }, + { url = "https://files.pythonhosted.org/packages/83/27/ab851ebd1b7172e3e161f5f8d39e315d54a91bea246f01f4d872d3376aef/propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660", size = 60645, upload-time = "2026-05-08T20:59:52.227Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/466b3d18022e9897cbda9c735c493c5bd747d7a4c6f5ea1480b4cec434b6/propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66", size = 61563, upload-time = "2026-05-08T20:59:53.866Z" }, + { url = "https://files.pythonhosted.org/packages/27/1b/16ab7f2cf2041da2f60d156ba64c2484eadf9168075b4ff43c3ef60045af/propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b", size = 58888, upload-time = "2026-05-08T20:59:55.457Z" }, + { url = "https://files.pythonhosted.org/packages/0a/67/bb777ffd907633563bf35fd859c4ce97b0512c32f4633cf5d1eb7c33512b/propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67", size = 59253, upload-time = "2026-05-08T20:59:57.075Z" }, + { url = "https://files.pythonhosted.org/packages/b9/42/64f8d90b73fd9cdc1499b48057ff6d9cd2a98a25734c9bb62ecf07e87061/propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f", size = 57558, upload-time = "2026-05-08T20:59:58.602Z" }, + { url = "https://files.pythonhosted.org/packages/eb/02/dba5bc03c9041f2092ea55a449caf5dfe68352c6654511b29ba0654ddb69/propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c", size = 55007, upload-time = "2026-05-08T20:59:59.837Z" }, + { url = "https://files.pythonhosted.org/packages/14/c0/43f649c7aa2a77a3b100d84e9dea3a483120ecb608bfe36ce49eaff517fe/propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0", size = 60355, upload-time = "2026-05-08T21:00:01.144Z" }, + { url = "https://files.pythonhosted.org/packages/83/c0/435dafd27f1cb4a495381dae60e25883ccfe4020bb72818e8184c1678092/propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6", size = 59057, upload-time = "2026-05-08T21:00:02.401Z" }, + { url = "https://files.pythonhosted.org/packages/53/ae/6e292df9135d659944e96cb3389258e4a663e5b2b5f6c217ef0ddc8d2f73/propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27", size = 61938, upload-time = "2026-05-08T21:00:03.638Z" }, + { url = "https://files.pythonhosted.org/packages/0b/42/314ebc50d8159055411fd6b0bda322ff510e4b1f7d2e4927940ad0f6af20/propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f", size = 59731, upload-time = "2026-05-08T21:00:04.881Z" }, + { url = "https://files.pythonhosted.org/packages/b8/9b/2da6dee38871c3c8772fabc2758325a5c9077d6d18c597737dc04dd884cd/propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0", size = 38966, upload-time = "2026-05-08T21:00:06.511Z" }, + { url = "https://files.pythonhosted.org/packages/42/4e/f17363fb58c0afe05b067361cb6d86ed2d29de6506779a27547c4d183075/propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82", size = 42135, upload-time = "2026-05-08T21:00:08.088Z" }, + { url = "https://files.pythonhosted.org/packages/c6/eb/6af6685077d22e8b33358d3c548e3282706a0b3cd85044ffba4e5dd08e3b/propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab", size = 38381, upload-time = "2026-05-08T21:00:09.692Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, +] + +[[package]] +name = "proto-plus" +version = "1.27.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/02/8832cde80e7380c600fbf55090b6ab7b62bd6825dbedde6d6657c15a1f8e/proto_plus-1.27.1.tar.gz", hash = "sha256:912a7460446625b792f6448bade9e55cd4e41e6ac10e27009ef71a7f317fa147", size = 56929, upload-time = "2026-02-02T17:34:49.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/79/ac273cbbf744691821a9cca88957257f41afe271637794975ca090b9588b/proto_plus-1.27.1-py3-none-any.whl", hash = "sha256:e4643061f3a4d0de092d62aa4ad09fa4756b2cbb89d4627f3985018216f9fefc", size = 50480, upload-time = "2026-02-02T17:34:47.339Z" }, +] + +[[package]] +name = "protobuf" +version = "4.23.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/1c/de86d82a5fc780feca36ef52c1231823bb3140266af8a04ed6286957aa6e/protobuf-4.23.4.tar.gz", hash = "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9", size = 400173, upload-time = "2023-07-06T23:28:22.071Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/a7/872807299eb114956c665fb1717ce106a8874db08a724651ac4f78c1198c/protobuf-4.23.4-cp310-abi3-win32.whl", hash = "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b", size = 402981, upload-time = "2023-07-06T23:27:57.401Z" }, + { url = "https://files.pythonhosted.org/packages/80/70/dc63d340d27b8ff22022d7dd14b8d6d68b479a003eacdc4507150a286d9a/protobuf-4.23.4-cp310-abi3-win_amd64.whl", hash = "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12", size = 422467, upload-time = "2023-07-06T23:28:00.387Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d3/a164038605494d49acc4f9cda1c0bc200b96382c53edd561387263bb181d/protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd", size = 400308, upload-time = "2023-07-06T23:28:02.356Z" }, + { url = "https://files.pythonhosted.org/packages/71/42/3a7fc57f360f728f38eca6656e8d00edaf22bc0ffc35dd2936f23e5fbb3e/protobuf-4.23.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a", size = 303455, upload-time = "2023-07-06T23:28:04.292Z" }, + { url = "https://files.pythonhosted.org/packages/01/cb/445b3e465abdb8042a41957dc8f60c54620dc7540dbcf9b458a921531ca2/protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597", size = 304498, upload-time = "2023-07-06T23:28:06.277Z" }, + { url = "https://files.pythonhosted.org/packages/b0/07/fb712cce15ba456f7c24b82b97c8a7db2233f07037ffe61c9011660c592a/protobuf-4.23.4-py3-none-any.whl", hash = "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff", size = 173332, upload-time = "2023-07-06T23:28:20.053Z" }, +] + +[[package]] +name = "psutil" +version = "5.9.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/7d/d05864a69e452f003c0d77e728e155a89a2a26b09e64860ddd70ad64fb26/psutil-5.9.4.tar.gz", hash = "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62", size = 485825, upload-time = "2022-11-07T19:53:36.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/73/35cea01aad1baf901c915dc95ea33a2f271c8ff8cf2f1c73b7f591f1bdf1/psutil-5.9.4-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7", size = 243468, upload-time = "2022-11-07T18:45:00.474Z" }, + { url = "https://files.pythonhosted.org/packages/5a/37/ef88eed265d93bc28c681316f68762c5e04167519e5627a0187c8878b409/psutil-5.9.4-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1", size = 277515, upload-time = "2022-11-07T18:45:05.428Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c8/784968329c1c67c28cce91991ef9af8a8913aa5a3399a6a8954b1380572f/psutil-5.9.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08", size = 280218, upload-time = "2022-11-07T18:45:11.831Z" }, + { url = "https://files.pythonhosted.org/packages/3e/af/fe14b984e8b0f778d502d387b789d846cb2fcc3989f63be942741266d8c8/psutil-5.9.4-cp36-abi3-win32.whl", hash = "sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff", size = 247217, upload-time = "2022-11-08T11:50:03.989Z" }, + { url = "https://files.pythonhosted.org/packages/25/6e/ba97809175c90cbdcd33b470e466ebf0854d15d1506e605cc0ddd284d5b6/psutil-5.9.4-cp36-abi3-win_amd64.whl", hash = "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4", size = 252462, upload-time = "2022-11-08T11:50:07.829Z" }, + { url = "https://files.pythonhosted.org/packages/79/26/f026804298b933b11640cc2d15155a545805df732e5ead3a2ad7cf45a38b/psutil-5.9.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e", size = 244234, upload-time = "2022-11-07T19:53:20.872Z" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + +[[package]] +name = "pyfakefs" +version = "4.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/a5/998d96ed6c26d1406ddc24c11dfdf24a4b6d5dd9d6834c889784976e7e70/pyfakefs-4.4.0.tar.gz", hash = "sha256:082d863e0e2a74351f697da404e329a91e18e5055942e59d1b836e8459b2c94c", size = 168586, upload-time = "2021-02-24T20:32:56.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/db/0b13c98f5eb13919b4bae3fe641b71310925bd7bac9465010e6d09fa8aeb/pyfakefs-4.4.0-py3-none-any.whl", hash = "sha256:1ac3b2845dabe69af56c20691b9347914581195ccdde352535fb7d4ff0055c19", size = 185060, upload-time = "2021-02-24T20:32:54.225Z" }, +] + +[[package]] +name = "pygithub" +version = "1.55" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "pyjwt" }, + { name = "pynacl" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/36/386d282903c572b18abc36de68aaf4146db4659c82dceee009ef88a86b67/PyGithub-1.55.tar.gz", hash = "sha256:1bbfff9372047ff3f21d5cd8e07720f3dbfdaf6462fcaed9d815f528f1ba7283", size = 158046, upload-time = "2021-04-26T04:45:21.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/1f/9dc4ba315eeea222473cf4c15d3e665f32d52f859d9d6e73219d0a408969/PyGithub-1.55-py3-none-any.whl", hash = "sha256:2caf0054ea079b71e539741ae56c5a95e073b81fa472ce222e81667381b9601b", size = 291677, upload-time = "2021-04-26T04:45:19.923Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/f0/9804c72e9a314360c135f42c434eb42eaabb5e7ebad760cbd8fc7023be38/PyJWT-2.7.0.tar.gz", hash = "sha256:bd6ca4a3c4285c1a2d4349e5a035fdf8fb94e04ccd0fcbe6ba289dae9cc3e074", size = 77902, upload-time = "2023-05-09T20:04:43.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/e8/01b2e35d81e618a8212e651e10c91660bdfda49c1d15ce66f4ca1ff43649/PyJWT-2.7.0-py3-none-any.whl", hash = "sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1", size = 22366, upload-time = "2023-05-09T20:04:39.903Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + +[[package]] +name = "pylint" +version = "2.17.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astroid" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "dill" }, + { name = "isort" }, + { name = "mccabe" }, + { name = "platformdirs" }, + { name = "tomlkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/e9/21f9ce3e4b81eef011be070a29f8a5c193e2488ed8713a898baa4e8b3362/pylint-2.17.7.tar.gz", hash = "sha256:f4fcac7ae74cfe36bc8451e931d8438e4a476c20314b1101c458ad0f05191fad", size = 434994, upload-time = "2023-09-30T21:25:10.328Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/49/cea450a83079445a84f16050e571a7c383d3f474b13c5caedfebd4e35def/pylint-2.17.7-py3-none-any.whl", hash = "sha256:27a8d4c7ddc8c2f8c18aa0050148f89ffc09838142193fdbe98f172781a3ff87", size = 537178, upload-time = "2023-09-30T21:25:07.527Z" }, +] + +[[package]] +name = "pylint-protobuf" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astroid" }, + { name = "protobuf" }, + { name = "pylint" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/6d/eece5070419848acf648153be2b561662abeae5428ef19e12f27a6888fd6/pylint-protobuf-0.21.0.tar.gz", hash = "sha256:c643f796db9d0c85fc597803bc92c920c0a3eb1f02acff210518589aef79112f", size = 14552, upload-time = "2023-10-06T08:27:54.68Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/00/a819c6295a92734fa9c9ea12a4ae28d17eb97a025a19eecf4302c9cc2380/pylint_protobuf-0.21.0-py2.py3-none-any.whl", hash = "sha256:b334ba32f40a579d1f8b44f3ab7a495f2553b204df6f628ee20e53db4d5a9b17", size = 15604, upload-time = "2023-10-06T08:27:52.485Z" }, +] + +[[package]] +name = "pymemcache" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/b6/4541b664aeaad025dfb8e851dcddf8e25ab22607e674dd2b562ea3e3586f/pymemcache-4.0.0.tar.gz", hash = "sha256:27bf9bd1bbc1e20f83633208620d56de50f14185055e49504f4f5e94e94aff94", size = 70176, upload-time = "2022-10-17T16:53:07.726Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/ba/2f7b22d8135b51c4fefb041461f8431e1908778e6539ff5af6eeaaee367a/pymemcache-4.0.0-py2.py3-none-any.whl", hash = "sha256:f507bc20e0dc8d562f8df9d872107a278df049fa496805c1431b926f3ddd0eab", size = 60772, upload-time = "2022-10-17T16:53:04.388Z" }, +] + +[[package]] +name = "pynacl" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c6/a3124dee667a423f2c637cfd262a54d67d8ccf3e160f3c50f622a85b7723/pynacl-1.6.0.tar.gz", hash = "sha256:cb36deafe6e2bce3b286e5d1f3e1c246e0ccdb8808ddb4550bb2792f2df298f2", size = 3505641, upload-time = "2025-09-10T23:39:22.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/37/87c72df19857c5b3b47ace6f211a26eb862ada495cc96daa372d96048fca/pynacl-1.6.0-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:f4b3824920e206b4f52abd7de621ea7a44fd3cb5c8daceb7c3612345dfc54f2e", size = 382610, upload-time = "2025-09-10T23:38:49.459Z" }, + { url = "https://files.pythonhosted.org/packages/0c/64/3ce958a5817fd3cc6df4ec14441c43fd9854405668d73babccf77f9597a3/pynacl-1.6.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:16dd347cdc8ae0b0f6187a2608c0af1c8b7ecbbe6b4a06bff8253c192f696990", size = 798744, upload-time = "2025-09-10T23:38:58.531Z" }, + { url = "https://files.pythonhosted.org/packages/e4/8a/3f0dd297a0a33fa3739c255feebd0206bb1df0b44c52fbe2caf8e8bc4425/pynacl-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:16c60daceee88d04f8d41d0a4004a7ed8d9a5126b997efd2933e08e93a3bd850", size = 1397879, upload-time = "2025-09-10T23:39:00.44Z" }, + { url = "https://files.pythonhosted.org/packages/41/94/028ff0434a69448f61348d50d2c147dda51aabdd4fbc93ec61343332174d/pynacl-1.6.0-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25720bad35dfac34a2bcdd61d9e08d6bfc6041bebc7751d9c9f2446cf1e77d64", size = 833907, upload-time = "2025-09-10T23:38:50.936Z" }, + { url = "https://files.pythonhosted.org/packages/52/bc/a5cff7f8c30d5f4c26a07dfb0bcda1176ab8b2de86dda3106c00a02ad787/pynacl-1.6.0-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bfaa0a28a1ab718bad6239979a5a57a8d1506d0caf2fba17e524dbb409441cf", size = 1436649, upload-time = "2025-09-10T23:38:52.783Z" }, + { url = "https://files.pythonhosted.org/packages/7a/20/c397be374fd5d84295046e398de4ba5f0722dc14450f65db76a43c121471/pynacl-1.6.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ef214b90556bb46a485b7da8258e59204c244b1b5b576fb71848819b468c44a7", size = 817142, upload-time = "2025-09-10T23:38:54.4Z" }, + { url = "https://files.pythonhosted.org/packages/12/30/5efcef3406940cda75296c6d884090b8a9aad2dcc0c304daebb5ae99fb4a/pynacl-1.6.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:49c336dd80ea54780bcff6a03ee1a476be1612423010472e60af83452aa0f442", size = 1401794, upload-time = "2025-09-10T23:38:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/be/e1/a8fe1248cc17ccb03b676d80fa90763760a6d1247da434844ea388d0816c/pynacl-1.6.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f3482abf0f9815e7246d461fab597aa179b7524628a4bc36f86a7dc418d2608d", size = 772161, upload-time = "2025-09-10T23:39:01.93Z" }, + { url = "https://files.pythonhosted.org/packages/a3/76/8a62702fb657d6d9104ce13449db221a345665d05e6a3fdefb5a7cafd2ad/pynacl-1.6.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:140373378e34a1f6977e573033d1dd1de88d2a5d90ec6958c9485b2fd9f3eb90", size = 1370720, upload-time = "2025-09-10T23:39:03.531Z" }, + { url = "https://files.pythonhosted.org/packages/6d/38/9e9e9b777a1c4c8204053733e1a0269672c0bd40852908c9ad6b6eaba82c/pynacl-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6b393bc5e5a0eb86bb85b533deb2d2c815666665f840a09e0aa3362bb6088736", size = 791252, upload-time = "2025-09-10T23:39:05.058Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d972ce3d92ae05c9091363cf185e8646933f91c376e97b8be79ea6e96c22/pynacl-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a25cfede801f01e54179b8ff9514bd7b5944da560b7040939732d1804d25419", size = 1362910, upload-time = "2025-09-10T23:39:06.924Z" }, + { url = "https://files.pythonhosted.org/packages/35/2c/ee0b373a1861f66a7ca8bdb999331525615061320dd628527a50ba8e8a60/pynacl-1.6.0-cp38-abi3-win32.whl", hash = "sha256:dcdeb41c22ff3c66eef5e63049abf7639e0db4edee57ba70531fc1b6b133185d", size = 226461, upload-time = "2025-09-10T23:39:11.894Z" }, + { url = "https://files.pythonhosted.org/packages/75/f7/41b6c0b9dd9970173b6acc026bab7b4c187e4e5beef2756d419ad65482da/pynacl-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:cf831615cc16ba324240de79d925eacae8265b7691412ac6b24221db157f6bd1", size = 238802, upload-time = "2025-09-10T23:39:08.966Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0f/462326910c6172fa2c6ed07922b22ffc8e77432b3affffd9e18f444dbfbb/pynacl-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:84709cea8f888e618c21ed9a0efdb1a59cc63141c403db8bf56c469b71ad56f2", size = 183846, upload-time = "2025-09-10T23:39:10.552Z" }, +] + +[[package]] +name = "pyopenssl" +version = "22.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/d3/d6a9610f19d943e198df502ae660c6b5acf84cc3bc421a2aa3c0fb6b21d1/pyOpenSSL-22.0.0.tar.gz", hash = "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf", size = 178438, upload-time = "2022-01-29T20:13:05.197Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/9f/9c0e3288b85f907a008f9d31318b0e4de31b2f67724a8745e633741f609c/pyOpenSSL-22.0.0-py2.py3-none-any.whl", hash = "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0", size = 55833, upload-time = "2022-01-29T20:13:02.874Z" }, +] + +[[package]] +name = "pyparsing" +version = "2.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/47/dfc9c342c9842bbe0036c7f763d2d6686bcf5eb1808ba3e170afdb282210/pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", size = 649718, upload-time = "2020-04-05T22:21:25.349Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/bb/488841f56197b13700afd5658fc279a2025a39e22449b7cf29864669b15d/pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b", size = 67842, upload-time = "2020-04-05T22:21:22.897Z" }, +] + +[[package]] +name = "pyright" +version = "1.1.411" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/ab/265f7dc69d28113ebba19092e57b075f41543b2ed048429c5f56e2b88eac/pyright-1.1.411.tar.gz", hash = "sha256:d885a0551f2e763b089a02702174e7f4ba77548cddabc972ab86d1f7f1b0f998", size = 4112861, upload-time = "2026-06-25T02:14:06.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/49/385be530a6a5b78d1cbcd5c2e38debc8959a2fc6bdb716f4e581002979fc/pyright-1.1.411-py3-none-any.whl", hash = "sha256:dc7c72a8e2700c55baa127554040e067041ea53ccfd50bf96308cc4291c7d5d9", size = 6181526, upload-time = "2026-06-25T02:14:04.691Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/ed/5bbc91f03fa4c839c4c7360375da77f9659af5f7086b7a7bdda65771c8e0/python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", size = 331745, upload-time = "2019-11-03T05:42:03.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/70/d60450c3dd48ef87586924207ae8907090de0b306af2bce5d134d78615cb/python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a", size = 227183, upload-time = "2019-11-03T05:42:01.643Z" }, +] + +[[package]] +name = "python-http-client" +version = "3.3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/fa/284e52a8c6dcbe25671f02d217bf2f85660db940088faf18ae7a05e97313/python_http_client-3.3.7.tar.gz", hash = "sha256:bf841ee45262747e00dec7ee9971dfb8c7d83083f5713596488d67739170cea0", size = 9377, upload-time = "2022-03-09T20:23:56.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/31/9b360138f4e4035ee9dac4fe1132b6437bd05751aaf1db2a2d83dc45db5f/python_http_client-3.3.7-py3-none-any.whl", hash = "sha256:ad371d2bbedc6ea15c26179c6222a78bc9308d272435ddf1d5c84f068f249a36", size = 8352, upload-time = "2022-03-09T20:23:54.862Z" }, +] + +[[package]] +name = "pytz" +version = "2023.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/32/12032aa8c673ee16707a9b6cdda2b09c0089131f35af55d443b6a9c69c1d/pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", size = 317095, upload-time = "2023-03-29T04:23:28.655Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/99/ad6bd37e748257dd70d6f85d916cafe79c0b0f5e2e95b11f7fbc82bf3110/pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb", size = 502345, upload-time = "2023-03-29T04:23:25.307Z" }, +] + +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/f5/10a6e845a00fc5e7afd0a988b744f403d4d57162a28d160a093c4d9322f0/pywin32-312-cp311-cp311-win32.whl", hash = "sha256:17948aeadbdb091f0ced6ef0841620794e68327b94ee415571c1203594b7215c", size = 6362659, upload-time = "2026-06-04T07:49:21.349Z" }, + { url = "https://files.pythonhosted.org/packages/35/c4/dcd2d62b5944b6d5db53413a5899016ccd57ffcb7278f3f81655d25d2027/pywin32-312-cp311-cp311-win_amd64.whl", hash = "sha256:d11417d84412f859b722fad0841b3614459ed0047f7542d8362e77884f6b6e8a", size = 6928825, upload-time = "2026-06-04T07:49:23.934Z" }, + { url = "https://files.pythonhosted.org/packages/b7/56/3cbb433fe4501cdba2eb9040f56a4e1a8243faa4186b25295564d1a7a79d/pywin32-312-cp311-cp311-win_arm64.whl", hash = "sha256:b2200a054ca6d6625c4842fc56a4976a4b47f96b73dbe5538c3f813a80359f47", size = 6721875, upload-time = "2026-06-04T07:49:26.416Z" }, +] + +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + +[[package]] +name = "pywinauto" +version = "0.6.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "comtypes" }, + { name = "pywin32" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a5/29/a08521d81d85c0703c59425adf3c3a6943d7be3330e7e080a8cab395fe08/pywinauto-0.6.8.tar.gz", hash = "sha256:de23f1e977cc51e7eddd95c8f365710343136433968d1e2ad377962d6bd6540a", size = 431976, upload-time = "2019-10-27T20:46:18.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/44/fba89cdd8330999f342f3560c779084c45b012805a53d38732edfe95f0ee/pywinauto-0.6.8-py2.py3-none-any.whl", hash = "sha256:931ce622d7f402b1892ab472987a1332e4c0681bf87e106f798390d16ca95e58", size = 362903, upload-time = "2019-10-27T20:46:16.126Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/e5/af35f7ea75cf72f2cd079c95ee16797de7cd71f29ea7c68ae5ce7be1eda0/PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", size = 125201, upload-time = "2023-07-18T00:00:23.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/0d/26fb23e8863e0aeaac0c64e03fd27367ad2ae3f3cccf3798ee98ce160368/PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", size = 187867, upload-time = "2023-07-17T23:57:34.35Z" }, + { url = "https://files.pythonhosted.org/packages/28/09/55f715ddbf95a054b764b547f617e22f1d5e45d83905660e9a088078fe67/PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", size = 167530, upload-time = "2023-07-17T23:57:36.975Z" }, + { url = "https://files.pythonhosted.org/packages/5e/94/7d5ee059dfb92ca9e62f4057dcdec9ac08a9e42679644854dc01177f8145/PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", size = 732244, upload-time = "2023-07-17T23:57:43.774Z" }, + { url = "https://files.pythonhosted.org/packages/06/92/e0224aa6ebf9dc54a06a4609da37da40bb08d126f5535d81bff6b417b2ae/PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", size = 752871, upload-time = "2023-07-17T23:57:51.921Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5e/efd033ab7199a0b2044dab3b9f7a4f6670e6a52c089de572e928d2873b06/PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", size = 757729, upload-time = "2023-07-17T23:57:59.865Z" }, + { url = "https://files.pythonhosted.org/packages/03/5c/c4671451b2f1d76ebe352c0945d4cd13500adb5d05f5a51ee296d80152f7/PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", size = 748528, upload-time = "2023-08-28T18:43:23.207Z" }, + { url = "https://files.pythonhosted.org/packages/73/9c/766e78d1efc0d1fca637a6b62cea1b4510a7fb93617eb805223294fef681/PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", size = 130286, upload-time = "2023-07-17T23:58:02.964Z" }, + { url = "https://files.pythonhosted.org/packages/b3/34/65bb4b2d7908044963ebf614fe0fdb080773fc7030d7e39c8d3eddcd4257/PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", size = 144699, upload-time = "2023-07-17T23:58:05.586Z" }, +] + +[[package]] +name = "readme-renderer" +version = "45.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "nh3" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/51/d3a6ea424652c60f05600d8c2e01a55c913755e7cdad64afabbd1aa16f44/readme_renderer-45.0.tar.gz", hash = "sha256:030a8fac74904f8fba11ad1bb6964e3f76e896dc7e5e71f16af190c9056696d1", size = 36172, upload-time = "2026-06-09T21:05:17.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl", hash = "sha256:3385ed220117104a2bceb4a9dac8c5fdf6d1f96890d7ea2a9c7174fd5c84091f", size = 14134, upload-time = "2026-06-09T21:05:15.85Z" }, +] + +[[package]] +name = "redis" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version <= '3.11.2'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/88/63d802c2b18dd9eaa5b846cbf18917c6b2882f20efda398cc16a7500b02c/redis-4.6.0.tar.gz", hash = "sha256:585dc516b9eb042a619ef0a39c3d7d55fe81bdb4df09a52c9cdde0d07bf1aa7d", size = 4561721, upload-time = "2023-06-25T13:13:57.139Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/2e/409703d645363352a20c944f5d119bdae3eb3034051a53724a7c5fee12b8/redis-4.6.0-py3-none-any.whl", hash = "sha256:e2b03db868160ee4591de3cb90d40ebb50a90dd302138775937f6a42b7ed183c", size = 241149, upload-time = "2023-06-25T13:13:54.563Z" }, +] + +[[package]] +name = "requests" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "chardet" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/2c/514e4ac25da2b08ca5a464c50463682126385c4272c18193876e91f4bc38/requests-2.21.0.tar.gz", hash = "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", size = 111528, upload-time = "2018-12-10T15:40:11.072Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/e3/20f3d364d6c8e5d2353c72a67778eb189176f08e873c9900e10c0287b84b/requests-2.21.0-py2.py3-none-any.whl", hash = "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b", size = 57987, upload-time = "2018-12-10T15:40:08.741Z" }, +] + +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "oauthlib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, +] + +[[package]] +name = "requests-toolbelt" +version = "0.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/30/7bf7e5071081f761766d46820e52f4b16c8a08fef02d2eb4682ca7534310/requests-toolbelt-0.9.1.tar.gz", hash = "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0", size = 207286, upload-time = "2019-01-30T01:29:54.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/ef/7681134338fc097acef8d9b2f8abe0458e4d87559c689a8c306d0957ece5/requests_toolbelt-0.9.1-py2.py3-none-any.whl", hash = "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", size = 54314, upload-time = "2019-01-30T01:29:52.656Z" }, +] + +[[package]] +name = "rfc3986" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c", size = 49026, upload-time = "2022-01-10T00:52:30.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", size = 31326, upload-time = "2022-01-10T00:52:29.594Z" }, +] + +[[package]] +name = "rsa" +version = "4.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, +] + +[[package]] +name = "secretstorage" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "jeepney" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, +] + +[[package]] +name = "selenium" +version = "3.141.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/9c/9030520bf6ff0b4c98988448a93c04fcbd5b13cd9520074d8ed53569ccfe/selenium-3.141.0.tar.gz", hash = "sha256:deaf32b60ad91a4611b98d8002757f29e6f2c2d5fcaf202e1c9ad06d6772300d", size = 854669, upload-time = "2018-11-01T09:01:07.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/d6/4294f0b4bce4de0abf13e17190289f9d0613b0a44e5dd6a7f5ca98459853/selenium-3.141.0-py2.py3-none-any.whl", hash = "sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c", size = 904577, upload-time = "2018-11-01T09:01:35.607Z" }, +] + +[[package]] +name = "sendgrid" +version = "6.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-http-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/62/2584a7020c04a2117ce0e7bb9ea4d2d3b9df3f9eb82cd62eb9d5d397ac99/sendgrid-6.0.4.tar.gz", hash = "sha256:351a7fc501d2b9d5afdcbc70a02490917057d6ce5cc22c558cadfc16229f157b", size = 43620, upload-time = "2019-04-06T03:56:35.19Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/40/770077b459ff962b3b2c4cde9ccc4417896d506b2fee453defa83e2542cd/sendgrid-6.0.4-py2.py3-none-any.whl", hash = "sha256:e1f93c72b3db3bd00d86f79ee926a093ee7e65533936a140855916569b08e0b0", size = 72387, upload-time = "2019-04-06T03:56:41.537Z" }, +] + +[[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/99/a6ca3beb3ccacb41fb3321d8a60e5566f9e6467601ef8eba6a17e1b89778/soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74", size = 122445, upload-time = "2026-08-07T00:57:24.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823", size = 37370, upload-time = "2026-08-07T00:57:23.524Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/96/e07752635b98536177fa1f37671c8f3cdde2e724c6bcf6034b2cfb571565/tomlkit-0.15.1.tar.gz", hash = "sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97", size = 180129, upload-time = "2026-07-17T01:48:04.562Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/bc/8c13eb66537dce1d2bd3a57132902f38d0e7f5bb46fa9f4daed9fe9d76ee/tomlkit-0.15.1-py3-none-any.whl", hash = "sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304", size = 49449, upload-time = "2026-07-17T01:48:05.728Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + +[[package]] +name = "twine" +version = "3.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, + { name = "importlib-metadata" }, + { name = "keyring" }, + { name = "pkginfo" }, + { name = "readme-renderer" }, + { name = "requests" }, + { name = "requests-toolbelt" }, + { name = "rfc3986" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/12/0e4c8df764d87c15b8256444d0b8b433c183ce3a986ffae3086df3f876ef/twine-3.7.1.tar.gz", hash = "sha256:28460a3db6b4532bde6a5db6755cf2dce6c5020bada8a641bb2c5c7a9b1f35b8", size = 231946, upload-time = "2021-12-07T11:46:02.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/aa/636b8eb9637944d2d94b766997d0420d1911abfd6392a6e3e2a75347949a/twine-3.7.1-py3-none-any.whl", hash = "sha256:8c120845fc05270f9ee3e9d7ebbed29ea840e41f48cd059e04733f7e1d401345", size = 35982, upload-time = "2021-12-07T11:46:01.03Z" }, +] + +[[package]] +name = "ty" +version = "0.0.69" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/5b/7a618632dfe9373b7df572ecd7a08c8f799d772fbc317da82dd3aa363207/ty-0.0.69.tar.gz", hash = "sha256:b65106e9ff24fa76e25e1142fb09c85244e815c40450e3021d2bf652c231bb43", size = 6565094, upload-time = "2026-08-06T10:04:25.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/60/6534092f4d2c15e2491807edd609c2e50d527c1fed957acf40b9f110b64a/ty-0.0.69-py3-none-linux_armv6l.whl", hash = "sha256:98bfd383b273540829af673e7f98b9c1c4bcc8547d12a1a3806cd0bec7f0e087", size = 12364185, upload-time = "2026-08-06T10:03:47.137Z" }, + { url = "https://files.pythonhosted.org/packages/34/2b/5c29689bd4f74c2e3394d983d85e4011b629f2ce3730c9442553b8554bf8/ty-0.0.69-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:964621ddd05771660017c51b4e74078d861d9fc863c21ef2a500db1ab62c9ccf", size = 12042510, upload-time = "2026-08-06T10:03:49.481Z" }, + { url = "https://files.pythonhosted.org/packages/09/46/fa085bde4d23516d7ef14b24736fc5dd7dc498f60f52b3d077e59ffdea20/ty-0.0.69-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3ffea4048dd0da4c9c97393b4be0901098a9065b06fa81be2477cbde65d8a151", size = 11549397, upload-time = "2026-08-06T10:03:51.747Z" }, + { url = "https://files.pythonhosted.org/packages/25/cc/97b9efb2061dcab6fef1e94a4ad99df0bb45bd2cc15d4f5794c787ee0552/ty-0.0.69-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8684d4a70aadd1eab0f41bdba835e3288ef49db8402a8e6ca81bab52ed5d610", size = 12115567, upload-time = "2026-08-06T10:03:53.79Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c1/a5e0404965093835f3e62544e661784ec0aa8ef0b006ed50af50b19c107e/ty-0.0.69-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:afaaba240ab4122e2069a796836d10be81b4ddb053ae268b3dff962a0b4ca5c7", size = 12149770, upload-time = "2026-08-06T10:03:55.993Z" }, + { url = "https://files.pythonhosted.org/packages/e2/39/8cad6b205a4abe8a044ca0c84aea71e8ccda29b07a75a5f090e310605580/ty-0.0.69-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11ea63ef07d4e33aeb1a775cf5f2c736b3ed22fa6f8b1b608591612c36795044", size = 12941278, upload-time = "2026-08-06T10:03:58.324Z" }, + { url = "https://files.pythonhosted.org/packages/d6/8b/8766d96b732c2a060d70dc8ccafcc4d6a54109a2a95f1deb0705de88892b/ty-0.0.69-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cb3730b1268e92a2907d7aea3afe8dd1b360ae65862f0557080cf479d481b424", size = 13426509, upload-time = "2026-08-06T10:04:00.621Z" }, + { url = "https://files.pythonhosted.org/packages/02/1f/e991b2cde953ea5b94d6a9a4c45c87937bd916bc09235f764407bf471c0a/ty-0.0.69-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a544ff57a752ef186ed40b5a2f44c17402af4cdefeb74a311ca02ebd57c4fca0", size = 13106582, upload-time = "2026-08-06T10:04:02.818Z" }, + { url = "https://files.pythonhosted.org/packages/ea/bb/73538f1b99e3558fd9db87b98698426f0f60fc8666da0b1efd0e70e275eb/ty-0.0.69-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ed2cbca20caddfdf8e3e14d213ce91b67e75feed78900f4aaf3ef884954028", size = 12708931, upload-time = "2026-08-06T10:04:05.233Z" }, + { url = "https://files.pythonhosted.org/packages/87/cd/484a5208d74c4ad1155933906295ccdce9aa81a257d8df2ab9e41bd60133/ty-0.0.69-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2684efcbce5b6fe45045faf610b377b50781b6d2aa7e61ea23ecf5b3d2bce421", size = 12985322, upload-time = "2026-08-06T10:04:07.587Z" }, + { url = "https://files.pythonhosted.org/packages/6e/81/b75003f0d4da9ab3bc8fd4f4802f836cb9921ff7e70f460604f7b769a0b5/ty-0.0.69-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:da9aeb26fdac1d2214937542b59e0d4d1ba94ec7a3f45444f33c846de1eb1d63", size = 12063910, upload-time = "2026-08-06T10:04:09.835Z" }, + { url = "https://files.pythonhosted.org/packages/8a/76/088469f547ef63dceefc4a75826aedee5014f9371dc5171cde931896a82c/ty-0.0.69-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:00e7677cd14ede381f705f71104ea7b8ea0ce217a8634e19a89781953de0e9ad", size = 12166823, upload-time = "2026-08-06T10:04:12.114Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c9/ce88a0bec0d46d8ae180b99c6ec014866fecc4cba1727b5feec8877b2765/ty-0.0.69-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d91965eb799649833d0d6042db09cd03d15289125245337cc46a2606effb7bda", size = 12483136, upload-time = "2026-08-06T10:04:14.33Z" }, + { url = "https://files.pythonhosted.org/packages/63/9e/6fae0ff225a0012642cf72c077e20f8f448c0a80771bc3360e8178fe2f32/ty-0.0.69-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1f03359cd8e5c412aa0c181118fa9b9061a4dddaedbb61bac0a424fb0814d402", size = 12799025, upload-time = "2026-08-06T10:04:16.445Z" }, + { url = "https://files.pythonhosted.org/packages/e4/43/78a658d18b2a4ccf35b053392f2213bf12e3c63b2abea512d3b6751d1f4c/ty-0.0.69-py3-none-win32.whl", hash = "sha256:ec460e01586b1eb91894c4a8403bee3e045a47e7a4ada943cc27ce8e348e88cf", size = 11787774, upload-time = "2026-08-06T10:04:18.622Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5e/88db1f674403f2b81316a853a44a81ed220621fa96f8f7ae586fb6ca7513/ty-0.0.69-py3-none-win_amd64.whl", hash = "sha256:18976ca26a4e28fc3249477f79a695d5502e670803f2e080d89ac905baef3c6e", size = 12864038, upload-time = "2026-08-06T10:04:20.748Z" }, + { url = "https://files.pythonhosted.org/packages/4d/7b/6fc6efd00c69103d70f2bdbe824343089cd70b17b3079170057d3e5a3ac0/ty-0.0.69-py3-none-win_arm64.whl", hash = "sha256:7d4ca3bb74d91cb9947ba3f3b4cb131ad6a2b3ecc76d34040c4ec6092d2e411d", size = 12196693, upload-time = "2026-08-06T10:04:22.902Z" }, +] + +[[package]] +name = "types-protobuf" +version = "7.34.1.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/59/e2b13b499d15e6720150c4b1a8d91e31fcacf716b432397475b3151ff7e4/types_protobuf-7.34.1.20260518.tar.gz", hash = "sha256:28cfaded25889cb83ebfb63cfb0a43628f0b6f3785767bec17287dc6468795f2", size = 68936, upload-time = "2026-05-18T06:01:47.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/1f/ec5caf72c2e3b688ca3927e0979a04ddad19e1afc4bf1c199bd743e0f419/types_protobuf-7.34.1.20260518-py3-none-any.whl", hash = "sha256:a0a5337413347166439c0e07cbc26c6164d091401c6f01b1dfd8cdb966c4dd8f", size = 85992, upload-time = "2026-05-18T06:01:45.696Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "uritemplate" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, +] + +[[package]] +name = "urllib3" +version = "1.24.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/3c/1bb7ef6c435dea026f06ed9f3ba16aa93f9f4f5d3857a51a35dfa00882f1/urllib3-1.24.3.tar.gz", hash = "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", size = 232237, upload-time = "2019-05-02T15:37:46.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/11/525b02e4acc0c747de8b6ccdab376331597c569c42ea66ab0a1dbd36eca2/urllib3-1.24.3-py2.py3-none-any.whl", hash = "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb", size = 118799, upload-time = "2019-05-02T15:37:44.733Z" }, +] + +[[package]] +name = "waitress" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/cb/04ddb054f45faa306a230769e868c28b8065ea196891f09004ebace5b184/waitress-3.0.2.tar.gz", hash = "sha256:682aaaf2af0c44ada4abfb70ded36393f0e307f4ab9456a215ce0020baefc31f", size = 179901, upload-time = "2024-11-16T20:02:35.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/57/a27182528c90ef38d82b636a11f606b0cbb0e17588ed205435f8affe3368/waitress-3.0.2-py3-none-any.whl", hash = "sha256:c56d67fd6e87c2ee598b76abdd4e96cfad1f24cacdea5078d382b1f9d7b5ed2e", size = 56232, upload-time = "2024-11-16T20:02:33.858Z" }, +] + +[[package]] +name = "webob" +version = "1.8.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e6/92/9329872e36be90e25ba616ac6e9ee8c1be4326a1ea3fdffcdabd0eb48efb/webob-1.8.11.tar.gz", hash = "sha256:aa8c27231070b135c025e567a9cd7eda03f4df71352ffaac740cb6a75f0f81a5", size = 286047, upload-time = "2026-08-02T06:26:26.063Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/71/370a65bc7726a1cbf478445c898c7affe907aa42e2e9e5bb38319db74115/webob-1.8.11-py2.py3-none-any.whl", hash = "sha256:4addd1d38d6a7fbe0eda22d45f25a40d74c8b290f3a99c0ac3d4023cf21f2da2", size = 118319, upload-time = "2026-08-02T06:26:24.765Z" }, +] + +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + +[[package]] +name = "webtest" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "waitress" }, + { name = "webob" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/c8/8ffba1782700eb06e9b9169156698c6ba95c05b66cda3fc9e025b6b3b649/WebTest-3.0.0.tar.gz", hash = "sha256:54bd969725838d9861a9fa27f8d971f79d275d94ae255f5c501f53bb6d9929eb", size = 75871, upload-time = "2021-08-19T12:40:07.234Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/c7/3897bd62366cb4a50bfb411d37efca9fa33bf07a7c1c22fce8f6ad2664ff/WebTest-3.0.0-py3-none-any.whl", hash = "sha256:2a001a9efa40d2a7e5d9cd8d1527c75f41814eb6afce2c3d207402547b1e5ead", size = 31858, upload-time = "2021-08-19T12:40:04.047Z" }, +] + +[[package]] +name = "werkzeug" +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/c1/1c8e539f040acd80f844c69a5ef8e2fccdf8b442dabb969e497b55d544e1/Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f", size = 844378, upload-time = "2022-08-08T21:44:15.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/27/be6ddbcf60115305205de79c29004a0c6bc53cec814f733467b1bb89386d/Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5", size = 232700, upload-time = "2022-08-08T21:44:13.251Z" }, +] + +[[package]] +name = "wrapt" +version = "1.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/4c/063a912e20bcef7124e0df97282a8af3ff3e4b603ce84c481d6d7346be0a/wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d", size = 53972, upload-time = "2023-11-09T06:33:30.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/03/c188ac517f402775b90d6f312955a5e53b866c964b32119f2ed76315697e/wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09", size = 37313, upload-time = "2023-11-09T06:31:52.168Z" }, + { url = "https://files.pythonhosted.org/packages/0f/16/ea627d7817394db04518f62934a5de59874b587b792300991b3c347ff5e0/wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d", size = 38164, upload-time = "2023-11-09T06:31:53.522Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a7/f1212ba098f3de0fd244e2de0f8791ad2539c03bef6c05a9fcb03e45b089/wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389", size = 80890, upload-time = "2023-11-09T06:31:55.247Z" }, + { url = "https://files.pythonhosted.org/packages/b7/96/bb5e08b3d6db003c9ab219c487714c13a237ee7dcc572a555eaf1ce7dc82/wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060", size = 73118, upload-time = "2023-11-09T06:31:57.023Z" }, + { url = "https://files.pythonhosted.org/packages/6e/52/2da48b35193e39ac53cfb141467d9f259851522d0e8c87153f0ba4205fb1/wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1", size = 80746, upload-time = "2023-11-09T06:31:58.686Z" }, + { url = "https://files.pythonhosted.org/packages/11/fb/18ec40265ab81c0e82a934de04596b6ce972c27ba2592c8b53d5585e6bcd/wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3", size = 85668, upload-time = "2023-11-09T06:31:59.992Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ef/0ecb1fa23145560431b970418dce575cfaec555ab08617d82eb92afc7ccf/wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956", size = 78556, upload-time = "2023-11-09T06:32:01.942Z" }, + { url = "https://files.pythonhosted.org/packages/25/62/cd284b2b747f175b5a96cbd8092b32e7369edab0644c45784871528eb852/wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d", size = 85712, upload-time = "2023-11-09T06:32:03.686Z" }, + { url = "https://files.pythonhosted.org/packages/e5/a7/47b7ff74fbadf81b696872d5ba504966591a3468f1bc86bca2f407baef68/wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362", size = 35327, upload-time = "2023-11-09T06:32:05.284Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c3/0084351951d9579ae83a3d9e38c140371e4c6b038136909235079f2e6e78/wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89", size = 37523, upload-time = "2023-11-09T06:32:07.17Z" }, + { url = "https://files.pythonhosted.org/packages/ff/21/abdedb4cdf6ff41ebf01a74087740a709e2edb146490e4d9beea054b0b7a/wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1", size = 23362, upload-time = "2023-11-09T06:33:28.271Z" }, +] + +[[package]] +name = "yapf" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/73/a34efe529fad3fb33c2d1f5bb357d1a8b9382cf0f3053c7468eb5862e9ce/yapf-0.22.0.tar.gz", hash = "sha256:a98a6eacca64d2b920558f4a2f78150db9474de821227e60deaa29f186121c63", size = 135605, upload-time = "2018-05-16T05:25:51.636Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/ae/9d2e8f43f2ce467991c8310e361bbf4f1e1bf32afc6441b4e3416685b7ef/yapf-0.22.0-py2.py3-none-any.whl", hash = "sha256:6567745f0b6656f9c33a73c56a393071c699e6284a70d793798ab6e3769d25ec", size = 166305, upload-time = "2018-05-16T05:25:50.045Z" }, +] + +[[package]] +name = "yarl" +version = "1.24.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/33/ebe9e3d1f86c7a0b51094c0a146392045ca1631d2664889539dec8088a33/yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f", size = 228679, upload-time = "2026-07-20T02:07:45.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/db/3cb5df059756a45761cc3dee8fd25ec82b83a6585ea3542b969fda850f99/yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3", size = 135043, upload-time = "2026-07-20T02:04:52.39Z" }, + { url = "https://files.pythonhosted.org/packages/44/f8/767d6bd5a03db63bc467df2fb56d6fafeae9667d74aea92cd6af399f828b/yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a", size = 96942, upload-time = "2026-07-20T02:04:54.26Z" }, + { url = "https://files.pythonhosted.org/packages/ce/97/10b939c44d7b28d1dbc389cfc7012306d1ea8dba01eaef44b39fffaee52a/yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840", size = 97046, upload-time = "2026-07-20T02:04:56.638Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7a/b410dbe39b6255c55fb2a2bcee96eb844d0789235ddc381a889a90dc72d6/yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966", size = 110512, upload-time = "2026-07-20T02:04:58.955Z" }, + { url = "https://files.pythonhosted.org/packages/83/c7/da591971f78a5617e1f21f5699858ebccd836fe181a6493788ffc91ba69b/yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723", size = 102454, upload-time = "2026-07-20T02:05:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8e/73b0ed4de47289a78a96045d76d1cfe5e41848bf0da59ce25b2ec87ee05d/yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb", size = 117617, upload-time = "2026-07-20T02:05:02.325Z" }, + { url = "https://files.pythonhosted.org/packages/cf/14/b744747bc4f57a8d55bd744df463457524583e1e9f7538b5ace0346ab92e/yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780", size = 116135, upload-time = "2026-07-20T02:05:04.05Z" }, + { url = "https://files.pythonhosted.org/packages/66/ca/95aa4d0e5b7ea4f20e4d577c42d001ed9df207569fdb063cc5ed4ebb496b/yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e", size = 111935, upload-time = "2026-07-20T02:05:05.738Z" }, + { url = "https://files.pythonhosted.org/packages/72/0d/d2ad8d6b147832d177a4e720ba1962fe686eb0913b74503b3eca094b8bba/yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2", size = 110010, upload-time = "2026-07-20T02:05:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/50/18/eb335e4120903903f4865041355ae46256a2406eb2865bc24827f4f27b61/yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58", size = 110058, upload-time = "2026-07-20T02:05:09.246Z" }, + { url = "https://files.pythonhosted.org/packages/44/70/97353add32c62ad6f206d948ac5a5ee84398225e534dc6ed6433d1b335b6/yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61", size = 103308, upload-time = "2026-07-20T02:05:11.31Z" }, + { url = "https://files.pythonhosted.org/packages/68/39/5e7398d4b6f6b3c9062823ebc60802df5b272e3fe9e788f9734c6ee46c85/yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6", size = 116898, upload-time = "2026-07-20T02:05:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c9/09e52f2239e8b96357eccca05915382e4ba5405ebfb623b6036040d99654/yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f", size = 109400, upload-time = "2026-07-20T02:05:14.821Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6a/e94133d4c2d1a14d2384310bf3e79d9cf32c9d1eae1c6f034fb80d098fa1/yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077", size = 115934, upload-time = "2026-07-20T02:05:17.78Z" }, + { url = "https://files.pythonhosted.org/packages/4e/3c/34955ed967b976fc38edcbb6d538dee79dbda4cb7fc7f72a0907a7c78e0f/yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd", size = 112178, upload-time = "2026-07-20T02:05:19.675Z" }, + { url = "https://files.pythonhosted.org/packages/f5/46/d7bd3a8859d47dcfaffd7127af7076032a7da278a9a02e17b5f37bfb6712/yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25", size = 97544, upload-time = "2026-07-20T02:05:21.523Z" }, + { url = "https://files.pythonhosted.org/packages/01/69/c1bfd21e32c638974ea2c542a0b8c53ef1fa9eff336020f5d014f9503ff2/yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a", size = 93359, upload-time = "2026-07-20T02:05:23.493Z" }, + { url = "https://files.pythonhosted.org/packages/61/02/962c1cbfc401a30c1d034dc67ff395f64b52302c6d62de556c1fca99acc0/yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7", size = 58612, upload-time = "2026-07-20T02:07:43.461Z" }, +] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, +]