Retry transient GitHub API timeouts

src/github_sdk.py

@@ -4,13 +4,23 @@
 import hashlib
 import io
 import logging
+import time
 import uuid
 from datetime import datetime
 
 import requests
 from sentry_sdk.envelope import Envelope
 from sentry_sdk.utils import format_timestamp
 
+GITHUB_REQUEST_MAX_ATTEMPTS = 3
+GITHUB_REQUEST_RETRY_BACKOFF_SECONDS = 0.5
+GITHUB_REQUEST_TIMEOUT_SECONDS = 10
+GITHUB_RETRYABLE_EXCEPTIONS = (
+    requests.exceptions.ConnectionError,
+    requests.exceptions.Timeout,
+    requests.exceptions.SSLError,
+)
+
 
 class GithubSentryError(Exception):
     pass
@@ -42,9 +52,25 @@
     def _fetch_github(self, url):
         headers = {"Authorization": f"token {self.token}"}
 
-        req = requests.get(url, headers=headers)
-        req.raise_for_status()
-        return req
+        for attempt in range(1, GITHUB_REQUEST_MAX_ATTEMPTS + 1):
+            try:
+                req = requests.get(
+                    url,
+                    headers=headers,
+                    timeout=GITHUB_REQUEST_TIMEOUT_SECONDS,
+                )
+                req.raise_for_status()
+                return req
+            except GITHUB_RETRYABLE_EXCEPTIONS:
+                if attempt == GITHUB_REQUEST_MAX_ATTEMPTS:
+                    raise
+
+                logging.warning(
+                    "Transient GitHub API request failed; retrying (%s/%s)",
+                    attempt,
+                    GITHUB_REQUEST_MAX_ATTEMPTS,
+                )
+                time.sleep(GITHUB_REQUEST_RETRY_BACKOFF_SECONDS * 2 ** (attempt - 1))
 
     def _get_extra_metadata(self, job):
         # XXX: This is the slowest call

tests/test_github_sdk.py

@@ -2,6 +2,8 @@
 
 import sys
 from datetime import datetime
+from unittest.mock import call
+from unittest.mock import Mock
 from unittest.mock import patch
 
 import pytest
@@ -59,6 +61,54 @@ def test_ensure_raise_error_on_github_api_failure():
     )
 
 
+@patch("src.github_sdk.time.sleep")
+@patch("src.github_sdk.requests.get")
+def test_fetch_github_retries_transient_connect_timeout(mock_get, mock_sleep):
+    url = "https://api.github.com/repos/example/repo/actions/runs/1"
+    response = Mock()
+    response.raise_for_status.return_value = None
+    mock_get.side_effect = [
+        requests.exceptions.ConnectTimeout("connect timed out"),
+        response,
+    ]
+
+    client = GithubClient(dsn=DSN, token=TOKEN)
+    resp = client._fetch_github(url)
+
+    assert resp is response
+    assert mock_get.call_count == 2
+    mock_get.assert_has_calls(
+        [
+            call(
+                url,
+                headers={"Authorization": f"token {TOKEN}"},
+                timeout=10,
+            ),
+            call(
+                url,
+                headers={"Authorization": f"token {TOKEN}"},
+                timeout=10,
+            ),
+        ]
+    )
+    mock_sleep.assert_called_once_with(0.5)
+
+
+@patch("src.github_sdk.time.sleep")
+@patch("src.github_sdk.requests.get")
+def test_fetch_github_raises_after_transient_retries(mock_get, mock_sleep):
+    url = "https://api.github.com/repos/example/repo/actions/runs/1"
+    error = requests.exceptions.ConnectTimeout("connect timed out")
+    mock_get.side_effect = error
+
+    client = GithubClient(dsn=DSN, token=TOKEN)
+    with pytest.raises(requests.exceptions.ConnectTimeout):
+        client._fetch_github(url)
+
+    assert mock_get.call_count == 3
+    mock_sleep.assert_has_calls([call(0.5), call(1.0)])
+
+
 @freeze_time()
 @responses.activate
 @patch("src.github_sdk.get_uuid")