From 9ccbdc1a0cf411eb175a5fc1a96a574c85615472 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Date: Mon, 27 Jul 2026 08:22:43 +0000 Subject: [PATCH 1/8] feat: add Insights Snowflake connectivity validation --- .../commands/check_insights_snowflake.py | 44 ++++++ analytics_data_api/snowflake_client.py | 126 ++++++++++++++++++ analyticsdataserver/settings/base.py | 14 ++ requirements/base.in | 4 +- requirements/base.txt | 90 +++++++++---- requirements/dev.txt | 91 +++++++++---- requirements/doc.txt | 105 ++++++++++----- requirements/production.txt | 99 +++++++++----- requirements/test.txt | 102 +++++++++----- 9 files changed, 519 insertions(+), 156 deletions(-) create mode 100644 analytics_data_api/management/commands/check_insights_snowflake.py create mode 100644 analytics_data_api/snowflake_client.py diff --git a/analytics_data_api/management/commands/check_insights_snowflake.py b/analytics_data_api/management/commands/check_insights_snowflake.py new file mode 100644 index 00000000..ae1a2017 --- /dev/null +++ b/analytics_data_api/management/commands/check_insights_snowflake.py @@ -0,0 +1,44 @@ +"""Validate the Insights Snowflake connection from the deployed Analytics API service.""" + +from django.core.management.base import BaseCommand, CommandError + +from analytics_data_api.snowflake_client import ( + DEFAULT_VALIDATION_TABLE, + SnowflakeConfigurationError, + get_insights_snowflake_config, + get_table_row_count, +) + + +class Command(BaseCommand): + """Run a manual read-only Snowflake connectivity check.""" + + help = 'Validate Insights Snowflake connectivity with a read-only query.' + + def add_arguments(self, parser): + parser.add_argument( + '--table', + default=DEFAULT_VALIDATION_TABLE, + help='Snowflake table to count for validation. Defaults to COURSE_ACTIVITY_WEEKLY.', + ) + + def handle(self, *args, **options): + table = options['table'] + + try: + config = get_insights_snowflake_config() + row_count = get_table_row_count(table) + except SnowflakeConfigurationError as exc: + raise CommandError(str(exc)) + except Exception as exc: # pylint: disable=broad-except + raise CommandError('Insights Snowflake connectivity check failed: {}'.format(exc)) + + self.stdout.write('Insights Snowflake connectivity check succeeded.') + self.stdout.write('Account: {}'.format(config['ACCOUNT'])) + self.stdout.write('User: {}'.format(config['USER'])) + self.stdout.write('Role: {}'.format(config['ROLE'])) + self.stdout.write('Warehouse: {}'.format(config['WAREHOUSE'])) + self.stdout.write('Database: {}'.format(config['DATABASE'])) + self.stdout.write('Schema: {}'.format(config['SCHEMA'])) + self.stdout.write('Table: {}'.format(table)) + self.stdout.write('Row count: {}'.format(row_count)) diff --git a/analytics_data_api/snowflake_client.py b/analytics_data_api/snowflake_client.py new file mode 100644 index 00000000..81d3f82a --- /dev/null +++ b/analytics_data_api/snowflake_client.py @@ -0,0 +1,126 @@ +""" +Helpers for the Insights Snowflake connection. + +This module is intentionally not used by existing API views. Phase 1 only uses +it from a manual management command to validate deployed Snowflake connectivity. +""" + +import re +from importlib import import_module + +from cryptography.hazmat.primitives import serialization +from django.conf import settings + + +QUERY_TAG = 'edx_analytics_api_insights_snowflake_phase1' +DEFAULT_VALIDATION_TABLE = 'COURSE_ACTIVITY_WEEKLY' +REQUIRED_CONFIG_KEYS = ( + 'ACCOUNT', + 'USER', + 'ROLE', + 'WAREHOUSE', + 'DATABASE', + 'SCHEMA', + 'PRIVATE_KEY', +) +IDENTIFIER_PATTERN = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') + + +class SnowflakeConfigurationError(Exception): + """Raised when Insights Snowflake configuration is missing or unsafe.""" + + +def get_insights_snowflake_config(): + """Return the configured Insights Snowflake settings.""" + config = getattr(settings, 'INSIGHTS_SNOWFLAKE', None) + if not isinstance(config, dict): + raise SnowflakeConfigurationError('INSIGHTS_SNOWFLAKE must be configured as a dictionary.') + + missing_keys = [key for key in REQUIRED_CONFIG_KEYS if not config.get(key)] + if missing_keys: + raise SnowflakeConfigurationError( + 'INSIGHTS_SNOWFLAKE is missing required settings: {}.'.format(', '.join(missing_keys)) + ) + + return config + + +def get_private_key_der(config): + """ + Convert the configured PEM private key string into DER bytes. + + edx-internal stores the Snowflake private key with literal "\\n" sequences. + The connector expects a loaded private key serialized as DER/PKCS8 bytes. + """ + private_key_value = config['PRIVATE_KEY'] + if not isinstance(private_key_value, str): + raise SnowflakeConfigurationError('INSIGHTS_SNOWFLAKE PRIVATE_KEY must be a string.') + + normalized_private_key = private_key_value.replace('\\n', '\n').encode('utf-8') + passphrase = config.get('PRIVATE_KEY_PASSPHRASE') + passphrase_bytes = passphrase.encode('utf-8') if passphrase else None + + private_key = serialization.load_pem_private_key( + normalized_private_key, + password=passphrase_bytes, + ) + + return private_key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + +def connect_to_insights_snowflake(): + """Open a Snowflake connection using the Insights Snowflake settings.""" + config = get_insights_snowflake_config() + snowflake_connector = import_module('snowflake.connector') + + return snowflake_connector.connect( + account=config['ACCOUNT'], + user=config['USER'], + role=config['ROLE'], + warehouse=config['WAREHOUSE'], + database=config['DATABASE'], + schema=config['SCHEMA'], + private_key=get_private_key_der(config), + session_parameters={ + 'QUERY_TAG': QUERY_TAG, + }, + ) + + +def validate_snowflake_identifier(identifier, label): + """Return a safe Snowflake identifier or raise a sanitized error.""" + if not isinstance(identifier, str) or not IDENTIFIER_PATTERN.match(identifier): + raise SnowflakeConfigurationError('Unsafe Snowflake {} identifier.'.format(label)) + return identifier.upper() + + +def run_read_only_query(sql): + """Run a read-only Snowflake query and return all rows.""" + if not sql.lstrip().upper().startswith('SELECT '): + raise SnowflakeConfigurationError('Only SELECT queries are allowed for Insights Snowflake validation.') + + connection = connect_to_insights_snowflake() + cursor = None + try: + cursor = connection.cursor() + cursor.execute(sql) + return cursor.fetchall() + finally: + if cursor is not None: + cursor.close() + connection.close() + + +def get_table_row_count(table_name=DEFAULT_VALIDATION_TABLE): + """Return the row count for a validated Snowflake table name.""" + config = get_insights_snowflake_config() + database = validate_snowflake_identifier(config['DATABASE'], 'database') + schema = validate_snowflake_identifier(config['SCHEMA'], 'schema') + table = validate_snowflake_identifier(table_name, 'table') + + rows = run_read_only_query('SELECT COUNT(*) AS row_count FROM {}.{}.{}'.format(database, schema, table)) + return rows[0][0] if rows else 0 diff --git a/analyticsdataserver/settings/base.py b/analyticsdataserver/settings/base.py index cf426db1..a420a97c 100644 --- a/analyticsdataserver/settings/base.py +++ b/analyticsdataserver/settings/base.py @@ -90,6 +90,20 @@ } ########## END DATABASE CONFIGURATION +########## INSIGHTS SNOWFLAKE CONFIGURATION +INSIGHTS_SNOWFLAKE = { + 'ENABLED': False, + 'ACCOUNT': None, + 'USER': None, + 'ROLE': None, + 'WAREHOUSE': None, + 'DATABASE': 'PROD', + 'SCHEMA': 'INSIGHTS', + 'PRIVATE_KEY': None, + 'PRIVATE_KEY_PASSPHRASE': None, +} +########## END INSIGHTS SNOWFLAKE CONFIGURATION + ########## GENERAL CONFIGURATION # See: https://docs.djangoproject.com/en/dev/ref/settings/#time-zone TIME_ZONE = 'UTC' diff --git a/requirements/base.in b/requirements/base.in index eb4b8ed6..af3d963c 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -9,10 +9,12 @@ Django # BSD License django-countries # MIT python-memcached # Python Software Foundation License v2 pymemcache +cryptography # Apache-2.0 OR BSD +snowflake-connector-python # Apache 2.0 djangorestframework # BSD djangorestframework-csv # BSD django-storages # BSD -html5lib # MIT +html5lib # MIT ordered-set # MIT tqdm # MIT urllib3 # MIT diff --git a/requirements/base.txt b/requirements/base.txt index aa620ee3..4171c7ef 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,36 +1,48 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # make upgrade # -asgiref==3.11.1 +asgiref==3.12.1 # via # django # django-cors-headers # django-countries +asn1crypto==1.5.1 + # via snowflake-connector-python boto==2.49.0 # via -r requirements/base.in -boto3==1.42.90 - # via -r requirements/base.in -botocore==1.42.90 +boto3==1.43.56 + # via + # -r requirements/base.in + # snowflake-connector-python +botocore==1.43.56 # via # boto3 # s3transfer -certifi==2026.2.25 - # via requests -cffi==2.0.0 + # snowflake-connector-python +certifi==2026.7.22 + # via + # requests + # snowflake-connector-python +cffi==2.1.0 # via # cryptography # pynacl -charset-normalizer==3.4.7 - # via requests -click==8.3.2 +charset-normalizer==3.4.9 + # via + # requests + # snowflake-connector-python +click==8.4.2 # via edx-django-utils -cryptography==46.0.7 +cryptography==49.0.0 # via + # -r requirements/base.in # django-fernet-fields-v2 # pyjwt + # pyopenssl + # snowflake-connector-python django==4.2.30 # via # -c requirements/constraints.txt @@ -53,7 +65,7 @@ django==4.2.30 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==8.2.0 +django-countries==9.0.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -128,12 +140,16 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.13.0 +faker==40.36.0 # via factory-boy +filelock==3.32.0 + # via snowflake-connector-python html5lib==1.1 # via -r requirements/base.in -idna==3.11 - # via requests +idna==3.18 + # via + # requests + # snowflake-connector-python inflection==0.5.1 # via drf-yasg jmespath==1.1.0 @@ -146,41 +162,51 @@ mysql-connector-python==9.5.0 # via edx-enterprise-data ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.1 - # via drf-yasg +packaging==26.2 + # via + # drf-yasg + # snowflake-connector-python +platformdirs==4.11.0 + # via snowflake-connector-python psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi -pyjwt[crypto]==2.12.1 +pyjwt[crypto]==2.13.0 # via # drf-jwt # edx-drf-extensions # edx-rest-api-client + # snowflake-connector-python pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.16.0 +pymongo==4.17.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils +pyopenssl==26.3.0 + # via snowflake-connector-python python-dateutil==2.9.0.post0 # via botocore python-memcached==1.62 # via -r requirements/base.in -pytz==2026.1.post1 - # via drf-yasg +pytz==2026.3.post1 + # via + # drf-yasg + # snowflake-connector-python pyyaml==6.0.3 # via # drf-yasg # edx-django-release-util -requests==2.33.1 +requests==2.34.2 # via # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client + # snowflake-connector-python rules==3.5 # via edx-enterprise-data -s3transfer==0.16.0 +s3transfer==0.19.2 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -191,18 +217,28 @@ six==1.17.0 # edx-rbac # html5lib # python-dateutil +snowflake-connector-python==4.7.1 + # via + # -r requirements/base.in + # edx-enterprise-data +sortedcontainers==2.4.0 + # via snowflake-connector-python sqlparse==0.5.5 # via django -stevedore==5.7.0 +stevedore==5.9.0 # via # edx-django-utils # edx-opaque-keys -tqdm==4.67.3 +tomlkit==0.15.1 + # via snowflake-connector-python +tqdm==4.69.1 # via -r requirements/base.in -typing-extensions==4.15.0 +typing-extensions==4.16.0 # via # django-countries # edx-opaque-keys + # pyopenssl + # snowflake-connector-python uritemplate==4.2.0 # via drf-yasg urllib3==1.26.20 diff --git a/requirements/dev.txt b/requirements/dev.txt index b0cf19be..f82826cb 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,36 +1,48 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # make upgrade # -asgiref==3.11.1 +asgiref==3.12.1 # via # django # django-cors-headers # django-countries +asn1crypto==1.5.1 + # via snowflake-connector-python boto==2.49.0 # via -r requirements/base.in -boto3==1.42.90 - # via -r requirements/base.in -botocore==1.42.90 +boto3==1.43.56 + # via + # -r requirements/base.in + # snowflake-connector-python +botocore==1.43.56 # via # boto3 # s3transfer -certifi==2026.2.25 - # via requests -cffi==2.0.0 + # snowflake-connector-python +certifi==2026.7.22 + # via + # requests + # snowflake-connector-python +cffi==2.1.0 # via # cryptography # pynacl -charset-normalizer==3.4.7 - # via requests -click==8.3.2 +charset-normalizer==3.4.9 + # via + # requests + # snowflake-connector-python +click==8.4.2 # via edx-django-utils -cryptography==46.0.7 +cryptography==49.0.0 # via + # -r requirements/base.in # django-fernet-fields-v2 # pyjwt + # pyopenssl + # snowflake-connector-python django==4.2.30 # via # -c requirements/constraints.txt @@ -53,7 +65,7 @@ django==4.2.30 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==8.2.0 +django-countries==9.0.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -112,7 +124,6 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac - edx-enterprise-data==10.22.10 # via -r requirements/base.in edx-opaque-keys==4.0.0 @@ -129,12 +140,16 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.13.0 +faker==40.36.0 # via factory-boy +filelock==3.32.0 + # via snowflake-connector-python html5lib==1.1 # via -r requirements/base.in -idna==3.11 - # via requests +idna==3.18 + # via + # requests + # snowflake-connector-python inflection==0.5.1 # via drf-yasg jmespath==1.1.0 @@ -149,41 +164,51 @@ mysqlclient==2.2.8 # via -r requirements/dev.in ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.1 - # via drf-yasg +packaging==26.2 + # via + # drf-yasg + # snowflake-connector-python +platformdirs==4.11.0 + # via snowflake-connector-python psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi -pyjwt[crypto]==2.12.1 +pyjwt[crypto]==2.13.0 # via # drf-jwt # edx-drf-extensions # edx-rest-api-client + # snowflake-connector-python pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.16.0 +pymongo==4.17.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils +pyopenssl==26.3.0 + # via snowflake-connector-python python-dateutil==2.9.0.post0 # via botocore python-memcached==1.62 # via -r requirements/base.in -pytz==2026.1.post1 - # via drf-yasg +pytz==2026.3.post1 + # via + # drf-yasg + # snowflake-connector-python pyyaml==6.0.3 # via # drf-yasg # edx-django-release-util -requests==2.33.1 +requests==2.34.2 # via # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client + # snowflake-connector-python rules==3.5 # via edx-enterprise-data -s3transfer==0.16.0 +s3transfer==0.19.2 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -194,18 +219,28 @@ six==1.17.0 # edx-rbac # html5lib # python-dateutil +snowflake-connector-python==4.7.1 + # via + # -r requirements/base.in + # edx-enterprise-data +sortedcontainers==2.4.0 + # via snowflake-connector-python sqlparse==0.5.5 # via django -stevedore==5.7.0 +stevedore==5.9.0 # via # edx-django-utils # edx-opaque-keys -tqdm==4.67.3 +tomlkit==0.15.1 + # via snowflake-connector-python +tqdm==4.69.1 # via -r requirements/base.in -typing-extensions==4.15.0 +typing-extensions==4.16.0 # via # django-countries # edx-opaque-keys + # pyopenssl + # snowflake-connector-python uritemplate==4.2.0 # via drf-yasg urllib3==1.26.20 diff --git a/requirements/doc.txt b/requirements/doc.txt index 0186eb5a..fcd480e4 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # make upgrade @@ -8,39 +8,51 @@ accessible-pygments==0.0.5 # via pydata-sphinx-theme alabaster==1.0.0 # via sphinx -asgiref==3.11.1 +asgiref==3.12.1 # via # django # django-cors-headers # django-countries +asn1crypto==1.5.1 + # via snowflake-connector-python babel==2.18.0 # via # pydata-sphinx-theme # sphinx -beautifulsoup4==4.14.3 +beautifulsoup4==4.15.0 # via pydata-sphinx-theme boto==2.49.0 # via -r requirements/base.in -boto3==1.42.90 - # via -r requirements/base.in -botocore==1.42.90 +boto3==1.43.56 + # via + # -r requirements/base.in + # snowflake-connector-python +botocore==1.43.56 # via # boto3 # s3transfer -certifi==2026.2.25 - # via requests -cffi==2.0.0 + # snowflake-connector-python +certifi==2026.7.22 + # via + # requests + # snowflake-connector-python +cffi==2.1.0 # via # cryptography # pynacl -charset-normalizer==3.4.7 - # via requests -click==8.3.2 +charset-normalizer==3.4.9 + # via + # requests + # snowflake-connector-python +click==8.4.2 # via edx-django-utils -cryptography==46.0.7 +cryptography==49.0.0 # via + # -r requirements/base.in # django-fernet-fields-v2 # pyjwt + # pyopenssl + # snowflake-connector-python django==4.2.30 # via # -c requirements/constraints.txt @@ -63,7 +75,7 @@ django==4.2.30 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==8.2.0 +django-countries==9.0.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -142,18 +154,24 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.13.0 +faker==40.36.0 # via factory-boy +filelock==3.32.0 + # via snowflake-connector-python html5lib==1.1 # via -r requirements/base.in -idna==3.11 - # via requests +idna==3.18 + # via + # requests + # snowflake-connector-python imagesize==2.0.0 # via sphinx inflection==0.5.1 # via drf-yasg jinja2==3.1.6 - # via sphinx + # via + # pydata-sphinx-theme + # sphinx jmespath==1.1.0 # via # boto3 @@ -166,57 +184,67 @@ mysql-connector-python==9.5.0 # via edx-enterprise-data ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.1 +packaging==26.2 # via # drf-yasg + # snowflake-connector-python # sphinx path==16.14.0 # via # -c requirements/constraints.txt # -r requirements/doc.in +platformdirs==4.11.0 + # via snowflake-connector-python psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi -pydata-sphinx-theme==0.16.1 +pydata-sphinx-theme==0.20.0 # via sphinx-book-theme pygments==2.20.0 # via # accessible-pygments # pydata-sphinx-theme # sphinx -pyjwt[crypto]==2.12.1 +pyjwt[crypto]==2.13.0 # via # drf-jwt # edx-drf-extensions # edx-rest-api-client + # snowflake-connector-python pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.16.0 +pymongo==4.17.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils +pyopenssl==26.3.0 + # via snowflake-connector-python python-dateutil==2.9.0.post0 # via botocore python-memcached==1.62 # via -r requirements/base.in -pytz==2026.1.post1 - # via drf-yasg +pytz==2026.3.post1 + # via + # drf-yasg + # snowflake-connector-python pyyaml==6.0.3 # via # drf-yasg # edx-django-release-util -requests==2.33.1 +requests==2.34.2 # via # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client + # pydata-sphinx-theme + # snowflake-connector-python # sphinx roman-numerals==4.1.0 # via sphinx rules==3.5 # via edx-enterprise-data -s3transfer==0.16.0 +s3transfer==0.19.2 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -227,16 +255,22 @@ six==1.17.0 # edx-rbac # html5lib # python-dateutil -snowballstemmer==3.0.1 +snowballstemmer==3.1.1 # via sphinx -soupsieve==2.8.3 +snowflake-connector-python==4.7.1 + # via + # -r requirements/base.in + # edx-enterprise-data +sortedcontainers==2.4.0 + # via snowflake-connector-python +soupsieve==2.9.1 # via beautifulsoup4 -sphinx==9.0.4 +sphinx==9.1.0 # via # -r requirements/doc.in # pydata-sphinx-theme # sphinx-book-theme -sphinx-book-theme==1.2.0 +sphinx-book-theme==1.4.0 # via -r requirements/doc.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -252,18 +286,21 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx sqlparse==0.5.5 # via django -stevedore==5.7.0 +stevedore==5.9.0 # via # edx-django-utils # edx-opaque-keys -tqdm==4.67.3 +tomlkit==0.15.1 + # via snowflake-connector-python +tqdm==4.69.1 # via -r requirements/base.in -typing-extensions==4.15.0 +typing-extensions==4.16.0 # via # beautifulsoup4 # django-countries # edx-opaque-keys - # pydata-sphinx-theme + # pyopenssl + # snowflake-connector-python uritemplate==4.2.0 # via drf-yasg urllib3==1.26.20 diff --git a/requirements/production.txt b/requirements/production.txt index 3df43b86..be0e5f5f 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,36 +1,48 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # make upgrade # -asgiref==3.11.1 +asgiref==3.12.1 # via # django # django-cors-headers # django-countries +asn1crypto==1.5.1 + # via snowflake-connector-python boto==2.49.0 # via -r requirements/base.in -boto3==1.42.90 - # via -r requirements/base.in -botocore==1.42.90 +boto3==1.43.56 + # via + # -r requirements/base.in + # snowflake-connector-python +botocore==1.43.56 # via # boto3 # s3transfer -certifi==2026.2.25 - # via requests -cffi==2.0.0 + # snowflake-connector-python +certifi==2026.7.22 + # via + # requests + # snowflake-connector-python +cffi==2.1.0 # via # cryptography # pynacl -charset-normalizer==3.4.7 - # via requests -click==8.3.2 +charset-normalizer==3.4.9 + # via + # requests + # snowflake-connector-python +click==8.4.2 # via edx-django-utils -cryptography==46.0.7 +cryptography==49.0.0 # via + # -r requirements/base.in # django-fernet-fields-v2 # pyjwt + # pyopenssl + # snowflake-connector-python django==4.2.30 # via # -c requirements/constraints.txt @@ -53,7 +65,7 @@ django==4.2.30 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==8.2.0 +django-countries==9.0.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -128,18 +140,22 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.13.0 +faker==40.36.0 # via factory-boy -gevent==26.4.0 +filelock==3.32.0 + # via snowflake-connector-python +gevent==26.7.0 # via -r requirements/production.in -greenlet==3.4.0 +greenlet==3.5.4 # via gevent -gunicorn==25.3.0 +gunicorn==26.0.0 # via -r requirements/production.in html5lib==1.1 # via -r requirements/base.in -idna==3.11 - # via requests +idna==3.18 + # via + # requests + # snowflake-connector-python inflection==0.5.1 # via drf-yasg jmespath==1.1.0 @@ -152,50 +168,59 @@ mysql-connector-python==9.5.0 # via edx-enterprise-data mysqlclient==2.2.8 # via -r requirements/production.in -newrelic==12.1.0 +newrelic==13.3.0 # via -r requirements/production.in ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.1 +packaging==26.2 # via # drf-yasg # gunicorn + # snowflake-connector-python path-py==8.2.1 # via -r requirements/production.in +platformdirs==4.11.0 + # via snowflake-connector-python psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi -pyjwt[crypto]==2.12.1 +pyjwt[crypto]==2.13.0 # via # drf-jwt # edx-drf-extensions # edx-rest-api-client + # snowflake-connector-python pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.16.0 +pymongo==4.17.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils +pyopenssl==26.3.0 + # via snowflake-connector-python python-dateutil==2.9.0.post0 # via botocore python-memcached==1.62 # via -r requirements/base.in -pytz==2026.1.post1 - # via drf-yasg +pytz==2026.3.post1 + # via + # drf-yasg + # snowflake-connector-python pyyaml==6.0.3 # via # -r requirements/production.in # drf-yasg # edx-django-release-util -requests==2.33.1 +requests==2.34.2 # via # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client + # snowflake-connector-python rules==3.5 # via edx-enterprise-data -s3transfer==0.16.0 +s3transfer==0.19.2 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -206,18 +231,28 @@ six==1.17.0 # edx-rbac # html5lib # python-dateutil +snowflake-connector-python==4.7.1 + # via + # -r requirements/base.in + # edx-enterprise-data +sortedcontainers==2.4.0 + # via snowflake-connector-python sqlparse==0.5.5 # via django -stevedore==5.7.0 +stevedore==5.9.0 # via # edx-django-utils # edx-opaque-keys -tqdm==4.67.3 +tomlkit==0.15.1 + # via snowflake-connector-python +tqdm==4.69.1 # via -r requirements/base.in -typing-extensions==4.15.0 +typing-extensions==4.16.0 # via # django-countries # edx-opaque-keys + # pyopenssl + # snowflake-connector-python uritemplate==4.2.0 # via drf-yasg urllib3==1.26.20 @@ -228,9 +263,9 @@ urllib3==1.26.20 # requests webencodings==0.5.1 # via html5lib -zope-event==6.1 +zope-event==6.2 # via gevent -zope-interface==8.3 +zope-interface==8.5 # via gevent # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/test.txt b/requirements/test.txt index 4d73a891..6f7f272d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,47 +1,59 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # make upgrade # -asgiref==3.11.1 +asgiref==3.12.1 # via # django # django-cors-headers # django-countries +asn1crypto==1.5.1 + # via snowflake-connector-python astroid==3.3.11 # via pylint boto==2.49.0 # via -r requirements/base.in -boto3==1.42.90 - # via -r requirements/base.in -botocore==1.42.90 +boto3==1.43.56 + # via + # -r requirements/base.in + # snowflake-connector-python +botocore==1.43.56 # via # boto3 # s3transfer -certifi==2026.2.25 - # via requests -cffi==2.0.0 + # snowflake-connector-python +certifi==2026.7.22 + # via + # requests + # snowflake-connector-python +cffi==2.1.0 # via # cryptography # pynacl chardet==7.4.3 # via diff-cover -charset-normalizer==3.4.7 - # via requests -click==8.3.2 +charset-normalizer==3.4.9 + # via + # requests + # snowflake-connector-python +click==8.4.2 # via edx-django-utils -coverage[toml]==7.13.5 +coverage[toml]==7.15.2 # via # -r requirements/test.in # pytest-cov -cryptography==46.0.7 +cryptography==49.0.0 # via + # -r requirements/base.in # django-fernet-fields-v2 # pyjwt + # pyopenssl + # snowflake-connector-python ddt==1.7.2 # via -r requirements/test.in -diff-cover==10.2.0 +diff-cover==10.4.1 # via -r requirements/test.in dill==0.4.1 # via pylint @@ -66,7 +78,7 @@ dill==0.4.1 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==8.2.0 +django-countries==9.0.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -143,14 +155,18 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.13.0 +faker==40.36.0 # via factory-boy +filelock==3.32.0 + # via snowflake-connector-python freezegun==1.5.5 # via -r requirements/test.in html5lib==1.1 # via -r requirements/base.in -idna==3.11 - # via requests +idna==3.18 + # via + # requests + # snowflake-connector-python inflection==0.5.1 # via drf-yasg iniconfig==2.3.0 @@ -173,12 +189,15 @@ mysql-connector-python==9.5.0 # via edx-enterprise-data ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.1 +packaging==26.2 # via # drf-yasg # pytest -platformdirs==4.9.6 - # via pylint + # snowflake-connector-python +platformdirs==4.11.0 + # via + # pylint + # snowflake-connector-python pluggy==1.6.0 # via # diff-cover @@ -196,22 +215,25 @@ pygments==2.20.0 # via # diff-cover # pytest -pyjwt[crypto]==2.12.1 +pyjwt[crypto]==2.13.0 # via # drf-jwt # edx-drf-extensions # edx-rest-api-client + # snowflake-connector-python pylint==3.3.8 # via # -c requirements/constraints.txt # -r requirements/test.in pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.16.0 +pymongo==4.17.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils -pytest==9.0.3 +pyopenssl==26.3.0 + # via snowflake-connector-python +pytest==9.1.1 # via # pytest-cov # pytest-django @@ -225,26 +247,28 @@ python-dateutil==2.9.0.post0 # freezegun python-memcached==1.62 # via -r requirements/base.in -pytz==2026.1.post1 +pytz==2026.3.post1 # via # -r requirements/test.in # drf-yasg + # snowflake-connector-python pyyaml==6.0.3 # via # drf-yasg # edx-django-release-util # responses -requests==2.33.1 +requests==2.34.2 # via # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client # responses -responses==0.26.0 + # snowflake-connector-python +responses==0.26.2 # via -r requirements/test.in rules==3.5 # via edx-enterprise-data -s3transfer==0.16.0 +s3transfer==0.19.2 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -255,22 +279,32 @@ six==1.17.0 # edx-rbac # html5lib # python-dateutil -snowballstemmer==3.0.1 +snowballstemmer==3.1.1 # via pydocstyle +snowflake-connector-python==4.7.1 + # via + # -r requirements/base.in + # edx-enterprise-data +sortedcontainers==2.4.0 + # via snowflake-connector-python sqlparse==0.5.5 # via django -stevedore==5.7.0 +stevedore==5.9.0 # via # edx-django-utils # edx-opaque-keys -tomlkit==0.14.0 - # via pylint -tqdm==4.67.3 +tomlkit==0.15.1 + # via + # pylint + # snowflake-connector-python +tqdm==4.69.1 # via -r requirements/base.in -typing-extensions==4.15.0 +typing-extensions==4.16.0 # via # django-countries # edx-opaque-keys + # pyopenssl + # snowflake-connector-python uritemplate==4.2.0 # via drf-yasg urllib3==1.26.20 From 1e0c5a149c8aacf2395c424ecf33952ba5c72f67 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Jul 2026 08:36:21 +0000 Subject: [PATCH 2/8] fix: pin Sphinx below 9.1 for Python 3.11 docs job --- requirements/doc.in | 2 +- requirements/doc.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/doc.in b/requirements/doc.in index 936c0cd0..5d996317 100644 --- a/requirements/doc.in +++ b/requirements/doc.in @@ -2,6 +2,6 @@ -r base.in # Core dependencies of edx-analytics-data-api -Sphinx # Developer documentation builder +Sphinx<9.1 # Developer documentation builder (9.1+ requires Python 3.12) path sphinx-book-theme \ No newline at end of file diff --git a/requirements/doc.txt b/requirements/doc.txt index fcd480e4..9d1933dc 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -265,7 +265,7 @@ sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.9.1 # via beautifulsoup4 -sphinx==9.1.0 +sphinx==9.0.4 # via # -r requirements/doc.in # pydata-sphinx-theme From 2c1e3d09bea58854427d34f610c3963762107dc9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Jul 2026 03:00:16 +0000 Subject: [PATCH 3/8] fix: sort snowflake client imports --- analytics_data_api/snowflake_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/analytics_data_api/snowflake_client.py b/analytics_data_api/snowflake_client.py index 81d3f82a..194f0647 100644 --- a/analytics_data_api/snowflake_client.py +++ b/analytics_data_api/snowflake_client.py @@ -11,7 +11,6 @@ from cryptography.hazmat.primitives import serialization from django.conf import settings - QUERY_TAG = 'edx_analytics_api_insights_snowflake_phase1' DEFAULT_VALIDATION_TABLE = 'COURSE_ACTIVITY_WEEKLY' REQUIRED_CONFIG_KEYS = ( From f20ab5d0236883081c8aa1ae28704179af96e969 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Date: Tue, 28 Jul 2026 03:26:26 +0000 Subject: [PATCH 4/8] fix: regenerate requirements with python 3.11 --- requirements/base.txt | 10 +++++----- requirements/dev.txt | 10 +++++----- requirements/doc.txt | 10 +++++----- requirements/production.txt | 10 +++++----- requirements/test.txt | 10 +++++----- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 4171c7ef..04816f76 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -13,11 +13,11 @@ asn1crypto==1.5.1 # via snowflake-connector-python boto==2.49.0 # via -r requirements/base.in -boto3==1.43.56 +boto3==1.43.57 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.56 +botocore==1.43.57 # via # boto3 # s3transfer @@ -124,7 +124,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.10 +edx-enterprise-data==10.22.11 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -231,7 +231,7 @@ stevedore==5.9.0 # edx-opaque-keys tomlkit==0.15.1 # via snowflake-connector-python -tqdm==4.69.1 +tqdm==4.70.0 # via -r requirements/base.in typing-extensions==4.16.0 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index f82826cb..99e3c9b4 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -13,11 +13,11 @@ asn1crypto==1.5.1 # via snowflake-connector-python boto==2.49.0 # via -r requirements/base.in -boto3==1.43.56 +boto3==1.43.57 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.56 +botocore==1.43.57 # via # boto3 # s3transfer @@ -124,7 +124,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.10 +edx-enterprise-data==10.22.11 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -233,7 +233,7 @@ stevedore==5.9.0 # edx-opaque-keys tomlkit==0.15.1 # via snowflake-connector-python -tqdm==4.69.1 +tqdm==4.70.0 # via -r requirements/base.in typing-extensions==4.16.0 # via diff --git a/requirements/doc.txt b/requirements/doc.txt index 9d1933dc..713882b9 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -23,11 +23,11 @@ beautifulsoup4==4.15.0 # via pydata-sphinx-theme boto==2.49.0 # via -r requirements/base.in -boto3==1.43.56 +boto3==1.43.57 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.56 +botocore==1.43.57 # via # boto3 # s3transfer @@ -138,7 +138,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.10 +edx-enterprise-data==10.22.11 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -292,7 +292,7 @@ stevedore==5.9.0 # edx-opaque-keys tomlkit==0.15.1 # via snowflake-connector-python -tqdm==4.69.1 +tqdm==4.70.0 # via -r requirements/base.in typing-extensions==4.16.0 # via diff --git a/requirements/production.txt b/requirements/production.txt index be0e5f5f..ebf0daea 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -13,11 +13,11 @@ asn1crypto==1.5.1 # via snowflake-connector-python boto==2.49.0 # via -r requirements/base.in -boto3==1.43.56 +boto3==1.43.57 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.56 +botocore==1.43.57 # via # boto3 # s3transfer @@ -124,7 +124,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.10 +edx-enterprise-data==10.22.11 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -245,7 +245,7 @@ stevedore==5.9.0 # edx-opaque-keys tomlkit==0.15.1 # via snowflake-connector-python -tqdm==4.69.1 +tqdm==4.70.0 # via -r requirements/base.in typing-extensions==4.16.0 # via diff --git a/requirements/test.txt b/requirements/test.txt index 6f7f272d..45d3d36f 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -15,11 +15,11 @@ astroid==3.3.11 # via pylint boto==2.49.0 # via -r requirements/base.in -boto3==1.43.56 +boto3==1.43.57 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.56 +botocore==1.43.57 # via # boto3 # s3transfer @@ -139,7 +139,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.10 +edx-enterprise-data==10.22.11 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -297,7 +297,7 @@ tomlkit==0.15.1 # via # pylint # snowflake-connector-python -tqdm==4.69.1 +tqdm==4.70.0 # via -r requirements/base.in typing-extensions==4.16.0 # via From 5cc6daf6e247409c41052737cada5cb96b3e5be3 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Date: Tue, 28 Jul 2026 03:58:51 +0000 Subject: [PATCH 5/8] fix: remove unused insights snowflake enabled setting --- analyticsdataserver/settings/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/analyticsdataserver/settings/base.py b/analyticsdataserver/settings/base.py index a420a97c..630f5f98 100644 --- a/analyticsdataserver/settings/base.py +++ b/analyticsdataserver/settings/base.py @@ -92,7 +92,6 @@ ########## INSIGHTS SNOWFLAKE CONFIGURATION INSIGHTS_SNOWFLAKE = { - 'ENABLED': False, 'ACCOUNT': None, 'USER': None, 'ROLE': None, From 4dd1c4a9b982cb3d8dd9b57f599c11c9fdfdb6e5 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Date: Tue, 28 Jul 2026 06:07:43 +0000 Subject: [PATCH 6/8] fix: narrow snowflake requirements changes --- requirements/base.txt | 46 +++++++++++++-------------- requirements/dev.txt | 46 +++++++++++++-------------- requirements/doc.in | 2 +- requirements/doc.txt | 62 ++++++++++++++++++------------------- requirements/production.txt | 58 +++++++++++++++++----------------- requirements/test.txt | 61 ++++++++++++++++++------------------ 6 files changed, 137 insertions(+), 138 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 04816f76..f6f1caec 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# make upgrade +# pip-compile --output-file=requirements/base.txt requirements/base.in # -asgiref==3.12.1 +asgiref==3.11.1 # via # django # django-cors-headers @@ -13,30 +13,30 @@ asn1crypto==1.5.1 # via snowflake-connector-python boto==2.49.0 # via -r requirements/base.in -boto3==1.43.57 +boto3==1.42.90 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.57 +botocore==1.42.90 # via # boto3 # s3transfer # snowflake-connector-python -certifi==2026.7.22 +certifi==2026.2.25 # via # requests # snowflake-connector-python -cffi==2.1.0 +cffi==2.0.0 # via # cryptography # pynacl -charset-normalizer==3.4.9 +charset-normalizer==3.4.7 # via # requests # snowflake-connector-python -click==8.4.2 +click==8.3.2 # via edx-django-utils -cryptography==49.0.0 +cryptography==46.0.7 # via # -r requirements/base.in # django-fernet-fields-v2 @@ -65,7 +65,7 @@ django==4.2.30 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==9.0.0 +django-countries==8.2.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -124,7 +124,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.11 +edx-enterprise-data==10.22.10 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -140,13 +140,13 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.36.0 +faker==40.13.0 # via factory-boy filelock==3.32.0 # via snowflake-connector-python html5lib==1.1 # via -r requirements/base.in -idna==3.18 +idna==3.11 # via # requests # snowflake-connector-python @@ -162,7 +162,7 @@ mysql-connector-python==9.5.0 # via edx-enterprise-data ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.2 +packaging==26.1 # via # drf-yasg # snowflake-connector-python @@ -172,7 +172,7 @@ psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi -pyjwt[crypto]==2.13.0 +pyjwt[crypto]==2.12.1 # via # drf-jwt # edx-drf-extensions @@ -180,17 +180,17 @@ pyjwt[crypto]==2.13.0 # snowflake-connector-python pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.17.0 +pymongo==4.16.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils -pyopenssl==26.3.0 +pyopenssl==26.2.0 # via snowflake-connector-python python-dateutil==2.9.0.post0 # via botocore python-memcached==1.62 # via -r requirements/base.in -pytz==2026.3.post1 +pytz==2026.1.post1 # via # drf-yasg # snowflake-connector-python @@ -198,7 +198,7 @@ pyyaml==6.0.3 # via # drf-yasg # edx-django-release-util -requests==2.34.2 +requests==2.33.1 # via # edx-drf-extensions # edx-enterprise-data @@ -206,7 +206,7 @@ requests==2.34.2 # snowflake-connector-python rules==3.5 # via edx-enterprise-data -s3transfer==0.19.2 +s3transfer==0.16.0 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -225,15 +225,15 @@ sortedcontainers==2.4.0 # via snowflake-connector-python sqlparse==0.5.5 # via django -stevedore==5.9.0 +stevedore==5.7.0 # via # edx-django-utils # edx-opaque-keys tomlkit==0.15.1 # via snowflake-connector-python -tqdm==4.70.0 +tqdm==4.67.3 # via -r requirements/base.in -typing-extensions==4.16.0 +typing-extensions==4.15.0 # via # django-countries # edx-opaque-keys diff --git a/requirements/dev.txt b/requirements/dev.txt index 99e3c9b4..d150ffc1 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# make upgrade +# pip-compile --output-file=requirements/dev.txt requirements/dev.in # -asgiref==3.12.1 +asgiref==3.11.1 # via # django # django-cors-headers @@ -13,30 +13,30 @@ asn1crypto==1.5.1 # via snowflake-connector-python boto==2.49.0 # via -r requirements/base.in -boto3==1.43.57 +boto3==1.42.90 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.57 +botocore==1.42.90 # via # boto3 # s3transfer # snowflake-connector-python -certifi==2026.7.22 +certifi==2026.2.25 # via # requests # snowflake-connector-python -cffi==2.1.0 +cffi==2.0.0 # via # cryptography # pynacl -charset-normalizer==3.4.9 +charset-normalizer==3.4.7 # via # requests # snowflake-connector-python -click==8.4.2 +click==8.3.2 # via edx-django-utils -cryptography==49.0.0 +cryptography==46.0.7 # via # -r requirements/base.in # django-fernet-fields-v2 @@ -65,7 +65,7 @@ django==4.2.30 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==9.0.0 +django-countries==8.2.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -124,7 +124,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.11 +edx-enterprise-data==10.22.10 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -140,13 +140,13 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.36.0 +faker==40.13.0 # via factory-boy filelock==3.32.0 # via snowflake-connector-python html5lib==1.1 # via -r requirements/base.in -idna==3.18 +idna==3.11 # via # requests # snowflake-connector-python @@ -164,7 +164,7 @@ mysqlclient==2.2.8 # via -r requirements/dev.in ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.2 +packaging==26.1 # via # drf-yasg # snowflake-connector-python @@ -174,7 +174,7 @@ psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi -pyjwt[crypto]==2.13.0 +pyjwt[crypto]==2.12.1 # via # drf-jwt # edx-drf-extensions @@ -182,17 +182,17 @@ pyjwt[crypto]==2.13.0 # snowflake-connector-python pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.17.0 +pymongo==4.16.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils -pyopenssl==26.3.0 +pyopenssl==26.2.0 # via snowflake-connector-python python-dateutil==2.9.0.post0 # via botocore python-memcached==1.62 # via -r requirements/base.in -pytz==2026.3.post1 +pytz==2026.1.post1 # via # drf-yasg # snowflake-connector-python @@ -200,7 +200,7 @@ pyyaml==6.0.3 # via # drf-yasg # edx-django-release-util -requests==2.34.2 +requests==2.33.1 # via # edx-drf-extensions # edx-enterprise-data @@ -208,7 +208,7 @@ requests==2.34.2 # snowflake-connector-python rules==3.5 # via edx-enterprise-data -s3transfer==0.19.2 +s3transfer==0.16.0 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -227,15 +227,15 @@ sortedcontainers==2.4.0 # via snowflake-connector-python sqlparse==0.5.5 # via django -stevedore==5.9.0 +stevedore==5.7.0 # via # edx-django-utils # edx-opaque-keys tomlkit==0.15.1 # via snowflake-connector-python -tqdm==4.70.0 +tqdm==4.67.3 # via -r requirements/base.in -typing-extensions==4.16.0 +typing-extensions==4.15.0 # via # django-countries # edx-opaque-keys diff --git a/requirements/doc.in b/requirements/doc.in index 5d996317..936c0cd0 100644 --- a/requirements/doc.in +++ b/requirements/doc.in @@ -2,6 +2,6 @@ -r base.in # Core dependencies of edx-analytics-data-api -Sphinx<9.1 # Developer documentation builder (9.1+ requires Python 3.12) +Sphinx # Developer documentation builder path sphinx-book-theme \ No newline at end of file diff --git a/requirements/doc.txt b/requirements/doc.txt index 713882b9..d1c55148 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -2,13 +2,13 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# make upgrade +# pip-compile --output-file=requirements/doc.txt requirements/doc.in # accessible-pygments==0.0.5 # via pydata-sphinx-theme alabaster==1.0.0 # via sphinx -asgiref==3.12.1 +asgiref==3.11.1 # via # django # django-cors-headers @@ -19,34 +19,34 @@ babel==2.18.0 # via # pydata-sphinx-theme # sphinx -beautifulsoup4==4.15.0 +beautifulsoup4==4.14.3 # via pydata-sphinx-theme boto==2.49.0 # via -r requirements/base.in -boto3==1.43.57 +boto3==1.42.90 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.57 +botocore==1.42.90 # via # boto3 # s3transfer # snowflake-connector-python -certifi==2026.7.22 +certifi==2026.2.25 # via # requests # snowflake-connector-python -cffi==2.1.0 +cffi==2.0.0 # via # cryptography # pynacl -charset-normalizer==3.4.9 +charset-normalizer==3.4.7 # via # requests # snowflake-connector-python -click==8.4.2 +click==8.3.2 # via edx-django-utils -cryptography==49.0.0 +cryptography==46.0.7 # via # -r requirements/base.in # django-fernet-fields-v2 @@ -75,7 +75,7 @@ django==4.2.30 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==9.0.0 +django-countries==8.2.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -138,7 +138,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.11 +edx-enterprise-data==10.22.10 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -154,13 +154,13 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.36.0 +faker==40.13.0 # via factory-boy filelock==3.32.0 # via snowflake-connector-python html5lib==1.1 # via -r requirements/base.in -idna==3.18 +idna==3.11 # via # requests # snowflake-connector-python @@ -169,9 +169,7 @@ imagesize==2.0.0 inflection==0.5.1 # via drf-yasg jinja2==3.1.6 - # via - # pydata-sphinx-theme - # sphinx + # via sphinx jmespath==1.1.0 # via # boto3 @@ -184,7 +182,7 @@ mysql-connector-python==9.5.0 # via edx-enterprise-data ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.2 +packaging==26.1 # via # drf-yasg # snowflake-connector-python @@ -199,14 +197,14 @@ psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi -pydata-sphinx-theme==0.20.0 +pydata-sphinx-theme==0.16.1 # via sphinx-book-theme pygments==2.20.0 # via # accessible-pygments # pydata-sphinx-theme # sphinx -pyjwt[crypto]==2.13.0 +pyjwt[crypto]==2.12.1 # via # drf-jwt # edx-drf-extensions @@ -214,17 +212,17 @@ pyjwt[crypto]==2.13.0 # snowflake-connector-python pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.17.0 +pymongo==4.16.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils -pyopenssl==26.3.0 +pyopenssl==26.2.0 # via snowflake-connector-python python-dateutil==2.9.0.post0 # via botocore python-memcached==1.62 # via -r requirements/base.in -pytz==2026.3.post1 +pytz==2026.1.post1 # via # drf-yasg # snowflake-connector-python @@ -232,19 +230,18 @@ pyyaml==6.0.3 # via # drf-yasg # edx-django-release-util -requests==2.34.2 +requests==2.33.1 # via # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client - # pydata-sphinx-theme # snowflake-connector-python # sphinx roman-numerals==4.1.0 # via sphinx rules==3.5 # via edx-enterprise-data -s3transfer==0.19.2 +s3transfer==0.16.0 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -255,7 +252,7 @@ six==1.17.0 # edx-rbac # html5lib # python-dateutil -snowballstemmer==3.1.1 +snowballstemmer==3.0.1 # via sphinx snowflake-connector-python==4.7.1 # via @@ -263,14 +260,14 @@ snowflake-connector-python==4.7.1 # edx-enterprise-data sortedcontainers==2.4.0 # via snowflake-connector-python -soupsieve==2.9.1 +soupsieve==2.8.3 # via beautifulsoup4 sphinx==9.0.4 # via # -r requirements/doc.in # pydata-sphinx-theme # sphinx-book-theme -sphinx-book-theme==1.4.0 +sphinx-book-theme==1.2.0 # via -r requirements/doc.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -286,19 +283,20 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx sqlparse==0.5.5 # via django -stevedore==5.9.0 +stevedore==5.7.0 # via # edx-django-utils # edx-opaque-keys tomlkit==0.15.1 # via snowflake-connector-python -tqdm==4.70.0 +tqdm==4.67.3 # via -r requirements/base.in -typing-extensions==4.16.0 +typing-extensions==4.15.0 # via # beautifulsoup4 # django-countries # edx-opaque-keys + # pydata-sphinx-theme # pyopenssl # snowflake-connector-python uritemplate==4.2.0 diff --git a/requirements/production.txt b/requirements/production.txt index ebf0daea..e144c090 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# make upgrade +# pip-compile --output-file=requirements/production.txt requirements/production.in # -asgiref==3.12.1 +asgiref==3.11.1 # via # django # django-cors-headers @@ -13,30 +13,30 @@ asn1crypto==1.5.1 # via snowflake-connector-python boto==2.49.0 # via -r requirements/base.in -boto3==1.43.57 +boto3==1.42.90 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.57 +botocore==1.42.90 # via # boto3 # s3transfer # snowflake-connector-python -certifi==2026.7.22 +certifi==2026.2.25 # via # requests # snowflake-connector-python -cffi==2.1.0 +cffi==2.0.0 # via # cryptography # pynacl -charset-normalizer==3.4.9 +charset-normalizer==3.4.7 # via # requests # snowflake-connector-python -click==8.4.2 +click==8.3.2 # via edx-django-utils -cryptography==49.0.0 +cryptography==46.0.7 # via # -r requirements/base.in # django-fernet-fields-v2 @@ -65,7 +65,7 @@ django==4.2.30 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==9.0.0 +django-countries==8.2.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -124,7 +124,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.11 +edx-enterprise-data==10.22.10 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -140,19 +140,19 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.36.0 +faker==40.13.0 # via factory-boy filelock==3.32.0 # via snowflake-connector-python -gevent==26.7.0 +gevent==26.4.0 # via -r requirements/production.in -greenlet==3.5.4 +greenlet==3.4.0 # via gevent -gunicorn==26.0.0 +gunicorn==25.3.0 # via -r requirements/production.in html5lib==1.1 # via -r requirements/base.in -idna==3.18 +idna==3.11 # via # requests # snowflake-connector-python @@ -168,11 +168,11 @@ mysql-connector-python==9.5.0 # via edx-enterprise-data mysqlclient==2.2.8 # via -r requirements/production.in -newrelic==13.3.0 +newrelic==12.1.0 # via -r requirements/production.in ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.2 +packaging==26.1 # via # drf-yasg # gunicorn @@ -185,7 +185,7 @@ psutil==7.2.2 # via edx-django-utils pycparser==3.0 # via cffi -pyjwt[crypto]==2.13.0 +pyjwt[crypto]==2.12.1 # via # drf-jwt # edx-drf-extensions @@ -193,17 +193,17 @@ pyjwt[crypto]==2.13.0 # snowflake-connector-python pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.17.0 +pymongo==4.16.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils -pyopenssl==26.3.0 +pyopenssl==26.2.0 # via snowflake-connector-python python-dateutil==2.9.0.post0 # via botocore python-memcached==1.62 # via -r requirements/base.in -pytz==2026.3.post1 +pytz==2026.1.post1 # via # drf-yasg # snowflake-connector-python @@ -212,7 +212,7 @@ pyyaml==6.0.3 # -r requirements/production.in # drf-yasg # edx-django-release-util -requests==2.34.2 +requests==2.33.1 # via # edx-drf-extensions # edx-enterprise-data @@ -220,7 +220,7 @@ requests==2.34.2 # snowflake-connector-python rules==3.5 # via edx-enterprise-data -s3transfer==0.19.2 +s3transfer==0.16.0 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -239,15 +239,15 @@ sortedcontainers==2.4.0 # via snowflake-connector-python sqlparse==0.5.5 # via django -stevedore==5.9.0 +stevedore==5.7.0 # via # edx-django-utils # edx-opaque-keys tomlkit==0.15.1 # via snowflake-connector-python -tqdm==4.70.0 +tqdm==4.67.3 # via -r requirements/base.in -typing-extensions==4.16.0 +typing-extensions==4.15.0 # via # django-countries # edx-opaque-keys @@ -263,9 +263,9 @@ urllib3==1.26.20 # requests webencodings==0.5.1 # via html5lib -zope-event==6.2 +zope-event==6.1 # via gevent -zope-interface==8.5 +zope-interface==8.3 # via gevent # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/test.txt b/requirements/test.txt index 45d3d36f..e54cfb63 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# make upgrade +# pip-compile --output-file=requirements/test.txt requirements/test.in # -asgiref==3.12.1 +asgiref==3.11.1 # via # django # django-cors-headers @@ -15,36 +15,36 @@ astroid==3.3.11 # via pylint boto==2.49.0 # via -r requirements/base.in -boto3==1.43.57 +boto3==1.42.90 # via # -r requirements/base.in # snowflake-connector-python -botocore==1.43.57 +botocore==1.42.90 # via # boto3 # s3transfer # snowflake-connector-python -certifi==2026.7.22 +certifi==2026.2.25 # via # requests # snowflake-connector-python -cffi==2.1.0 +cffi==2.0.0 # via # cryptography # pynacl chardet==7.4.3 # via diff-cover -charset-normalizer==3.4.9 +charset-normalizer==3.4.7 # via # requests # snowflake-connector-python -click==8.4.2 +click==8.3.2 # via edx-django-utils -coverage[toml]==7.15.2 +coverage[toml]==7.13.5 # via # -r requirements/test.in # pytest-cov -cryptography==49.0.0 +cryptography==46.0.7 # via # -r requirements/base.in # django-fernet-fields-v2 @@ -53,10 +53,11 @@ cryptography==49.0.0 # snowflake-connector-python ddt==1.7.2 # via -r requirements/test.in -diff-cover==10.4.1 +diff-cover==10.2.0 # via -r requirements/test.in dill==0.4.1 # via pylint +django==4.2.30 # via # -c requirements/constraints.txt # -r requirements/base.in @@ -78,7 +79,7 @@ dill==0.4.1 # edx-rbac django-cors-headers==4.9.0 # via -r requirements/base.in -django-countries==9.0.0 +django-countries==8.2.0 # via -r requirements/base.in django-crum==0.7.9 # via @@ -139,7 +140,7 @@ edx-drf-extensions==10.6.0 # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==10.22.11 +edx-enterprise-data==10.22.10 # via -r requirements/base.in edx-opaque-keys==4.0.0 # via @@ -155,7 +156,7 @@ edx-rest-api-client==7.0.0 # edx-enterprise-data factory-boy==3.3.3 # via edx-enterprise-data -faker==40.36.0 +faker==40.13.0 # via factory-boy filelock==3.32.0 # via snowflake-connector-python @@ -163,7 +164,7 @@ freezegun==1.5.5 # via -r requirements/test.in html5lib==1.1 # via -r requirements/base.in -idna==3.18 +idna==3.11 # via # requests # snowflake-connector-python @@ -189,12 +190,12 @@ mysql-connector-python==9.5.0 # via edx-enterprise-data ordered-set==4.1.0 # via -r requirements/base.in -packaging==26.2 +packaging==26.1 # via # drf-yasg # pytest # snowflake-connector-python -platformdirs==4.11.0 +platformdirs==4.9.6 # via # pylint # snowflake-connector-python @@ -215,7 +216,7 @@ pygments==2.20.0 # via # diff-cover # pytest -pyjwt[crypto]==2.13.0 +pyjwt[crypto]==2.12.1 # via # drf-jwt # edx-drf-extensions @@ -227,13 +228,13 @@ pylint==3.3.8 # -r requirements/test.in pymemcache==4.0.0 # via -r requirements/base.in -pymongo==4.17.0 +pymongo==4.16.0 # via edx-opaque-keys pynacl==1.6.2 # via edx-django-utils -pyopenssl==26.3.0 +pyopenssl==26.2.0 # via snowflake-connector-python -pytest==9.1.1 +pytest==9.0.3 # via # pytest-cov # pytest-django @@ -247,7 +248,7 @@ python-dateutil==2.9.0.post0 # freezegun python-memcached==1.62 # via -r requirements/base.in -pytz==2026.3.post1 +pytz==2026.1.post1 # via # -r requirements/test.in # drf-yasg @@ -257,18 +258,18 @@ pyyaml==6.0.3 # drf-yasg # edx-django-release-util # responses -requests==2.34.2 +requests==2.33.1 # via # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client # responses # snowflake-connector-python -responses==0.26.2 +responses==0.26.0 # via -r requirements/test.in rules==3.5 # via edx-enterprise-data -s3transfer==0.19.2 +s3transfer==0.16.0 # via boto3 semantic-version==2.10.0 # via edx-drf-extensions @@ -279,7 +280,7 @@ six==1.17.0 # edx-rbac # html5lib # python-dateutil -snowballstemmer==3.1.1 +snowballstemmer==3.0.1 # via pydocstyle snowflake-connector-python==4.7.1 # via @@ -289,17 +290,17 @@ sortedcontainers==2.4.0 # via snowflake-connector-python sqlparse==0.5.5 # via django -stevedore==5.9.0 +stevedore==5.7.0 # via # edx-django-utils # edx-opaque-keys -tomlkit==0.15.1 +tomlkit==0.14.0 # via # pylint # snowflake-connector-python -tqdm==4.70.0 +tqdm==4.67.3 # via -r requirements/base.in -typing-extensions==4.16.0 +typing-extensions==4.15.0 # via # django-countries # edx-opaque-keys From e03226df66fd48bf76dc110d2758730d5674b4f2 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Date: Tue, 28 Jul 2026 07:18:13 +0000 Subject: [PATCH 7/8] test: cover insights snowflake validation helpers --- .../commands/check_insights_snowflake.py | 7 +- .../tests/test_check_insights_snowflake.py | 67 +++++++++ .../tests/test_snowflake_client.py | 127 ++++++++++++++++++ 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 analytics_data_api/management/commands/tests/test_check_insights_snowflake.py create mode 100644 analytics_data_api/tests/test_snowflake_client.py diff --git a/analytics_data_api/management/commands/check_insights_snowflake.py b/analytics_data_api/management/commands/check_insights_snowflake.py index ae1a2017..48935ab8 100644 --- a/analytics_data_api/management/commands/check_insights_snowflake.py +++ b/analytics_data_api/management/commands/check_insights_snowflake.py @@ -1,4 +1,9 @@ -"""Validate the Insights Snowflake connection from the deployed Analytics API service.""" +""" +Validate the Insights Snowflake connection from the deployed Analytics API service. + +Temporary Phase 1 smoke-test command. Remove it after Snowflake connectivity +validation is complete and real endpoint migration work owns the connection path. +""" from django.core.management.base import BaseCommand, CommandError diff --git a/analytics_data_api/management/commands/tests/test_check_insights_snowflake.py b/analytics_data_api/management/commands/tests/test_check_insights_snowflake.py new file mode 100644 index 00000000..b46480b1 --- /dev/null +++ b/analytics_data_api/management/commands/tests/test_check_insights_snowflake.py @@ -0,0 +1,67 @@ +"""Tests for the temporary Insights Snowflake smoke-test command.""" + +from io import StringIO +from unittest.mock import patch + +from django.core.management import CommandError, call_command +from django.test import SimpleTestCase + +from analytics_data_api.snowflake_client import SnowflakeConfigurationError + + +VALID_CONFIG = { + 'ACCOUNT': 'edx.us-east-1', + 'USER': 'INSIGHTS_API_SERVICE_USER', + 'ROLE': 'INSIGHTS_API_SERVICE_ROLE', + 'WAREHOUSE': 'INSIGHTS_API_SERVICE', + 'DATABASE': 'PROD', + 'SCHEMA': 'INSIGHTS', + 'PRIVATE_KEY': 'secret-private-key', +} + + +class CheckInsightsSnowflakeCommandTests(SimpleTestCase): + """ + Cover the Phase 1 smoke-test command. + + Remove these tests with the command after Snowflake connectivity validation + is complete and endpoint migration owns the connection path. + """ + + @patch('analytics_data_api.management.commands.check_insights_snowflake.get_table_row_count') + @patch('analytics_data_api.management.commands.check_insights_snowflake.get_insights_snowflake_config') + def test_command_prints_non_secret_connection_context(self, mock_get_config, mock_get_table_row_count): + mock_get_config.return_value = VALID_CONFIG + mock_get_table_row_count.return_value = 42 + stdout = StringIO() + + call_command('check_insights_snowflake', '--table', 'COURSE_ACTIVITY_WEEKLY', stdout=stdout) + + output = stdout.getvalue() + self.assertIn('Insights Snowflake connectivity check succeeded.', output) + self.assertIn('Account: edx.us-east-1', output) + self.assertIn('User: INSIGHTS_API_SERVICE_USER', output) + self.assertIn('Role: INSIGHTS_API_SERVICE_ROLE', output) + self.assertIn('Warehouse: INSIGHTS_API_SERVICE', output) + self.assertIn('Database: PROD', output) + self.assertIn('Schema: INSIGHTS', output) + self.assertIn('Table: COURSE_ACTIVITY_WEEKLY', output) + self.assertIn('Row count: 42', output) + self.assertNotIn('secret-private-key', output) + mock_get_table_row_count.assert_called_once_with('COURSE_ACTIVITY_WEEKLY') + + @patch('analytics_data_api.management.commands.check_insights_snowflake.get_insights_snowflake_config') + def test_command_raises_config_errors(self, mock_get_config): + mock_get_config.side_effect = SnowflakeConfigurationError('missing config') + + with self.assertRaisesRegex(CommandError, 'missing config'): + call_command('check_insights_snowflake') + + @patch('analytics_data_api.management.commands.check_insights_snowflake.get_table_row_count') + @patch('analytics_data_api.management.commands.check_insights_snowflake.get_insights_snowflake_config') + def test_command_wraps_unexpected_errors(self, mock_get_config, mock_get_table_row_count): + mock_get_config.return_value = VALID_CONFIG + mock_get_table_row_count.side_effect = RuntimeError('snowflake unavailable') + + with self.assertRaisesRegex(CommandError, 'Insights Snowflake connectivity check failed'): + call_command('check_insights_snowflake') diff --git a/analytics_data_api/tests/test_snowflake_client.py b/analytics_data_api/tests/test_snowflake_client.py new file mode 100644 index 00000000..ad353eee --- /dev/null +++ b/analytics_data_api/tests/test_snowflake_client.py @@ -0,0 +1,127 @@ +"""Tests for the temporary Insights Snowflake connectivity helper.""" + +from unittest.mock import Mock, patch + +from django.test import SimpleTestCase, override_settings + +from analytics_data_api.snowflake_client import ( + QUERY_TAG, + SnowflakeConfigurationError, + connect_to_insights_snowflake, + get_insights_snowflake_config, + get_private_key_der, + get_table_row_count, + run_read_only_query, + validate_snowflake_identifier, +) + + +VALID_CONFIG = { + 'ACCOUNT': 'edx.us-east-1', + 'USER': 'INSIGHTS_API_SERVICE_USER', + 'ROLE': 'INSIGHTS_API_SERVICE_ROLE', + 'WAREHOUSE': 'INSIGHTS_API_SERVICE', + 'DATABASE': 'PROD', + 'SCHEMA': 'INSIGHTS', + 'PRIVATE_KEY': '-----BEGIN PRIVATE KEY-----\\nkey\\n-----END PRIVATE KEY-----', + 'PRIVATE_KEY_PASSPHRASE': None, +} + + +class SnowflakeClientTests(SimpleTestCase): + """Cover the connection helper without making real Snowflake calls.""" + + @override_settings(INSIGHTS_SNOWFLAKE=VALID_CONFIG) + def test_get_insights_snowflake_config(self): + self.assertEqual(get_insights_snowflake_config(), VALID_CONFIG) + + @override_settings(INSIGHTS_SNOWFLAKE=None) + def test_get_insights_snowflake_config_requires_dictionary(self): + with self.assertRaisesRegex(SnowflakeConfigurationError, 'must be configured as a dictionary'): + get_insights_snowflake_config() + + @override_settings(INSIGHTS_SNOWFLAKE={}) + def test_get_insights_snowflake_config_requires_values(self): + with self.assertRaisesRegex(SnowflakeConfigurationError, 'missing required settings'): + get_insights_snowflake_config() + + def test_get_private_key_der_requires_string_key(self): + config = dict(VALID_CONFIG, PRIVATE_KEY=None) + + with self.assertRaisesRegex(SnowflakeConfigurationError, 'PRIVATE_KEY must be a string'): + get_private_key_der(config) + + @patch('analytics_data_api.snowflake_client.serialization.load_pem_private_key') + def test_get_private_key_der_converts_key_and_passphrase(self, mock_load_pem_private_key): + private_key = Mock() + private_key.private_bytes.return_value = b'der-key' + mock_load_pem_private_key.return_value = private_key + config = dict(VALID_CONFIG, PRIVATE_KEY='line1\\nline2', PRIVATE_KEY_PASSPHRASE='passphrase') + + self.assertEqual(get_private_key_der(config), b'der-key') + + mock_load_pem_private_key.assert_called_once_with(b'line1\nline2', password=b'passphrase') + + @override_settings(INSIGHTS_SNOWFLAKE=VALID_CONFIG) + @patch('analytics_data_api.snowflake_client.get_private_key_der') + @patch('analytics_data_api.snowflake_client.import_module') + def test_connect_to_insights_snowflake_uses_config(self, mock_import_module, mock_get_private_key_der): + connector = Mock() + mock_import_module.return_value = connector + mock_get_private_key_der.return_value = b'der-key' + + connect_to_insights_snowflake() + + connector.connect.assert_called_once_with( + account='edx.us-east-1', + user='INSIGHTS_API_SERVICE_USER', + role='INSIGHTS_API_SERVICE_ROLE', + warehouse='INSIGHTS_API_SERVICE', + database='PROD', + schema='INSIGHTS', + private_key=b'der-key', + session_parameters={'QUERY_TAG': QUERY_TAG}, + ) + + def test_validate_snowflake_identifier(self): + self.assertEqual(validate_snowflake_identifier('course_activity_weekly', 'table'), 'COURSE_ACTIVITY_WEEKLY') + + def test_validate_snowflake_identifier_rejects_unsafe_values(self): + with self.assertRaisesRegex(SnowflakeConfigurationError, 'Unsafe Snowflake table identifier'): + validate_snowflake_identifier('course_activity_weekly;drop table users', 'table') + + def test_run_read_only_query_rejects_non_select_sql(self): + with self.assertRaisesRegex(SnowflakeConfigurationError, 'Only SELECT queries are allowed'): + run_read_only_query('DELETE FROM PROD.INSIGHTS.COURSE_ACTIVITY_WEEKLY') + + @patch('analytics_data_api.snowflake_client.connect_to_insights_snowflake') + def test_run_read_only_query_closes_resources(self, mock_connect_to_insights_snowflake): + cursor = Mock() + cursor.fetchall.return_value = [(3,)] + connection = Mock() + connection.cursor.return_value = cursor + mock_connect_to_insights_snowflake.return_value = connection + + self.assertEqual(run_read_only_query('SELECT 1'), [(3,)]) + + cursor.execute.assert_called_once_with('SELECT 1') + cursor.close.assert_called_once_with() + connection.close.assert_called_once_with() + + @override_settings(INSIGHTS_SNOWFLAKE=VALID_CONFIG) + @patch('analytics_data_api.snowflake_client.run_read_only_query') + def test_get_table_row_count(self, mock_run_read_only_query): + mock_run_read_only_query.return_value = [(12,)] + + self.assertEqual(get_table_row_count('course_activity_weekly'), 12) + + mock_run_read_only_query.assert_called_once_with( + 'SELECT COUNT(*) AS row_count FROM PROD.INSIGHTS.COURSE_ACTIVITY_WEEKLY' + ) + + @override_settings(INSIGHTS_SNOWFLAKE=VALID_CONFIG) + @patch('analytics_data_api.snowflake_client.run_read_only_query') + def test_get_table_row_count_handles_empty_result(self, mock_run_read_only_query): + mock_run_read_only_query.return_value = [] + + self.assertEqual(get_table_row_count(), 0) From 146ea6f78529339424834465f8de0b04750bf25a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:13:38 +0000 Subject: [PATCH 8/8] fix: remove extra blank lines causing isort CI failure --- .../management/commands/tests/test_check_insights_snowflake.py | 1 - analytics_data_api/tests/test_snowflake_client.py | 1 - 2 files changed, 2 deletions(-) diff --git a/analytics_data_api/management/commands/tests/test_check_insights_snowflake.py b/analytics_data_api/management/commands/tests/test_check_insights_snowflake.py index b46480b1..26a88bdb 100644 --- a/analytics_data_api/management/commands/tests/test_check_insights_snowflake.py +++ b/analytics_data_api/management/commands/tests/test_check_insights_snowflake.py @@ -8,7 +8,6 @@ from analytics_data_api.snowflake_client import SnowflakeConfigurationError - VALID_CONFIG = { 'ACCOUNT': 'edx.us-east-1', 'USER': 'INSIGHTS_API_SERVICE_USER', diff --git a/analytics_data_api/tests/test_snowflake_client.py b/analytics_data_api/tests/test_snowflake_client.py index ad353eee..4c1c4bbb 100644 --- a/analytics_data_api/tests/test_snowflake_client.py +++ b/analytics_data_api/tests/test_snowflake_client.py @@ -15,7 +15,6 @@ validate_snowflake_identifier, ) - VALID_CONFIG = { 'ACCOUNT': 'edx.us-east-1', 'USER': 'INSIGHTS_API_SERVICE_USER',