diff --git a/core/collections/serializers.py b/core/collections/serializers.py index f6dc3ab4..35ced7eb 100644 --- a/core/collections/serializers.py +++ b/core/collections/serializers.py @@ -11,7 +11,8 @@ from core.client_configs.serializers import ClientConfigSerializer from core.collections.models import Collection, CollectionReference, Expansion from core.common.constants import HEAD, DEFAULT_ACCESS_TYPE, NAMESPACE_REGEX, ACCESS_TYPE_CHOICES, INCLUDE_SUMMARY, \ - INCLUDE_CLIENT_CONFIGS, INVALID_EXPANSION_URL, INCLUDE_STATES, INCLUDE_TASKS, INCLUDE_RESOLVED_REPO_VERSIONS + INCLUDE_CLIENT_CONFIGS, INVALID_EXPANSION_URL, INCLUDE_STATES, INCLUDE_TASKS, INCLUDE_RESOLVED_REPO_VERSIONS, \ + INCLUDE_EXTERNAL_EXPORTS from core.common.serializers import AbstractRepoResourcesSerializer, AbstractResourceSerializer from core.common.utils import get_truthy_values from core.orgs.models import Organization @@ -106,19 +107,42 @@ class CollectionVersionListSerializer(ModelSerializer): autoexpand = BooleanField(source='should_auto_expand') expansion_url = CharField(source='expansion_uri', read_only=True) checksums = SerializerMethodField() + external_exports = SerializerMethodField() class Meta: model = Collection fields = ( 'type', 'short_code', 'name', 'url', 'canonical_url', 'owner', 'owner_type', 'owner_url', 'version', 'created_at', 'id', 'collection_type', 'updated_at', 'released', 'retired', 'version_url', - 'previous_version_url', 'autoexpand', 'expansion_url', 'checksums' + 'previous_version_url', 'autoexpand', 'expansion_url', 'checksums', 'external_exports' ) + def __init__(self, *args, **kwargs): + params = get(kwargs, 'context.request.query_params') + + self.query_params = {} + if params: + self.query_params = params if isinstance(params, dict) else params.dict() + self.include_external_exports = self.query_params.get(INCLUDE_EXTERNAL_EXPORTS) in TRUTHY + + try: + if not self.include_external_exports: + self.fields.pop('external_exports', None) + except: # pylint: disable=bare-except + pass + + super().__init__(*args, **kwargs) + @staticmethod def get_checksums(obj): return obj.get_all_checksums() + @staticmethod + def get_external_exports(obj): + from core.repos.serializers import RepoExternalExportSerializer + queryset = obj.external_exports.filter() + return RepoExternalExportSerializer(queryset, many=True).data + class CollectionCreateOrUpdateSerializer(ModelSerializer): canonical_url = CharField(allow_blank=True, allow_null=True, required=False) @@ -442,6 +466,7 @@ class CollectionVersionDetailSerializer(CollectionCreateOrUpdateSerializer, Abst expansion_url = CharField(source='expansion_uri', allow_null=True, allow_blank=True) states = SerializerMethodField() tasks = SerializerMethodField() + external_exports = SerializerMethodField() class Meta: model = Collection @@ -454,7 +479,7 @@ class Meta: 'version', 'concepts_url', 'mappings_url', 'expansions_url', 'is_processing', 'released', 'retired', 'canonical_url', 'identifier', 'publisher', 'contact', 'jurisdiction', 'purpose', 'copyright', 'meta', 'immutable', 'revision_date', 'summary', 'text', 'experimental', 'locked_date', - 'autoexpand', 'expansion_url', 'checksums', 'states', 'tasks' + 'autoexpand', 'expansion_url', 'checksums', 'states', 'tasks', 'external_exports' ) + AbstractRepoResourcesSerializer.Meta.fields def __init__(self, *args, **kwargs): @@ -467,6 +492,7 @@ def __init__(self, *args, **kwargs): self.include_summary = self.query_params.get(INCLUDE_SUMMARY) in TRUTHY self.include_states = self.query_params.get(INCLUDE_STATES) in TRUTHY self.include_tasks = self.query_params.get(INCLUDE_TASKS) in TRUTHY + self.include_external_exports = self.query_params.get(INCLUDE_EXTERNAL_EXPORTS) in TRUTHY try: if not self.include_summary: @@ -475,6 +501,8 @@ def __init__(self, *args, **kwargs): self.fields.pop('states', None) if not self.include_tasks: self.fields.pop('tasks', None) + if not self.include_external_exports: + self.fields.pop('external_exports', None) except: # pylint: disable=bare-except pass @@ -508,6 +536,12 @@ def get_tasks(self, obj): def get_autoexpand(obj): return obj.should_auto_expand + @staticmethod + def get_external_exports(obj): + from core.repos.serializers import RepoExternalExportSerializer + queryset = obj.external_exports.filter() + return RepoExternalExportSerializer(queryset, many=True).data + class CollectionReferenceSerializer(ModelSerializer): reference_type = CharField(read_only=True) diff --git a/core/collections/urls.py b/core/collections/urls.py index 0c152913..edce68f2 100644 --- a/core/collections/urls.py +++ b/core/collections/urls.py @@ -46,6 +46,11 @@ views.CollectionVersionExportView.as_view(), name='collectionversion-latest-export-detail' ), + path( + '/latest/export//', + views.CollectionVersionExternalExportView.as_view(), + name='collectionversion-latest-external-export-detail' + ), path( "/concepts//mappings/", views.CollectionVersionConceptMappingsView.as_view(), @@ -215,6 +220,10 @@ '//export/', views.CollectionVersionExportView.as_view(), name='collectionversion-export' ), + path( + '//export//', + views.CollectionVersionExternalExportView.as_view(), name='collectionversion-external-export' + ), path( "//extras/", views.CollectionVersionExtrasView.as_view(), diff --git a/core/collections/views.py b/core/collections/views.py index 170257e1..959292ec 100644 --- a/core/collections/views.py +++ b/core/collections/views.py @@ -45,6 +45,7 @@ ConceptDictionaryCreateMixin, ListWithHeadersMixin, ConceptDictionaryUpdateMixin, ConceptContainerExportMixin, ConceptContainerProcessingMixin) +from core.repos.mixins import RepoExternalExportMixin from core.common.permissions import ( CanViewConceptDictionary, CanEditConceptDictionary, HasAccessToVersionedObject, CanViewConceptDictionaryVersion @@ -1184,6 +1185,10 @@ def handle_export_version(self): return status.HTTP_409_CONFLICT +class CollectionVersionExternalExportView(RepoExternalExportMixin, CollectionVersionBaseView): + permission_classes = (CanViewConceptDictionary, IsAuthenticated) + + class CollectionSummaryView(SummaryMixin, CollectionBaseView, RetrieveAPIView): serializer_class = CollectionSummaryDetailSerializer permission_classes = (CanViewConceptDictionary,) diff --git a/core/common/constants.py b/core/common/constants.py index 70bfc814..906c5b0d 100644 --- a/core/common/constants.py +++ b/core/common/constants.py @@ -60,6 +60,7 @@ INCLUDE_CREATOR_PINS = 'includeCreatorPins' INCLUDE_HIERARCHY_ROOT = 'includeHierarchyRoot' INCLUDE_SUMMARY = 'includeSummary' +INCLUDE_EXTERNAL_EXPORTS = 'includeExternalExports' INCLUDE_LOGS = 'includeLogs' INCLUDE_STATES = 'includeStates' INCLUDE_TASKS = 'includeTasks' diff --git a/core/common/models.py b/core/common/models.py index 889d9fb4..0117ff04 100644 --- a/core/common/models.py +++ b/core/common/models.py @@ -11,6 +11,7 @@ from django.db.models.expressions import CombinedExpression, F from django.utils import timezone from django.utils.functional import cached_property +from django.utils.text import get_valid_filename from django_elasticsearch_dsl.registries import registry from django_elasticsearch_dsl.signals import RealTimeSignalProcessor from elasticsearch import TransportError @@ -481,6 +482,9 @@ class ConceptContainerModel(VersionedModel, ChecksumModel): 'url_registry.URLRegistry', object_id_field='repo_id', content_type_field='repo_type' ) followers = GenericRelation('users.Follow', object_id_field='following_id', content_type_field='following_type') + external_exports = GenericRelation( + 'repos.RepoExternalExport', object_id_field='resource_id', content_type_field='resource_type' + ) class Meta: abstract = True @@ -1002,6 +1006,11 @@ def get_version_export_path(self, suffix='*'): return path + def get_external_export_path(self, key, filename): + base_path = self.get_version_export_path(suffix=None).rstrip('.') + safe_filename = get_valid_filename(filename) + return f"{base_path}/external/{key}_{safe_filename}" + def get_export_path(self): if self.is_head: return self.version_export_path @@ -1287,6 +1296,10 @@ def get_tasks(self): 'exported': export_task, } + def upload_external_export(self, key, file, user, description=None): + from core.repos.models import RepoExternalExport + return RepoExternalExport.upsert(self, key, file, user, description) + class CelerySignalProcessor(RealTimeSignalProcessor): def handle_save(self, sender, instance, **kwargs): diff --git a/core/integration_tests/tests_collections.py b/core/integration_tests/tests_collections.py index bee2282d..5427c2be 100644 --- a/core/integration_tests/tests_collections.py +++ b/core/integration_tests/tests_collections.py @@ -2,6 +2,7 @@ import zipfile from celery_once import AlreadyQueued +from django.core.files.uploadedfile import SimpleUploadedFile from mock import patch, Mock, ANY from mock.mock import PropertyMock from rest_framework.exceptions import ErrorDetail @@ -2831,6 +2832,131 @@ def test_delete_204_head(self, s3_remove_mock, has_export_mock, export_path_mock s3_remove_mock.assert_called_once_with('head/export/path') +class CollectionVersionExternalExportViewTest(OCLAPITestCase): + def setUp(self): + super().setUp() + self.admin = UserProfile.objects.get(username='ocladmin') + self.admin_token = self.admin.get_token() + self.user = UserProfileFactory(username='username') + self.token = self.user.get_token() + self.collection_v1 = UserCollectionFactory(version='v1', mnemonic='coll', user=self.user) + + def test_get_404_unknown_key(self): + response = self.client.get( + self.collection_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + self.token, + format='json' + ) + + self.assertEqual(response.status_code, 404) + + @patch('core.services.storages.cloud.aws.S3.remove') + @patch('core.services.storages.cloud.aws.S3.upload') + def test_post_201_create_then_get_302_and_delete_204(self, s3_upload_mock, s3_remove_mock): + uploaded_file = SimpleUploadedFile('openmrs23.sql.zip', b'content', content_type='application/zip') + + response = self.client.post( + self.collection_v1.uri + 'export/openmrs23-sql/', + {'file': uploaded_file}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + + self.assertEqual(response.status_code, 201) + self.assertEqual(response.data['key'], 'openmrs23-sql') + self.assertEqual(response.data['url'], self.collection_v1.uri + 'export/openmrs23-sql/') + s3_upload_mock.assert_called_once() + + from core.repos.models import RepoExternalExport + instance = RepoExternalExport.objects.get(key='openmrs23-sql') + + with patch('core.services.storages.cloud.aws.S3.url_for') as s3_url_for_mock: + s3_url_for_mock.return_value = 'https://signed.example/openmrs23.sql.zip' + response = self.client.get( + self.collection_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + self.token, + format='json' + ) + + self.assertEqual(response.status_code, 302) + self.assertEqual(response['Location'], 'https://signed.example/openmrs23.sql.zip') + + response = self.client.delete( + self.collection_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + self.token, + format='json' + ) + + self.assertEqual(response.status_code, 204) + s3_remove_mock.assert_called_once_with(instance.file_path) + self.assertFalse(RepoExternalExport.objects.filter(key='openmrs23-sql').exists()) + + def test_post_400_no_file(self): + response = self.client.post( + self.collection_v1.uri + 'export/openmrs23-sql/', + {}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + + self.assertEqual(response.status_code, 400) + + def test_post_403_non_admin(self): + random_user = UserProfileFactory() + uploaded_file = SimpleUploadedFile('openmrs23.sql.zip', b'content', content_type='application/zip') + + response = self.client.post( + self.collection_v1.uri + 'export/openmrs23-sql/', + {'file': uploaded_file}, + HTTP_AUTHORIZATION='Token ' + random_user.get_token(), + ) + + self.assertEqual(response.status_code, 403) + + @patch('core.services.storages.cloud.aws.S3.upload') + def test_export_serializer_includes_external_exports(self, s3_upload_mock): # pylint: disable=unused-argument + uploaded_file = SimpleUploadedFile('openmrs23.sql.zip', b'content', content_type='application/zip') + self.client.post( + self.collection_v1.uri + 'export/openmrs23-sql/', + {'file': uploaded_file}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + + self.collection_v1.refresh_from_db() + external_exports = CollectionVersionExportSerializer(self.collection_v1).data['external_exports'] + + self.assertEqual(len(external_exports), 1) + self.assertEqual(external_exports[0]['key'], 'openmrs23-sql') + self.assertEqual(external_exports[0]['url'], self.collection_v1.uri + 'export/openmrs23-sql/') + + def test_delete_404_unknown_key(self): + response = self.client.delete( + self.collection_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + self.token, + format='json' + ) + + self.assertEqual(response.status_code, 404) + + @patch('core.services.storages.cloud.aws.S3.remove') + @patch('core.services.storages.cloud.aws.S3.upload') + def test_delete_403_non_admin(self, s3_upload_mock, s3_remove_mock): # pylint: disable=unused-argument + uploaded_file = SimpleUploadedFile('openmrs23.sql.zip', b'content', content_type='application/zip') + self.client.post( + self.collection_v1.uri + 'export/openmrs23-sql/', + {'file': uploaded_file}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + + random_user = UserProfileFactory() + response = self.client.delete( + self.collection_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + random_user.get_token(), + format='json' + ) + + self.assertEqual(response.status_code, 403) + s3_remove_mock.assert_not_called() + + class CollectionVersionListViewTest(OCLAPITestCase): def setUp(self): super().setUp() diff --git a/core/integration_tests/tests_sources.py b/core/integration_tests/tests_sources.py index df58d6d9..375d5c71 100644 --- a/core/integration_tests/tests_sources.py +++ b/core/integration_tests/tests_sources.py @@ -5,6 +5,7 @@ from celery_once import AlreadyQueued from django.conf import settings +from django.core.files.uploadedfile import SimpleUploadedFile from django.db import transaction from mock import patch, Mock, ANY, PropertyMock from mock.mock import call @@ -1283,6 +1284,157 @@ def test_delete_401_anonymous(self, s3_remove_mock, has_export_mock, export_path s3_remove_mock.assert_not_called() +class SourceVersionExternalExportViewTest(OCLAPITestCase): + def setUp(self): + super().setUp() + self.admin = UserProfile.objects.get(username='ocladmin') + self.admin_token = self.admin.get_token() + self.user = UserProfileFactory(username='username') + self.token = self.user.get_token() + self.source_v1 = UserSourceFactory(version='v1', mnemonic='source1', user=self.user) + + def test_get_404_unknown_key(self): + response = self.client.get( + self.source_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + self.token, + format='json' + ) + + self.assertEqual(response.status_code, 404) + + @patch('core.services.storages.cloud.aws.S3.remove') + @patch('core.services.storages.cloud.aws.S3.upload') + def test_post_201_create_then_get_302_and_delete_204(self, s3_upload_mock, s3_remove_mock): + uploaded_file = SimpleUploadedFile('openmrs23.sql.zip', b'content', content_type='application/zip') + + response = self.client.post( + self.source_v1.uri + 'export/openmrs23-sql/', + {'file': uploaded_file}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + + self.assertEqual(response.status_code, 201) + self.assertEqual(response.data['key'], 'openmrs23-sql') + self.assertEqual(response.data['url'], self.source_v1.uri + 'export/openmrs23-sql/') + s3_upload_mock.assert_called_once() + + from core.repos.models import RepoExternalExport + instance = RepoExternalExport.objects.get(key='openmrs23-sql') + + with patch('core.services.storages.cloud.aws.S3.url_for') as s3_url_for_mock: + s3_url_for_mock.return_value = 'https://signed.example/openmrs23.sql.zip' + response = self.client.get( + self.source_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + self.token, + format='json' + ) + + self.assertEqual(response.status_code, 302) + self.assertEqual(response['Location'], 'https://signed.example/openmrs23.sql.zip') + + response = self.client.delete( + self.source_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + self.token, + format='json' + ) + + self.assertEqual(response.status_code, 204) + s3_remove_mock.assert_called_once_with(instance.file_path) + self.assertFalse(RepoExternalExport.objects.filter(key='openmrs23-sql').exists()) + + @patch('core.services.storages.cloud.aws.S3.remove') + @patch('core.services.storages.cloud.aws.S3.upload') + def test_post_200_update_existing(self, s3_upload_mock, s3_remove_mock): + from core.repos.models import RepoExternalExport + + first_file = SimpleUploadedFile('openmrs23.sql.zip', b'v1', content_type='application/zip') + response = self.client.post( + self.source_v1.uri + 'export/openmrs23-sql/', + {'file': first_file}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + self.assertEqual(response.status_code, 201) + first_file_path = RepoExternalExport.objects.get(key='openmrs23-sql').file_path + + second_file = SimpleUploadedFile('openmrs23-v2.sql.zip', b'v2-content', content_type='application/zip') + response = self.client.post( + self.source_v1.uri + 'export/openmrs23-sql/', + {'file': second_file}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(RepoExternalExport.objects.filter(key='openmrs23-sql').count(), 1) + s3_remove_mock.assert_called_once_with(first_file_path) + s3_upload_mock.assert_called() + + def test_post_400_no_file(self): + response = self.client.post( + self.source_v1.uri + 'export/openmrs23-sql/', + {}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + + self.assertEqual(response.status_code, 400) + + def test_post_403_non_admin(self): + random_user = UserProfileFactory() + uploaded_file = SimpleUploadedFile('openmrs23.sql.zip', b'content', content_type='application/zip') + + response = self.client.post( + self.source_v1.uri + 'export/openmrs23-sql/', + {'file': uploaded_file}, + HTTP_AUTHORIZATION='Token ' + random_user.get_token(), + ) + + self.assertEqual(response.status_code, 403) + + @patch('core.services.storages.cloud.aws.S3.upload') + def test_export_serializer_includes_external_exports(self, s3_upload_mock): # pylint: disable=unused-argument + uploaded_file = SimpleUploadedFile('openmrs23.sql.zip', b'content', content_type='application/zip') + self.client.post( + self.source_v1.uri + 'export/openmrs23-sql/', + {'file': uploaded_file}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + + self.source_v1.refresh_from_db() + external_exports = SourceVersionExportSerializer(self.source_v1).data['external_exports'] + + self.assertEqual(len(external_exports), 1) + self.assertEqual(external_exports[0]['key'], 'openmrs23-sql') + self.assertEqual(external_exports[0]['url'], self.source_v1.uri + 'export/openmrs23-sql/') + + def test_delete_404_unknown_key(self): + response = self.client.delete( + self.source_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + self.token, + format='json' + ) + + self.assertEqual(response.status_code, 404) + + @patch('core.services.storages.cloud.aws.S3.remove') + @patch('core.services.storages.cloud.aws.S3.upload') + def test_delete_403_non_admin(self, s3_upload_mock, s3_remove_mock): # pylint: disable=unused-argument + uploaded_file = SimpleUploadedFile('openmrs23.sql.zip', b'content', content_type='application/zip') + self.client.post( + self.source_v1.uri + 'export/openmrs23-sql/', + {'file': uploaded_file}, + HTTP_AUTHORIZATION='Token ' + self.token, + ) + + random_user = UserProfileFactory() + response = self.client.delete( + self.source_v1.uri + 'export/openmrs23-sql/', + HTTP_AUTHORIZATION='Token ' + random_user.get_token(), + format='json' + ) + + self.assertEqual(response.status_code, 403) + s3_remove_mock.assert_not_called() + + class ExportSourceTaskTest(OCLAPITestCase): @patch('core.common.utils.get_export_service') def test_export_source(self, export_service_mock): # pylint: disable=too-many-locals diff --git a/core/repos/migrations/0001_initial.py b/core/repos/migrations/0001_initial.py new file mode 100644 index 00000000..06b110f1 --- /dev/null +++ b/core/repos/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 5.1.15 on 2026-06-16 10:29 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='RepoExternalExport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.CharField(max_length=255)), + ('description', models.TextField(blank=True, null=True)), + ('file_path', models.CharField(max_length=512)), + ('resource_id', models.PositiveIntegerField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('created_by', models.ForeignKey(default=1, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='%(app_label)s_%(class)s_related_created_by', related_query_name='%(app_label)s_%(class)ss_created_by', to=settings.AUTH_USER_MODEL)), + ('resource_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), + ('updated_by', models.ForeignKey(default=1, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='%(app_label)s_%(class)s_related_updated_by', related_query_name='%(app_label)s_%(class)ss_updated_by', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'db_table': 'repo_external_exports', + 'unique_together': {('resource_type', 'resource_id', 'key')}, + }, + ), + ] diff --git a/core/repos/migrations/__init__.py b/core/repos/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/repos/mixins.py b/core/repos/mixins.py new file mode 100644 index 00000000..442db219 --- /dev/null +++ b/core/repos/mixins.py @@ -0,0 +1,79 @@ +from django.http import Http404, HttpResponseForbidden +from django.shortcuts import redirect +from rest_framework import status +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response + +from core.common.permissions import CanViewConceptDictionaryVersion +from core.common.utils import get_export_service +from core.repos.serializers import RepoExternalExportSerializer + + +class RepoExternalExportMixin: + permission_classes = (CanViewConceptDictionaryVersion, IsAuthenticated) + + def get_object(self): + queryset = self.get_queryset() + if 'version' not in self.kwargs: + queryset = queryset.filter(released=True) + + instance = queryset.first() + + if not instance: + raise Http404() + + self.check_object_permissions(self.request, instance) + + return instance + + def get_external_export(self, version, required=True): + instance = version.external_exports.filter(key=self.kwargs.get('external_export_key')).first() + + if required and not instance: + raise Http404() + + return instance + + @staticmethod + def is_permitted(user, version): + return user.is_staff or user.is_superuser or user.is_admin_for(version) + + def get(self, request, *args, **kwargs): # pylint: disable=unused-argument + version = self.get_object() + instance = self.get_external_export(version) + + export_url = instance.file_url + if not export_url: + return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + return redirect(export_url) + + def post(self, request, *args, **kwargs): # pylint: disable=unused-argument + version = self.get_object() + user = request.user + + if not self.is_permitted(user, version): + return HttpResponseForbidden() + + uploaded_file = request.data.get('file') + if not uploaded_file: + return Response({'file': ['This field is required.']}, status=status.HTTP_400_BAD_REQUEST) + + instance, is_create = version.upload_external_export( + self.kwargs.get('external_export_key'), uploaded_file, user, request.data.get('description')) + + serializer = RepoExternalExportSerializer(instance) + return Response(serializer.data, status=status.HTTP_201_CREATED if is_create else status.HTTP_200_OK) + + def delete(self, request, *args, **kwargs): # pylint: disable=unused-argument + version = self.get_object() + user = request.user + + if not self.is_permitted(user, version): + return HttpResponseForbidden() + + instance = self.get_external_export(version) + get_export_service().remove(instance.file_path) + instance.delete() + + return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/core/repos/models.py b/core/repos/models.py index b1fcbb90..9379bbae 100644 --- a/core/repos/models.py +++ b/core/repos/models.py @@ -1,4 +1,10 @@ +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.db import models + from core.collections.models import Collection +from core.common.constants import SUPER_ADMIN_USER_ID +from core.common.utils import get_export_service from core.sources.models import Source @@ -11,3 +17,64 @@ def get(cls, criteria): repo = Collection.objects.filter(criteria).first() return repo + + +class RepoExternalExport(models.Model): + class Meta: + db_table = 'repo_external_exports' + unique_together = ('resource_type', 'resource_id', 'key') + + key = models.CharField(max_length=255) + description = models.TextField(null=True, blank=True) + file_path = models.CharField(max_length=512) + + resource_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + resource_id = models.PositiveIntegerField() + resource = GenericForeignKey('resource_type', 'resource_id') + + created_by = models.ForeignKey( + 'users.UserProfile', default=SUPER_ADMIN_USER_ID, on_delete=models.SET_DEFAULT, + related_name='%(app_label)s_%(class)s_related_created_by', + related_query_name='%(app_label)s_%(class)ss_created_by', + ) + updated_by = models.ForeignKey( + 'users.UserProfile', default=SUPER_ADMIN_USER_ID, on_delete=models.SET_DEFAULT, + related_name='%(app_label)s_%(class)s_related_updated_by', + related_query_name='%(app_label)s_%(class)ss_updated_by', + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + @property + def uri(self): + return f"{self.resource.uri}export/{self.key}/" + + @property + def file_url(self): + return get_export_service().url_for(self.file_path) + + @classmethod + def upsert(cls, repo_version, key, file, user, description=None): # pylint: disable=too-many-arguments + instance = repo_version.external_exports.filter(key=key).first() + is_create = instance is None + + file_path = repo_version.get_external_export_path(key, file.name) + if instance and instance.file_path != file_path: + get_export_service().remove(instance.file_path) + + get_export_service().upload( + key=file_path, file_content=file, + headers={'content-type': file.content_type}, + metadata={'ContentType': file.content_type} + ) + + if is_create: + instance = cls(resource=repo_version, key=key, created_by=user) + + if description: + instance.description = description + + instance.file_path = file_path + instance.updated_by = user + instance.save() + return instance, is_create diff --git a/core/repos/serializers.py b/core/repos/serializers.py index e9e37820..d131e3c2 100644 --- a/core/repos/serializers.py +++ b/core/repos/serializers.py @@ -2,10 +2,19 @@ from core.collections.models import Collection from core.collections.serializers import CollectionListSerializer +from core.repos.models import RepoExternalExport from core.sources.models import Source from core.sources.serializers import SourceListSerializer +class RepoExternalExportSerializer(serializers.ModelSerializer): + url = serializers.CharField(source='uri', read_only=True) + + class Meta: + model = RepoExternalExport + fields = ('key', 'description', 'url', 'created_at', 'updated_at', 'file_path') + + class RepoListSerializer(serializers.Serializer): # pylint: disable=abstract-method def to_representation(self, instance): data = self.get_data(instance) diff --git a/core/repos/tests.py b/core/repos/tests.py index 31a58524..b9387a1c 100644 --- a/core/repos/tests.py +++ b/core/repos/tests.py @@ -1,8 +1,9 @@ from core.collections.documents import CollectionDocument from core.collections.models import Collection from core.collections.tests.factories import OrganizationCollectionFactory, UserCollectionFactory -from core.common.tests import OCLAPITestCase +from core.common.tests import OCLAPITestCase, OCLTestCase from core.orgs.tests.factories import OrganizationFactory +from core.repos.models import RepoExternalExport from core.sources.documents import SourceDocument from core.sources.models import Source from core.sources.tests.factories import OrganizationSourceFactory, UserSourceFactory @@ -68,6 +69,44 @@ def test_get_200(self): self.assertEqual(len(response.data), 1) +class RepoExternalExportTest(OCLTestCase): + def test_uri(self): + source_v1 = UserSourceFactory(version='v1', mnemonic='source1') + instance = RepoExternalExport(resource=source_v1, key='openmrs23-sql', file_path='foo/bar.zip',) + + self.assertEqual(instance.uri, source_v1.uri + 'export/openmrs23-sql/') + + def test_get_external_export_path(self): + source_v1 = UserSourceFactory(version='v1', mnemonic='source1') + owner_mnemonic = source_v1.parent.mnemonic + + path = source_v1.get_external_export_path('openmrs23-sql', 'openmrs23.sql.zip') + + self.assertEqual( + path, f"users/{owner_mnemonic}/{owner_mnemonic}_source1_v1/external/openmrs23-sql_openmrs23.sql.zip" + ) + + def test_get_external_export_path_sanitizes_filename(self): + source_v1 = UserSourceFactory(version='v1', mnemonic='source1') + owner_mnemonic = source_v1.parent.mnemonic + + path = source_v1.get_external_export_path('openmrs23-sql', '../etc/passwd') + + self.assertEqual( + path, f"users/{owner_mnemonic}/{owner_mnemonic}_source1_v1/external/openmrs23-sql_..etcpasswd" + ) + + def test_get_external_export_path_normalizes_invalid_filename_characters(self): + source_v1 = UserSourceFactory(version='v1', mnemonic='source1') + owner_mnemonic = source_v1.parent.mnemonic + + path = source_v1.get_external_export_path('openmrs23-sql', 'foo bar:baz*.zip') + + self.assertEqual( + path, f"users/{owner_mnemonic}/{owner_mnemonic}_source1_v1/external/openmrs23-sql_foo_barbaz.zip" + ) + + class UserOrganizationRepoListViewTest(OCLAPITestCase): def test_get(self): CollectionDocument._index.delete() # pylint: disable=protected-access diff --git a/core/sources/serializers.py b/core/sources/serializers.py index 454c0921..e27dc487 100644 --- a/core/sources/serializers.py +++ b/core/sources/serializers.py @@ -9,7 +9,8 @@ from core.client_configs.serializers import ClientConfigSerializer from core.common.constants import DEFAULT_ACCESS_TYPE, NAMESPACE_REGEX, ACCESS_TYPE_CHOICES, HEAD, \ - INCLUDE_SUMMARY, INCLUDE_CLIENT_CONFIGS, INCLUDE_HIERARCHY_ROOT, INCLUDE_STATES, INCLUDE_TASKS + INCLUDE_SUMMARY, INCLUDE_CLIENT_CONFIGS, INCLUDE_HIERARCHY_ROOT, INCLUDE_STATES, INCLUDE_TASKS, \ + INCLUDE_EXTERNAL_EXPORTS from core.common.serializers import AbstractRepoResourcesSerializer, AbstractResourceSerializer from core.common.utils import get_truthy_values from core.orgs.models import Organization @@ -17,7 +18,6 @@ from core.sources.models import Source from core.users.models import UserProfile - TRUTHY = get_truthy_values() @@ -97,19 +97,42 @@ class SourceVersionListSerializer(ModelSerializer): url = CharField(source='versioned_object_url') previous_version_url = CharField(source='prev_version_uri') checksums = SerializerMethodField() + external_exports = SerializerMethodField() class Meta: model = Source fields = ( 'type', 'short_code', 'name', 'url', 'canonical_url', 'owner', 'owner_type', 'owner_url', 'version', 'created_at', 'id', 'source_type', 'updated_at', 'released', 'retired', 'version_url', - 'previous_version_url', 'checksums', 'match_algorithms' + 'previous_version_url', 'checksums', 'match_algorithms', 'external_exports' ) + def __init__(self, *args, **kwargs): + params = get(kwargs, 'context.request.query_params') + + self.query_params = {} + if params: + self.query_params = params if isinstance(params, dict) else params.dict() + self.include_external_exports = self.query_params.get(INCLUDE_EXTERNAL_EXPORTS) in TRUTHY + + try: + if not self.include_external_exports: + self.fields.pop('external_exports', None) + except: # pylint: disable=bare-except + pass + + super().__init__(*args, **kwargs) + @staticmethod def get_checksums(obj): return obj.get_all_checksums() + @staticmethod + def get_external_exports(obj): + from core.repos.serializers import RepoExternalExportSerializer + queryset = obj.external_exports.filter() + return RepoExternalExportSerializer(queryset, many=True).data + class SourceCreateOrUpdateSerializer(ModelSerializer): canonical_url = CharField(allow_blank=True, allow_null=True, required=False) @@ -458,6 +481,7 @@ class SourceVersionDetailSerializer(SourceCreateOrUpdateSerializer, AbstractRepo tasks = SerializerMethodField() hierarchy_root_url = CharField(source='hierarchy_root.url', required=False, allow_blank=True, allow_null=True) filters = ListField(required=False, allow_null=True) + external_exports = SerializerMethodField() class Meta: model = Source @@ -472,7 +496,7 @@ class Meta: 'content_type', 'revision_date', 'summary', 'text', 'meta', 'experimental', 'case_sensitive', 'collection_reference', 'hierarchy_meaning', 'compositional', 'version_needed', 'hierarchy_root_url', 'checksums', 'states', 'tasks', 'properties', 'filters', - 'match_algorithms' + 'match_algorithms', 'external_exports' ) + AbstractRepoResourcesSerializer.Meta.fields def __init__(self, *args, **kwargs): @@ -485,6 +509,7 @@ def __init__(self, *args, **kwargs): self.include_summary = self.query_params.get(INCLUDE_SUMMARY) in TRUTHY self.include_states = self.query_params.get(INCLUDE_STATES) in TRUTHY self.include_tasks = self.query_params.get(INCLUDE_TASKS) in TRUTHY + self.include_external_exports = self.query_params.get(INCLUDE_EXTERNAL_EXPORTS) in TRUTHY try: if not self.include_summary: @@ -493,6 +518,8 @@ def __init__(self, *args, **kwargs): self.fields.pop('states', None) if not self.include_tasks: self.fields.pop('tasks', None) + if not self.include_external_exports: + self.fields.pop('external_exports', None) except: # pylint: disable=bare-except pass @@ -522,6 +549,12 @@ def get_tasks(self, obj): return tasks + @staticmethod + def get_external_exports(obj): + from core.repos.serializers import RepoExternalExportSerializer + queryset = obj.external_exports.filter() + return RepoExternalExportSerializer(queryset, many=True).data + def to_representation(self, instance): # used to be to_native ret = super().to_representation(instance) ret.update({"supported_locales": instance.get_supported_locales()}) diff --git a/core/sources/urls.py b/core/sources/urls.py index 7e4e97d7..a1eefb10 100644 --- a/core/sources/urls.py +++ b/core/sources/urls.py @@ -68,6 +68,11 @@ views.SourceVersionExportView.as_view(), name='sourceversion-latest-export-detail' ), + path( + '/latest/export//', + views.SourceVersionExternalExportView.as_view(), + name='sourceversion-latest-external-export-detail' + ), path("/concepts/$clone/", views.SourceConceptsCloneView.as_view()), path("/concepts/indexes/", views.SourceConceptsIndexView.as_view()), path("/mappings/indexes/", views.SourceMappingsIndexView.as_view()), @@ -109,6 +114,10 @@ '//export/', views.SourceVersionExportView.as_view(), name='sourceversion-export' ), + path( + '//export//', + views.SourceVersionExternalExportView.as_view(), name='sourceversion-external-export' + ), path( "//extras/", views.SourceVersionExtrasView.as_view(), diff --git a/core/sources/views.py b/core/sources/views.py index a87223bf..b1365ca6 100644 --- a/core/sources/views.py +++ b/core/sources/views.py @@ -20,6 +20,7 @@ from core.common.exceptions import Http405, Http400 from core.common.mixins import ListWithHeadersMixin, ConceptDictionaryCreateMixin, ConceptDictionaryUpdateMixin, \ ConceptContainerExportMixin, ConceptContainerProcessingMixin +from core.repos.mixins import RepoExternalExportMixin from core.common.permissions import CanViewConceptDictionary, CanEditConceptDictionary, HasAccessToVersionedObject, \ CanViewConceptDictionaryVersion from core.common.serializers import TaskSerializer @@ -559,6 +560,10 @@ def handle_export_version(self): return status.HTTP_409_CONFLICT +class SourceVersionExternalExportView(RepoExternalExportMixin, SourceVersionBaseView): + permission_classes = (CanViewConceptDictionary, IsAuthenticated) + + class SourceHierarchyView(SourceBaseView, RetrieveAPIView): serializer_class = SourceSummaryDetailSerializer permission_classes = (CanViewConceptDictionary,)