diff --git a/addons/weko/client.py b/addons/weko/client.py index f7b00fcf459..634f62f9cda 100644 --- a/addons/weko/client.py +++ b/addons/weko/client.py @@ -15,6 +15,19 @@ def _flatten_indices(indices): return r +def _get_truncated_index_ids(descs): + # WEKO replaces the children beyond an index's display number with a "more" entry + # unless the index is listed in more_ids + r = [] + for desc in descs: + if 'children' not in desc: + continue + if any(child['id'] == 'more' for child in desc['children']): + r.append(str(desc['id'])) + r += _get_truncated_index_ids(desc['children']) + return r + + def _is_valid_index(desc): if 'name' in desc and 'id' in desc: return True @@ -58,11 +71,24 @@ def get_login_user(self, default_user=None): results = resp.json().get('results', {}) return results.get('subitem_mail_address', default_user) - def get_indices(self): + def get_indices(self, accept_language=None): """ Get all indices from the WEKO. """ - root = self._get('api/tree?action=browsing') + headers = None if accept_language is None else {'Accept-Language': accept_language} + more_ids = [] + while True: + path = 'api/tree?action=browsing' + if more_ids: + path += '&more_ids=' + '/'.join(more_ids) + root = self._get(path, headers=headers) + truncated_ids = _get_truncated_index_ids(root) + if not truncated_ids: + break + unexpanded_ids = set(truncated_ids) & set(more_ids) + if unexpanded_ids: + raise ValueError(f'WEKO did not expand indices: {unexpanded_ids}') + more_ids += truncated_ids indices = [] for desc in root: if not _is_valid_index(desc): @@ -97,8 +123,11 @@ def _base_host(self): return self.host return self.host[:-6] - def _get(self, path): - resp = requests.get(self._base_host + path, **self._requests_args()) + def _get(self, path, headers=None): + resp = requests.get( + self._base_host + path, + **self._requests_args(headers=headers) + ) resp.raise_for_status() return resp.json() diff --git a/addons/weko/serializer.py b/addons/weko/serializer.py index a191f3f94b1..add5743edb4 100644 --- a/addons/weko/serializer.py +++ b/addons/weko/serializer.py @@ -1,6 +1,11 @@ +import logging + +from flask_babel import get_locale + from addons.base.serializer import OAuthAddonSerializer from . import settings as weko_settings from .apps import SHORT_NAME +from .client import _flatten_indices from website.util import api_url_for, web_url_for from admin.rdm_addons.utils import get_rdm_addon_option @@ -8,6 +13,9 @@ from requests import exceptions as requests_exceptions +logger = logging.getLogger(__name__) + + def get_repository_options(user): repos = list(weko_settings.REPOSITORY_IDS) for institution_id in user.affiliated_institutions.all(): @@ -90,10 +98,14 @@ def serialized_node_settings(self): if self.node_settings.has_auth: c = self.node_settings.create_client() indices = c.get_indices() + # WEKO names indices per language; `title` stays its default name because that is what gets saved. + # The browser's Accept-Language is not forwarded: WEKO matches tags exactly and answers "ja-JP" in English + localized_indices = c.get_indices(accept_language=get_locale().language) + localized_titles = {index.identifier: index.title for index in _flatten_indices(localized_indices)} result.update({ 'validCredentials': True, - 'indices': [self._serialize_index(index) for index in indices], + 'indices': [self._serialize_index(index, localized_titles) for index in indices], 'savedIndex': { 'title': self.node_settings.index_title, 'id': self.node_settings.index_id, @@ -136,9 +148,17 @@ def serialize_settings(self, node_settings, current_user, client=None): result.update(self.serialized_node_settings) return result - def _serialize_index(self, index): + def _serialize_index(self, index, localized_titles): + if index.identifier in localized_titles: + localized_title = localized_titles[index.identifier] + else: + # WEKO caches the tree per language and refreshes only the languages registered in its + # admin settings, so the two trees can disagree; the name is the only thing at stake + logger.warning('WEKO returned no localized name for index %s; showing its default name', index.identifier) + localized_title = index.title return { 'title': index.title, + 'localizedTitle': localized_title, 'id': index.identifier, - 'children': [self._serialize_index(i) for i in index.children], + 'children': [self._serialize_index(i, localized_titles) for i in index.children], } diff --git a/addons/weko/static/wekoNodeConfig.js b/addons/weko/static/wekoNodeConfig.js index 7b1fcd0a430..03b4b096c81 100644 --- a/addons/weko/static/wekoNodeConfig.js +++ b/addons/weko/static/wekoNodeConfig.js @@ -8,6 +8,7 @@ var bootbox = require('bootbox'); var Raven = require('raven-js'); var $osf = require('js/osfHelpers'); +require('js/rdmSelect2'); var $modal = $('#wekoInputCredentials'); @@ -29,31 +30,53 @@ function _getIndexById(indices, id) { return null; } -function _getIndexDisplayTitle(index, level) { - if (level === 0) { - return index.title; - } - var prefix = ''; - for (var i = 0; i < level; i ++) { - prefix += ' '; - } - return prefix + '- ' + index.title; -} - -function _flattenIndices(indices, level) { +function _flattenIndices(indices, ancestorTitles, ancestorLocalizedTitles) { const r = []; indices.forEach(function(data) { - const displayTitle = _getIndexDisplayTitle(data, level); + const titles = ancestorTitles.concat([data.title]); + const localizedTitles = ancestorLocalizedTitles.concat([data.localizedTitle]); r.push(Object.assign({ - displayTitle: displayTitle, + displayTitle: titles.join(' / '), + label: localizedTitles.join(' / ') + ' (ID:' + data.id + ')', }, data)); - _flattenIndices(data.children, level + 1).forEach(function(child) { + _flattenIndices(data.children, titles, localizedTitles).forEach(function(child) { r.push(child); }); }); return r; } +ko.bindingHandlers.wekoIndexSelect2 = { + // select2 mirrors the relies on it + $(element).select2({ + width: '100%', + // The + value: selectedIndexId, + wekoIndexSelect2: {value: selectedIndexId, labels: indexLabels}"> diff --git a/addons/weko/tests/test_client.py b/addons/weko/tests/test_client.py index d616196ec50..a624e293adc 100644 --- a/addons/weko/tests/test_client.py +++ b/addons/weko/tests/test_client.py @@ -19,6 +19,36 @@ def mock_requests_get(url, **kwargs): return utils.mock_response_404 +def _fake_truncated_indices(expanded_ids): + grandchildren = [{'id': '300', 'name': 'Grandchild 1', 'children': []}] + if '201' in expanded_ids: + grandchildren.append({'id': '301', 'name': 'Grandchild 2', 'children': []}) + else: + grandchildren.append({'id': 'more'}) + children = [{'id': '200', 'name': 'Child 1', 'children': []}] + if '100' in expanded_ids: + children.append({'id': '201', 'name': 'Child 2', 'children': grandchildren}) + else: + children.append({'id': 'more'}) + return [{'id': '100', 'name': 'Sample Index', 'children': children}] + + +def mock_requests_get_truncated(url, **kwargs): + prefix = 'https://test.sample.nii.ac.jp/api/tree?action=browsing' + if url == prefix: + return utils.MockResponse(_fake_truncated_indices([]), 200) + if url.startswith(prefix + '&more_ids='): + expanded_ids = url[len(prefix + '&more_ids='):].split('/') + return utils.MockResponse(_fake_truncated_indices(expanded_ids), 200) + return utils.mock_response_404 + + +def mock_requests_get_never_expanded(url, **kwargs): + if url.startswith('https://test.sample.nii.ac.jp/api/tree?action=browsing'): + return utils.MockResponse(_fake_truncated_indices([]), 200) + return utils.mock_response_404 + + class TestWEKOClient(OsfTestCase): def setUp(self): self.host = utils.fake_weko_host @@ -35,6 +65,37 @@ def test_weko_get_indices(self, get_req_mock): assert_equal(indices[0].title, 'Sample Index') assert_equal(indices[0].identifier, 100) + @mock.patch('requests.get', side_effect=mock_requests_get_truncated) + def test_weko_get_indices_expands_truncated_children(self, get_req_mock): + indices = self.conn.get_indices() + + titles = [i.title for i in client._flatten_indices(indices)] + assert_equal(titles, ['Sample Index', 'Child 1', 'Child 2', 'Grandchild 1', 'Grandchild 2']) + assert_equal( + [c[0][0] for c in get_req_mock.call_args_list], + [ + 'https://test.sample.nii.ac.jp/api/tree?action=browsing', + 'https://test.sample.nii.ac.jp/api/tree?action=browsing&more_ids=100', + 'https://test.sample.nii.ac.jp/api/tree?action=browsing&more_ids=100/201', + ] + ) + + @mock.patch('requests.get', side_effect=mock_requests_get_never_expanded) + def test_weko_get_indices_fails_when_weko_keeps_children_truncated(self, get_req_mock): + with assert_raises(ValueError): + self.conn.get_indices() + + @mock.patch('requests.get', side_effect=mock_requests_get) + def test_weko_get_indices_asks_for_the_given_language(self, get_req_mock): + self.conn.get_indices(accept_language='ja') + + get_req_mock.assert_called_once_with( + 'https://test.sample.nii.ac.jp/api/tree?action=browsing', + auth=(None, None), + headers={'Accept-Language': 'ja'}, + timeout=client.DEFAULT_TIMEOUT, + ) + @mock.patch('requests.get', side_effect=mock_requests_get) def test_weko_get_index_by_id(self, get_req_mock): index = self.conn.get_index_by_id(100) diff --git a/addons/weko/tests/test_serializer.py b/addons/weko/tests/test_serializer.py index 51d56e40930..6fbf5e5a1fe 100644 --- a/addons/weko/tests/test_serializer.py +++ b/addons/weko/tests/test_serializer.py @@ -82,6 +82,42 @@ def test_serialize_settings_authorized(self): assert_equal(serialized['ownerName'], self.user_settings.owner.fullname) assert_in('savedIndex', serialized) + def test_serialize_settings_names_indices_in_the_page_language(self): + def requests_get(url, **kwargs): + if kwargs.get('headers', {}).get('Accept-Language') == 'ja': + return utils.MockResponse([{'id': 100, 'name': 'サンプルインデックス'}], 200) + return utils.MockResponse([{'id': 100, 'name': 'Sample Index'}], 200) + + with mock.patch.object(type(self.node_settings), 'has_auth', return_value=True), \ + mock.patch.object(WEKOSerializer, 'credentials_are_valid', return_value=True), \ + mock.patch('requests.get', side_effect=requests_get), \ + self.app.app.test_request_context(headers={'Accept-Language': 'ja-JP'}): + serialized = self.ser.serialize_settings(self.node_settings, self.user, self.client) + + assert_equal(serialized['indices'], [{ + 'title': 'Sample Index', + 'localizedTitle': 'サンプルインデックス', + 'id': 100, + 'children': [], + }]) + + def test_serialize_settings_falls_back_to_the_default_name_and_warns(self): + def requests_get(url, **kwargs): + if kwargs.get('headers', {}).get('Accept-Language') == 'ja': + return utils.MockResponse([{'id': 100, 'name': 'サンプルインデックス'}], 200) + return utils.MockResponse([{'id': 100, 'name': 'Sample Index'}, {'id': 101, 'name': 'New Index'}], 200) + + with mock.patch.object(type(self.node_settings), 'has_auth', return_value=True), \ + mock.patch.object(WEKOSerializer, 'credentials_are_valid', return_value=True), \ + mock.patch('requests.get', side_effect=requests_get), \ + self.app.app.test_request_context(headers={'Accept-Language': 'ja'}), \ + self.assertLogs('addons.weko.serializer', level='WARNING') as logs: + serialized = self.ser.serialize_settings(self.node_settings, self.user, self.client) + + assert_equal([i['localizedTitle'] for i in serialized['indices']], ['サンプルインデックス', 'New Index']) + assert_equal(len(logs.records), 1) + assert_in('101', logs.records[0].getMessage()) + def test_serialize_settings_authorized_folder_is_set(self): pass diff --git a/addons/weko/tests/utils.py b/addons/weko/tests/utils.py index 076064d897a..f04a75ae656 100644 --- a/addons/weko/tests/utils.py +++ b/addons/weko/tests/utils.py @@ -11,9 +11,6 @@ 'id': 100, 'name': 'Sample Index', 'children': [ - { - 'id': 'more', - }, { 'id': 'dummy', }