diff --git a/modules/invenio-oaiserver/invenio_oaiserver/query.py b/modules/invenio-oaiserver/invenio_oaiserver/query.py index 9c492bca1d..e7946f1b16 100644 --- a/modules/invenio-oaiserver/invenio_oaiserver/query.py +++ b/modules/invenio-oaiserver/invenio_oaiserver/query.py @@ -152,7 +152,7 @@ def get_descendant_ids(index_id): descendant_ids = db.session.query(cte.c.descendant_id).all() return [descendant_id[0] for descendant_id in descendant_ids] - + page_ = kwargs.get('resumptionToken', {}).get('page', 1) size_ = current_app.config['OAISERVER_PAGE_SIZE'] @@ -187,20 +187,26 @@ def get_descendant_ids(index_id): #search = search.query('match', **{'path': kwargs['set']}) #search = search.query('match', **{'_oai.sets': sets}) #search = search.query('terms', **{'_oai.sets': sets}) - + if not sets: search = search.query('match_none') else: index_ids = [sets] + get_descendant_ids(sets) search = search.query('terms', **{'_oai.sets': index_ids}) - time_range = {} - if 'from_' in kwargs: - time_range['gte'] = kwargs['from_'] - if 'until' in kwargs: - time_range['lte'] = kwargs['until'] - if time_range: - search = search.filter('range', **{'_updated': time_range}) + if current_app.config.get('WEKO_SEARCH_FIX_ACCESSRIGHTS', False): + if 'from_' in kwargs or 'until' in kwargs: + rq = range_query(kwargs.get('from_'), kwargs.get('until')) + if rq is not None: + search = search.filter(rq) + else: + time_range = {} + if 'from_' in kwargs: + time_range['gte'] = kwargs['from_'] + if 'until' in kwargs: + time_range['lte'] = kwargs['until'] + if time_range: + search = search.filter('range', **{'_updated': time_range}) search = search.query('match', **{'relation_version_is_last': 'true'}) search = search.query('terms', **{'publish_status': [ @@ -288,3 +294,175 @@ def items(self): } return Pagination(response) + +def range_query(_from=None, _until=None): + """Generate a search query considering update date changes. + + Args: + _from (datetime or str or None): + Lower bound of update date. + _until (datetime or str or None): + Upper bound of update date. + + Returns: + elasticsearch_dsl.query.Q or None: + The generated query object, or None if no range is specified. + """ + + if isinstance(_from, datetime): + from_date = _from.strftime('%Y-%m-%d') + _from = _from.isoformat() + elif isinstance(_from, str) and len(_from) >= 10: + from_date = _from[:10] + else: + _from = None + + if isinstance(_until, datetime): + until_date = _until.strftime('%Y-%m-%d') + _until = _until.isoformat() + elif isinstance(_until, str) and len(_until) >= 10: + until_date = _until[:10] + else: + _until = None + + if _from is None and _until is None: + return None + + now = datetime.now().strftime('%Y-%m-%d') + + # First should condition + + must_not_embargoed = Q( + 'bool', must_not=[Q('term', accessRights='embargoed access')] + ) + must_not_content_accessrole = Q( + 'bool', must_not=[ + Q( + 'nested', + path='content', + query=Q('exists', field='content.accessrole.raw') + ) + ] + ) + must_not_open_access = Q( + 'nested', + path='content', + query=Q( + 'bool', + must_not=[ + Q('term', **{'content.accessrole.raw': 'open_access'}), + Q( + 'bool', + must=[ + Q('term', **{'content.accessrole.raw': 'open_date'}), + Q('range', **{'content.date.dateValue.raw': {'lte': now}}) + ] + ) + ] + ) + ) + should1 = Q( + 'bool', + must=[ + Q( + 'bool', + should=[ + must_not_embargoed, + must_not_content_accessrole, + must_not_open_access + ] + ), + Q( + 'range', + **{ + '_updated': { + **({'gte': _from} if _from else {}), + **({'lte': _until} if _until else {}) + } + } + ) + ] + ) + + # Second should condition + must_not_open_access2 = Q( + 'nested', + path='content', + query=Q( + 'bool', + must_not=[ + Q('term', **{'content.accessrole.raw': 'open_access'}), + Q( + 'bool', + must=[ + Q('term', **{'content.accessrole.raw': 'open_date'}), + Q('range', **{'content.date.dateValue.raw': {'lte': now}}) + ] + ) + ] + ) + ) + + # from condition + from_should = [] + if _from: + from_should.append( + Q( + 'nested', + path='content', + query=Q( + 'bool', + must=[ + Q('term', **{'content.accessrole.raw': 'open_date'}), + Q('range', **{'content.date.dateValue.raw': {'gte': from_date}}) + ] + ) + ) + ) + from_should.append(Q('range', **{'_updated': {'gte': _from}})) + + # until condition + until_must = [] + if _until: + until_must.append( + Q( + 'bool', + must_not=[ + Q( + 'nested', + path='content', + query=Q( + 'bool', + must=[ + Q('term', **{'content.accessrole.raw': 'open_date'}), + Q('range', **{'content.date.dateValue.raw': {'gt': until_date}}) + ] + ) + ) + ] + ) + ) + until_must.append(Q('range', **{'_updated': {'lte': _until}})) + + must2 = [ + Q('term', accessRights='embargoed access'), + Q( + 'nested', + path='content', + query=Q('exists', field='content.accessrole.raw') + ), + Q('bool', must_not=[must_not_open_access2]) + ] + if from_should: + must2.append(Q('bool', should=from_should)) + if until_must: + must2.extend(until_must) + + should2 = Q('bool', must=must2) + + # Overall should + return Q( + 'bool', + should=[should1, should2], + minimum_should_match=1 + ) diff --git a/modules/invenio-oaiserver/tests/test_query.py b/modules/invenio-oaiserver/tests/test_query.py index 63c18b2515..e62d2283db 100644 --- a/modules/invenio-oaiserver/tests/test_query.py +++ b/modules/invenio-oaiserver/tests/test_query.py @@ -1,4 +1,3 @@ - import pytest from mock import patch import uuid @@ -15,7 +14,8 @@ from invenio_oaiserver.query import ( query_string_parser, get_affected_records, - get_records + get_records, + range_query ) # .tox/c1/bin/pytest --cov=invenio_oaiserver tests/test_query.py -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/invenio-oaiserver/.tox/c1/tmp @@ -28,7 +28,7 @@ def test_query_string_parser(es_app): assert type(result) == QueryString assert result.name == "query_string" assert result.to_dict() == {"query_string":{"query":"test_path"}} - + # current_oaiserver not have query_parse, config is not str current_app.config.update(OAISERVER_QUERY_PARSER=Q) delattr(current_oaiserver,"query_parser") @@ -36,7 +36,7 @@ def test_query_string_parser(es_app): assert type(result) == QueryString assert result.name == "query_string" assert result.to_dict() == {"query_string":{"query":"test_path"}} - + # current_oaiserver have query_parse result = query_string_parser("test_path") assert type(result) == QueryString @@ -53,23 +53,23 @@ def test_get_affected_records(es_app): result = get_affected_records(None,None) for i in result: pass - + spec="1671155386910" search_path = 'path:"1671155386910"' # exist spec, not exist search_path result = get_affected_records(spec,None) for i in result: assert i - + # not exist spec, exist search_path result = get_affected_records(None,search_path) for i in result: assert i - + result = get_affected_records(spec,search_path) for i in result: assert i - + #def get_records(**kwargs): # .tox/c1/bin/pytest --cov=invenio_oaiserver tests/test_query.py::test_get_records -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiserver/.tox/c1/tmp @@ -102,9 +102,9 @@ def test_get_records(es_app,db, mock_execute): db.session.add_all(indexes) db.session.add(rec1) db.session.add(rec2) - + db.session.commit() - + es_info = dict(id=str(rec_uuid1), index=current_app.config['INDEXER_DEFAULT_INDEX'], doc_type=current_app.config['INDEXER_DEFAULT_DOCTYPE']) @@ -119,21 +119,21 @@ def test_get_records(es_app,db, mock_execute): version_type='external_gte', body=rec_data2) current_search_client.index(**{**es_info,**body}) - + # not scroll_id, ":" not in set data = { "set":"12345" } result = get_records(**data) assert result - + # not scroll_id, ":" in set data = { "set":"12345:6789" } result = get_records(**data) assert result - + # not scroll_id, "set" not in data, exist "from_","until" in data data = { "from_":"2022-01-01", @@ -141,7 +141,7 @@ def test_get_records(es_app,db, mock_execute): } result = get_records(**data) assert result - + # in scroll_id data = { "resumptionToken":{"page":1,"scroll_id":"DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAVfgWYmVhQ3BkbEdSSm0wS3pTaEdQeHQ1QQ=="} @@ -201,7 +201,7 @@ def test_get_records_with_set(es_app,db, users): browsing_role="3,-99" ) ) - + rec_uuid1 = uuid.uuid4() rec_data1 = {"title":["test_item1"], "path":["123"], @@ -212,7 +212,7 @@ def test_get_records_with_set(es_app,db, users): "_updated": "2022-01-01T00:00:00" } rec1 = RecordMetadata(id=rec_uuid1,json=rec_data1) - + rec_uuid2 = uuid.uuid4() rec_data2 = {"title":["test_item2"], "path":["456"], @@ -223,7 +223,7 @@ def test_get_records_with_set(es_app,db, users): "_updated": "2022-01-01T00:00:00" } rec2 = RecordMetadata(id=rec_uuid2,json=rec_data2) - + rec_uuid3 = uuid.uuid4() rec_data3 = {"title":["test_item3"], "path":["789"], @@ -234,13 +234,13 @@ def test_get_records_with_set(es_app,db, users): "_updated": "2022-01-01T00:00:00" } rec3 = RecordMetadata(id=rec_uuid3,json=rec_data3) - + db.session.add_all(indexes) db.session.add(rec1) db.session.add(rec2) db.session.add(rec3) db.session.commit() - + es_info = dict(index=current_app.config['INDEXER_DEFAULT_INDEX'], doc_type=current_app.config['INDEXER_DEFAULT_DOCTYPE'], version=1, @@ -252,14 +252,14 @@ def test_get_records_with_set(es_app,db, users): current_search_client.index(**es_info,**body1) current_search_client.index(**es_info,**body2) current_search_client.index(**es_info,**body3) - + comm1 = Community.create(community_id="test_comm", role_id=users[0]["id"], id_user=users[0]["id"], title="test community", description="this is test community", root_node_id=indexes[0].id) db.session.add(comm1) db.session.commit() - + data = {"set":"123"} result = get_records(**data) assert result.total == 3 @@ -267,20 +267,20 @@ def test_get_records_with_set(es_app,db, users): assert result_items[0]["json"]["_source"] == rec_data1 assert result_items[1]["json"]["_source"] == rec_data2 assert result_items[2]["json"]["_source"] == rec_data3 - + data = {"set":"123:456"} result = get_records(**data) assert result.total == 2 result_items = [r for r in result.items] assert result_items[0]["json"]["_source"] == rec_data2 assert result_items[1]["json"]["_source"] == rec_data3 - + data = {"set":"123:456:789"} result = get_records(**data) assert result.total == 1 result_items = [r for r in result.items] assert result_items[0]["json"]["_source"] == rec_data3 - + data = {"set":"user-test_comm"} result = get_records(**data) assert result.total == 3 @@ -288,7 +288,7 @@ def test_get_records_with_set(es_app,db, users): assert result_items[0]["json"]["_source"] == rec_data1 assert result_items[1]["json"]["_source"] == rec_data2 assert result_items[2]["json"]["_source"] == rec_data3 - + data = {"set":"test_comm"} result = get_records(**data) assert result.total == 3 @@ -296,11 +296,167 @@ def test_get_records_with_set(es_app,db, users): assert result_items[0]["json"]["_source"] == rec_data1 assert result_items[1]["json"]["_source"] == rec_data2 assert result_items[2]["json"]["_source"] == rec_data3 - + data = {"set":"999"} result = get_records(**data) assert result.total == 0 - + data = {"set":"aaa"} result = get_records(**data) assert result.total == 0 + +# .tox/c1/bin/pytest --cov=invenio_oaiserver tests/test_query.py::test_range_query -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-oaiserver/.tox/c1/tmp +def test_range_query(): + # Case: _from is datetime + from datetime import datetime + _from = datetime(2026, 1, 1, 12, 0, 0) + result = range_query(_from, None) + assert result is not None + d = result.to_dict() + should2 = d['bool']['should'][1]['bool']['must'] + assert any('gte' in str(x) for x in should2) + + # Case: _until is datetime + _until = datetime(2026, 12, 31, 23, 59, 59) + result = range_query(None, _until) + assert result is not None + d = result.to_dict() + should2 = d['bool']['should'][1]['bool']['must'] + assert any('lte' in str(x) for x in should2) + + # Case: _from is invalid type (int) + _from = 123456 + result = range_query(_from, None) + assert result is None + + # Case: _until is invalid type (int) + _until = 123456 + result = range_query(None, _until) + assert result is None + + # Case: _from is short string (invalid) + _from = '2026-01' + result = range_query(_from, None) + assert result is None + + # Case: _until is short string (invalid) + _until = '2026-12' + result = range_query(None, _until) + assert result is None + + # Case: both _from and _until are None + result = range_query(None, None) + assert result is None + + # Case: only _from is specified + _from = '2026-01-01' + result = range_query(_from, None) + assert result is not None + d = result.to_dict() + # Check that 'gte' appears in should2 (from_should is active) + should2 = d['bool']['should'][1]['bool']['must'] + assert any('gte' in str(x) for x in should2) + + # Case: only _until is specified + _until = '2026-12-31' + result = range_query(None, _until) + assert result is not None + d = result.to_dict() + # Check that 'lte' appears in should2 (until_must is active) + should2 = d['bool']['should'][1]['bool']['must'] + assert any('lte' in str(x) for x in should2) + + # Case: both _from and _until are specified + _from = '2026-01-01' + _until = '2026-12-31' + result = range_query(_from, _until) + assert result is not None + d = result.to_dict() + should2 = d['bool']['should'][1]['bool']['must'] + # Both from_should and until_must should be active + assert any('gte' in str(x) for x in should2) + assert any('lte' in str(x) for x in should2) + +@pytest.mark.parametrize("fix_access, from_, until, expect_range, expect_rq, rangequery_none", [ + (False, None, None, False, False, False), + (False, "2026-01-01", None, True, False, False), + (False, None, "2026-12-31", True, False, False), + (False, "2026-01-01", "2026-12-31", True, False, False), + (True, None, None, False, False, False), + (True, "2026-01-01", None, True, True, False), + (True, None, "2026-12-31", True, True, False), + (True, "2026-01-01", "2026-12-31", True, True, False), + (True, "2026-01-01", "2026-12-31", False, True, True), +]) +# .tox/c1/bin/pytest --cov=invenio_oaiserver tests/test_query.py::test_get_records_range_branch -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/invenio-oaiserver/.tox/c1/tmp +def test_get_records_range_branch(es_app, db, monkeypatch, fix_access, from_, until, expect_range, expect_rq, rangequery_none): + es_app.config['WEKO_SEARCH_FIX_ACCESSRIGHTS'] = fix_access + + index = Index( + id=30, + parent=0, + position=1, + index_name_english="range_index", + index_link_name_english="range_index_link", + harvest_public_state=True, + public_state=True, + public_date=datetime(2100,1,1), + browsing_role="3,-99" + ) + db.session.add(index) + rec_uuid = uuid.uuid4() + PersistentIdentifier.create('doi', "https://doi.org/00030", object_type='rec', object_uuid=rec_uuid, status=PIDStatus.REGISTERED) + rec_data = { + "title": ["range_item"], + "path": ["30"], + "_oai": {"id": "oai:test:00030", "sets": ["30"]}, + "set": ["30"], + "relation_version_is_last": "true", + "control_number": "30", + "publish_status": "0", + "_updated": "2100-01-01T00:00:00" + } + rec = RecordMetadata(id=rec_uuid, json=rec_data) + db.session.add(rec) + db.session.commit() + es_info = dict( + id=str(rec_uuid), + index=es_app.config['INDEXER_DEFAULT_INDEX'], + doc_type=es_app.config['INDEXER_DEFAULT_DOCTYPE'], + refresh="wait_for" + ) + body = dict(version=1, version_type="external_gte", body=rec_data) + from invenio_search import current_search_client + current_search_client.index(**{**es_info, **body}) + current_search_client.indices.refresh(index=es_app.config['INDEXER_DEFAULT_INDEX']) + + called = {"filter": False, "rq": False} + import invenio_oaiserver.query as query_mod + + orig_filter = query_mod.OAIServerSearch.filter + def filter_spy(self, *args, **kwargs): + called["filter"] = True + return orig_filter(self, *args, **kwargs) + monkeypatch.setattr(query_mod.OAIServerSearch, "filter", filter_spy) + + orig_rq = query_mod.range_query + if rangequery_none: + def rq_spy(_from, _until): + called["rq"] = True + return None + monkeypatch.setattr(query_mod, "range_query", rq_spy) + else: + def rq_spy(_from, _until): + called["rq"] = True + return orig_rq(_from, _until) + monkeypatch.setattr(query_mod, "range_query", rq_spy) + + with es_app.app_context(): + kwargs = {"set": "30"} + if from_: + kwargs["from_"] = from_ + if until: + kwargs["until"] = until + get_records(**kwargs) + assert called["filter"] == expect_range + assert called["rq"] == expect_rq diff --git a/modules/invenio-records-rest/invenio_records_rest/facets.py b/modules/invenio-records-rest/invenio_records_rest/facets.py index 280c8f7321..5d30f92e17 100644 --- a/modules/invenio-records-rest/invenio_records_rest/facets.py +++ b/modules/invenio-records-rest/invenio_records_rest/facets.py @@ -94,13 +94,23 @@ def inner(values): def _create_filter_dsl(urlkwargs, definitions): """Create a filter DSL expression.""" filters = [] + weko_search_fix_accessrights = current_app.config.get("WEKO_SEARCH_FIX_ACCESSRIGHTS", False) for name, filter_factory in definitions.items(): values = request.values.getlist(name, type=text_type) if values: - filters.append(filter_factory(values)) - for v in values: - urlkwargs.add(name, v) - + if name in ("Access", "accessRights") and "new_accessRights" in definitions and weko_search_fix_accessrights: + new_accessrights_filters = definitions["new_accessRights"]["filters"]["filters"] + access_rights_queries = [] + for v in values: + if v in new_accessrights_filters: + access_rights_queries.append(Q(new_accessrights_filters[v])) + urlkwargs.add(name, v) + if access_rights_queries: + filters.append(Q('bool', should=access_rights_queries)) + else: + filters.append(filter_factory(values)) + for v in values: + urlkwargs.add(name, v) return (filters, urlkwargs) @@ -139,7 +149,7 @@ def default_facets_factory(search, index): all fields and values used. """ urlkwargs = MultiDict() - + from weko_admin.utils import get_facet_search_query from weko_search_ui.permissions import search_permission facets = get_facet_search_query(search_permission.can()).get(index) diff --git a/modules/invenio-records-rest/invenio_records_rest/views.py b/modules/invenio-records-rest/invenio_records_rest/views.py index fff588810d..dd073f60a4 100644 --- a/modules/invenio-records-rest/invenio_records_rest/views.py +++ b/modules/invenio-records-rest/invenio_records_rest/views.py @@ -853,8 +853,9 @@ def url_args_check(): if size * page < search_result.hits.total and \ size * page < self.max_result_window: links['next'] = url_for(endpoint, page=page + 1, **urlkwargs) - from weko_search_ui.utils import combine_aggs + from weko_search_ui.utils import combine_aggs, fix_aggregations_accessrights search_result = combine_aggs(search_result.to_dict()) + search_result = fix_aggregations_accessrights(search_result) return self.make_response( pid_fetcher=self.pid_fetcher, search_result=search_result, diff --git a/modules/invenio-records-rest/tests/test_facets.py b/modules/invenio-records-rest/tests/test_facets.py index 27f25a5716..bc2843e33a 100644 --- a/modules/invenio-records-rest/tests/test_facets.py +++ b/modules/invenio-records-rest/tests/test_facets.py @@ -85,6 +85,58 @@ def test_create_filter_dsl(): assert not filters assert args == kwargs +def test_create_filter_dsl_accessrights(): + app = Flask('testapp') + app.config['WEKO_SEARCH_FIX_ACCESSRIGHTS'] = True + # Definition of new_accessRights + definitions = { + "accessRights": lambda values: Q('terms', accessRights=values), + "new_accessRights": { + "filters": { + "filters": { + "open": {"term": {"accessRights": "open"}}, + "closed": {"term": {"accessRights": "closed"}}, + } + } + } + } + # Values included in new_accessrights_filters + with app.test_request_context('?accessRights=open&accessRights=closed'): + kwargs = MultiDict() + filters, args = _create_filter_dsl(kwargs, definitions) + assert len(filters) == 1 + assert filters[0].to_dict() == { + "bool": { + "should": [ + {"term": {"accessRights": "open"}}, + {"term": {"accessRights": "closed"}} + ] + } + } + assert args.getlist('accessRights') == ['open', 'closed'] + + # Values not included in new_accessrights_filters + with app.test_request_context('?accessRights=unknown'): + kwargs = MultiDict() + filters, args = _create_filter_dsl(kwargs, definitions) + assert filters == [] + assert args.getlist('accessRights') == [] + + # When the request value is empty + with app.test_request_context(''): + kwargs = MultiDict() + filters, args = _create_filter_dsl(kwargs, definitions) + assert filters == [] + assert args == kwargs + + definitions = { + "other": lambda values: Q('terms', other=values), + } + with app.test_request_context('?other=open'): + kwargs = MultiDict() + filters, args = _create_filter_dsl(kwargs, definitions) + assert filters == [Q('terms', other=['open'])] + assert args.getlist('other') == ['open'] def test_post_filter(app): """Test post filter.""" @@ -242,4 +294,4 @@ def test_default_facets_factory(app, db, search_user, redis_connect): assert 'aggs' not in search.to_dict() assert 'post_filter' not in search.to_dict() assert 'bool' not in search.to_dict()['query'] - redis_connect.delete(test_redis_key) \ No newline at end of file + redis_connect.delete(test_redis_key) diff --git a/modules/invenio-records/invenio_records/api.py b/modules/invenio-records/invenio_records/api.py index 96d6a3ec8d..d73f5fac3a 100644 --- a/modules/invenio-records/invenio_records/api.py +++ b/modules/invenio-records/invenio_records/api.py @@ -17,6 +17,7 @@ from jsonpatch import apply_patch from sqlalchemy.orm.attributes import flag_modified from werkzeug.local import LocalProxy +from datetime import datetime, time from .errors import MissingModelError from .models import RecordMetadata @@ -214,10 +215,11 @@ def __custom_record_metadata(cls, record_metadata: dict): Args: record_metadata (dict): Record metadata. """ - from weko_records.utils import replace_fqdn_of_file_metadata + from weko_records.utils import replace_fqdn_of_file_metadata, update_embargo_rights for k, v in record_metadata.items(): if isinstance(v, dict) and v.get('attribute_type') == 'file': replace_fqdn_of_file_metadata(v.get("attribute_value_mlt", [])) + update_embargo_rights(record_metadata) @classmethod def get_records(cls, ids, with_deleted=False): @@ -376,7 +378,83 @@ def revisions(self): return RevisionsIterator(self.model) + @property + def updated(self): + if not self.model: + return None + original_updated = self.model.updated + if not current_app.config.get('WEKO_SEARCH_FIX_ACCESSRIGHTS', False): + return original_updated + metadata = db.session.query(RecordMetadata.json).filter_by(id=self.model.id).scalar() + item_type_id = metadata.get("item_type_id") if metadata else None + from weko_records.serializers.utils import get_mapping + mapping = get_mapping(item_type_id, "jpcoar_mapping") if item_type_id else None + access_path = mapping.get("accessRights.@value") if mapping else None + + def _get_nested_value(data, path): + keys = path.split('.') + key = keys[0] + rest = '.'.join(keys[1:]) + if isinstance(data, dict): + if key in data: + if rest: + return _get_nested_value(data[key], rest) + else: + return data[key] + elif 'attribute_value_mlt' in data: + for item in data['attribute_value_mlt']: + found = _get_nested_value(item, '.'.join(keys)) + if found is not None: + return found + return None + else: + return None + elif isinstance(data, list): + for item in data: + found = _get_nested_value(item, '.'.join(keys)) + if found is not None: + return found + return None + else: + return None + + access_rights = ( + _get_nested_value(metadata, access_path) + if metadata and access_path else None + ) + accessrole_date = [] + for v in metadata.values() if metadata else []: + if isinstance(v, dict) and v.get("attribute_type") == "file": + for file_info in v.get("attribute_value_mlt", []): + date_list = file_info.get("date", []) + date_value = date_list[0].get("dateValue") if date_list else None + accessrole = file_info.get("accessrole") + if date_value and accessrole: + date_obj = datetime.strptime(date_value, "%Y-%m-%d").date() + accessrole_date.append((accessrole, date_obj)) + + today = datetime.now().date() + + from weko_records.utils import check_embargo_rights + is_update, change_value = check_embargo_rights( + access_rights, today, accessrole_date + ) + if not (is_update and change_value == "open access"): + return original_updated + + open_dates = [] + for role, date_str in accessrole_date: + dt = datetime.combine(date_str, time(0, 0, 0)) + dt = datetime.combine(dt.date(), time(0, 0, 0)) + open_dates.append(dt) + + latest_open_date = max(open_dates) + if original_updated: + result = max(original_updated, latest_open_date) + return result + else: + return latest_open_date class RecordRevision(RecordBase): """API for record revisions.""" diff --git a/modules/invenio-records/tests/test_api.py b/modules/invenio-records/tests/test_api.py index 0298c35caa..071657de17 100644 --- a/modules/invenio-records/tests/test_api.py +++ b/modules/invenio-records/tests/test_api.py @@ -392,3 +392,299 @@ def test_validate_partial(app, db): record['a'] = 1 with pytest.raises(ValidationError) as exc_info: record.commit(validator=PartialDraft4Validator) + +@pytest.mark.parametrize( + "fix_accessrights, access_path, updated, meta_patch, expected", + [ + # Empty data + (True, None, None, {}, None), + + # Only updated is set + (True, None, datetime(2026, 3, 1, 0, 0, 0), None, datetime(2026, 3, 1, 0, 0, 0)), + + # item_type_id is None + (True, None, datetime(2026, 3, 1, 0, 0, 0), {"item_type_id": None}, datetime(2026, 3, 1, 0, 0, 0)), + + # Only item_type_id is set + (True, None, datetime(2026, 3, 1, 0, 0, 0), {"item_type_id": 1}, datetime(2026, 3, 1, 0, 0, 0)), + + # Empty file attribute + (True, None, datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [] + } + }, datetime(2026, 3, 1, 0, 0, 0)), + + # Empty file attribute (updated=None) + (True, None, None, {"item_type_id": 1, "item_foo": {"attribute_type": "file", "attribute_value_mlt": []}}, None), + + # open_date is in the future + (True, "item_1736146823660.attribute_value_mlt.subitem_access_right", datetime(2027, 1, 1, 0, 0, 0), { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"date": [{"dateType": "Available", "dateValue": "2026-12-31"}], "accessrole": "open_date"} + ] + } + }, datetime(2027, 1, 1, 0, 0, 0)), + + # open_date is in the past + (True, "item_1736146823660.attribute_value_mlt.subitem_access_right", datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"date": [{"dateType": "Available", "dateValue": "2026-01-01"}], "accessrole": "open_date"} + ] + } + }, datetime(2026, 3, 1, 0, 0, 0)), + + # open_access file + (True, "item_1736146823660.attribute_value_mlt.subitem_access_right", datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"date": [{"dateType": "Available", "dateValue": "2026-04-01"}], "accessrole": "open_access"} + ] + } + }, datetime(2026, 4, 1, 0, 0, 0)), + + # date exists but no accessrole + (True, "item_1736146823660.attribute_value_mlt.subitem_access_right", datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"date": [{"dateType": "Available", "dateValue": "2027-01-01"}]} + ] + } + }, datetime(2026, 3, 1, 0, 0, 0)), + + # accessrole exists but no date + (True, "item_1736146823660.attribute_value_mlt.subitem_access_right", datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"accessrole": "open_date"} + ] + } + }, datetime(2026, 3, 1, 0, 0, 0)), + + # Multiple open_dates (with access rights info) + (True, "item_1736146823660.attribute_value_mlt.subitem_access_right", datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"date": [{"dateType": "Available", "dateValue": "2026-03-31"}], "accessrole": "open_date"}, + {"date": [{"dateType": "Available", "dateValue": "2026-03-30"}], "accessrole": "open_date"} + ] + } + }, datetime(2026, 3, 31, 0, 0, 0)), + + # Multiple open_dates (with access rights info), original_updated > max(open_dates) + (True, "item_1736146823660.attribute_value_mlt.subitem_access_right", datetime(2026, 4, 2, 0, 0, 0), { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"date": [{"dateType": "Available", "dateValue": "2026-03-31"}], "accessrole": "open_date"}, + {"date": [{"dateType": "Available", "dateValue": "2026-03-30"}], "accessrole": "open_date"} + ] + } + }, datetime(2026, 4, 2, 0, 0, 0)), + # WEKO_SEARCH_FIX_ACCESSRIGHTS is False + (False, "item_1736146823660.attribute_value_mlt.subitem_access_right", datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"date": [{"dateType": "Available", "dateValue": "2026-03-31"}], "accessrole": "open_date"}, + {"date": [{"dateType": "Available", "dateValue": "2026-03-30"}], "accessrole": "open_date"} + ] + } + }, datetime(2026, 3, 1, 0, 0, 0)), + + # _get_nested_value list branch (does not contain subitem_access_right) + (True, "item_foo.subitem_access_right", datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_foo": [ + {"not_target": 1} + ] + }, datetime(2026, 3, 1, 0, 0, 0)), + + #Case where attribute_mlt is not in the path + (True, "item_foo.subitem_access_right",datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_foo": { + "attribute_type": "アクセス権", + "attribute_value_mlt": [ + {"not_target": 1}, + {"subitem_access_right": "embargoed access"} + ] + } + }, + datetime(2026, 3, 1, 0, 0, 0) + ), + + # Case where the path key does not exist + (True,"item_foo.subitem_access_right", datetime(2026, 3, 1, 0, 0, 0), { + "item_type_id": 1, + "item_foo": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"not_target": 1} + ] + } + }, + datetime(2026, 3, 1, 0, 0, 0) + ), + + # Case where value is neither list nor dict + (True, "item_foo.subitem_access_right" ,datetime(2026, 3, 1, 0, 0, 0),{ + "item_type_id": 1, + "item_foo": { + "attribute_type": "アクセス権", + "attribute_value_mlt": "invalid" + } + }, + datetime(2026, 3, 1, 0, 0, 0) + ), + + # Case where updated is None but opendate exists + (True, "item_1736146823660.attribute_value_mlt.subitem_access_right" ,None, { + "item_type_id": 1, + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_type": "file", + "attribute_value_mlt": [ + {"date": [{"dateType": "Available", "dateValue": "2026-03-31"}], "accessrole": "open_date"} + ] + } + }, + datetime(2026, 3, 31, 0, 0, 0) + ), + ] +) +# .tox/c1/bin/pytest --cov=invenio_records tests/test_api.py::test_record_updated -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-records/.tox/c1/tmp +def test_record_updated(app, monkeypatch, access_path, fix_accessrights, updated, meta_patch, expected): + base_meta = { + "item_type_id": 1 + } + meta = copy.deepcopy(base_meta) + if meta_patch is not None: + meta.update(copy.deepcopy(meta_patch)) + else: + meta = None + + class DummyQuery: + def filter_by(self, id): + return self + def scalar(self): + return meta + + class DummySession: + def __init__(self): + self.session = self + def query(self, *args, **kwargs): + return DummyQuery() + + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + with app.app_context(): + from flask import current_app + current_app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = fix_accessrights + record = Record({}) + class DummyModel: + def __init__(self, updated, json, id=1): + self.updated = updated + self.json = json + self.id = id + record.model = DummyModel(updated=updated, json=meta, id=1) + monkeypatch.setattr("invenio_records.api.db", DummySession()) + result = record.updated + assert result == expected diff --git a/modules/invenio-resourcesyncserver/invenio_resourcesyncserver/query.py b/modules/invenio-resourcesyncserver/invenio_resourcesyncserver/query.py index f2d381b557..c7ee054d33 100644 --- a/modules/invenio-resourcesyncserver/invenio_resourcesyncserver/query.py +++ b/modules/invenio-resourcesyncserver/invenio_resourcesyncserver/query.py @@ -28,6 +28,8 @@ from weko_index_tree.api import Indexes from weko_schema_ui.models import PublishStatus from weko_search_ui.utils import execute_search_with_pagination +from invenio_oaiserver.query import range_query +from datetime import datetime, timedelta from .config import WEKO_ROOT_INDEX @@ -262,14 +264,31 @@ def _get_index_search_query(_date_from: str, _date_until: str) -> dict: "path": list_path } } - post_filter['bool']['must'].append({ - "range": { - "_updated": { - "lte": _date_until, - "gte": _date_from + if current_app.config.get('WEKO_SEARCH_FIX_ACCESSRIGHTS', False): + if len(_date_until) == 19 and 'T' in _date_until: + _date_until = ( + datetime.strptime(_date_until, '%Y-%m-%dT%H:%M:%S') + - timedelta(seconds=1) + ) + _date_until = _date_until.isoformat() + elif len(_date_until) == 10: + _date_until = ( + datetime.strptime(_date_until, '%Y-%m-%d') + - timedelta(seconds=1) + ) + _date_until = _date_until.isoformat() + rq = range_query(_date_from, _date_until) + if rq is not None: + post_filter['bool']['must'].append(rq.to_dict()) + else: + post_filter['bool']['must'].append({ + "range": { + "_updated": { + "lte": _date_until, + "gte": _date_from + } } - } - }) + }) # create search query try: query_q = json.dumps(query_q).replace("@index", q) @@ -285,14 +304,32 @@ def _get_index_search_query(_date_from: str, _date_until: str) -> dict: "path": list_path } }) - post_filter['bool']['must'].append({ - "range": { - "_updated": { - "lte": _date_until, - "gte": _date_from + if current_app.config.get('WEKO_SEARCH_FIX_ACCESSRIGHTS', False): + if len(_date_until) == 19 and 'T' in _date_until: + _date_until = ( + datetime.strptime(_date_until, '%Y-%m-%dT%H:%M:%S') + - timedelta(seconds=1) + ) + _date_until = _date_until.isoformat() + elif len(_date_until) == 10: + _date_until = ( + datetime.strptime(_date_until, '%Y-%m-%d') + - timedelta(seconds=1) + ) + _date_until = _date_until.isoformat() + rq = range_query(_date_from, _date_until) + if rq is not None: + post_filter['bool']['must'].append(rq.to_dict()) + else: + post_filter['bool']['must'].append({ + "range": { + "_updated": { + "lte": _date_until, + "gte": _date_from + } } - } - }) + }) + # create search query wild_card = [] child_list = Indexes.get_child_list(q) @@ -320,7 +357,6 @@ def _get_index_search_query(_date_from: str, _date_until: str) -> dict: # create a index search query query_q = _get_index_search_query(date_from, date_until) - try: search.update_from_dict(query_q) except SyntaxError: diff --git a/modules/invenio-resourcesyncserver/tests/test_query.py b/modules/invenio-resourcesyncserver/tests/test_query.py index 20a423e029..a9cfd0dbd8 100644 --- a/modules/invenio-resourcesyncserver/tests/test_query.py +++ b/modules/invenio-resourcesyncserver/tests/test_query.py @@ -6,7 +6,7 @@ import datetime from elasticsearch import helpers from mock import patch, MagicMock, Mock -from flask import current_app, make_response, request +from flask import current_app, make_response, request, Flask from flask_login import current_user from flask_babelex import Babel @@ -68,7 +68,7 @@ def test_item_path_search_factory(i18n_app, indices): with patch("weko_index_tree.api.Indexes.get_list_path_publish", return_value="test"): with patch("weko_index_tree.api.Indexes.get_child_list", return_value=[MagicMock()]): assert item_path_search_factory(data_1, index_id="Root Index") - + assert item_path_search_factory(data_1, index_id=33) @@ -81,3 +81,116 @@ def test_item_changes_search_factory(i18n_app, indices): with patch("weko_index_tree.api.Indexes.get_list_path_publish", return_value="test"): with patch("weko_index_tree.api.Indexes.get_child_list", return_value=[MagicMock()]): assert item_changes_search_factory(search, index_id="Root Index") + +@pytest.mark.parametrize("fix_access, is_root, date_until, rq_none, expect_range, expect_updated", [ + # fix_access, is_root, date_until, rq_none, expect_range, expect_updated + (True, False, '2020-12-31T23:59:59', False, True, False), # 19 digits, range_query enabled + (True, False, '2020-12-31', False, True, False), # 10 digits, range_query enabled + (True, False, 'invalid', False, True, False), # invalid value, range_query enabled + (True, False, '2020-12-31T23:59:59', True, False, False), # 19 digits, range_query is None + (False, False, '2020-12-31T23:59:59', False, False, True), # fix_access=False, _updated + (True, True, '2020-12-31T23:59:59', False, True, False), # root, range_query enabled + (False, True, '2020-12-31T23:59:59', False, False, True), # root, fix_access=False + (True, True, '2020-12-31', False, True, False), # root, 10 digits, range_query enabled + (True, True, 'invalid', True, False, False), # root, invalid value, range_query is None +]) +# .tox/c1/bin/pytest --cov=invenio_resourcesyncserver tests/test_query.py::test_item_changes_search_factory_branch -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/invenio-resourcesyncserver/.tox/c1/tmp +def test_item_changes_search_factory_branch(monkeypatch, fix_access, is_root, date_until, rq_none, expect_range, expect_updated): + # Flask Apps and Config + app = Flask(__name__) + app.config['WEKO_SEARCH_FIX_ACCESSRIGHTS'] = fix_access + app.config['WEKO_ROOT_INDEX'] = 0 + + # mock + class DummySearch: + def __init__(self): + self.query = None + def update_from_dict(self, q): + self.query = q + + class DummyItem: + cid = 'dummy' + + class DummyIndexes: + @staticmethod + def get_list_path_publish(index_id): + return [1, 2, 3] + @staticmethod + def get_child_list(q): + return [DummyItem()] + + called = {'range_query': False} + def fake_range_query(*args, **kwargs): + called['range_query'] = True + if rq_none: + return None + class DummyRQ: + def to_dict(self): + return {'RANGE_QUERY': True} + return DummyRQ() + + # Get the modules to be imported + from invenio_resourcesyncserver import query as query_mod + monkeypatch.setattr(query_mod, "range_query", fake_range_query) + monkeypatch.setattr(query_mod, "Indexes", DummyIndexes) + + with app.app_context(): + search = DummySearch() + index_id = 0 if is_root else 1 + result = query_mod.item_changes_search_factory( + search, index_id=index_id, date_from='2020-01-01', date_until=date_until + ) + q = result.query + if expect_range: + assert any( + (isinstance(m, dict) and 'RANGE_QUERY' in m) or + (hasattr(m, 'to_dict') and 'RANGE_QUERY' in m.to_dict()) + for m in q['post_filter']['bool']['must'] + ) + assert called['range_query'] + else: + # If range_query is None, RANGE_QUERY should not be included in must + assert not any( + (isinstance(m, dict) and 'RANGE_QUERY' in m) or + (hasattr(m, 'to_dict') and 'RANGE_QUERY' in m.to_dict()) + for m in q['post_filter']['bool']['must'] + ) + if expect_updated: + assert any( + (isinstance(m, dict) and '_updated' in m.get('range', {})) or + (hasattr(m, 'to_dict') and '_updated' in m.to_dict().get('range', {})) + for m in q['post_filter']['bool']['must'] + ) + +def test_item_changes_search_factory_except_baseexception(monkeypatch): + from invenio_resourcesyncserver import query as query_mod + from flask import Flask + class DummySearch: + def update_from_dict(self, q): + pass + # Mock json.dumps to raise TypeError + monkeypatch.setattr(query_mod.json, "dumps", lambda *a, **k: (_ for _ in ()).throw(TypeError("dummy"))) + # Mock Indexes.get_list_path_publish + monkeypatch.setattr(query_mod.Indexes, "get_list_path_publish", lambda idx: ["dummy"]) + app = Flask(__name__) + app.config["WEKO_ROOT_INDEX"] = 0 + with app.app_context(): + search = DummySearch() + # Confirm that the exception is suppressed and the function returns normally + result = query_mod.item_changes_search_factory(search, index_id=1) + assert result is not None + +def test_item_changes_search_factory_except_syntaxerror(monkeypatch): + from invenio_resourcesyncserver import query as query_mod + from flask import Flask + class DummySearch: + def update_from_dict(self, q): + raise SyntaxError("dummy syntax error") + # Mock Indexes.get_list_path_publish + monkeypatch.setattr(query_mod.Indexes, "get_list_path_publish", lambda idx: ["dummy"]) + app = Flask(__name__) + app.config["WEKO_ROOT_INDEX"] = 0 + with app.app_context(): + search = DummySearch() + with pytest.raises(query_mod.InvalidQueryRESTError): + query_mod.item_changes_search_factory(search, index_id=1) diff --git a/modules/weko-admin/tests/test_utils.py b/modules/weko-admin/tests/test_utils.py index 97d0fa57da..8a8abf686b 100755 --- a/modules/weko-admin/tests/test_utils.py +++ b/modules/weko-admin/tests/test_utils.py @@ -2,7 +2,7 @@ import uuid from mock import patch from datetime import datetime, timedelta -from flask import current_app, Markup +from flask import current_app, Markup, Flask from io import StringIO import pytest import json @@ -1927,7 +1927,7 @@ def test_update_restricted_access(admin_settings,mocker): result = update_restricted_access(data) mock_called.assert_not_called() mock_called.reset_mock() - + data = { "edit_mail_templates_enable": False } @@ -2171,6 +2171,252 @@ def test_create_facet_search_query(facet_search_settings): assert has_permission == test_has_permission assert no_permission == test_no_permission +class DummyFacet: + def __init__(self, name_en, mapping, aggregations): + self.name_en = name_en + self.mapping = mapping + self.aggregations = aggregations + +# .tox/c1/bin/pytest --cov=weko_admin tests/test_utils.py::test_create_aggregations_branch -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp +def test_create_aggregations_branch(mocker): + ACCESS_RIGHTS_QUERY_TEMPLATE = { + "open access": { + "bool": { + "should": [ + {"term": {"accessRights": "open access"}}, + { + "bool": { + "must": [ + {"term": {"accessRights": "embargoed access"}}, + {"nested": {"path": "content", "query": {"exists": {"field": "content.accessrole.raw"}}}}, + { + "bool": { + "must_not": [ + { + "nested": { + "path": "content", + "query": { + "bool": { + "must_not": [ + {"term": {"content.accessrole.raw": "open_access"}}, + { + "bool": { + "must": [ + {"term": {"content.accessrole.raw": "open_date"}}, + {"range": {"content.date.dateValue.raw": {"lte": "@date"}}} + ] + } + } + ] + } + } + } + } + ] + } + } + ] + } + } + ] + } + }, + "embargoed access": { + "bool": { + "must": [ + {"term": {"accessRights": "embargoed access"}}, + { + "bool": { + "should": [ + { + "nested": { + "path": "content", + "query": { + "bool": { + "must": [ + {"term": {"content.accessrole.raw": "open_date"}}, + {"range": {"content.date.dateValue.raw": {"gt": "@date"}}} + ] + } + } + } + }, + { + "bool": { + "must": [ + {"nested": { + "path": "content", + "query": {"term": {"content.accessrole.raw": "open_no"}} + }} + ], + "must_not": [ + {"nested": { + "path": "content", + "query": {"term": {"content.accessrole.raw": "open_login"}} + }} + ] + } + }, + { + "bool": { + "must_not": [ + {"nested": { + "path": "content", + "query": {"exists": {"field": "content.accessrole.raw"}} + }} + ] + } + } + ] + } + } + ], + "must_not": [ + { + "nested": { + "path": "content", + "query": {"term": {"content.accessrole.raw": "open_restricted"}} + } + } + ] + } + }, + "restricted access": { + "bool": { + "should": [ + {"term": {"accessRights": "restricted access"}}, + { + "bool": { + "must": [ + {"term": {"accessRights": "embargoed access"}}, + { + "nested": { + "path": "content", + "query": { + "term": {"content.accessrole.raw": "open_login"} + } + } + }, + { + "bool": { + "must_not": [ + { + "nested": { + "path": "content", + "query": { + "bool": { + "must": [ + {"term": {"content.accessrole.raw": "open_date"}}, + {"range": {"content.date.dateValue.raw": {"gt": "@date"}}} + ] + } + } + } + } + ] + } + } + ] + } + }, + { + "bool": { + "must": [ + {"term": {"accessRights": "embargoed access"}}, + { + "nested": { + "path": "content", + "query": { + "term": {"content.accessrole.raw": "open_restricted"} + } + } + } + ] + } + } + ] + } + }, + "metadata only access": { + "bool": { + "must": [ + {"term": {"accessRights": "metadata only access"}} + ] + } + } + } + # 1. ACCESSRIGHTS_FIX_ENABLED=False + app = Flask(__name__) + app.config["SEARCH_UI_SEARCH_INDEX"] = "testidx" + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = False + mocker.patch("weko_admin.models.FacetSearchSetting.get_activated_facets", return_value=[DummyFacet("type", "type", [])]) + with app.app_context(): + has_permission, no_permission = create_facet_search_query() + aggs = has_permission["testidx"]["aggs"] + assert "new_accessRights" not in aggs + + # 2. ACCESSRIGHTS_FIX_ENABLED=True, no accessRights facet + app = Flask(__name__) + app.config["SEARCH_UI_SEARCH_INDEX"] = "testidx" + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + mocker.patch("weko_admin.models.FacetSearchSetting.get_activated_facets", return_value=[DummyFacet("type", "type", [])]) + with app.app_context(): + has_permission, no_permission = create_facet_search_query() + aggs = has_permission["testidx"]["aggs"] + assert "new_accessRights" not in aggs + + # 3. ACCESSRIGHTS_FIX_ENABLED=True, with accessRights facet + app = Flask(__name__) + app.config["SEARCH_UI_SEARCH_INDEX"] = "testidx" + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + app.config["WEKO_ACCESS_RIGHTS_CHOICES"] = [ + "open access", "embargoed access", "restricted access", "metadata only access" + ] + mocker.patch("weko_admin.models.FacetSearchSetting.get_activated_facets", return_value=[ + DummyFacet("accessRights", "accessRights", [ + {"agg_mapping": "publish_status", "agg_value": 0} + ]) + ]) + with app.app_context(): + has_permission, no_permission = create_facet_search_query() + aggs = has_permission["testidx"]["aggs"] + assert "new_accessRights" in aggs + filters = aggs["new_accessRights"]["filters"]["filters"] + for access_type in app.config["WEKO_ACCESS_RIGHTS_CHOICES"]: + assert access_type in filters + assert "bool" in filters[access_type] + import copy + for access_type in app.config["WEKO_ACCESS_RIGHTS_CHOICES"]: + template = copy.deepcopy(ACCESS_RIGHTS_QUERY_TEMPLATE[access_type]) + actual_bool = filters[access_type]["bool"] + # Replace @date in the template with today's date + def replace_date(obj, today): + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, str) and v == "@date": + obj[k] = today + else: + replace_date(v, today) + elif isinstance(obj, list): + for v in obj: + replace_date(v, today) + today = datetime.now().strftime("%Y-%m-%d") + replace_date(template, today) + # After the publish_status term condition, check if the template structure matches + must_list = actual_bool.get("must", []) + access_query = must_list[2] if len(must_list) > 2 else must_list[-1] + assert access_query == template + + # When facet mapping includes “fields.raw” + app = Flask(__name__) + app.config["SEARCH_UI_SEARCH_INDEX"] = "testidx" + mocker.patch("weko_admin.models.FacetSearchSetting.get_activated_facets", return_value=[ + DummyFacet("raw_test", "test.fields.raw", []) + ]) + with app.app_context(): + has_permission, no_permission = create_facet_search_query() + post_filters = has_permission["testidx"]["post_filters"] + assert post_filters["raw_test"] == "test.raw" # def store_facet_search_query_in_redis(): # .tox/c1/bin/pytest --cov=weko_admin tests/test_utils.py::test_store_facet_search_query_in_redis -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp @@ -2189,7 +2435,12 @@ def test_store_facet_search_query_in_redis(mocker): 'aggs': {'Data Language': {'aggs': {'Data Language': {'terms': {'field': 'language','size': 1000}}}, 'filter': {'bool': {'must': [{'term': {'publish_status': '0'}}]}}}, 'Data Type': {'aggs': {'Data Type': {'terms': {'field': 'description.value','size': 1000}}}, - 'filter': {'bool': {'must': [{'term': {'description.descriptionType': 'Other'}},{'term': {'publish_status': '0'}}]}}}}, + 'filter': {'bool': {'must': [{'term': {'description.descriptionType': 'Other'}},{'term': {'publish_status': '0'}}]}}}, + 'Time Period(s)': {'aggs': {'Time Period(s)': {'terms':{'field': 'temporal','size':1000}}}, + 'filter':{'bool':{'must':[{'term':{'publish_status':'0'}}]}}}, + 'raw_test': {'aggs': {'raw_test': {'terms':{'field': 'fields.raw','size':1000}}}, + 'filter':{'bool':{'must':[{'term':{'publish_status':'0'}}]}}} + }, 'post_filters': {'Data Language': 'language', 'Data Type': 'description.value'}}, } diff --git a/modules/weko-admin/weko_admin/config.py b/modules/weko-admin/weko_admin/config.py index 6cc81bebc2..11d67a15b9 100644 --- a/modules/weko-admin/weko_admin/config.py +++ b/modules/weko-admin/weko_admin/config.py @@ -1113,7 +1113,30 @@ 'mappingName':'', 'item_value':{'1': {'path': {'type': '', 'coordinates': ''}, 'path_type': {'type': 'json', 'coordinates': 'json'}} } - } + }, + { + "id":"accessrights", + "contents":"", + "contents_value":{"en":"Access Rights","ja":"アクセス権"}, + "useable_status":True, + "mapping":[ + "embargoed access", + "metadata only access", + "open access", + "restricted access", + ], + "check_val":[ + {"id":"embargoed access","contents":"embargoed access","checkStus":False}, + {"id":"metadata only access","contents":"metadata only access","checkStus":False}, + {"id":"open access","contents":"open access","checkStus":False}, + {"id":"restricted access","contents":"restricted access","checkStus":False}, + ], + "default_display":True, + "inputType":"checkbox_list", + "inputVal":"", + "mappingFlg":False, + "mappingName":"" + } ] } @@ -1335,7 +1358,7 @@ """Default restricted access settings.""" WEKO_ADMIN_RESTRICTED_ACCESS_MAX_INTEGER = 9999999 -"""max value of expiration_date and download_limit. +"""max value of expiration_date and download_limit. Any more than this and the datetime may overflow. """ WEKO_ADMIN_ITEMS_PER_PAGE_USAGE_REPORT_REMINDER = 25 diff --git a/modules/weko-admin/weko_admin/utils.py b/modules/weko-admin/weko_admin/utils.py index dc9e4a1194..838f1a3fc2 100755 --- a/modules/weko-admin/weko_admin/utils.py +++ b/modules/weko-admin/weko_admin/utils.py @@ -25,6 +25,7 @@ import os import traceback import zipfile +import copy from datetime import datetime, timedelta from io import BytesIO, StringIO from typing import Dict, Optional, Tuple, Union @@ -2198,12 +2199,239 @@ def create_aggregations(facets): val = facet.mapping # Update agg query for has permission. agg_has_permission_query.update( - create_agg_by_aggregations(facet.aggregations, key, val)) + create_agg_by_aggregations(facet.aggregations, key, val) + ) # Update agg query for no permission. facet.aggregations.append( - {'agg_mapping': 'publish_status', 'agg_value': PublishStatus.PUBLIC.value}) + { + 'agg_mapping': 'publish_status', + 'agg_value': PublishStatus.PUBLIC.value + } + ) agg_no_permission_query.update( - create_agg_by_aggregations(facet.aggregations, key, val)) + create_agg_by_aggregations(facet.aggregations, key, val) + ) + # Add aggregation conditions for accessRights + ACCESSRIGHTS_FIX_ENABLED = current_app.config.get( + "WEKO_SEARCH_FIX_ACCESSRIGHTS", False + ) + ACCESS_RIGHTS_CHOICES = current_app.config.get( + "WEKO_ACCESS_RIGHTS_CHOICES", + [ + "open access", + "embargoed access", + "restricted access", + "metadata only access", + ] + ) + ACCESS_RIGHTS_QUERY_TEMPLATE = { + "open access": { + "bool": { + "should": [ + {"term": {"accessRights": "open access"}}, + { + "bool": { + "must": [ + {"term": {"accessRights": "embargoed access"}}, + {"nested": {"path": "content", "query": {"exists": {"field": "content.accessrole.raw"}}}}, + { + "bool": { + "must_not": [ + { + "nested": { + "path": "content", + "query": { + "bool": { + "must_not": [ + {"term": {"content.accessrole.raw": "open_access"}}, + { + "bool": { + "must": [ + {"term": {"content.accessrole.raw": "open_date"}}, + {"range": {"content.date.dateValue.raw": {"lte": "@date"}}} + ] + } + } + ] + } + } + } + } + ] + } + } + ] + } + } + ] + } + }, + "embargoed access": { + "bool": { + "must": [ + {"term": {"accessRights": "embargoed access"}}, + { + "bool": { + "should": [ + { + "nested": { + "path": "content", + "query": { + "bool": { + "must": [ + {"term": {"content.accessrole.raw": "open_date"}}, + {"range": {"content.date.dateValue.raw": {"gt": "@date"}}} + ] + } + } + } + }, + { + "bool": { + "must": [ + {"nested": { + "path": "content", + "query": {"term": {"content.accessrole.raw": "open_no"}} + }} + ], + "must_not": [ + {"nested": { + "path": "content", + "query": {"term": {"content.accessrole.raw": "open_login"}} + }} + ] + } + }, + { + "bool": { + "must_not": [ + {"nested": { + "path": "content", + "query": {"exists": {"field": "content.accessrole.raw"}} + }} + ] + } + } + ] + } + } + ], + "must_not": [ + { + "nested": { + "path": "content", + "query": {"term": {"content.accessrole.raw": "open_restricted"}} + } + } + ] + } + }, + "restricted access": { + "bool": { + "should": [ + {"term": {"accessRights": "restricted access"}}, + { + "bool": { + "must": [ + {"term": {"accessRights": "embargoed access"}}, + { + "nested": { + "path": "content", + "query": { + "term": {"content.accessrole.raw": "open_login"} + } + } + }, + { + "bool": { + "must_not": [ + { + "nested": { + "path": "content", + "query": { + "bool": { + "must": [ + {"term": {"content.accessrole.raw": "open_date"}}, + {"range": {"content.date.dateValue.raw": {"gt": "@date"}}} + ] + } + } + } + } + ] + } + } + ] + } + }, + { + "bool": { + "must": [ + {"term": {"accessRights": "embargoed access"}}, + { + "nested": { + "path": "content", + "query": { + "term": {"content.accessrole.raw": "open_restricted"} + } + } + } + ] + } + } + ] + } + }, + "metadata only access": { + "bool": { + "must": [ + {"term": {"accessRights": "metadata only access"}} + ] + } + } + } + def _replace_date(obj, now): + """Recursively replace @date with current date.""" + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, str) and v == "@date": + obj[k] = now + else: + _replace_date(v, now) + elif isinstance(obj, list): + for v in obj: + _replace_date(v, now) + if ACCESSRIGHTS_FIX_ENABLED: + access_rights = next( + (facet for facet in facets if facet.mapping == "accessRights"), + None + ) + if access_rights: + now = datetime.now().strftime("%Y-%m-%d") + must = [ + dict(term={d["agg_mapping"]: d["agg_value"]}) + for d in access_rights.aggregations + ] + new_access_rights = { + "new_accessRights": { + "filters": { + "filters": {} + } + } + } + for access_type in ACCESS_RIGHTS_CHOICES: + query_template = copy.deepcopy( + ACCESS_RIGHTS_QUERY_TEMPLATE.get(access_type, {}) + ) + _replace_date(query_template, now) + must_copy = copy.deepcopy(must) + if query_template: + must_copy.append(query_template) + new_access_rights["new_accessRights"]["filters"]["filters"][ + access_type + ] = {"bool": {"must": must_copy}} + agg_has_permission_query.update(new_access_rights) + agg_no_permission_query.update(new_access_rights) return agg_has_permission_query, agg_no_permission_query def create_post_filters(facets): @@ -2224,6 +2452,11 @@ def create_post_filters(facets): agg_has_permission, agg_no_permission = create_aggregations( activated_facets) post_filters = create_post_filters(activated_facets) + ACCESSRIGHTS_FIX_ENABLED = current_app.config.get( + "WEKO_SEARCH_FIX_ACCESSRIGHTS", False + ) + if ACCESSRIGHTS_FIX_ENABLED and "new_accessRights" in agg_has_permission: + post_filters["new_accessRights"] = agg_has_permission["new_accessRights"] # Create facet search query for has permission. has_permission_query[search_index] = dict( aggs=agg_has_permission, @@ -2275,6 +2508,8 @@ def get_facet_search_query(has_permission=True): from weko_admin.utils import get_title_facets titles, order, uiTypes, isOpens, displayNumbers, searchConditions = get_title_facets() for k, v in post_filters.items(): + if k == "new_accessRights": + continue if v == 'temporal': # If the mapping name is [template], it is assumed to be a Filter to date_range1. post_filters.update({k: range_filter('date_range1', False, False)}) diff --git a/modules/weko-records/tests/test_utils.py b/modules/weko-records/tests/test_utils.py index f92e6db48e..b781302290 100644 --- a/modules/weko-records/tests/test_utils.py +++ b/modules/weko-records/tests/test_utils.py @@ -7,6 +7,7 @@ import copy from mock import patch, MagicMock from tests.helpers import json_data +import types from invenio_accounts import testutils from weko_admin.models import AdminSettings @@ -58,7 +59,9 @@ replace_fqdn, replace_fqdn_of_file_metadata, get_author_link, - set_file_date) + set_file_date, + check_embargo_rights, + update_embargo_rights) from weko_records.api import ItemTypes, Mapping from weko_records.models import ItemTypeName from weko_workflow.models import Activity @@ -122,7 +125,7 @@ def dumps(self): app.config['WEKO_SCHEMA_JPCOAR_V1_SCHEMA_NAME'] = 'jpcoar_v1_mapping' app.config['WEKO_SCHEMA_DDI_SCHEMA_NAME'] = 'ddi_mapping' dc, jrc, is_edit = json_loader(data3,_pid) - assert dc == OrderedDict([('item_1', {'attribute_name': 'item_1', 'attribute_value': 'item_1_v'}), ('item_2', {'attribute_name': 'item_2', 'attribute_value': ''}), ('item_3', {'attribute_name': 'item_3', 'attribute_type': 'creator', 'attribute_value_mlt': [{'item_3_1': 'item_3_1_v'}]}), ('item_4', {'attribute_name': 'item_4', 'attribute_value_mlt': [{'item_4_1': 'item_4_1_v'}]}), ('item_5', {'attribute_name': 'item_5', 'attribute_type': 'file', 'attribute_value_mlt': [{'filename': 'item_5'}]}), ('item_6', {'attribute_name': 'item_6', 'attribute_value_mlt': [{}]}), ('item_7', {'attribute_name': 'item_7', 'attribute_value_mlt': [{}, {'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '1234'}]}]}), ('item_8', {'attribute_name': 'item_8', 'attribute_value_mlt': [{'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '5678'}]}]}), ('item_title', 'test_item1'), ('item_type_id', '3'), ('control_number', '1'), + assert dc == OrderedDict([('item_1', {'attribute_name': 'item_1', 'attribute_value': 'item_1_v'}), ('item_2', {'attribute_name': 'item_2', 'attribute_value': ''}), ('item_3', {'attribute_name': 'item_3', 'attribute_type': 'creator', 'attribute_value_mlt': [{'item_3_1': 'item_3_1_v'}]}), ('item_4', {'attribute_name': 'item_4', 'attribute_value_mlt': [{'item_4_1': 'item_4_1_v'}]}), ('item_5', {'attribute_name': 'item_5', 'attribute_type': 'file', 'attribute_value_mlt': [{'filename': 'item_5'}]}), ('item_6', {'attribute_name': 'item_6', 'attribute_value_mlt': [{}]}), ('item_7', {'attribute_name': 'item_7', 'attribute_value_mlt': [{}, {'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '1234'}]}]}), ('item_8', {'attribute_name': 'item_8', 'attribute_value_mlt': [{'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '5678'}]}]}), ('item_title', 'test_item1'), ('item_type_id', '3'), ('control_number', '1'), ('author_link', ['1234', '5678']), ('weko_shared_ids', []), ('_oai', {'id': '1'}), ('owner', 1), ('owners', [1])]) assert jrc == {'item_4': ['item_4_1_v'], 'creator1': {'nameIdentifier': ['1234', '5678']}, 'item_5': ['item_5'], 'item_3': ['item_3_1_v'], 'control_number': '1', '_oai': {'id': '1'}, '_item_metadata': OrderedDict([('item_1', {'attribute_name': 'item_1', 'attribute_value': 'item_1_v'}), ('item_2', {'attribute_name': 'item_2', 'attribute_value': ''}), ('item_3', {'attribute_name': 'item_3', 'attribute_type': 'creator', 'attribute_value_mlt': [{'item_3_1': 'item_3_1_v'}]}), ('item_4', {'attribute_name': 'item_4', 'attribute_value_mlt': [{'item_4_1': 'item_4_1_v'}]}), ('item_5', {'attribute_name': 'item_5', 'attribute_type': 'file', 'attribute_value_mlt': [{'filename': 'item_5'}]}), ('item_6', {'attribute_name': 'item_6', 'attribute_value_mlt': [{}]}), ('item_7', {'attribute_name': 'item_7', 'attribute_value_mlt': [{}, {'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '1234'}]}]}), ('item_8', {'attribute_name': 'item_8', 'attribute_value_mlt': [{'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '5678'}]}]}), ('item_title', 'test_item1'), ('item_type_id', '3'), ('control_number', '1'), ('author_link', ['1234', '5678']), ('weko_shared_ids', []), ('_oai', {'id': '1'}), ('owner', 1), ('owners', [1])]), 'itemtype': 'test10', 'publish_date': None, @@ -175,18 +178,18 @@ class MockSM: dc, jrc, is_edit = json_loader(data5, _pid) assert dc == OrderedDict([('item_1', {'attribute_name': 'item_1', 'attribute_value': 'item_1_v'}), ('item_2', {'attribute_name': 'item_2', 'attribute_value': 'item_2_v'}), ('item_3', {'attribute_name': 'item_3', 'attribute_type': 'creator', 'attribute_value_mlt': [{'item_3_1': 'item_3_1_v'}]}), ('item_4', {'attribute_name': 'item_4', 'attribute_value_mlt': [{'item_4_1': 'item_4_1_v'}]}), ('item_5', {'attribute_name': 'item_5', 'attribute_type': 'file', 'attribute_value_mlt': [{'filename': 'item_5'}]}), ('item_6', {'attribute_name': 'item_6', 'attribute_value_mlt': [{'item_6_1': 'item_6_1_v'}]}), ('item_7', {'attribute_name': 'item_7', 'attribute_value_mlt': [{}, {'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '1234'}]}]}), ('item_8', {'attribute_name': 'item_8', 'attribute_value_mlt': [{'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '5678'}]}]}), ('item_title', 'test_item2'), ('item_type_id', '3'), ('control_number', '1'), ('author_link', ['1234', '5678']), ('weko_shared_ids', [2]), ('owner', 5), ('owners', [5])]) - assert jrc == {'item_5': ['item_5'], 'item_4': ['item_4_1_v'], 'creator1': {'nameIdentifier': ['1234', '5678']}, 'item_6': ['item_6_1_v'], 'item_3': ['item_3_1_v'], 'control_number': '1', '_oai': {'id': '1'}, '_item_metadata': - OrderedDict([('item_1', {'attribute_name': 'item_1', 'attribute_value': 'item_1_v'}), - ('item_2', {'attribute_name': 'item_2', 'attribute_value': 'item_2_v'}), + assert jrc == {'item_5': ['item_5'], 'item_4': ['item_4_1_v'], 'creator1': {'nameIdentifier': ['1234', '5678']}, 'item_6': ['item_6_1_v'], 'item_3': ['item_3_1_v'], 'control_number': '1', '_oai': {'id': '1'}, '_item_metadata': + OrderedDict([('item_1', {'attribute_name': 'item_1', 'attribute_value': 'item_1_v'}), + ('item_2', {'attribute_name': 'item_2', 'attribute_value': 'item_2_v'}), ('item_3', {'attribute_name': 'item_3', 'attribute_type': 'creator', 'attribute_value_mlt': [{'item_3_1': 'item_3_1_v'}]}), ('item_4', {'attribute_name': 'item_4', 'attribute_value_mlt': [{'item_4_1': 'item_4_1_v'}]}), ('item_5', {'attribute_name': 'item_5', 'attribute_type': 'file', 'attribute_value_mlt': [{'filename': 'item_5'}]}), ('item_6', {'attribute_name': 'item_6', 'attribute_value_mlt': [{'item_6_1': 'item_6_1_v'}]}), ('item_7', {'attribute_name': 'item_7', 'attribute_value_mlt': [{}, {'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '1234'}]}]}), ('item_8', {'attribute_name': 'item_8', 'attribute_value_mlt': [{'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '5678'}]}]}), - ('item_title', 'test_item2'), ('item_type_id', '3'), ('control_number', '1'), + ('item_title', 'test_item2'), ('item_type_id', '3'), ('control_number', '1'), ('author_link', ['1234', '5678']),('weko_shared_ids', [2]), ('owner', 5), ('owners', [5])]), - 'itemtype': 'test10', 'publish_date': None, + 'itemtype': 'test10', 'publish_date': None, 'author_link': ['1234', '5678'], 'weko_creator_id': '5', 'weko_shared_ids': [2]} assert is_edit == True @@ -836,6 +839,111 @@ def dumps(self): assert jrc == {'weko_creator_id': '1', 'item_5': ['item_5'], 'item_6': ['item_6_1_v'], 'item_3': ['item_3_1_v'], 'item_4': ['item_4_1_v'], 'creator1': {'nameIdentifier': ['1234', '5678']}, 'control_number': '1', '_oai': {'id': '1'}, '_item_metadata': OrderedDict([('item_1', {'attribute_name': 'item_1', 'attribute_value': 'item_1_v'}), ('item_2', {'attribute_name': 'item_2', 'attribute_value': 'item_2_v'}), ('item_3', {'attribute_name': 'item_3', 'attribute_type': 'creator', 'attribute_value_mlt': [{'item_3_1': 'item_3_1_v'}]}), ('item_4', {'attribute_name': 'item_4', 'attribute_value_mlt': [{'item_4_1': 'item_4_1_v'}]}), ('item_5', {'attribute_name': 'item_5', 'attribute_type': 'file', 'attribute_value_mlt': [{'filename': 'item_5'}]}), ('item_6', {'attribute_name': 'item_6', 'attribute_value': ['item_6_1', 'item_6_1_v']}), ('item_7', {'attribute_name': 'item_7', 'attribute_value_mlt': [{}, {'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '1234'}]}]}), ('item_8', {'attribute_name': 'item_8', 'attribute_value_mlt': [{'nameIdentifiers': [{'nameIdentifierScheme': 'WEKO', 'nameIdentifier': '5678'}]}]}), ('item_title', 'test_item2'), ('item_type_id', '3'), ('control_number', '1'), ('author_link', ['1234', '5678']), ('weko_shared_ids', 2), ('_oai', {'id': '1'}), ('owner', 1), ('owners', [1])]), 'itemtype': 'test10', 'publish_date': None, 'author_link': ['1234', '5678'], 'weko_shared_ids': 2} assert is_edit == False +# .tox/c1/bin/pytest --cov=weko_records tests/test_utils.py::test_json_loader_access_rights -v -s -vv --cov-branch --cov-report=html --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp +def test_json_loader_access_rights(app, db, monkeypatch): + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + + minimal_item_data = { + "$schema": "test_schema/1", + "item_type_id": "test", + "item_1": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "open access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + } + minimal_item_data2 = { + "$schema": "test_schema/1", + "item_type_id": "test", + "item_1": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [{}] + }, + } + minimal_item_data3 = { + "$schema": "test_schema/1", + "item_type_id": "test", + "item_1": { + "attribute_name": "アクセス権", + "attribute_value_mlt": "test_data" + }, + } + class DummyPid: + object_uuid = "dummy-uuid" + pid_value = "dummy-pid" + minimal_pid = DummyPid() + + # ItemTypes, Mapping, SchemaTree, PersistentIdentifier, current_pidstore などをモック + from collections import UserDict + class DummyOjson(UserDict): + pass + + dummy_ojson = DummyOjson() + dummy_ojson["properties"] = { + "item_1": { + "type": "array", + "items": {"properties": {}} + } + } + dummy_ojson.model = types.SimpleNamespace() + dummy_ojson.model.item_type_name = types.SimpleNamespace() + dummy_ojson.model.item_type_name.name = "dummy_type" + dummy_mjson = types.SimpleNamespace() + dummy_mjson.dumps = lambda: {"item_1": {}} + dummy_schema_tree = types.SimpleNamespace() + dummy_schema_tree.get_jpcoar_json = lambda jpcoar, replace_field=True: { + "item_1": { + "attribute_value_mlt": [ + {"subitem_access_right": "open access"} + ] + } + } + monkeypatch.setattr("weko_records.utils.ItemTypes", types.SimpleNamespace(get_record=lambda *a, **k: dummy_ojson)) + monkeypatch.setattr("weko_records.utils.Mapping", types.SimpleNamespace(get_record=lambda *a, **k: dummy_mjson)) + monkeypatch.setattr("weko_records.utils.SchemaTree", types.SimpleNamespace(get_jpcoar_json=dummy_schema_tree.get_jpcoar_json)) + monkeypatch.setattr("weko_records.utils.PersistentIdentifier", types.SimpleNamespace(get_by_object=lambda *a, **k: types.SimpleNamespace(pid_value="oaiid"), get=lambda *a, **k: types.SimpleNamespace(object_uuid="dummy-uuid"))) + monkeypatch.setattr("weko_records.utils.current_pidstore", types.SimpleNamespace(minters={"oaiid": lambda *a, **k: types.SimpleNamespace(pid_value="oaiid")})) + monkeypatch.setattr("weko_records.utils.current_user", types.SimpleNamespace(get_id=lambda: "1")) + monkeypatch.setattr("weko_records.utils.COPY_NEW_FIELD", False) + monkeypatch.setattr("weko_records.utils.sm", types.SimpleNamespace(get=lambda: None)) + monkeypatch.setattr("weko_records.utils.ad_config", types.SimpleNamespace(WEKO_ADMIN_MANAGEMENT_OPTIONS={"detail_condition": []})) + + access_path = "item_1.attribute_value_mlt.subitem_access_right" + with app.app_context(): + # When access_path is set (accessRights should be found) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr("weko_records.utils.update_embargo_rights", lambda meta: None) + from weko_records.utils import json_loader + dc, jrc, is_edit = json_loader(minimal_item_data, minimal_pid) + assert "accessRights" in jrc + assert "open access" in jrc["accessRights"] + + # When access_path is not set (accessRights should not exist) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {}) + dc, jrc, is_edit = json_loader(minimal_item_data, minimal_pid) + assert "accessRights" not in jrc + + # When config is False (accessRights should not exist) + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = False + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + dc, jrc, is_edit = json_loader(minimal_item_data, minimal_pid) + assert "accessRights" not in jrc + + # When access_path is set but value cannot be found (accessRights should not exist) + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + dc, jrc, is_edit = json_loader(minimal_item_data2, minimal_pid) + assert "accessRights" not in jrc + + # When access_path is set but value cannot be found (accessRights should not exist) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + dc, jrc, is_edit = json_loader(minimal_item_data3, minimal_pid) + assert "accessRights" not in jrc + # def get_author_link(author_link, value) # .tox/c1/bin/pytest --cov=weko_records tests/test_utils.py::test_get_author_link -v -s -vv --cov-branch --cov-report=term --cov-report=html --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp @@ -2649,3 +2757,361 @@ def test_replace_fqdn_of_file_metadata(app): replace_fqdn_of_file_metadata(_file_metadata_list2) assert _file_metadata_list2==[{'url': {'url': 'https://localhost/a'}, 'version_id': '1'}, {'url': {'url': 'https://localhost/b'}, 'version_id': '1'}] +import datetime +# .tox/c1/bin/pytest --cov=weko_records tests/test_utils.py::test_check_embargo_rights -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp +def test_check_embargo_rights(): + # Do nothing except for 'embargoed access' + result = check_embargo_rights("open_access", datetime.date.today(), []) + assert result == (False, None) + + # If there is at least one 'open_restricted', return 'restricted access' + today = datetime.date.today() + accessrole_date = [("open_restricted", None), ("open_access", None)] + result = check_embargo_rights("embargoed access", today, accessrole_date) + assert result == (True, "restricted access") + + # If there is a future date in 'open_date', do nothing + today = datetime.date.today() + future = today + datetime.timedelta(days=1) + accessrole_date = [("open_date", future)] + result = check_embargo_rights("embargoed access", today, accessrole_date) + assert result == (False, None) + + # If there is at least one 'open_login', return 'restricted access' + today = datetime.date.today() + accessrole_date = [("open_login", None)] + result = check_embargo_rights("embargoed access", today, accessrole_date) + assert result == (True, "restricted access") + + # If all are 'open_access', return 'open access' + today = datetime.date.today() + accessrole_date = [("open_access", None), ("open_access", None)] + result = check_embargo_rights("embargoed access", today, accessrole_date) + assert result == (True, "open access") + + # If all are 'open_date' and the date is in the past, return 'open access' + today = datetime.date.today() + past = today - datetime.timedelta(days=1) + accessrole_date = [("open_date", past), ("open_date", past)] + result = check_embargo_rights("embargoed access", today, accessrole_date) + assert result == (True, "open access") + + # If 'open_access' and 'open_date' (past) are mixed, return 'open access' + today = datetime.date.today() + past = today - datetime.timedelta(days=1) + accessrole_date = [("open_access", None), ("open_date", past)] + result = check_embargo_rights("embargoed access", today, accessrole_date) + assert result == (True, "open access") + + # If accessrole_date is empty, do nothing + today = datetime.date.today() + accessrole_date = [] + result = check_embargo_rights("embargoed access", today, accessrole_date) + assert result == (False, None) + +from unittest.mock import MagicMock +import weko_records.utils as utils + +# .tox/c1/bin/pytest --cov=weko_records tests/test_utils.py::test_update_embargo_rights -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp +def test_update_embargo_rights(app, monkeypatch): + base_meta = { + "item_type_id": "test", + "item_1736146823660": { + "attribute_name": "アクセス権", + "attribute_value_mlt": [ + { + "subitem_access_right": "embargoed access", + "subitem_access_right_uri": "http://purl.org/coar/access_right/c_f1cf" + } + ] + }, + "item_1736148125517": { + "attribute_name": "ファイル情報", + "attribute_type": "file", + "attribute_value_mlt": [ + { + "url": { + "url": "https://weko3.example.org/record/2000088/files/image141.png" + }, + "date": [ + { + "dateType": "Available", + "dateValue": "2026-03-31" + } + ], + "format": "image/png", + "filename": "image141.png", + "filesize": [ + { + "value": "11 KB" + } + ], + "accessrole": "open_date", + "version_id": "55995df6-6d1c-4bbf-8530-d1c0fb5a4219", + } + ] + } + } + + # Embargo is released and access becomes open_access + cfg = True + meta = copy.deepcopy(base_meta) + access_path = "item_1736146823660.attribute_value_mlt.subitem_access_right" + access_right_type_uri = { + "embargoed access": "http://purl.org/coar/access_right/c_f1cf", + "metadata only access": "http://purl.org/coar/access_right/c_14cb", + "open access": "http://purl.org/coar/access_right/c_abf2", + "restricted access": "http://purl.org/coar/access_right/c_16ec", + } + monkeypatch.setattr( + utils, + "current_app", + MagicMock(config={ + "WEKO_SEARCH_FIX_ACCESSRIGHTS": cfg, + "ACCESS_RIGHT_TYPE_URI": access_right_type_uri, + }) + ) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (True, "open access")) + utils.update_embargo_rights(meta) + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right"] == "open access" + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right_uri"] == "http://purl.org/coar/access_right/c_abf2" + + # Embargo is released and access becomes restricted_access + meta = copy.deepcopy(base_meta) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (True, "restricted access")) + utils.update_embargo_rights(meta) + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right"] == "restricted access" + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right_uri"] == "http://purl.org/coar/access_right/c_16ec" + + # Embargo is not released (returns None), access remains embargoed + meta = copy.deepcopy(base_meta) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (False, None)) + utils.update_embargo_rights(meta) + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right"] == "embargoed access" + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right_uri"] == "http://purl.org/coar/access_right/c_f1cf" + + # File information with accessrole=None is handled correctly + meta = copy.deepcopy(base_meta) + meta["item_1736146823660"]["attribute_value_mlt"][0]["accessrole"] = None + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (False, None)) + utils.update_embargo_rights(meta) + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right"] == "embargoed access" + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right_uri"] == "http://purl.org/coar/access_right/c_f1cf" + + # Access rights information with None is handled correctly + meta = copy.deepcopy(base_meta) + meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right"] = None + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (False, None)) + utils.update_embargo_rights(meta) + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right"] == None + + # Empty meta dict does not change anything + meta = {} + utils.update_embargo_rights(meta) + assert "subitem_access_right" not in meta + + # No change if access_path does not exist in mapping + meta = copy.deepcopy(base_meta) + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {}) # accessRights.@valueなし + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (True, "open access")) + utils.update_embargo_rights(meta) + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right"] == "embargoed access" + + # No change if WEKO_SEARCH_FIX_ACCESSRIGHTS is False + meta = copy.deepcopy(base_meta) + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": False})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (True, "open access")) + utils.update_embargo_rights(meta) + assert meta["item_1736146823660"]["attribute_value_mlt"][0]["subitem_access_right"] == "embargoed access" + + # Only the second element in attribute_value_mlt list is the target key + meta = { + "item_type_id": "test", + "item_1": { + "attribute_value_mlt": [ + {"dummy": 1}, + {"subitem_access_right": "embargoed access"} + ] + } + } + access_path = "item_1.attribute_value_mlt.subitem_access_right" + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (True, "open access")) + utils.update_embargo_rights(meta) + assert meta["item_1"]["attribute_value_mlt"][1]["subitem_access_right"] == "open access" + + # Deep nesting in attribute_value_mlt is handled correctly + meta = { + "item_type_id": "test", + "item_1": { + "attribute_value_mlt": [ + {"attribute_value_mlt": [ + {"subitem_access_right": "embargoed access"} + ]} + ] + } + } + access_path = "item_1.attribute_value_mlt.attribute_value_mlt.subitem_access_right" + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (False, None)) + utils.update_embargo_rights(meta) + assert meta["item_1"]["attribute_value_mlt"][0]["attribute_value_mlt"][0]["subitem_access_right"] == "embargoed access" + + # Tests recursive list branch in _set_nested_value; all nested rights are updated. + meta = { + "item_type_id": "test", + "item_1": { + "attribute_value_mlt": [ + {"attribute_value_mlt": [ + {"subitem_access_right": "embargoed access"}, + {"subitem_access_right": "embargoed access"} + ]}, + {"attribute_value_mlt": [ + {"subitem_access_right": "embargoed access"} + ]} + ] + } + } + access_path = "item_1.attribute_value_mlt.attribute_value_mlt.subitem_access_right" + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (True, "open access")) + utils.update_embargo_rights(meta) + for item in meta["item_1"]["attribute_value_mlt"]: + for subitem in item["attribute_value_mlt"]: + assert subitem["subitem_access_right"] == "open access" + + # File information date (dateValue) is None + meta = { + "item_type_id": "test", + "item_1": { + "attribute_type": "file", + "attribute_value_mlt": [ + { + "date": [ + {"dateType": "Available", "dateValue": None} + ], + "accessrole": "open_date", + "subitem_access_right": "embargoed access" + } + ] + } + } + access_path = "item_1.attribute_value_mlt.subitem_access_right" + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (False, None)) + utils.update_embargo_rights(meta) + assert meta["item_1"]["attribute_value_mlt"][0]["subitem_access_right"] == "embargoed access" + + # File information date (dateValue) is invalid format + meta = { + "item_type_id": "test", + "item_1": { + "attribute_type": "file", + "attribute_value_mlt": [ + { + "date": [ + {"dateType": "Available", "dateValue": "test_invalid"} + ], + "accessrole": "open_date", + "subitem_access_right": "embargoed access" + } + ] + } + } + access_path = "item_1.attribute_value_mlt.subitem_access_right" + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (False, None)) + utils.update_embargo_rights(meta) + assert meta["item_1"]["attribute_value_mlt"][0]["subitem_access_right"] == "embargoed access" + + # Update value if mapping path points to a key inside attribute_value_mlt list element, even if not directly in the dict. + meta = { + "item_type_id": "test", + "item_1": { + "attribute_value_mlt": [ + {"subitem_access_right": "embargoed access"} + ] + } + } + access_path = "item_1.subitem_access_right" + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (True, "open access")) + utils.update_embargo_rights(meta) + assert meta["item_1"]["attribute_value_mlt"][0]["subitem_access_right"] == "open access" + + # Test: Do nothing if mapping path points to a key that does not exist in any attribute_value_mlt list element. + meta = { + "item_type_id": "test", + "item_1": { + "attribute_value_mlt": [ + {"dummy": 1} + ] + } + } + access_path = "item_1.subitem_access_right" + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (False, None)) + utils.update_embargo_rights(meta) + assert "subitem_access_right" not in meta["item_1"]["attribute_value_mlt"][0] + + # If mapping points to a non-dict/list (e.g. int), no error and value stays unchanged. + class DummyMapping(dict): + def get(self, key, default=None): + if key == "accessRights.@value": + return "item1.item2.item3" + return super().get(key, default) + def dummy_get_mapping(item_type_id, mapping_type): + return DummyMapping({"accessRights.@value": "item1.item2.item3"}) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", dummy_get_mapping) + with app.app_context(): + import flask + flask.current_app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + metadata = { + "item_type_id": 1, + "item1": {"item2": 12345} + } + update_embargo_rights(metadata) + assert metadata["item1"]["item2"] == 12345 + + # access_path points to a nested key inside a list + meta = { + "item_type_id": "test", + "item_1": { + "attribute_value_mlt": [ + {"subitem_access_right": "embargoed access"}, + {"subitem_access_right": "embargoed access"} + ] + } + } + access_path = "item_1.attribute_value_mlt.subitem_access_right" + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (True, "open access")) + utils.update_embargo_rights(meta) + # Both elements in the list should be updated + assert meta["item_1"]["attribute_value_mlt"][0]["subitem_access_right"] == "open access" + assert meta["item_1"]["attribute_value_mlt"][1]["subitem_access_right"] == "open access" + + # When the list is empty + meta = { + "item_type_id": "test", + "item_1": { + "attribute_value_mlt": [] + } + } + access_path = "item_1.attribute_value_mlt.subitem_access_right" + monkeypatch.setattr(utils, "current_app", MagicMock(config={"WEKO_SEARCH_FIX_ACCESSRIGHTS": True})) + monkeypatch.setattr("weko_records.serializers.utils.get_mapping", lambda i, t: {"accessRights.@value": access_path}) + monkeypatch.setattr(utils, "check_embargo_rights", lambda a, t, d: (True, "open access")) + utils.update_embargo_rights(meta) + assert meta["item_1"]["attribute_value_mlt"] == [] diff --git a/modules/weko-records/weko_records/utils.py b/modules/weko-records/weko_records/utils.py index d78c5978e6..47493cc89b 100644 --- a/modules/weko-records/weko_records/utils.py +++ b/modules/weko-records/weko_records/utils.py @@ -280,6 +280,42 @@ def _set_shared_ids(data): dc.update(dict(owner=int(owner_id))) dc.update(dict(owners=[int(owner_id)])) + if current_app.config.get("WEKO_SEARCH_FIX_ACCESSRIGHTS", False): + update_embargo_rights(jrc["_item_metadata"]) + + from weko_records.serializers.utils import get_mapping + mapping = get_mapping(item_type_id, "jpcoar_mapping") + access_path = mapping.get("accessRights.@value") + def _get_nested_value(data, path): + keys = path.split('.') + key = keys[0] + rest = '.'.join(keys[1:]) + if isinstance(data, dict): + if key in data: + if rest: + return _get_nested_value(data[key], rest) + else: + return data[key] + elif 'attribute_value_mlt' in data: + for item in data['attribute_value_mlt']: + found = _get_nested_value(item, '.'.join(keys)) + if found is not None: + return found + return None + else: + return None + elif isinstance(data, list): + for item in data: + found = _get_nested_value(item, '.'.join(keys)) + if found is not None: + return found + return None + else: + return None + access_right = _get_nested_value(jrc["_item_metadata"], access_path) if access_path else None + if access_right: + jrc["accessRights"] = [access_right] + del ojson, mjson, item return dc, jrc, is_edit @@ -2864,3 +2900,173 @@ def replace_fqdn_of_file_metadata(file_metadata_lst: list, file_url: list = None file["url"]["url"] = replace_fqdn(file["url"]["url"]) elif isinstance(file_url, list): file_url.append(file["url"]["url"]) + +def check_embargo_rights(access_right: str, today, accessrole_date: list = []): + """ + Determines whether the accessrights value needs to be updated based on + the mapped item values and file information, and what value it should be + changed to. + Args: + access_right (str): The value mapped to the item's accessRight. + today (date): The current date. + accessrole_date (list): List of (accessrole, date) tuples for + registered files. + Returns: + is_update_required (bool): Whether an update is required. + change_value (str): The value after the update. + """ + # Do nothing if not 'embargoed access' + if access_right != "embargoed access": + return False, None + + # 1. If there is at least one 'open_restricted', set to 'restricted access' + if any( + role == "open_restricted" for role, _ in accessrole_date + ): + return True, "restricted access" + + # 2. If there is at least one 'open_date' with a future date, no update required + if any( + role == "open_date" and date and today and date > today + for role, date in accessrole_date + ): + return False, None + + # 3. If there is at least one 'open_login', set to 'restricted access' + if any( + role == "open_login" for role, _ in accessrole_date + ): + return True, "restricted access" + + # 4. If all are 'open_access' or 'open_date' with date <= today, + # set to 'open access' + if accessrole_date and all( + (role == "open_access") or + (role == "open_date" and date and today and date <= today) + for role, date in accessrole_date + ): + return True, "open access" + + return False, None + +def update_embargo_rights(metadata: dict) -> None: + """ + Update accessrights value in item metadata in-place. + Args: + metadata (dict): Item metadata to update. + Returns: + None (modifies metadata in-place) + """ + # Skip if config disables accessrights fix + if not current_app.config.get("WEKO_SEARCH_FIX_ACCESSRIGHTS", False): + return + + item_type_id = metadata.get("item_type_id") + if not item_type_id: + return + + from weko_records.serializers.utils import get_mapping + mapping = get_mapping(item_type_id, "jpcoar_mapping") + access_path = mapping.get("accessRights.@value") + if not access_path: + return + + if access_path.endswith("subitem_access_right"): + access_uri_path = ( + access_path[:-len("subitem_access_right")] + + "subitem_access_right_uri" + ) + else: + access_uri_path = None + + def _get_nested_value(data, path): + keys = path.split('.') + key = keys[0] + rest = '.'.join(keys[1:]) + if isinstance(data, dict): + if key in data: + if rest: + return _get_nested_value(data[key], rest) + else: + return data[key] + elif 'attribute_value_mlt' in data: + for item in data['attribute_value_mlt']: + found = _get_nested_value(item, '.'.join(keys)) + if found is not None: + return found + return None + else: + return None + elif isinstance(data, list): + for item in data: + found = _get_nested_value(item, '.'.join(keys)) + if found is not None: + return found + return None + else: + return None + + access_right_value = _get_nested_value(metadata, access_path) + if not access_right_value: + return + + from datetime import datetime + accessrole_date = [] + today = datetime.now().date() + + for v in metadata.values(): + if ( + isinstance(v, dict) and v.get("attribute_type") == "file" + ): + mlt = v.get("attribute_value_mlt", []) + for data in mlt: + date_val = None + accessrole_val = data.get("accessrole") + if ( + "date" in data and + isinstance(data["date"], list) and + data["date"] + ): + date_val = data["date"][0].get("dateValue") + if date_val: + if re.match(r"^\d{4}-\d{2}-\d{2}$", date_val): + date_val = datetime.strptime(date_val, "%Y-%m-%d").date() + else: + date_val = None + if accessrole_val: + accessrole_date.append((accessrole_val, date_val)) + + from .utils import check_embargo_rights + is_update, change_value = check_embargo_rights( + access_right_value, today, accessrole_date + ) + + access_right_type_uri = current_app.config.get("ACCESS_RIGHT_TYPE_URI", {}) + access_right_type_uri_value = access_right_type_uri.get(change_value, "") + + def _set_nested_value(data, path, value): + keys = path.split('.') + key = keys[0] + rest = '.'.join(keys[1:]) + if isinstance(data, list): + for item in data: + _set_nested_value(item, path, value) + return + if len(keys) == 1: + if isinstance(data, dict) and key in data: + data[key] = value + if isinstance(data, dict) and 'attribute_value_mlt' in data: + for item in data['attribute_value_mlt']: + _set_nested_value(item, key, value) + else: + if isinstance(data, dict): + if key in data: + _set_nested_value(data[key], rest, value) + if 'attribute_value_mlt' in data: + for item in data['attribute_value_mlt']: + _set_nested_value(item, rest, value) + + if is_update and change_value: + _set_nested_value(metadata, access_path, change_value) + if access_uri_path and access_right_type_uri_value: + _set_nested_value(metadata, access_uri_path, access_right_type_uri_value) diff --git a/modules/weko-search-ui/tests/test_query.py b/modules/weko-search-ui/tests/test_query.py index 34c8e11d1f..431d1b4da4 100644 --- a/modules/weko-search-ui/tests/test_query.py +++ b/modules/weko-search-ui/tests/test_query.py @@ -1,5 +1,7 @@ import json import copy + +import pytest from elasticsearch_dsl.query import Match, Range, Terms, Bool from mock import patch, MagicMock from werkzeug.datastructures import MultiDict @@ -947,3 +949,51 @@ def test_split_text_by_or(): assert _split_text_by_or("AAA OR　BBB | CCC") == ["AAA", "BBB", "CCC"] assert _split_text_by_or("AAA OR OR BBB") == ["AAA", "", "BBB"] assert _split_text_by_or("OR AAA |") == ["OR AAA |"] + +@pytest.mark.parametrize("fix_accessrights, accessrights, expected_key", [ + (True, "", None), # empty + (True, "open access", "open access"), # single value + (True, "embargoed access", "embargoed access"), + (True, "restricted access", "restricted access"), + (True, "metadata only access", "metadata only access"), + (True, "open access,embargoed access", "open access"), # multiple values + (True, "invalid access", None), # invalid value + (False, "open access", "open access"), # config disabled +]) +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_query.py::test_accessrights_query_param -vv -s --cov-branch --cov-report=xml --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp +def test_accessrights_query_param(app, users, fix_accessrights, accessrights, expected_key, mocker): + ACCESS_RIGHTS_CHOICES = [ + 'embargoed access', + 'metadata only access', + 'open access', + 'restricted access', + ] + _data = { + 'search_type': '0', + 'accessrights': accessrights, + 'sort': 'controlnumber', + 'page': '1', + 'size': '20', + } + with app.test_client() as client: + login_user_via_session(client, email=users[3]["email"]) + search = RecordsSearch() + + with app.test_request_context(headers=[('Accept-Language','en')], data=_data): + app.config['WEKO_SEARCH_KEYWORDS_DICT'] = WEKO_SEARCH_KEYWORDS_DICT + app.config['WEKO_ADMIN_MANAGEMENT_OPTIONS'] = WEKO_ADMIN_MANAGEMENT_OPTIONS + app.config['WEKO_SEARCH_FIX_ACCESSRIGHTS'] = fix_accessrights + app.config['WEKO_ACCESS_RIGHTS_CHOICES'] = ACCESS_RIGHTS_CHOICES + mocker.patch("weko_search_ui.query.search_permission",side_effect=MockSearchPerm) + mocker.patch("weko_search_ui.permissions.search_permission",side_effect=MockSearchPerm) + app.extensions['invenio-oauth2server'] = 1 + app.extensions['invenio-queues'] = 1 + search_result, urlkwargs = default_search_factory(self=None, search=search) + query = search_result.query().to_dict() + must_result = query["query"]["bool"]["filter"][0]["bool"]["must"] + if expected_key: + assert "accessRights" in str(must_result) + assert "accessRights" in str(urlkwargs) + else: + assert "accessRights" not in str(must_result) + assert "accessRights" not in str(urlkwargs) diff --git a/modules/weko-search-ui/tests/test_utils.py b/modules/weko-search-ui/tests/test_utils.py index d8209889c5..1018cbd81d 100644 --- a/modules/weko-search-ui/tests/test_utils.py +++ b/modules/weko-search-ui/tests/test_utils.py @@ -44,7 +44,6 @@ from weko_workflow.headless.activity import HeadlessActivity from weko_workflow.models import Activity, WorkFlow - from weko_search_ui.config import ( ACCESS_RIGHT_TYPE_URI, RESOURCE_TYPE_URI, @@ -76,6 +75,7 @@ delete_records, execute_search_with_pagination, export_all, + fix_aggregations_accessrights, get_retry_info, generate_metadata_from_jpcoar, get_change_identifier_mode_content, @@ -6216,3 +6216,69 @@ def test_none_scheme(self, mock_settings, app, db): handle_check_authors_affiliation(list_record) assert "errors" not in list_record[0] + +# .tox/c1/bin/pytest --cov=weko_search_ui tests/test_utils.py::test_fix_aggregations_accessrights -v -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-search-ui/.tox/c1/tmp +def test_fix_aggregations_accessrights(app, mocker): + # 1. When the flag is OFF, the aggregation result is returned as is. + with app.app_context(): + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = False + app.config["WEKO_ACCESS_RIGHTS_CHOICES"] = ["embargoed access", "metadata only access", "open access", "restricted access"] + mocker.patch("weko_search_ui.utils.FacetSearchSetting.get_activated_facets_mapping", return_value={"access": "accessRights"}) + aggs = {"access": {"buckets": []}} + data = {"aggregations": aggs.copy()} + result = fix_aggregations_accessrights(data.copy()) + assert result["aggregations"] == aggs # No change when the flag is OFF + + # 2. If the mapping is not accessRights, new_accessRights remains. + with app.app_context(): + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + app.config["WEKO_ACCESS_RIGHTS_CHOICES"] = ["embargoed access", "metadata only access", "open access", "restricted access"] + mocker.patch("weko_search_ui.utils.FacetSearchSetting.get_activated_facets_mapping", return_value={"other": "notAccessRights"}) + aggs = {"access": {"buckets": []}, "new_accessRights": {"buckets": {}}} + data = {"aggregations": aggs.copy()} + result = fix_aggregations_accessrights(data.copy()) + assert "new_accessRights" in result["aggregations"] # new_accessRights remains if mapping does not match + assert result["aggregations"]["access"] == {"buckets": []} + + # 3. If new_accessRights does not exist, return as is. + with app.app_context(): + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + app.config["WEKO_ACCESS_RIGHTS_CHOICES"] = ["embargoed access", "metadata only access", "open access", "restricted access"] + mocker.patch("weko_search_ui.utils.FacetSearchSetting.get_activated_facets_mapping", return_value={"access": "accessRights"}) + aggs = {"access": {"buckets": []}} + data = {"aggregations": aggs.copy()} + result = fix_aggregations_accessrights(data.copy()) + assert result["aggregations"] == aggs # No change if new_accessRights does not exist + + # 4. If ACCESS_RIGHTS_CHOICES are not in new_accessRights buckets, new_accessRights remains. + with app.app_context(): + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + app.config["WEKO_ACCESS_RIGHTS_CHOICES"] = ["embargoed access", "metadata only access", "open access", "restricted access"] + mocker.patch("weko_search_ui.utils.FacetSearchSetting.get_activated_facets_mapping", return_value={"access": "accessRights"}) + aggs = {"access": {"buckets": []}, "new_accessRights": {"buckets": {}}} + data = {"aggregations": aggs.copy()} + result = fix_aggregations_accessrights(data.copy()) + assert "new_accessRights" in result["aggregations"] # new_accessRights remains if ACCESS_RIGHTS_CHOICES not in buckets + assert result["aggregations"]["access"] == {"buckets": []} + + # 5. If doc_count is 0, new_accessRights remains. + with app.app_context(): + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + app.config["WEKO_ACCESS_RIGHTS_CHOICES"] = ["embargoed access", "metadata only access", "open access", "restricted access"] + mocker.patch("weko_search_ui.utils.FacetSearchSetting.get_activated_facets_mapping", return_value={"access": "accessRights"}) + aggs = {"access": {"buckets": []}, "new_accessRights": {"buckets": {"open access": {"doc_count": 0}}}} + data = {"aggregations": aggs.copy()} + result = fix_aggregations_accessrights(data.copy()) + assert "new_accessRights" in result["aggregations"] # new_accessRights remains if doc_count is 0 + assert result["aggregations"]["access"] == {"buckets": []} + + # 6. Normal case: If doc_count > 0, accessrights are set in buckets and new_accessRights is removed. + with app.app_context(): + app.config["WEKO_SEARCH_FIX_ACCESSRIGHTS"] = True + app.config["WEKO_ACCESS_RIGHTS_CHOICES"] = ["embargoed access", "metadata only access", "open access", "restricted access"] + mocker.patch("weko_search_ui.utils.FacetSearchSetting.get_activated_facets_mapping", return_value={"access": "accessRights"}) + aggs = {"access": {"buckets": []}, "new_accessRights": {"buckets": {"open access": {"doc_count": 2}, "embargoed access": {"doc_count": 1}}}} + data = {"aggregations": aggs.copy()} + result = fix_aggregations_accessrights(data.copy()) + assert "new_accessRights" not in result["aggregations"] # new_accessRights is removed in normal case + assert result["aggregations"]["access"] == {"buckets": [{"key": "embargoed access", "doc_count": 1}, {"key": "open access", "doc_count": 2}]} diff --git a/modules/weko-search-ui/weko_search_ui/config.py b/modules/weko-search-ui/weko_search_ui/config.py index 6869fca640..d200c02707 100644 --- a/modules/weko-search-ui/weko_search_ui/config.py +++ b/modules/weko-search-ui/weko_search_ui/config.py @@ -780,3 +780,8 @@ ROCRATE_METADATA_WK_CONTEXT_V1 = "http://purl.org/wk/v1/wk-context.jsonld" """ Metadata context file name for RO-Crate+Bagit. """ + +WEKO_ACCESS_RIGHTS_CHOICES = [ "embargoed access", "metadata only access", "open access", "restricted access"] + +WEKO_SEARCH_FIX_ACCESSRIGHTS = False +""" If true, the value of accessrights will be modified. """ diff --git a/modules/weko-search-ui/weko_search_ui/query.py b/modules/weko-search-ui/weko_search_ui/query.py index 678a66d8e7..fac34f7bea 100644 --- a/modules/weko-search-ui/weko_search_ui/query.py +++ b/modules/weko-search-ui/weko_search_ui/query.py @@ -603,6 +603,139 @@ def _get_geo_shape_query(k, v, params): return qry + def __get_accessrights_query(params): + """ + Build accessrights search query from request params. + Args: + params (dict): request parameters + Returns: + dict: accessrights search query + """ + weko_search_fix_accessrights = current_app.config.get( + 'WEKO_SEARCH_FIX_ACCESSRIGHTS', False + ) + + accessrights_value = params.get('accessrights') + if not accessrights_value: + return None + + accessrights_list = [v.strip() for v in accessrights_value.split(',') if v.strip()] + weko_access_rights_choices = current_app.config.get( + 'WEKO_ACCESS_RIGHTS_CHOICES', [ + 'embargoed access', + 'metadata only access', + 'open access', + 'restricted access', + ] + ) + accessrights_list = [ + v for v in accessrights_list if v in weko_access_rights_choices + ] + if not accessrights_list: + return None + + if not weko_search_fix_accessrights: + if len(accessrights_list) == 1: + return Q('term', accessRights=accessrights_list[0]) + else: + return Q('terms', accessRights=accessrights_list) + + now = datetime.now().isoformat() + + def open_access_query(now): + """Query for open access.""" + return Q( + 'bool', + should=[ + Q('term', accessRights='open access'), + Q('bool', must=[ + Q('term', accessRights='embargoed access'), + Q('nested', path='content', query=Q('exists', field='content.accessrole.raw')), + Q('bool', must_not=[ + Q('nested', path='content', query=Q('bool', must_not=[ + Q('term', **{'content.accessrole.raw': 'open_access'}), + Q('bool', must=[ + Q('term', **{'content.accessrole.raw': 'open_date'}), + Q('range', **{'content.date.dateValue.raw': {'lte': now}}) + ]) + ])) + ]) + ]) + ] + ) + + def embargoed_access_query(now): + """Query for embargoed access.""" + return Q( + 'bool', + must=[ + Q('term', accessRights='embargoed access'), + Q('bool', should=[ + Q('nested', path='content', query=Q('bool', must=[ + Q('term', **{'content.accessrole.raw': 'open_date'}), + Q('range', **{'content.date.dateValue.raw': {'gt': now}}) + ])), + Q('bool', must=[ + Q('nested', path='content', query=Q('term', **{'content.accessrole.raw': 'open_no'})) + ], must_not=[ + Q('nested', path='content', query=Q('term', **{'content.accessrole.raw': 'open_login'})) + ]), + Q('bool', must_not=[ + Q('nested', path='content', query=Q('exists', field='content.accessrole.raw')) + ]) + ]) + ], + must_not=[ + Q('nested', path='content', query=Q('term', **{'content.accessrole.raw': 'open_restricted'})), + ] + ) + + def restricted_access_query(now): + """Query for restricted access.""" + return Q( + 'bool', + should=[ + Q('term', accessRights='restricted access'), + Q('bool', must=[ + Q('term', accessRights='embargoed access'), + Q( + 'nested', path='content', + query=Q('term', **{'content.accessrole.raw': 'open_login'}) + ), + Q('bool', must_not=[ + Q('nested', path='content', query=Q('bool', must=[ + Q('term', **{'content.accessrole.raw': 'open_date'}), + Q('range', **{'content.date.dateValue.raw': {'gt': now}}) + ])) + ]) + ]), + Q('bool', must=[ + Q('term', accessRights='embargoed access'), + Q('nested', path='content', query=Q('term', **{'content.accessrole.raw': 'open_restricted'})) + ]) + ] + ) + + def metadata_only_query(): + """Query for metadata only access.""" + return Q('term', accessRights='metadata only access') + + queries = [] + for accessright in accessrights_list: + if accessright == 'open access': + queries.append(open_access_query(now)) + elif accessright == 'embargoed access': + queries.append(embargoed_access_query(now)) + elif accessright == 'restricted access': + queries.append(restricted_access_query(now)) + elif accessright == 'metadata only access': + queries.append(metadata_only_query()) + + queries = [q for q in queries if q is not None] + if len(queries) == 1: + return queries[0] + return Q('bool', should=queries, minimum_should_match=1) + params = request.values.to_dict() if additional_params: params.update(additional_params) @@ -665,6 +798,10 @@ def _get_geo_shape_query(k, v, params): if qy: mut.append(qy) + accessrights_q = __get_accessrights_query(params) + if accessrights_q: + mut.append(accessrights_q) + except Exception as e: current_app.logger.exception( "Detail search query parser failed. err:{0}".format(e) @@ -1381,4 +1518,4 @@ def _split_text_by_or(text): pattern = r'(?<= )(?:OR|\|)(?= )' split_text_list = re.split(pattern, text) split_text_list = [item.strip() for item in split_text_list] - return split_text_list \ No newline at end of file + return split_text_list diff --git a/modules/weko-search-ui/weko_search_ui/utils.py b/modules/weko-search-ui/weko_search_ui/utils.py index 3c02cde998..6e021b6da9 100644 --- a/modules/weko-search-ui/weko_search_ui/utils.py +++ b/modules/weko-search-ui/weko_search_ui/utils.py @@ -73,7 +73,7 @@ from sqlalchemy import func as _func from sqlalchemy.exc import SQLAlchemyError -from weko_admin.models import AdminSettings, SessionLifetime +from weko_admin.models import AdminSettings, SessionLifetime, FacetSearchSetting from weko_admin.utils import get_redis_cache, reset_redis_cache, get_restricted_access from weko_admin.api import TempDirInfo from weko_authors.models import AuthorsAffiliationSettings, AuthorsPrefixSettings @@ -6149,3 +6149,59 @@ def check_provide_in_system(key, item): is_provide_exist[idx] = False break return all(is_provide_exist) + +def fix_aggregations_accessrights(data): + """ + Refactor accessrights aggregation in search result for compliance. + Args: + data (dict): Aggregation result from search response. + Returns: + dict: Modified aggregation result. + """ + ACCESSRIGHTS_FIX_ENABLED = current_app.config.get( + "WEKO_SEARCH_FIX_ACCESSRIGHTS", False + ) + ACCESS_RIGHTS_CHOICES = current_app.config.get( + "WEKO_ACCESS_RIGHTS_CHOICES", + [ + "embargoed access", + "metadata only access", + "open access", + "restricted access", + ] + ) + aggs = data['aggregations'] + if not ACCESSRIGHTS_FIX_ENABLED: + return data + + # Get mapping for accessRights facets + mapping = FacetSearchSetting.get_activated_facets_mapping() + accessrights_keys = [ + k for k, v in mapping.items() if v == "accessRights" + ] + if "new_accessRights" not in aggs or not accessrights_keys: + return data + + new_accessrights = aggs["new_accessRights"] + buckets_dict = new_accessrights.get("buckets", {}) + if not any( + right in buckets_dict for right in ACCESS_RIGHTS_CHOICES + ): + return data + + buckets = [] + for right in ACCESS_RIGHTS_CHOICES: + value = buckets_dict.get(right) + if not value: + continue + doc_count = value.get("doc_count", 0) + if doc_count == 0: + continue + buckets.append({"key": right, "doc_count": doc_count}) + if buckets: + for key in accessrights_keys: + if key in aggs: + aggs[key]["buckets"] = buckets + aggs.pop("new_accessRights", None) + data['aggregations'] = aggs + return data diff --git a/scripts/demo/search_management_update.sql b/scripts/demo/search_management_update.sql new file mode 100644 index 0000000000..242daad44c --- /dev/null +++ b/scripts/demo/search_management_update.sql @@ -0,0 +1,44 @@ +DO $$ +DECLARE + rec RECORD; + accessrights_json jsonb := '{ + "id":"accessrights", + "contents":"", + "contents_value":{"en":"Access Rights","ja":"アクセス権"}, + "useable_status":true, + "mapping":["embargoed access","metadata only access","open access","restricted access"], + "check_val":[ + {"id":"embargoed access","contents":"embargoed access","checkStus":false}, + {"id":"metadata only access","contents":"metadata only access","checkStus":false}, + {"id":"open access","contents":"open access","checkStus":false}, + {"id":"restricted access","contents":"restricted access","checkStus":false} + ], + "default_display":true, + "inputType":"checkbox_list", + "inputVal":"", + "mappingFlg":false, + "mappingName":"" + }'::jsonb; +BEGIN + FOR rec IN SELECT id, search_conditions, search_setting_all FROM search_management LOOP + -- search_conditions + IF rec.search_conditions IS NOT NULL THEN + IF NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(rec.search_conditions) elem + WHERE elem->>'id' = 'accessrights' + ) THEN + UPDATE search_management SET search_conditions = rec.search_conditions || accessrights_json WHERE id = rec.id; + END IF; + END IF; + -- search_setting_all + IF rec.search_setting_all IS NOT NULL AND rec.search_setting_all->'detail_condition' IS NOT NULL THEN + IF NOT (rec.search_setting_all->'detail_condition')::jsonb @> '[{"id": "accessrights"}]'::jsonb THEN + UPDATE search_management SET search_setting_all = jsonb_set( + rec.search_setting_all, + '{detail_condition}', + (rec.search_setting_all->'detail_condition')::jsonb || accessrights_json + ) WHERE id = rec.id; + END IF; + END IF; + END LOOP; +END $$; diff --git a/scripts/demo/update_search_management.sh b/scripts/demo/update_search_management.sh new file mode 100644 index 0000000000..6fc11c208a --- /dev/null +++ b/scripts/demo/update_search_management.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# This script executes SQL only if records exist in the search_management table. + +## Database connection information +DB_USER="invenio" +DB_NAME="invenio" +COMPOSE_FILE="docker-compose2.yml" +POSTGRES_CONTAINER=$(docker-compose -f $COMPOSE_FILE ps -q postgresql) # Get PostgreSQL container ID + +## Get record count from search_management table +record_count=$(docker-compose -f $COMPOSE_FILE exec postgresql psql -U $DB_USER -d $DB_NAME -t -c 'SELECT count(*) FROM search_management;') +record_count=$(echo $record_count | tr -d ' ') + +## If records exist, execute SQL update +if [ "$record_count" -ge 1 ]; then + echo "Records exist in search_management table. Executing SQL." + # Copy SQL file to container + docker cp scripts/demo/search_management_update.sql $POSTGRES_CONTAINER:/tmp/ams_search_management_update.sql + # Execute SQL file + docker-compose -f $COMPOSE_FILE exec postgresql psql -U $DB_USER -d $DB_NAME -f /tmp/ams_search_management_update.sql +else + echo "search_management table is empty. SQL will not be executed." +fi