feature/W2025-33 add fix static report for autosend and opensearch for issn

modules/invenio-records-rest/invenio_records_rest/sorter.py

@@ -23,7 +23,7 @@
 import pickle
 
 import six
-from flask import current_app, request
+from flask import current_app, request, has_request_context
 
 from .config import RECORDS_REST_DEFAULT_SORT
 
@@ -126,12 +126,14 @@ def default_sorter_factory(search, index):
     :returns: Tuple of (query, URL arguments).
     """
     sort_arg_name = 'sort'
-    urlfield = request.values.get(sort_arg_name, '', type=str)
+    urlfield = request.values.get(sort_arg_name, '', type=str) \
+        if has_request_context() else ''
 
     # Get default sorting if sort is not specified.
     if not urlfield:
         # cast to six.text_type to handle unicodes in Python 2
-        has_query = request.values.get('q', type=six.text_type)
+        has_query = request.values.get('q', type=six.text_type) \
+            if has_request_context() else None
         if current_app.config.get('RECORDS_REST_DEFAULT_SORT'):
             urlfield = current_app.config['RECORDS_REST_DEFAULT_SORT'].get(
                 index, {}).get('query' if has_query else 'noquery', '')

modules/invenio-records-rest/tests/test_sorter.py

@@ -13,6 +13,7 @@
 import pytest
 from elasticsearch_dsl import Search
 
+import invenio_records_rest.sorter as sorter
 from invenio_records_rest.sorter import default_sorter_factory, eval_field, \
     geolocation_sort, parse_sort_field, reverse_order
 
@@ -101,14 +102,16 @@ def test_default_sorter_factory(app):
     with app.test_request_context("?sort=myfield"):
         query, urlargs = default_sorter_factory(Search(), 'myindex')
         assert query.to_dict()['sort'] == \
-            [{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}]
+            [{'field1': {'order': 'asc', 'unmapped_type': 'long'}},
+             {'field2': {'order': 'desc', 'unmapped_type': 'long'}}]
         assert urlargs['sort'] == 'myfield'
 
     # Reverse sort
     with app.test_request_context("?sort=-myfield"):
         query, urlargs = default_sorter_factory(Search(), 'myindex')
         assert query.to_dict()['sort'] == \
-            [{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}]
+            [{'field1': {'order': 'desc', 'unmapped_type': 'long'}},
+             {'field2': {'order': 'asc', 'unmapped_type': 'long'}}]
         assert urlargs['sort'] == '-myfield'
 
     # Invalid sort key
@@ -121,24 +124,72 @@ def test_default_sorter_factory(app):
     with app.test_request_context("/?q="):
         query, urlargs = default_sorter_factory(Search(), 'myindex')
         assert query.to_dict()['sort'] == \
-            [{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}]
+            [{'field1': {'order': 'asc', 'unmapped_type': 'long'}},
+             {'field2': {'order': 'desc', 'unmapped_type': 'long'}}]
         assert urlargs == dict(sort='myfield')
 
     # Default sort with query
     with app.test_request_context("/?q=test"):
         query, urlargs = default_sorter_factory(Search(), 'myindex')
         assert query.to_dict()['sort'] == \
-            [{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}]
+            [{'field1': {'order': 'desc', 'unmapped_type': 'long'}},
+             {'field2': {'order': 'asc', 'unmapped_type': 'long'}}]
         assert urlargs == dict(sort='-myfield')
 
     # Default sort with query that includes unicodes
     with app.test_request_context("/?q=tést"):
         query, urlargs = default_sorter_factory(Search(), 'myindex')
         assert query.to_dict()['sort'] == \
-            [{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}]
+            [{'field1': {'order': 'desc', 'unmapped_type': 'long'}},
+             {'field2': {'order': 'asc', 'unmapped_type': 'long'}}]
         assert urlargs == dict(sort='-myfield')
 
     # Default sort another index
     with app.test_request_context("/?q=test"):
         query, urlargs = default_sorter_factory(Search(), 'aidx')
         assert 'sort' not in query.to_dict()
+
+
+def test_default_sorter_factory_without_request_context(app, monkeypatch):
+    """Test default sorter factory without request context."""
+    app.config["RECORDS_REST_SORT_OPTIONS"] = dict(
+        myindex=dict(
+            myfield=dict(
+                fields=['field1', '-field2'],
+            )
+        ),
+    )
+    app.config["RECORDS_REST_DEFAULT_SORT"] = dict(
+        myindex=dict(
+            query='-myfield',
+            noquery='myfield',
+        ),
+    )
+
+    with app.app_context():
+        query, urlargs = default_sorter_factory(Search(), 'myindex')
+
+    assert query.to_dict()['sort'] == \
+        [{'field1': {'order': 'asc', 'unmapped_type': 'long'}},
+         {'field2': {'order': 'desc', 'unmapped_type': 'long'}}]
+    assert urlargs == dict(sort='myfield')
+
+    app.config["RECORDS_REST_DEFAULT_SORT"] = {}
+    monkeypatch.setattr(
+        sorter,
+        "RECORDS_REST_DEFAULT_SORT",
+        dict(
+            myindex=dict(
+                query='-myfield',
+                noquery='myfield',
+            ),
+        ),
+    )
+
+    with app.test_request_context("/?q=test"):
+        query, urlargs = default_sorter_factory(Search(), 'myindex')
+
+    assert query.to_dict()['sort'] == \
+        [{'field1': {'order': 'desc', 'unmapped_type': 'long'}},
+         {'field2': {'order': 'asc', 'unmapped_type': 'long'}}]
+    assert urlargs == dict(sort='-myfield')

modules/invenio-stats/invenio_stats/utils.py

@@ -1079,7 +1079,7 @@ def get(cls, **kwargs):
                 query_month = str(year) + '-' + str(month).zfill(2)
                 _, lastday = calendar.monthrange(year, month)
                 start_date = query_month + '-01'
-                end_date = query_month + '-' + str(lastday).zfill(2) + 'T23:59:59'
+                end_date = query_month + '-' + str(lastday).zfill(2)
             else:
                 query_month = f"{start_date}-{end_date}"
 
@@ -1089,6 +1089,7 @@ def get(cls, **kwargs):
             total = 1
             count = 0
             while count < total and (after_key or first_search):
+                end_date = end_date + 'T23:59:59'
                 agg_query = cls.build_query(start_date, end_date, after_key)
                 current_app.logger.debug(agg_query.to_dict())
                 temp_res = agg_query.execute().to_dict()

modules/invenio-stats/invenio_stats/utils_search.py

@@ -11,7 +11,7 @@ def billing_file_search_factory(search):
     from invenio_records_rest.sorter import default_sorter_factory
 
     # add  Permission filter by publish date and status
-    mst, _ = get_permission_filter()
+    mst = get_permission_filter()
 
     # billing file search filter
     query = Q('bool', must=[{'terms': {'content.billing.raw': ['billing_file']}}])
@@ -53,59 +53,22 @@ def get_permission_filter(index_id: str = None):
         List: Query command.
 
     """
-    from weko_index_tree.api import Indexes
-    is_perm = Permission(action_factory("search-access")).can()
     match = Q("match", publish_status="0")
     version = Q("match", relation_version_is_last="true")
     rng = Q("range", **{"publish_date": {"lte": "now/d"}})
-    term_list = []
     mst = []
-    is_perm_paths = Indexes.get_browsing_tree_paths()
-    is_perm_indexes = [item.split("/")[-1] for item in is_perm_paths]
-    search_type = request.values.get("search_type")
-
-    if index_id:
-        index_id = str(index_id)
-        if search_type == "0":
-            should_path = []
-            if index_id in is_perm_indexes:
-                should_path.append(Q("terms", path=index_id))
-
-            mst.append(match)
-            mst.append(rng)
-            terms = Q("bool", should=should_path)
-        else:  # In case search_type is keyword or index
-            if index_id in is_perm_indexes:
-                term_list.append(index_id)
-
-            mst.append(match)
-            mst.append(rng)
-            terms = Q("terms", path=term_list)
-    else:
-        mst.append(match)
-        mst.append(rng)
-        terms = Q("terms", path=is_perm_indexes)
+
+    mst.append(match)
+    mst.append(rng)
 
     mut = []
+    shuld= []
+
+    shuld.append(Q("bool", must=mst))
+    mut.append(Q("bool", should=shuld))
+    mut.append(Q("bool", must=version))
 
-    if is_perm:
-        user_id, result = check_admin_user()
-
-        if result:
-            shuld = [
-                Q("match", weko_creator_id=user_id),
-                Q("match", weko_shared_id=user_id),
-            ]
-            shuld.append(Q("bool", must=mst))
-            mut.append(Q("bool", should=shuld, must=[terms]))
-            mut.append(Q("bool", must=version))
-    else:
-        mut = mst
-        mut.append(terms)
-        base_mut = [match, version]
-        mut.append(Q("bool", must=base_mut))
-
-    return mut, is_perm_paths
+    return mut
 
 def check_admin_user():
     """

modules/invenio-stats/tests/data/billing_file_query.json

@@ -1,6 +1,7 @@
 {
     "_source": {
         "includes": [
+            "path",
             "content.filename",
             "_item_metadata.owner",
             "_oai.id"
@@ -14,29 +15,7 @@
                         "must": [
                             {
                                 "bool": {
-                                    "must": [
-                                        {
-                                            "terms": {
-                                                "path": [
-                                                    "1",
-                                                    "2",
-                                                    "3",
-                                                    "4"
-                                                ]
-                                            }
-                                        }
-                                    ],
                                     "should": [
-                                        {
-                                            "match": {
-                                                "weko_creator_id": "3"
-                                            }
-                                        },
-                                        {
-                                            "match": {
-                                                "weko_shared_id": "3"
-                                            }
-                                        },
                                         {
                                             "bool": {
                                                 "must": [

modules/invenio-stats/tests/test_utils.py

@@ -131,7 +131,7 @@ def test_agg_bucket_sort(app):
 # .tox/c1/bin/pytest --cov=invenio_stats tests/test_utils.py::test_parse_bucket_response -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/invenio-stats/.tox/c1/tmp
 def test_parse_bucket_response(app):
     _raw_res = {'buckets': [{'key': 'test_value'}], 'field': 'test_name'}
-    
+
     res = parse_bucket_response(_raw_res, {})
     assert res=={'test_name': 'test_value'}
 
@@ -145,7 +145,7 @@ def test_get_doctype(app):
 def test_is_valid_access(app):
     res = is_valid_access()
     assert res==True
-    
+
     with patch("invenio_stats.utils.get_remote_addr", return_value='0.0.0.0'):
         app.config['STATS_EXCLUDED_ADDRS'] = ['0.0.0.0']
         res = is_valid_access()
@@ -315,14 +315,14 @@ def test_query_file_reports_helper(i18n_app, roles, mock_es_execute, index):
         'get-file-download-per-user-report': None,
         'get-file-preview-per-user-report': None},
         _data_list)
-    assert _data_list=={} 
+    assert _data_list=={}
     QueryFileReportsHelper.Calculation(_report_res, _data_list)
     assert _data_list=={
         1: {'cur_user_id': 1, 'total_download': 2, 'total_preview': 5},
         2: {'cur_user_id': 2, 'total_download': 3},
         3: {'cur_user_id': 3, 'total_download': 4},
         4: {'cur_user_id': 4, 'total_preview': 1}}
-    
+
     expect_all_groups = []
     def mock_Calculation(res, data_list, all_groups=set(), event=None):
         if event == 'billing_file_download':
@@ -340,7 +340,7 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None):
     # get_file_stats_report
     with patch("invenio_stats.utils.QueryFileReportsHelper.Calculation", side_effect=mock_Calculation):
         with patch("invenio_stats.queries.ESTermsQuery.run", return_value=_res):
-            res = QueryFileReportsHelper.get_file_stats_report(event='file_download', year=2022, month=10) 
+            res = QueryFileReportsHelper.get_file_stats_report(event='file_download', year=2022, month=10)
             assert res=={
                 'all': _expect_data_list,
                 'all_groups': expect_all_groups,
@@ -356,7 +356,7 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None):
                 'open_access': _expect_data_list
             }
         with patch("invenio_stats.queries.ESTermsQuery.run", return_value=_billing_res):
-            res = QueryFileReportsHelper.get_file_stats_report(event='billing_file_download', year=2022, month=10) 
+            res = QueryFileReportsHelper.get_file_stats_report(event='billing_file_download', year=2022, month=10)
             assert res=={
                 'all': _expect_billing_data_list,
                 'all_groups': expect_all_groups,
@@ -378,9 +378,9 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None):
 
     # get
     res = QueryFileReportsHelper.get(year=2022, month=10, event='file_download')
-    assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} 
+    assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []}
     res = QueryFileReportsHelper.get(year=2022, month=10, event='billing_file_download')
-    assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} 
+    assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []}
     res = QueryFileReportsHelper.get(year=2022, month=10, event='file_using_per_user')
     assert res=={'all': {}, 'date': '2022-10'}
     res = QueryFileReportsHelper.get(year=2022, month=10, event='test')
@@ -398,9 +398,9 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None):
 def test_query_file_reports_helper_error(app):
     # get
     res = QueryFileReportsHelper.get(year=2022, month=10, event='file_download')
-    assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} 
+    assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []}
     res = QueryFileReportsHelper.get(year=2022, month=10, event='billing_file_download')
-    assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} 
+    assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []}
     res = QueryFileReportsHelper.get(year=2022, month=10, event='file_using_per_user')
     assert res=={'all': {}, 'date': '2022-10'}
     res = QueryFileReportsHelper.get(year=2022, month=10, event='test')
@@ -611,7 +611,7 @@ def test_query_access_counter_helper(app, es):
 }
     with patch('invenio_stats.queries.ESTermsQuery.run', return_value=_res):
         res = QueryAccessCounterHelper.get_top_page_access_counter(year=2022, month=10, start_date='2022-10-01', end_date='2022-10-10')
-        assert res=={'date': '2022-10-01-2022-10-10', 'all':{"count":152}}    
+        assert res=={'date': '2022-10-01-2022-10-10', 'all':{"count":152}}
 # .tox/c1/bin/pytest --cov=invenio_stats tests/test_utils.py::test_query_access_counter_helper_error -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/invenio-stats/.tox/c1/tmp
 def test_query_access_counter_helper_error(app):
     # get_top_page_access_counter
@@ -666,6 +666,26 @@ def test_query_record_view_per_index_report_helper(app, es):
     res = QueryRecordViewPerIndexReportHelper.get(start_date='2022-10-01', end_date='2022-10-31')
     assert res=={'all': [], 'date': '2022-10-01-2022-10-31', 'total': 0}
 
+    # end_dateに時間が追加されているか
+    with patch.object(QueryRecordViewPerIndexReportHelper, 'build_query')\
+            as mock_build_query:
+        mock_agg_query = MagicMock()
+        mock_agg_query.execute.return_value.to_dict.return_value = {
+            'aggregations': {
+                QueryRecordViewPerIndexReportHelper.nested_path: {
+                    'my_buckets': {'buckets': []},
+                    'doc_count': 0
+                }
+            }
+        }
+        mock_build_query.return_value = mock_agg_query
+        with patch.object(QueryRecordViewPerIndexReportHelper,
+                          'parse_bucket_response', return_value=0):
+            QueryRecordViewPerIndexReportHelper.get(
+                start_date='2022-10-01', end_date='2022-10-31')
+            called_args = mock_build_query.call_args[0]
+            assert called_args[1] == '2022-10-31T23:59:59'
+
 def test_query_record_view_per_index_report_helper_error(app):
     # get
     res = QueryRecordViewPerIndexReportHelper.get(year=2022, month=10)
@@ -1086,7 +1106,7 @@ def test_StatsCliUtil(app, db):
     )
     assert not stats_cli.delete_data(True)
 
-    
+
 
     stats_cli = StatsCliUtil(
         StatsCliUtil.EVENTS_TYPE, _empty_types, verbose=False