diff --git a/src/execution/logging.py b/src/execution/logging.py index 56f007de..0bd7b10a 100644 --- a/src/execution/logging.py +++ b/src/execution/logging.py @@ -117,6 +117,7 @@ def __init__(self, output_folder, log_name_creator, authorizer): self._visited_files = set() self._ids_to_file_map = {} + self._entries_cache = {} self._output_loggers = {} file_utils.prepare_folder(output_folder) @@ -183,33 +184,34 @@ def write_post_execution_info(self, execution_id, exit_code): log_file_path = os.path.join(self._output_folder, filename) - logger.set_close_callback(lambda: self._write_post_execution_info(log_file_path, exit_code)) + def close_cb(): + self._write_post_execution_info(log_file_path, exit_code) + cached_entry = self._entries_cache.get(execution_id) + if cached_entry: + cached_entry.exit_code = int(exit_code) + + logger.set_close_callback(close_cb) def get_history_entries(self, user_id, *, system_call=False): self._renew_files_cache() result = [] - for file in self._ids_to_file_map.values(): - history_entry = self._extract_history_entry(file) - if history_entry is not None and self._can_access_entry(history_entry, user_id, system_call): - result.append(history_entry) + for entry in self._entries_cache.values(): + if entry is not None and self._can_access_entry(entry, user_id, system_call): + result.append(entry) return result def find_history_entry(self, execution_id, user_id): self._renew_files_cache() - file = self._ids_to_file_map.get(execution_id) - if file is None: + entry = self._entries_cache.get(execution_id) + if entry is None: LOGGER.warning('find_history_entry: file for %s id not found', execution_id) return None - entry = self._extract_history_entry(file) - if entry is None: - LOGGER.warning('find_history_entry: cannot parse file for %s', execution_id) - - elif not self._can_access_entry(entry, user_id): + if not self._can_access_entry(entry, user_id): message = 'User ' + user_id + ' has no access to execution #' + str(execution_id) LOGGER.warning('%s. Original user: %s', message, entry.user_id) raise AccessProhibitedException(message) @@ -261,6 +263,7 @@ def _renew_files_cache(self): for obsolete_id in obsolete_ids: LOGGER.info('Logs for execution #' + obsolete_id + ' were deleted') del cache[obsolete_id] + self._entries_cache.pop(obsolete_id, None) for file in os.listdir(self._output_folder): if not file.lower().endswith('.log'): @@ -276,6 +279,7 @@ def _renew_files_cache(self): continue cache[entry.id] = file + self._entries_cache[entry.id] = entry @staticmethod def _create_log_identifier(audit_name, script_name, start_time): diff --git a/src/tests/web/server_test.py b/src/tests/web/server_test.py index 59fdceb6..35d4bd3f 100644 --- a/src/tests/web/server_test.py +++ b/src/tests/web/server_test.py @@ -313,6 +313,174 @@ def start_server(self, port, address, *, xsrf_protection=XSRF_PROTECTION_TOKEN): self._admin_session.cookies['username'] = create_signed_value(cookie_secret, 'username', 'admin_user') \ .decode('utf8') + def _create_mock_history_entry(self, entry_id, user_name='normal_user', script_name='script1', start_time_ms=None, exit_code=0): + from execution.logging import HistoryEntry + from datetime import datetime, timezone + entry = HistoryEntry() + entry.id = str(entry_id) + entry.user_name = user_name + entry.user_id = user_name + entry.script_name = script_name + entry.command = 'python script.py' + entry.output_format = 'terminal' + entry.exit_code = exit_code + if start_time_ms is not None: + entry.start_time = datetime.fromtimestamp(start_time_ms / 1000.0, tz=timezone.utc) + return entry + + def test_history_short_log_pagination_empty(self): + self.start_server(12345, '127.0.0.1') + server._tornado_app.execution_logging_service.get_history_entries.return_value = [] + + response = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=1&size=25') + self.assertEqual(response, { + 'records': [], + 'total': 0, + 'page': 1, + 'pageSize': 25, + 'totalPages': 1 + }) + + def test_history_short_log_pagination_slicing_and_sorting(self): + self.start_server(12345, '127.0.0.1') + entries = [ + self._create_mock_history_entry('e1', start_time_ms=1000), + self._create_mock_history_entry('e2', start_time_ms=3000), + self._create_mock_history_entry('e3', start_time_ms=2000), + ] + server._tornado_app.execution_logging_service.get_history_entries.return_value = entries + + page1 = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=1&size=10') + self.assertEqual(page1['total'], 3) + self.assertEqual(page1['page'], 1) + self.assertEqual(page1['pageSize'], 10) + self.assertEqual(page1['totalPages'], 1) + self.assertEqual([r['id'] for r in page1['records']], ['e2', 'e3', 'e1']) + + def test_history_short_log_pagination_multiple_pages(self): + self.start_server(12345, '127.0.0.1') + entries = [self._create_mock_history_entry(f'e{i}', start_time_ms=i * 1000) for i in range(1, 35)] + server._tornado_app.execution_logging_service.get_history_entries.return_value = entries + + res_p1 = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=1&size=10') + self.assertEqual(res_p1['total'], 34) + self.assertEqual(res_p1['page'], 1) + self.assertEqual(res_p1['pageSize'], 10) + self.assertEqual(res_p1['totalPages'], 4) + self.assertEqual(len(res_p1['records']), 10) + self.assertEqual(res_p1['records'][0]['id'], 'e34') + + res_p4 = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=4&size=10') + self.assertEqual(res_p4['page'], 4) + self.assertEqual(len(res_p4['records']), 4) + self.assertEqual(res_p4['records'][-1]['id'], 'e1') + + def test_history_short_log_pagination_invalid_size_fallback(self): + self.start_server(12345, '127.0.0.1') + server._tornado_app.execution_logging_service.get_history_entries.return_value = [] + + response = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=1&size=999') + self.assertEqual(response['pageSize'], 25) + + response_abc = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=1&size=abc') + self.assertEqual(response_abc['pageSize'], 25) + + def test_history_short_log_pagination_invalid_page_fallback(self): + self.start_server(12345, '127.0.0.1') + server._tornado_app.execution_logging_service.get_history_entries.return_value = [] + + response = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=-5&size=25') + self.assertEqual(response['page'], 1) + + response_invalid = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=xyz&size=25') + self.assertEqual(response_invalid['page'], 1) + + def test_history_short_log_legacy_unpaginated(self): + self.start_server(12345, '127.0.0.1') + entries = [self._create_mock_history_entry('e1', start_time_ms=1000)] + server._tornado_app.execution_logging_service.get_history_entries.return_value = entries + + response = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short') + self.assertIsInstance(response, list) + self.assertEqual(len(response), 1) + self.assertEqual(response[0]['id'], 'e1') + + def test_history_short_log_pagination_page_out_of_bounds(self): + self.start_server(12345, '127.0.0.1') + entries = [self._create_mock_history_entry(f'e{i}', start_time_ms=i * 1000) for i in range(1, 31)] + server._tornado_app.execution_logging_service.get_history_entries.return_value = entries + + response = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=99&size=10') + self.assertEqual(response['page'], 3) + self.assertEqual(response['totalPages'], 3) + self.assertEqual(len(response['records']), 10) + + def test_history_short_log_pagination_with_none_start_time(self): + self.start_server(12345, '127.0.0.1') + entries = [ + self._create_mock_history_entry('e_none', start_time_ms=None), + self._create_mock_history_entry('e_valid', start_time_ms=5000) + ] + server._tornado_app.execution_logging_service.get_history_entries.return_value = entries + + response = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=1&size=10') + self.assertEqual([r['id'] for r in response['records']], ['e_valid', 'e_none']) + + def test_history_short_log_pagination_all_supported_sizes(self): + self.start_server(12345, '127.0.0.1') + server._tornado_app.execution_logging_service.get_history_entries.return_value = [] + + for size in [10, 25, 50, 100, 250, 500]: + response = self.request('GET', f'http://127.0.0.1:12345/history/execution_log/short?page=1&size={size}') + self.assertEqual(response['pageSize'], size) + + def test_history_short_log_pagination_running_script_status(self): + self.start_server(12345, '127.0.0.1') + entries = [ + self._create_mock_history_entry('running_1', start_time_ms=2000, exit_code=None), + self._create_mock_history_entry('finished_1', start_time_ms=1000, exit_code=0) + ] + server._tornado_app.execution_logging_service.get_history_entries.return_value = entries + server._tornado_app.execution_service.is_running.side_effect = lambda entry_id, user: entry_id == 'running_1' + + response = self.request('GET', 'http://127.0.0.1:12345/history/execution_log/short?page=1&size=10') + records = response['records'] + self.assertEqual(records[0]['id'], 'running_1') + self.assertEqual(records[0]['status'], 'running') + self.assertEqual(records[1]['id'], 'finished_1') + self.assertEqual(records[1]['status'], 'finished') + + def test_isolated_paginate_empty(self): + result = server.paginate_history_entries([], '1', '25') + self.assertEqual(result, { + 'records': [], + 'total': 0, + 'page': 1, + 'pageSize': 25, + 'totalPages': 1 + }) + + def test_isolated_paginate_legacy_unpaginated_returns_list(self): + entries = [self._create_mock_history_entry('e1', start_time_ms=1000)] + result = server.paginate_history_entries(entries, None, None) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + + def test_isolated_paginate_running_checker_only_called_for_sliced_page(self): + entries = [self._create_mock_history_entry(f'e{i}', start_time_ms=i * 1000) for i in range(1, 100)] + checked_ids = [] + + def mock_checker(entry_id): + checked_ids.append(entry_id) + return entry_id == 'e99' + + result = server.paginate_history_entries(entries, '1', '10', is_running_checker=mock_checker) + self.assertEqual(result['total'], 99) + self.assertEqual(result['page'], 1) + self.assertEqual(len(result['records']), 10) + self.assertEqual(len(checked_ids), 10) + self.assertEqual(checked_ids[0], 'e99') + def start_loop(self): io_loop = IOLoop.current() self.ioloop_thread = threading.Thread(target=io_loop.start) diff --git a/src/web/server.py b/src/web/server.py index d037e5fc..7fba9513 100755 --- a/src/web/server.py +++ b/src/web/server.py @@ -2,11 +2,13 @@ import asyncio import json import logging.config +import math import os import signal import ssl import time import urllib +from datetime import datetime, timezone from urllib.parse import urlencode import tornado.concurrent @@ -679,18 +681,74 @@ def post(self): file_utils.write_file(file_path, value) +ALLOWED_PAGE_SIZES = {10, 25, 50, 100, 250, 500} +DEFAULT_PAGE_SIZE = 25 + + +def paginate_history_entries(entries, page_arg, size_arg, is_running_checker=None): + def _get_sort_key(entry): + if entry.start_time is None: + return datetime.min.replace(tzinfo=timezone.utc) + return entry.start_time + + entries.sort(key=_get_sort_key, reverse=True) + + if page_arg is None and size_arg is None: + running_ids = [e.id for e in entries if is_running_checker and is_running_checker(e.id)] + return to_short_execution_log(entries, running_ids) + + try: + size = int(size_arg) if size_arg is not None else DEFAULT_PAGE_SIZE + if size not in ALLOWED_PAGE_SIZES: + size = DEFAULT_PAGE_SIZE + except (ValueError, TypeError): + size = DEFAULT_PAGE_SIZE + + try: + page = int(page_arg) if page_arg is not None else 1 + if page < 1: + page = 1 + except (ValueError, TypeError): + page = 1 + + total_count = len(entries) + total_pages = math.ceil(total_count / size) if total_count > 0 else 1 + + if page > total_pages and total_pages > 0: + page = total_pages + + start_idx = (page - 1) * size + end_idx = start_idx + size + page_entries = entries[start_idx:end_idx] + + running_ids = [e.id for e in page_entries if is_running_checker and is_running_checker(e.id)] + + return { + 'records': to_short_execution_log(page_entries, running_ids), + 'total': total_count, + 'page': page, + 'pageSize': size, + 'totalPages': total_pages + } + + class GetShortHistoryEntriesHandler(BaseRequestHandler): @check_authorization @inject_user def get(self, user): + page_arg = self.get_argument('page', None) + size_arg = self.get_argument('size', None) + history_entries = self.application.execution_logging_service.get_history_entries(user.user_id) - running_script_ids = [] - for entry in history_entries: - if self.application.execution_service.is_running(entry.id, user): - running_script_ids.append(entry.id) - short_logs = to_short_execution_log(history_entries, running_script_ids) - self.write(json.dumps(short_logs)) + result = paginate_history_entries( + history_entries, + page_arg, + size_arg, + is_running_checker=lambda entry_id: self.application.execution_service.is_running(entry_id, user) + ) + + self.write(json.dumps(result)) class GetLongHistoryEntryHandler(BaseRequestHandler): @@ -895,7 +953,8 @@ def init(server_config: ServerConfig, asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) io_loop = tornado.ioloop.IOLoop.current() - global _http_server + global _http_server, _tornado_app + _tornado_app = application _http_server = httpserver.HTTPServer( application, ssl_options=ssl_context, diff --git a/web-src/src/common/components/history/Paginator.vue b/web-src/src/common/components/history/Paginator.vue new file mode 100644 index 00000000..12ee37e4 --- /dev/null +++ b/web-src/src/common/components/history/Paginator.vue @@ -0,0 +1,217 @@ + + + + + diff --git a/web-src/src/common/components/history/executions-log-table.vue b/web-src/src/common/components/history/executions-log-table.vue index 2d3483c7..64669071 100644 --- a/web-src/src/common/components/history/executions-log-table.vue +++ b/web-src/src/common/components/history/executions-log-table.vue @@ -32,7 +32,8 @@ -

History will appear here

+

Loading history...

+

No entries found

@@ -168,11 +169,13 @@ export default { width: 15%; } -.loading-text { +.loading-text, +.empty-text { color: var(--font-color-medium); font-size: 1.2em; text-align: center; - margin-top: 1em; + margin-top: 1.5em; + margin-bottom: 1.5em; } .executions-log-table .sorted:after { diff --git a/web-src/src/common/components/history/executions-log.vue b/web-src/src/common/components/history/executions-log.vue index fef05f76..2f0e0ddb 100644 --- a/web-src/src/common/components/history/executions-log.vue +++ b/web-src/src/common/components/history/executions-log.vue @@ -1,14 +1,27 @@