From 297e52f3c213dd660ea9dfbcae5da15a02e91cdb Mon Sep 17 00:00:00 2001 From: GitInno <86991526+gitnnolabs@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:51:29 -0300 Subject: [PATCH 1/4] To train reference markup. --- reference/data_utils.py | 3 --- reference/prompts.py | 42 ++++++++++++++++++++++++++++++++ reference/tests/test_coverage.py | 14 +++++++++++ reference/tests/test_marking.py | 21 +++++++++++++++- 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/reference/data_utils.py b/reference/data_utils.py index 3569a24..8d9c622 100644 --- a/reference/data_utils.py +++ b/reference/data_utils.py @@ -106,7 +106,6 @@ def append_citation_pages(root, pages): etree.SubElement(root, "elocation-id").text = value return etree.SubElement(root, "fpage").text = value - etree.SubElement(root, "lpage").text = value def append_fpage_lpage(root, json_reference): @@ -114,8 +113,6 @@ def append_fpage_lpage(root, json_reference): etree.SubElement(root, "fpage").text = str(json_reference["fpage"]) if "lpage" in json_reference: etree.SubElement(root, "lpage").text = str(json_reference["lpage"]) - else: - etree.SubElement(root, "lpage").text = str(json_reference["fpage"]) return True if "pages" in json_reference: append_citation_pages(root, json_reference["pages"]) diff --git a/reference/prompts.py b/reference/prompts.py index cb5c512..c363e15 100644 --- a/reference/prompts.py +++ b/reference/prompts.py @@ -26,6 +26,10 @@ "fpage/lpage (do not use pages for journal page ranges); pages only " "for elocation-id. Volume+pages after a periodical name means " "journal, not book. " + "Abbreviated end pages (Vancouver: 1751-2, 1105-10, 83-9) → expand " + "lpage with the fpage prefix (1751/1752, 1105/1110, 83/89); never " + "leave lpage truncated. Single page → fpage only (no lpage, no " + "pages like 237-237). " "book: whole work uses source only (do not use title); chapter uses " "chapter + source; publisher → organization; thesis: source=title " "(no title field). " @@ -83,6 +87,44 @@ '"doi":"10.1127/0941-2948/2013/0507"}' ), }, + { + "role": "user", + "content": ( + "Silva, A. B., & Costa, C. D. (2020). Abbreviated pagination example. " + "Journal of Examples, 12(3), 1751-2. " + "https://doi.org/10.1234/example.1751" + ), + }, + { + "role": "assistant", + "content": ( + '{"reftype":"journal",' + '"authors":[{"surname":"Silva","fname":"A. B."},' + '{"surname":"Costa","fname":"C. D."}],' + '"date":"2020",' + '"title":"Abbreviated pagination example",' + '"source":"Journal of Examples",' + '"vol":12,"num":3,"fpage":"1751","lpage":"1752",' + '"doi":"10.1234/example.1751"}' + ), + }, + { + "role": "user", + "content": ( + "Oliveira, M. (2019). Single page note. Rev. Exemplo, 5(1), 237. " + "https://doi.org/10.1234/example.237" + ), + }, + { + "role": "assistant", + "content": ( + '{"reftype":"journal",' + '"authors":[{"surname":"Oliveira","fname":"M."}],' + '"date":"2019","title":"Single page note","source":"Rev. Exemplo",' + '"vol":5,"num":1,"fpage":"237",' + '"doi":"10.1234/example.237"}' + ), + }, { "role": "user", "content": ( diff --git a/reference/tests/test_coverage.py b/reference/tests/test_coverage.py index a217b1b..b647bae 100644 --- a/reference/tests/test_coverage.py +++ b/reference/tests/test_coverage.py @@ -17,6 +17,7 @@ from reference.data_utils import ( append_access_date, append_citation_pages, + append_fpage_lpage, build_ref_list, get_number_of_month, get_reference, @@ -51,6 +52,19 @@ def test_append_citation_pages_empty(): assert list(root) == [] +def test_append_citation_pages_single_page_omits_lpage(): + root = etree.Element("element-citation") + append_citation_pages(root, "244") + assert root.find("fpage").text == "244" + assert root.find("lpage") is None + + +def test_append_fpage_lpage_single_fpage_omits_lpage(): + root = etree.Element("element-citation") + assert append_fpage_lpage(root, {"fpage": "237"}) is True + assert root.find("fpage").text == "237" + assert root.find("lpage") is None + def test_append_access_date_with_and_without_year(): with_year = etree.Element("element-citation") append_access_date(with_year, "cited 2025") diff --git a/reference/tests/test_marking.py b/reference/tests/test_marking.py index 56546a7..4f0d726 100644 --- a/reference/tests/test_marking.py +++ b/reference/tests/test_marking.py @@ -201,6 +201,8 @@ def test_prompt_instructs_skip_for_figures(): assert "Responsibility" in system or "contribution" in system.lower() assert "fpage" in system and "lpage" in system assert "do not use pages for journal page ranges" in system + assert "Abbreviated end pages" in system + assert "Single page" in system assert "whole work uses source only" in system assert "bare id" in system.lower() or "without https://doi.org/" in system assert "do not emit uri" in system @@ -258,6 +260,23 @@ def test_prompt_instructs_skip_for_figures(): assert '"pages"' not in zoo_example assert "https://doi.org/" not in zoo_example + abbreviated = next( + assistant["content"] + for user, assistant in pairs + if "1751-2" in user["content"] + ) + assert '"fpage":"1751"' in abbreviated + assert '"lpage":"1752"' in abbreviated + + single_page = next( + assistant["content"] + for user, assistant in pairs + if ", 237." in user["content"] or " 237." in user["content"] + ) + assert '"fpage":"237"' in single_page + assert '"lpage"' not in single_page + assert "237-237" not in single_page + def test_get_xml_journal(): sample_json = json.dumps( @@ -489,7 +508,7 @@ def test_get_xml_journal_pages_and_elocation(): ) ) assert single.find("fpage").text == "244" - assert single.find("lpage").text == "244" + assert single.find("lpage") is None elocation = get_xml( json.dumps( From f0015dd332ac760d579849298eb14c6b80087f00 Mon Sep 17 00:00:00 2001 From: GitInno <86991526+gitnnolabs@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:26:30 -0300 Subject: [PATCH 2/4] =?UTF-8?q?Reduz=20few-shots=20de=20refer=C3=AAncias?= =?UTF-8?q?=20a=206=20exemplos=20densos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corta prefill no Ollama ao fundir journals e skips redundantes e adicionar legal-doc com collab, sem alterar o system prompt. Co-authored-by: Cursor --- reference/prompts.py | 155 +++++--------------------------- reference/tests/test_marking.py | 52 +++++------ 2 files changed, 44 insertions(+), 163 deletions(-) diff --git a/reference/prompts.py b/reference/prompts.py index c363e15..1956e03 100644 --- a/reference/prompts.py +++ b/reference/prompts.py @@ -40,34 +40,12 @@ "confproc/webpage/software/legal-doc: use fields when present." ), }, - { - "role": "user", - "content": ( - "Bachman, S., J. Moat, A. W. Hill, J. de la Torre and B. Scott. 2011. " - "Supporting Red List threat assessments with GeoCAT: geospatial " - "conservation assessment tool. ZooKeys 150: 117-126. DOI: " - "https://doi.org/10.3897/zookeys.150.2109" - ), - }, - { - "role": "assistant", - "content": ( - '{"reftype":"journal","authors":[{"surname":"Bachman","fname":"S."},' - '{"surname":"Moat","fname":"J."},{"surname":"Hill","fname":"A. W."},' - '{"surname":"de la Torre","fname":"J."},' - '{"surname":"Scott","fname":"B."}],"date":"2011",' - '"title":"Supporting Red List threat assessments with GeoCAT: ' - 'geospatial conservation assessment tool","source":"ZooKeys",' - '"vol":150,"fpage":"117","lpage":"126",' - '"doi":"10.3897/zookeys.150.2109"}' - ), - }, { "role": "user", "content": ( "Alvares, C. A., Stape, J. L., Sentelhas, P. C., Gonçalves, J. L. M., " "& Sparovek, G. (2013b). Köppen’s climate classification map for " - "Brazil. Meteorologische Zeitschrift, 22(6), 711–728. " + "Brazil. Meteorologische Zeitschrift, 22(6), 1751-2. " "https://doi.org/10.1127/0941-2948/2013/0507" ), }, @@ -83,48 +61,10 @@ '"date":"2013b",' '"title":"Köppen’s climate classification map for Brazil",' '"source":"Meteorologische Zeitschrift",' - '"vol":22,"num":6,"fpage":"711","lpage":"728",' + '"vol":22,"num":6,"fpage":"1751","lpage":"1752",' '"doi":"10.1127/0941-2948/2013/0507"}' ), }, - { - "role": "user", - "content": ( - "Silva, A. B., & Costa, C. D. (2020). Abbreviated pagination example. " - "Journal of Examples, 12(3), 1751-2. " - "https://doi.org/10.1234/example.1751" - ), - }, - { - "role": "assistant", - "content": ( - '{"reftype":"journal",' - '"authors":[{"surname":"Silva","fname":"A. B."},' - '{"surname":"Costa","fname":"C. D."}],' - '"date":"2020",' - '"title":"Abbreviated pagination example",' - '"source":"Journal of Examples",' - '"vol":12,"num":3,"fpage":"1751","lpage":"1752",' - '"doi":"10.1234/example.1751"}' - ), - }, - { - "role": "user", - "content": ( - "Oliveira, M. (2019). Single page note. Rev. Exemplo, 5(1), 237. " - "https://doi.org/10.1234/example.237" - ), - }, - { - "role": "assistant", - "content": ( - '{"reftype":"journal",' - '"authors":[{"surname":"Oliveira","fname":"M."}],' - '"date":"2019","title":"Single page note","source":"Rev. Exemplo",' - '"vol":5,"num":1,"fpage":"237",' - '"doi":"10.1234/example.237"}' - ), - }, { "role": "user", "content": ( @@ -167,33 +107,20 @@ { "role": "user", "content": ( - "Brunel, J. F. 1987. Sur le genre Phyllanthus L. Thèse de doctorat " - "de l’Université L. Pasteur. Strasbourg, France. 760 pp." - ), - }, - { - "role": "assistant", - "content": ( - '{"reftype":"thesis","authors":[{"surname":"Brunel","fname":"J. F."}],' - '"date":"1987","source":"Sur le genre Phyllanthus L.",' - '"degree":"doctorat","organization":"l’Université L. Pasteur",' - '"location":"Strasbourg, France","num_pages":760}' - ), - }, - { - "role": "user", - "content": ( - "Felix Ribeiro, K. A. (2025). Replication data for: Mauritia flexuosa. " - "SciELO Data. https://doi.org/10.48331/SCIELODATA.RIVAW4" + "Brasil. (2024). Decreto nº 91.886, de 05 de novembro de 1985. " + "Diário Oficial da União. " + "https://www.planalto.gov.br/cCivil_03/Atos/decretos/1985/D91886" ), }, { "role": "assistant", "content": ( - '{"reftype":"data",' - '"authors":[{"surname":"Felix Ribeiro","fname":"K. A."}],' - '"date":"2025","title":"Replication data for: Mauritia flexuosa",' - '"source":"SciELO Data","doi":"10.48331/SCIELODATA.RIVAW4"}' + '{"reftype":"legal-doc","authors":[{"collab":"Brasil"}],' + '"date":"2024",' + '"source":"Decreto nº 91.886, de 05 de novembro de 1985. ' + 'Diário Oficial da União",' + '"uri":"https://www.planalto.gov.br/cCivil_03/Atos/decretos/' + '1985/D91886"}' ), }, { @@ -216,66 +143,26 @@ { "role": "user", "content": ( - "1. Bachman S et al. 2011. Supporting Red List. ZooKeys 150:117-126. " - "DOI: 10.3897/zookeys.150.2109\n" - "2. Figure 1. Map of the study area." + "1. Felix Ribeiro, K. A. (2025). Replication data for: Mauritia " + "flexuosa. SciELO Data. " + "https://doi.org/10.48331/SCIELODATA.RIVAW4\n" + "2. Figure 1. Map of the study area.\n" + "3. https://orcid.org/0000-0003-4872-7252" ), }, { "role": "assistant", "content": ( '{"results":[' - '{"reftype":"journal","authors":[{"surname":"Bachman","fname":"S"}],' - '"date":"2011","title":"Supporting Red List","source":"ZooKeys",' - '"vol":150,"fpage":"117","lpage":"126",' - '"doi":"10.3897/zookeys.150.2109"},' + '{"reftype":"data",' + '"authors":[{"surname":"Felix Ribeiro","fname":"K. A."}],' + '"date":"2025","title":"Replication data for: Mauritia flexuosa",' + '"source":"SciELO Data","doi":"10.48331/SCIELODATA.RIVAW4"},' + '{"is_reference": false},' '{"is_reference": false}' "]}" ), }, - { - "role": "user", - "content": "Figure 1. Map of the study area.", - }, - { - "role": "assistant", - "content": '{"is_reference": false}', - }, - { - "role": "user", - "content": "Figura 2. Densidade populacional na Amazônia.", - }, - { - "role": "assistant", - "content": '{"is_reference": false}', - }, - { - "role": "user", - "content": "https://orcid.org/0000-0003-4872-7252", - }, - { - "role": "assistant", - "content": '{"is_reference": false}', - }, - { - "role": "user", - "content": "SCIENTIFIC EDITOR", - }, - { - "role": "assistant", - "content": '{"is_reference": false}', - }, - { - "role": "user", - "content": ( - "Responsibility for all aspects of the content and the integrity of " - "the published article. Camila Lima Ribeiro, Marcelle Miranda da Silva." - ), - }, - { - "role": "assistant", - "content": '{"is_reference": false}', - }, ] ITEM_PROPERTIES = { diff --git a/reference/tests/test_marking.py b/reference/tests/test_marking.py index 4f0d726..e757654 100644 --- a/reference/tests/test_marking.py +++ b/reference/tests/test_marking.py @@ -226,16 +226,7 @@ def test_prompt_instructs_skip_for_figures(): assert key in ITEM_PROPERTIES pairs = list(zip(MESSAGES[1::2], MESSAGES[2::2])) - skip_examples = [ - user["content"] - for user, assistant in pairs - if assistant["content"] == '{"is_reference": false}' - ] - assert any("Figure 1" in text for text in skip_examples) - assert any("Figura" in text for text in skip_examples) - assert any("orcid.org" in text for text in skip_examples) - assert any("SCIENTIFIC EDITOR" in text for text in skip_examples) - assert any("Responsibility for" in text for text in skip_examples) + assert len(pairs) == 6 journal_example = next( assistant["content"] @@ -246,36 +237,39 @@ def test_prompt_instructs_skip_for_figures(): assert '"date":"2013b"' in journal_example assert '"num":6' in journal_example assert '"doi":"10.1127/0941-2948/2013/0507"' in journal_example - assert '"fpage":"711"' in journal_example - assert '"lpage":"728"' in journal_example + assert '"fpage":"1751"' in journal_example + assert '"lpage":"1752"' in journal_example + assert '"pages"' not in journal_example assert "https://doi.org/" not in journal_example - - zoo_example = next( - assistant["content"] + journal_user = next( + user["content"] for user, assistant in pairs - if "ZooKeys" in assistant["content"] and '"results"' not in assistant["content"] + if "1751-2" in user["content"] and "2013b" in user["content"] ) - assert '"fpage":"117"' in zoo_example - assert '"lpage":"126"' in zoo_example - assert '"pages"' not in zoo_example - assert "https://doi.org/" not in zoo_example + assert "1751-2" in journal_user - abbreviated = next( + legal_example = next( assistant["content"] for user, assistant in pairs - if "1751-2" in user["content"] + if '"reftype":"legal-doc"' in assistant["content"] ) - assert '"fpage":"1751"' in abbreviated - assert '"lpage":"1752"' in abbreviated + assert '"collab":"Brasil"' in legal_example + assert '"uri"' in legal_example - single_page = next( + batch_example = next( assistant["content"] for user, assistant in pairs - if ", 237." in user["content"] or " 237." in user["content"] + if '"results"' in assistant["content"] + ) + assert '"reftype":"data"' in batch_example + assert batch_example.count('{"is_reference": false}') == 2 + batch_user = next( + user["content"] + for user, assistant in pairs + if '"results"' in assistant["content"] ) - assert '"fpage":"237"' in single_page - assert '"lpage"' not in single_page - assert "237-237" not in single_page + assert "Figure 1" in batch_user + assert "orcid.org" in batch_user def test_get_xml_journal(): From 1088450b208ecd5507db4d17f544cc326d64340a Mon Sep 17 00:00:00 2001 From: GitInno <86991526+gitnnolabs@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:57:02 -0300 Subject: [PATCH 3/4] =?UTF-8?q?Acelera=20marca=C3=A7=C3=A3o=20Llama=20com?= =?UTF-8?q?=20format=20json=20e=20keep=5Falive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deixa de enviar JSON Schema no Ollama (descodificação constrangida lenta no 8B) e mantém o modelo residente com keep_alive=-1. Co-authored-by: Cursor --- .envs.example/.local/.django | 1 + .envs.example/.production/.django | 1 + config/settings/base.py | 1 + reference/providers/http.py | 13 ++++++++-- reference/tests/test_http_provider.py | 35 +++++++++++++++++++++++++-- 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/.envs.example/.local/.django b/.envs.example/.local/.django index 32b0242..24d33c9 100644 --- a/.envs.example/.local/.django +++ b/.envs.example/.local/.django @@ -18,3 +18,4 @@ REFERENCE_MODEL=llama3.2:3b # REFERENCE_TIMEOUT=300 # REFERENCE_BATCH_SIZE=10 # REFERENCE_NUM_CTX=8192 +# REFERENCE_KEEP_ALIVE=-1 diff --git a/.envs.example/.production/.django b/.envs.example/.production/.django index c03a932..d3de475 100644 --- a/.envs.example/.production/.django +++ b/.envs.example/.production/.django @@ -29,3 +29,4 @@ REFERENCE_MODEL=llama3.2:3b # REFERENCE_TIMEOUT=300 # REFERENCE_BATCH_SIZE=10 # REFERENCE_NUM_CTX=8192 +# REFERENCE_KEEP_ALIVE=-1 diff --git a/config/settings/base.py b/config/settings/base.py index 7fa8bd4..5516075 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -325,3 +325,4 @@ REFERENCE_TOKEN = env("REFERENCE_TOKEN", default="") REFERENCE_BATCH_SIZE = env.int("REFERENCE_BATCH_SIZE", default=10) REFERENCE_NUM_CTX = env.int("REFERENCE_NUM_CTX", default=8192) +REFERENCE_KEEP_ALIVE = env("REFERENCE_KEEP_ALIVE", default="-1") diff --git a/reference/providers/http.py b/reference/providers/http.py index ff7241f..6b5476a 100644 --- a/reference/providers/http.py +++ b/reference/providers/http.py @@ -42,6 +42,14 @@ def __init__( self.timeout = getattr(settings, "REFERENCE_TIMEOUT", 300) self.token = getattr(settings, "REFERENCE_TOKEN", "") or "" self.num_ctx = int(getattr(settings, "REFERENCE_NUM_CTX", 8192) or 8192) + raw_keep_alive = getattr(settings, "REFERENCE_KEEP_ALIVE", "-1") + if raw_keep_alive is None or raw_keep_alive == "": + self.keep_alive = None + else: + try: + self.keep_alive = int(raw_keep_alive) + except (TypeError, ValueError): + self.keep_alive = str(raw_keep_alive) def run(self, user_input): messages = self.messages.copy() @@ -72,8 +80,9 @@ def chat(self, messages): "stream": False, } if self.response_format and self.response_format.get("type") == "json_object": - schema = self.response_format.get("schema") - payload["format"] = schema if schema else "json" + payload["format"] = "json" + if self.keep_alive is not None: + payload["keep_alive"] = self.keep_alive headers = {} if self.token: diff --git a/reference/tests/test_http_provider.py b/reference/tests/test_http_provider.py index d9dc65c..0453b89 100644 --- a/reference/tests/test_http_provider.py +++ b/reference/tests/test_http_provider.py @@ -19,6 +19,7 @@ def llama_settings(settings): settings.REFERENCE_TIMEOUT = 30 settings.REFERENCE_TOKEN = "" settings.REFERENCE_NUM_CTX = 8192 + settings.REFERENCE_KEEP_ALIVE = "-1" return settings @@ -69,8 +70,8 @@ def test_http_provider_chat_success(llama_settings): assert args[0] == "http://llama.example:11434/api/chat" assert kwargs["json"]["model"] == "llama3.2:3b" assert kwargs["json"]["options"]["num_ctx"] == 8192 - assert kwargs["json"]["format"]["type"] == "object" - assert kwargs["json"]["format"]["required"] == ["reftype"] + assert kwargs["json"]["format"] == "json" + assert kwargs["json"]["keep_alive"] == -1 assert kwargs["json"]["messages"][-1]["content"] == "Smith J. Nature. 2024." assert kwargs["headers"] == {} @@ -88,6 +89,36 @@ def test_http_provider_sends_bearer_token(llama_settings): provider.chat([{"role": "user", "content": "hi"}]) assert post.call_args.kwargs["headers"]["Authorization"] == "Bearer secret-token" + assert "format" not in post.call_args.kwargs["json"] + assert post.call_args.kwargs["json"]["keep_alive"] == -1 + + +def test_http_provider_keep_alive_duration(llama_settings): + llama_settings.REFERENCE_KEEP_ALIVE = "30m" + mock_response = MagicMock() + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = {"message": {"content": "{}"}} + + with patch( + "reference.providers.http.requests.post", return_value=mock_response + ) as post: + Provider([], None).chat([{"role": "user", "content": "hi"}]) + + assert post.call_args.kwargs["json"]["keep_alive"] == "30m" + + +def test_http_provider_omits_keep_alive_when_empty(llama_settings): + llama_settings.REFERENCE_KEEP_ALIVE = "" + mock_response = MagicMock() + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = {"message": {"content": "{}"}} + + with patch( + "reference.providers.http.requests.post", return_value=mock_response + ) as post: + Provider([], None).chat([{"role": "user", "content": "hi"}]) + + assert "keep_alive" not in post.call_args.kwargs["json"] def test_http_provider_raises_on_http_error(llama_settings): From 277da857e7a0c425e33ec3e38e417b5a0e8a7765 Mon Sep 17 00:00:00 2001 From: GitInno <86991526+gitnnolabs@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:54:28 -0300 Subject: [PATCH 4/4] Adiciona API REST para marcar o front SPS 1.10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Novo app front, independente, com POST /api/v1/front/ e /docx/, cache por checksum e XML só com tags SciELO PS. --- .envs.example/.local/.django | 10 + .envs.example/.production/.django | 10 + README.md | 1 + config/api_router.py | 2 + config/settings/base.py | 9 + front/__init__.py | 0 front/api/__init__.py | 0 front/api/v1/__init__.py | 0 front/api/v1/serializers.py | 47 +++ front/api/v1/views.py | 96 +++++ front/apps.py | 6 + front/data_utils.py | 602 ++++++++++++++++++++++++++++++ front/exceptions.py | 14 + front/marking.py | 29 ++ front/migrations/0001_initial.py | 73 ++++ front/migrations/__init__.py | 0 front/models.py | 37 ++ front/prompts.py | 92 +++++ front/providers/__init__.py | 5 + front/providers/http.py | 112 ++++++ front/tests/__init__.py | 0 front/tests/test_api.py | 159 ++++++++ front/tests/test_docx.py | 126 +++++++ front/tests/test_provider.py | 77 ++++ front/tests/test_xml.py | 249 ++++++++++++ front/utils.py | 81 ++++ 26 files changed, 1837 insertions(+) create mode 100644 front/__init__.py create mode 100644 front/api/__init__.py create mode 100644 front/api/v1/__init__.py create mode 100644 front/api/v1/serializers.py create mode 100644 front/api/v1/views.py create mode 100644 front/apps.py create mode 100644 front/data_utils.py create mode 100644 front/exceptions.py create mode 100644 front/marking.py create mode 100644 front/migrations/0001_initial.py create mode 100644 front/migrations/__init__.py create mode 100644 front/models.py create mode 100644 front/prompts.py create mode 100644 front/providers/__init__.py create mode 100644 front/providers/http.py create mode 100644 front/tests/__init__.py create mode 100644 front/tests/test_api.py create mode 100644 front/tests/test_docx.py create mode 100644 front/tests/test_provider.py create mode 100644 front/tests/test_xml.py create mode 100644 front/utils.py diff --git a/.envs.example/.local/.django b/.envs.example/.local/.django index 24d33c9..2c309c1 100644 --- a/.envs.example/.local/.django +++ b/.envs.example/.local/.django @@ -19,3 +19,13 @@ REFERENCE_MODEL=llama3.2:3b # REFERENCE_BATCH_SIZE=10 # REFERENCE_NUM_CTX=8192 # REFERENCE_KEEP_ALIVE=-1 + +# Front (Llama via HTTP — serviço ollama no local.yml) +# ------------------------------------------------------------------------------ +FRONT_ENABLED=true +FRONT_URL=http://ollama:11434 +FRONT_MODEL=llama3.2:3b +# FRONT_TOKEN= +# FRONT_TIMEOUT=300 +# FRONT_NUM_CTX=8192 +# FRONT_KEEP_ALIVE=-1 diff --git a/.envs.example/.production/.django b/.envs.example/.production/.django index d3de475..80768db 100644 --- a/.envs.example/.production/.django +++ b/.envs.example/.production/.django @@ -30,3 +30,13 @@ REFERENCE_MODEL=llama3.2:3b # REFERENCE_BATCH_SIZE=10 # REFERENCE_NUM_CTX=8192 # REFERENCE_KEEP_ALIVE=-1 + +# Front (Llama via HTTP — Ollama-compatible API) +# ------------------------------------------------------------------------------ +FRONT_ENABLED=true +FRONT_URL= +FRONT_MODEL=llama3.2:3b +# FRONT_TOKEN= +# FRONT_TIMEOUT=300 +# FRONT_NUM_CTX=8192 +# FRONT_KEEP_ALIVE=-1 diff --git a/README.md b/README.md index 2388595..f893ef6 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ make down # para os containers | `core/` | Modelos base, Wagtail home, templates e static | | `core_settings/` | Configurações editáveis do site (nome, logo, favicon) | | `users/` | `CustomUser` (`AUTH_USER_MODEL`) | +| `front/` | Marcação do front do artigo (API REST, Llama local, XML SPS 1.10) | | `compose/` | Dockerfiles e scripts de inicialização | | `requirements/` | Dependências Python (base, local, production) | diff --git a/config/api_router.py b/config/api_router.py index 341dcad..c6f3a91 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -1,6 +1,7 @@ from django.conf import settings from rest_framework.routers import DefaultRouter, SimpleRouter +from front.api.v1.views import FrontViewSet from reference.api.v1.views import ReferenceViewSet if settings.DEBUG: @@ -9,5 +10,6 @@ router = SimpleRouter() router.register("reference", ReferenceViewSet, basename="reference") +router.register("front", FrontViewSet, basename="front") urlpatterns = router.urls diff --git a/config/settings/base.py b/config/settings/base.py index 5516075..7bec0f4 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -81,6 +81,7 @@ "core_settings", "xml_manager", "reference", + "front", ] INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS + WAGTAIL @@ -326,3 +327,11 @@ REFERENCE_BATCH_SIZE = env.int("REFERENCE_BATCH_SIZE", default=10) REFERENCE_NUM_CTX = env.int("REFERENCE_NUM_CTX", default=8192) REFERENCE_KEEP_ALIVE = env("REFERENCE_KEEP_ALIVE", default="-1") + +FRONT_ENABLED = env.bool("FRONT_ENABLED", default=True) +FRONT_URL = env("FRONT_URL", default="") +FRONT_MODEL = env("FRONT_MODEL", default="llama3.2:3b") +FRONT_TIMEOUT = env.int("FRONT_TIMEOUT", default=300) +FRONT_TOKEN = env("FRONT_TOKEN", default="") +FRONT_NUM_CTX = env.int("FRONT_NUM_CTX", default=8192) +FRONT_KEEP_ALIVE = env("FRONT_KEEP_ALIVE", default="-1") diff --git a/front/__init__.py b/front/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/front/api/__init__.py b/front/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/front/api/v1/__init__.py b/front/api/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/front/api/v1/serializers.py b/front/api/v1/serializers.py new file mode 100644 index 0000000..f02d958 --- /dev/null +++ b/front/api/v1/serializers.py @@ -0,0 +1,47 @@ +from rest_framework import serializers + +OUTPUT_TYPE_CHOICES = ["json", "xml"] + + +class FrontMarkRequestSerializer(serializers.Serializer): + front = serializers.CharField( + allow_blank=False, + trim_whitespace=True, + help_text="Texto do front do artigo a marcar.", + ) + type = serializers.ChoiceField( + choices=OUTPUT_TYPE_CHOICES, + default="json", + required=False, + help_text="Formato de saída: json ou xml.", + ) + language = serializers.CharField( + required=False, + allow_blank=True, + help_text="Idioma principal do artigo (ex.: pt, en, es).", + ) + + +class FrontDocxRequestSerializer(serializers.Serializer): + file = serializers.FileField( + help_text="Arquivo .docx com o front do artigo.", + ) + type = serializers.ChoiceField( + choices=OUTPUT_TYPE_CHOICES, + default="json", + required=False, + help_text="Formato de saída: json ou xml.", + ) + language = serializers.CharField( + required=False, + allow_blank=True, + help_text="Idioma principal do artigo (ex.: pt, en, es).", + ) + + def validate_file(self, value): + name = (getattr(value, "name", "") or "").lower() + if not name.endswith(".docx"): + raise serializers.ValidationError("Only .docx files are accepted.") + if getattr(value, "size", None) == 0: + raise serializers.ValidationError("Empty file.") + return value diff --git a/front/api/v1/views.py b/front/api/v1/views.py new file mode 100644 index 0000000..ec25580 --- /dev/null +++ b/front/api/v1/views.py @@ -0,0 +1,96 @@ +from collections.abc import Mapping + +from django.http import JsonResponse +from rest_framework.decorators import action +from rest_framework.parsers import FormParser, MultiPartParser +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.viewsets import GenericViewSet + +from front.api.v1.serializers import ( + FrontDocxRequestSerializer, + FrontMarkRequestSerializer, +) +from front.data_utils import resolve_front_result +from front.exceptions import ( + FrontDocxError, + FrontLlamaDisabledError, + FrontLlamaMisconfiguredError, + FrontLlamaUnavailableError, +) +from front.utils import front_from_docx_upload + + +class FrontViewSet(GenericViewSet): + serializer_class = FrontMarkRequestSerializer + permission_classes = [IsAuthenticated] + http_method_names = [ + "get", + "post", + "head", + "options", + ] + + def get_serializer_class(self): + if getattr(self, "action", None) == "docx": + return FrontDocxRequestSerializer + return FrontMarkRequestSerializer + + def create(self, request, *args, **kwargs): + data = request.data + if not isinstance(data, Mapping): + return JsonResponse({"error": "Error processing"}, status=400) + + serializer = self.get_serializer(data=data) + if not serializer.is_valid(): + return JsonResponse(serializer.errors, status=400) + + return self.mark_and_respond( + serializer.validated_data["front"], + serializer.validated_data.get("type", "json"), + serializer.validated_data.get("language") or None, + ) + + @action( + detail=False, + methods=["get", "post"], + url_path="docx", + parser_classes=[MultiPartParser, FormParser], + ) + def docx(self, request): + if request.method == "GET": + return Response({}) + + serializer = self.get_serializer(data=request.data) + if not serializer.is_valid(): + return JsonResponse(serializer.errors, status=400) + + uploaded = serializer.validated_data["file"] + output_type = serializer.validated_data.get("type", "json") + language = serializer.validated_data.get("language") or None + try: + front_text = front_from_docx_upload(uploaded) + except FrontDocxError as exc: + return JsonResponse({"error": str(exc)}, status=400) + return self.mark_and_respond(front_text, output_type, language) + + def mark_and_respond(self, front_text, output_type, language): + if not str(front_text or "").strip(): + return JsonResponse({"error": "No front provided"}, status=400) + try: + result = resolve_front_result( + front_text, + user=self.request.user, + output_type=output_type, + language=language, + ) + except ( + FrontLlamaDisabledError, + FrontLlamaMisconfiguredError, + FrontLlamaUnavailableError, + ) as exc: + return JsonResponse( + {"error": f"Llama model is not available: {exc}"}, + status=503, + ) + return JsonResponse(result) diff --git a/front/apps.py b/front/apps.py new file mode 100644 index 0000000..a4acdb9 --- /dev/null +++ b/front/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class FrontConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "front" diff --git a/front/data_utils.py b/front/data_utils.py new file mode 100644 index 0000000..177a86e --- /dev/null +++ b/front/data_utils.py @@ -0,0 +1,602 @@ +import hashlib +import json +import re + +from django.db import IntegrityError +from lxml import etree + +from front.exceptions import FrontLlamaUnavailableError +from front.marking import mark_front +from front.models import Front +from front.utils import normalize_front_text + +XLINK_NS = "http://www.w3.org/1999/xlink" + +COUNTRY_CODES = { + "brasil": "BR", + "brazil": "BR", + "argentina": "AR", + "chile": "CL", + "colombia": "CO", + "mexico": "MX", + "méxico": "MX", + "peru": "PE", + "perú": "PE", + "portugal": "PT", + "spain": "ES", + "españa": "ES", + "espanha": "ES", + "united states": "US", + "usa": "US", + "estados unidos": "US", + "france": "FR", + "frança": "FR", + "francia": "FR", + "germany": "DE", + "alemanha": "DE", + "alemania": "DE", + "united kingdom": "GB", + "uk": "GB", + "reino unido": "GB", + "italy": "IT", + "itália": "IT", + "italia": "IT", + "canada": "CA", + "canadá": "CA", + "uruguay": "UY", + "uruguai": "UY", + "paraguay": "PY", + "paraguai": "PY", + "bolivia": "BO", + "bolívia": "BO", + "ecuador": "EC", + "equador": "EC", + "venezuela": "VE", + "cuba": "CU", + "costa rica": "CR", +} + +ORCID_RE = re.compile( + r"(?:https?://orcid\.org/)?(\d{4}-\d{4}-\d{4}-\d{3}[\dX])", + re.IGNORECASE, +) +DOI_PREFIXES = ( + "https://doi.org/", + "http://doi.org/", + "https://dx.doi.org/", + "http://dx.doi.org/", + "doi:", +) + + +def _blank(value): + return value is None or str(value).strip() == "" + + +def _text_el(parent, tag, value, attrib=None): + if _blank(value): + return None + element = etree.SubElement(parent, tag, attrib=attrib or {}) + element.text = str(value).strip() + return element + + +def _normalize_orcid(value): + if _blank(value): + return "" + match = ORCID_RE.search(str(value).strip()) + return match.group(1) if match else str(value).strip() + + +def _normalize_doi(value): + if _blank(value): + return "" + text = str(value).strip() + lower = text.lower() + for prefix in DOI_PREFIXES: + if lower.startswith(prefix): + text = text[len(prefix) :].strip() + break + return text.rstrip(".,;:)]}»\"'") + + +def _country_code(affiliation): + code = str(affiliation.get("country_code") or "").strip().upper() + if len(code) == 2: + return code + name = str(affiliation.get("country") or "").strip().lower() + return COUNTRY_CODES.get(name, "") + + +def _date_parts(item): + if not isinstance(item, dict): + return {} + parts = { + "day": item.get("day"), + "month": item.get("month"), + "year": item.get("year"), + "season": item.get("season"), + } + raw = item.get("date") + if _blank(parts["year"]) and not _blank(raw): + match = re.match(r"^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?$", str(raw).strip()) + if match: + parts["year"] = match.group(1) + parts["month"] = match.group(2) or parts["month"] + parts["day"] = match.group(3) or parts["day"] + return parts + + +def _append_date_parts(parent, item): + parts = _date_parts(item) + _text_el(parent, "season", parts.get("season")) + _text_el(parent, "day", parts.get("day")) + _text_el(parent, "month", parts.get("month")) + _text_el(parent, "year", parts.get("year")) + return bool(list(parent)) + + +def _append_abstract(parent, tag, item, with_lang=False): + if not isinstance(item, dict): + return + text = item.get("text") + sections = item.get("sections") or [] + title = item.get("title") + if _blank(text) and not sections and _blank(title): + return + attrib = {} + if with_lang and not _blank(item.get("language")): + attrib["{http://www.w3.org/XML/1998/namespace}lang"] = str( + item["language"] + ).strip() + abstract_type = item.get("abstract_type") + if abstract_type == "key-points": + attrib["abstract-type"] = "key-points" + node = etree.SubElement(parent, tag, attrib=attrib) + _text_el(node, "title", title) + if sections: + for section in sections: + if not isinstance(section, dict): + continue + if _blank(section.get("title")) and _blank(section.get("text")): + continue + sec = etree.SubElement(node, "sec") + _text_el(sec, "title", section.get("title")) + _text_el(sec, "p", section.get("text")) + elif not _blank(text): + _text_el(node, "p", text) + + +def parse_marked(choice): + if isinstance(choice, dict): + return choice + try: + parsed = json.loads(choice) + except (TypeError, json.JSONDecodeError): + return None + return parsed if isinstance(parsed, dict) else None + + +def apply_language_fallback(data, language): + if not isinstance(data, dict) or _blank(language): + return data if isinstance(data, dict) else {} + lang = str(language).strip() + marked = dict(data) + keywords = [] + for group in marked.get("keywords") or []: + if not isinstance(group, dict): + continue + item = dict(group) + if _blank(item.get("language")): + item["language"] = lang + keywords.append(item) + if keywords: + marked["keywords"] = keywords + return marked + + +def get_front_xml(data): + if not isinstance(data, dict): + data = {} + + front = etree.Element("front", nsmap={"xlink": XLINK_NS}) + journal = data.get("journal") if isinstance(data.get("journal"), dict) else {} + + journal_ids = journal.get("journal_ids") or [] + issns = journal.get("issns") or [] + has_journal = any( + [ + journal_ids, + not _blank(journal.get("journal_title")), + not _blank(journal.get("abbrev_journal_title")), + issns, + not _blank(journal.get("publisher_name")), + ] + ) + if has_journal: + journal_meta = etree.SubElement(front, "journal-meta") + for item in journal_ids: + if not isinstance(item, dict): + continue + jtype = str(item.get("type") or "").strip() + if jtype not in ("publisher-id", "nlm-ta"): + continue + _text_el( + journal_meta, + "journal-id", + item.get("value"), + attrib={"journal-id-type": jtype}, + ) + if not _blank(journal.get("journal_title")) or not _blank( + journal.get("abbrev_journal_title") + ): + title_group = etree.SubElement(journal_meta, "journal-title-group") + _text_el(title_group, "journal-title", journal.get("journal_title")) + _text_el( + title_group, + "abbrev-journal-title", + journal.get("abbrev_journal_title"), + ) + for item in issns: + if not isinstance(item, dict): + continue + pub_type = str(item.get("pub_type") or "").strip() + attrib = {} + if pub_type in ("epub", "ppub"): + attrib["pub-type"] = pub_type + _text_el(journal_meta, "issn", item.get("value"), attrib=attrib) + if not _blank(journal.get("publisher_name")): + publisher = etree.SubElement(journal_meta, "publisher") + _text_el(publisher, "publisher-name", journal.get("publisher_name")) + + article_meta = etree.SubElement(front, "article-meta") + + for item in data.get("article_ids") or []: + if not isinstance(item, dict): + continue + pub_id_type = str(item.get("pub_id_type") or "").strip() + if pub_id_type not in ("doi", "publisher-id", "other"): + continue + value = item.get("value") + if pub_id_type == "doi": + value = _normalize_doi(value) + _text_el( + article_meta, + "article-id", + value, + attrib={"pub-id-type": pub_id_type}, + ) + + categories = [ + item + for item in (data.get("categories") or []) + if isinstance(item, dict) and not _blank(item.get("subject")) + ] + if categories: + article_categories = etree.SubElement(article_meta, "article-categories") + for item in categories: + group = etree.SubElement( + article_categories, + "subj-group", + attrib={"subj-group-type": "heading"}, + ) + _text_el(group, "subject", item.get("subject")) + + titles = [item for item in (data.get("titles") or []) if isinstance(item, dict)] + main_titles = [ + item + for item in titles + if item.get("kind") == "main" and not _blank(item.get("text")) + ] + trans_titles = [ + item + for item in titles + if item.get("kind") == "translated" + and not _blank(item.get("text")) + and not _blank(item.get("language")) + ] + if not main_titles: + main_titles = [ + item + for item in titles + if item.get("kind") not in ("translated",) and not _blank(item.get("text")) + ] + if main_titles or trans_titles: + title_group = etree.SubElement(article_meta, "title-group") + if main_titles: + _text_el(title_group, "article-title", main_titles[0].get("text")) + for item in trans_titles: + group = etree.SubElement( + title_group, + "trans-title-group", + attrib={ + "{http://www.w3.org/XML/1998/namespace}lang": str( + item["language"] + ).strip() + }, + ) + _text_el(group, "trans-title", item.get("text")) + + authors = [item for item in (data.get("authors") or []) if isinstance(item, dict)] + if authors: + contrib_group = etree.SubElement(article_meta, "contrib-group") + for author in authors: + contrib_type = str(author.get("contrib_type") or "author").strip() + allowed = ( + "author", + "compiler", + "editor", + "illustrator", + "translator", + "research-assistant", + "reviewer", + ) + if contrib_type not in allowed: + contrib_type = "author" + contrib = etree.SubElement( + contrib_group, "contrib", attrib={"contrib-type": contrib_type} + ) + orcid = _normalize_orcid(author.get("orcid")) + if orcid: + _text_el( + contrib, + "contrib-id", + orcid, + attrib={"contrib-id-type": "orcid"}, + ) + if ( + not _blank(author.get("collab")) + and _blank(author.get("surname")) + and _blank(author.get("given_names")) + ): + _text_el(contrib, "collab", author.get("collab")) + elif not _blank(author.get("surname")) or not _blank( + author.get("given_names") + ): + name = etree.SubElement(contrib, "name") + _text_el(name, "surname", author.get("surname")) + _text_el(name, "given-names", author.get("given_names")) + for aff_id in author.get("affiliations") or []: + if _blank(aff_id): + continue + xref = etree.SubElement( + contrib, + "xref", + attrib={"ref-type": "aff", "rid": str(aff_id).strip()}, + ) + label = None + for affiliation in data.get("affiliations") or []: + if ( + isinstance(affiliation, dict) + and affiliation.get("id") == aff_id + ): + label = affiliation.get("label") + break + if not _blank(label): + _text_el(xref, "sup", label) + if author.get("corresp"): + etree.SubElement( + contrib, "xref", attrib={"ref-type": "corresp", "rid": "c01"} + ) + for role in author.get("roles") or []: + _text_el(contrib, "role", role) + + for affiliation in data.get("affiliations") or []: + if not isinstance(affiliation, dict): + continue + aff_id = str(affiliation.get("id") or "").strip() + if not aff_id: + continue + aff = etree.SubElement(article_meta, "aff", attrib={"id": aff_id}) + _text_el(aff, "label", affiliation.get("label")) + _text_el( + aff, + "institution", + affiliation.get("original"), + attrib={"content-type": "original"}, + ) + _text_el( + aff, + "institution", + affiliation.get("orgname"), + attrib={"content-type": "orgname"}, + ) + _text_el( + aff, + "institution", + affiliation.get("orgdiv1"), + attrib={"content-type": "orgdiv1"}, + ) + _text_el( + aff, + "institution", + affiliation.get("orgdiv2"), + attrib={"content-type": "orgdiv2"}, + ) + addr_fields = ( + affiliation.get("city"), + affiliation.get("state"), + affiliation.get("postal_code"), + ) + if any(not _blank(field) for field in addr_fields): + addr_line = etree.SubElement(aff, "addr-line") + _text_el(addr_line, "city", affiliation.get("city")) + _text_el(addr_line, "state", affiliation.get("state")) + _text_el(addr_line, "postal-code", affiliation.get("postal_code")) + country_name = affiliation.get("country") + code = _country_code(affiliation) + if not _blank(country_name) or code: + attrib = {"country": code} if code else {} + _text_el(aff, "country", country_name or code, attrib=attrib) + _text_el(aff, "email", affiliation.get("email")) + + author_notes = data.get("author_notes") + if isinstance(author_notes, dict): + notes = etree.Element("author-notes") + _text_el(notes, "corresp", author_notes.get("corresp"), attrib={"id": "c01"}) + for fn_item in author_notes.get("fns") or []: + text = fn_item.get("text") if isinstance(fn_item, dict) else fn_item + _text_el(notes, "fn", text) + if list(notes): + article_meta.append(notes) + + for item in data.get("pub_dates") or []: + if not isinstance(item, dict): + continue + date_type = str(item.get("type") or "").strip() + if date_type not in ("pub", "collection"): + continue + pub_date = etree.SubElement( + article_meta, + "pub-date", + attrib={ + "date-type": date_type, + "publication-format": "electronic", + }, + ) + if not _append_date_parts(pub_date, item): + article_meta.remove(pub_date) + + _text_el(article_meta, "volume", data.get("volume")) + _text_el(article_meta, "issue", data.get("issue")) + if not _blank(data.get("elocation_id")): + _text_el(article_meta, "elocation-id", data.get("elocation_id")) + else: + _text_el(article_meta, "fpage", data.get("fpage")) + _text_el(article_meta, "lpage", data.get("lpage")) + + history_items = [ + item + for item in (data.get("history") or data.get("dates") or []) + if isinstance(item, dict) + and str(item.get("type") or "").strip() in ("received", "rev-recd", "accepted") + ] + if history_items: + history = etree.SubElement(article_meta, "history") + for item in history_items: + date_el = etree.SubElement( + history, + "date", + attrib={"date-type": str(item.get("type")).strip()}, + ) + if not _append_date_parts(date_el, item): + history.remove(date_el) + if not list(history): + article_meta.remove(history) + + permissions = data.get("permissions") + if isinstance(permissions, dict): + perm = etree.Element("permissions") + _text_el(perm, "copyright-statement", permissions.get("copyright_statement")) + _text_el(perm, "copyright-year", permissions.get("copyright_year")) + _text_el(perm, "copyright-holder", permissions.get("copyright_holder")) + href = permissions.get("license_href") + license_p = permissions.get("license_p") + if not _blank(href) or not _blank(license_p): + attrib = {"license-type": "open-access"} + if not _blank(href): + attrib["{%s}href" % XLINK_NS] = str(href).strip() + license_el = etree.SubElement(perm, "license", attrib=attrib) + _text_el(license_el, "license-p", license_p) + if list(perm): + article_meta.append(perm) + + abstracts = [ + item for item in (data.get("abstracts") or []) if isinstance(item, dict) + ] + for item in abstracts: + kind = item.get("kind") + if kind == "translated": + if _blank(item.get("language")): + continue + _append_abstract(article_meta, "trans-abstract", item, with_lang=True) + else: + _append_abstract(article_meta, "abstract", item, with_lang=False) + + for group in data.get("keywords") or []: + if not isinstance(group, dict): + continue + words = [word for word in (group.get("keywords") or []) if not _blank(word)] + if not words: + continue + attrib = {} + if not _blank(group.get("language")): + attrib["{http://www.w3.org/XML/1998/namespace}lang"] = str( + group["language"] + ).strip() + kwd_group = etree.SubElement(article_meta, "kwd-group", attrib=attrib) + _text_el(kwd_group, "title", group.get("title")) + for word in words: + _text_el(kwd_group, "kwd", word) + + funding = data.get("funding") + if isinstance(funding, dict): + funding_group = etree.Element("funding-group") + for award in funding.get("awards") or []: + if not isinstance(award, dict): + continue + if _blank(award.get("funding_source")) and _blank(award.get("award_id")): + continue + award_group = etree.SubElement(funding_group, "award-group") + _text_el(award_group, "funding-source", award.get("funding_source")) + _text_el(award_group, "award-id", award.get("award_id")) + _text_el(funding_group, "funding-statement", funding.get("funding_statement")) + if list(funding_group): + article_meta.append(funding_group) + + counts = data.get("counts") + if isinstance(counts, dict): + counts_el = etree.Element("counts") + for key, tag in ( + ("fig_count", "fig-count"), + ("table_count", "table-count"), + ("equation_count", "equation-count"), + ("ref_count", "ref-count"), + ): + if _blank(counts.get(key)): + continue + etree.SubElement( + counts_el, tag, attrib={"count": str(counts.get(key)).strip()} + ) + if list(counts_el): + article_meta.append(counts_el) + + return etree.tostring(front, pretty_print=True, encoding="unicode") + + +def resolve_front_result(front_text, user=None, output_type="json", language=None): + normalized = normalize_front_text(front_text) + checksum = hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + try: + record = Front.objects.get(checksum=checksum) + except Front.DoesNotExist: + raw = mark_front(front_text) + marked = parse_marked(raw) + if marked is None: + raise FrontLlamaUnavailableError("Front Llama returned invalid JSON") + marked = apply_language_fallback(marked, language) + xml = get_front_xml(marked) + try: + record, created = Front.objects.get_or_create( + checksum=checksum, + defaults={ + "source_text": front_text, + "marked": marked, + "marked_xml": xml, + "creator": user, + }, + ) + except IntegrityError: + record = Front.objects.get(checksum=checksum) + created = False + if not created and not record.marked: + record.marked = marked + record.marked_xml = xml + record.save(update_fields=["marked", "marked_xml", "updated"]) + + if output_type == "xml": + data = record.marked_xml + else: + data = record.marked + return {"data": data} diff --git a/front/exceptions.py b/front/exceptions.py new file mode 100644 index 0000000..fc62eef --- /dev/null +++ b/front/exceptions.py @@ -0,0 +1,14 @@ +class FrontLlamaDisabledError(Exception): + pass + + +class FrontLlamaMisconfiguredError(Exception): + pass + + +class FrontLlamaUnavailableError(Exception): + pass + + +class FrontDocxError(Exception): + pass diff --git a/front/marking.py b/front/marking.py new file mode 100644 index 0000000..31abf3c --- /dev/null +++ b/front/marking.py @@ -0,0 +1,29 @@ +import logging + +from front.exceptions import ( + FrontLlamaDisabledError, + FrontLlamaMisconfiguredError, + FrontLlamaUnavailableError, +) +from front.prompts import MESSAGES, RESPONSE_FORMAT +from front.providers import get_provider + +logger = logging.getLogger(__name__) + + +def mark_front(front_text): + try: + marker = get_provider(MESSAGES, RESPONSE_FORMAT) + output = marker.run(front_text) + for item in output.get("choices", []): + return item.get("message", {}).get("content", "") + return "" + except ( + FrontLlamaDisabledError, + FrontLlamaMisconfiguredError, + FrontLlamaUnavailableError, + ): + raise + except Exception: + logger.exception("Unexpected error marking front") + raise FrontLlamaUnavailableError("Front Llama returned an unexpected error") diff --git a/front/migrations/0001_initial.py b/front/migrations/0001_initial.py new file mode 100644 index 0000000..f3800c7 --- /dev/null +++ b/front/migrations/0001_initial.py @@ -0,0 +1,73 @@ +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Front", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("source_text", models.TextField(verbose_name="Source text")), + ( + "normalized_text", + models.TextField( + blank=True, db_index=True, verbose_name="Normalized text" + ), + ), + ( + "checksum", + models.CharField( + blank=True, max_length=64, unique=True, verbose_name="SHA256" + ), + ), + ( + "marked", + models.JSONField(blank=True, default=dict, verbose_name="Marked"), + ), + ("marked_xml", models.TextField(blank=True, verbose_name="Marked XML")), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "creator", + models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + ], + options={ + "verbose_name": "Front", + "verbose_name_plural": "Fronts", + }, + ), + ] diff --git a/front/migrations/__init__.py b/front/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/front/models.py b/front/models.py new file mode 100644 index 0000000..d36c487 --- /dev/null +++ b/front/models.py @@ -0,0 +1,37 @@ +import hashlib + +from django.conf import settings +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from front.utils import normalize_front_text + + +class Front(models.Model): + source_text = models.TextField(_("Source text")) + normalized_text = models.TextField(_("Normalized text"), blank=True, db_index=True) + checksum = models.CharField(_("SHA256"), max_length=64, blank=True, unique=True) + marked = models.JSONField(_("Marked"), default=dict, blank=True) + marked_xml = models.TextField(_("Marked XML"), blank=True) + created = models.DateTimeField(verbose_name=_("Creation date"), auto_now_add=True) + updated = models.DateTimeField(verbose_name=_("Last update date"), auto_now=True) + creator = models.ForeignKey( + settings.AUTH_USER_MODEL, + verbose_name=_("Creator"), + related_name="%(class)s_creator", + editable=False, + on_delete=models.SET_NULL, + null=True, + ) + + def __str__(self): + return self.checksum + + def save(self, *args, **kwargs): + self.normalized_text = normalize_front_text(self.source_text) + self.checksum = hashlib.sha256(self.normalized_text.encode("utf-8")).hexdigest() + super().save(*args, **kwargs) + + class Meta: + verbose_name = _("Front") + verbose_name_plural = _("Fronts") diff --git a/front/prompts.py b/front/prompts.py new file mode 100644 index 0000000..aa821ed --- /dev/null +++ b/front/prompts.py @@ -0,0 +1,92 @@ +MESSAGES = [ + { + "role": "system", + "content": ( + "You extract SciELO SPS 1.10 metadata from article front matter. " + "Respond ONLY with one JSON object. Omit keys that are not in the text. " + "Never invent DOI, ISSN, ORCID, dates, volume, issue, pages, funding, " + "license, counts, or journal identifiers. " + "JSON keys: journal, article_ids, categories, titles, authors, " + "affiliations, author_notes, pub_dates, volume, issue, fpage, lpage, " + "elocation_id, abstracts, keywords, history, permissions, counts, funding. " + "journal: journal_ids [{type: publisher-id|nlm-ta, value}], " + "journal_title, abbrev_journal_title, issns [{pub_type: epub|ppub, value}], " + "publisher_name. " + "article_ids: [{pub_id_type: doi|publisher-id|other, value}]; doi is the " + "bare id without https://doi.org/. " + "categories: [{subject}]. " + "titles: [{kind: main|translated, text, language?}]; language only on " + "translated. " + "authors: [{contrib_type (default author), given_names, surname, collab, " + "orcid (bare 0000-0000-0000-0000), affiliations [aff ids], corresp bool, " + "roles []}]. " + "affiliations: [{id, label, original, orgname, orgdiv1, orgdiv2, city, " + "state, postal_code, country, country_code (ISO 3166-1 alpha-2), email}]. " + "author_notes: {corresp, fns:[{text}]}. " + "pub_dates: [{type: pub|collection, day, month, year, season}]. " + "history: [{type: received|rev-recd|accepted, day, month, year}]. " + "abstracts: [{kind: main|translated, title, text, language?, " + "abstract_type?, sections:[{title, text}]}]; language only on translated; " + "abstract_type only key-points when the text is key points. " + "keywords: [{language, title, keywords:[]}]. " + "permissions: {copyright_statement, copyright_year, copyright_holder, " + "license_href, license_p}. " + "counts: {fig_count, table_count, equation_count, ref_count} as strings " + "only when stated. " + "funding: {awards:[{funding_source, award_id}], funding_statement}." + ), + }, + { + "role": "user", + "content": ( + "Biota Neotropica\n" + "ISSN 1676-0611\n" + "What do scientific collections reveal about the past of the Amazon?\n" + "O que as coleções científicas revelam sobre o passado da Amazônia?\n" + "Jéssica S. de Lima https://orcid.org/0000-0002-3193-9315\n" + "1 Instituto de Pesquisas Ambientais, São Paulo, SP, Brasil.\n" + "* Correspondence: jessica@example.com\n" + "Received: 10 January 2025. Accepted: 21 April 2026.\n" + "DOI: 10.1590/1676-0611-BN-2025-1870\n" + "Abstract\nThis study reports bryophyte species.\n" + "Resumo\nEste estudo relata espécies de briófitas.\n" + "Keywords: Amazon flora; Herbarium\n" + "Palavras-chave: Flora amazônica; Herbário\n" + "FAPESP 2024/23894-1" + ), + }, + { + "role": "assistant", + "content": ( + '{"article_ids":[{"pub_id_type":"doi",' + '"value":"10.1590/1676-0611-BN-2025-1870"}],' + '"journal":{"journal_title":"Biota Neotropica",' + '"issns":[{"pub_type":"epub","value":"1676-0611"}]},' + '"titles":[{"kind":"main","text":"What do scientific collections reveal ' + 'about the past of the Amazon?"},' + '{"kind":"translated","language":"pt",' + '"text":"O que as coleções científicas revelam sobre o passado da Amazônia?"}],' + '"authors":[{"contrib_type":"author","given_names":"Jéssica S. de",' + '"surname":"Lima","orcid":"0000-0002-3193-9315",' + '"affiliations":["aff1"],"corresp":true}],' + '"affiliations":[{"id":"aff1","label":"1",' + '"orgname":"Instituto de Pesquisas Ambientais",' + '"city":"São Paulo","state":"SP","country":"Brasil","country_code":"BR"}],' + '"author_notes":{"corresp":"* Correspondence: jessica@example.com"},' + '"history":[{"type":"received","day":"10","month":"01","year":"2025"},' + '{"type":"accepted","day":"21","month":"04","year":"2026"}],' + '"abstracts":[{"kind":"main","title":"Abstract",' + '"text":"This study reports bryophyte species."},' + '{"kind":"translated","language":"pt","title":"Resumo",' + '"text":"Este estudo relata espécies de briófitas."}],' + '"keywords":[{"language":"en","title":"Keywords",' + '"keywords":["Amazon flora","Herbarium"]},' + '{"language":"pt","title":"Palavras-chave",' + '"keywords":["Flora amazônica","Herbário"]}],' + '"funding":{"awards":[{"funding_source":"FAPESP",' + '"award_id":"2024/23894-1"}]}}' + ), + }, +] + +RESPONSE_FORMAT = {"type": "json_object"} diff --git a/front/providers/__init__.py b/front/providers/__init__.py new file mode 100644 index 0000000..beb49f8 --- /dev/null +++ b/front/providers/__init__.py @@ -0,0 +1,5 @@ +from front.providers.http import Provider + + +def get_provider(messages, response_format, **kwargs): + return Provider(messages, response_format, **kwargs) diff --git a/front/providers/http.py b/front/providers/http.py new file mode 100644 index 0000000..2651650 --- /dev/null +++ b/front/providers/http.py @@ -0,0 +1,112 @@ +import logging +import time + +import requests + +from front.exceptions import ( + FrontLlamaDisabledError, + FrontLlamaMisconfiguredError, + FrontLlamaUnavailableError, +) + +logger = logging.getLogger(__name__) + + +class Provider: + def __init__( + self, + messages, + response_format, + temperature=0.0, + top_p=0.1, + max_tokens=8000, + ): + from django.conf import settings + + self.messages = messages or [] + self.response_format = response_format + self.temperature = temperature + self.top_p = top_p + self.max_tokens = max_tokens + + if not getattr(settings, "FRONT_ENABLED", True): + raise FrontLlamaDisabledError("Front Llama is disabled.") + + self.url = (getattr(settings, "FRONT_URL", "") or "").rstrip("/") + if not self.url: + raise FrontLlamaMisconfiguredError( + "FRONT_URL is required when Front Llama is enabled." + ) + + self.model = getattr(settings, "FRONT_MODEL", "") or "llama3.2:3b" + self.timeout = getattr(settings, "FRONT_TIMEOUT", 300) + self.token = getattr(settings, "FRONT_TOKEN", "") or "" + self.num_ctx = int(getattr(settings, "FRONT_NUM_CTX", 8192) or 8192) + raw_keep_alive = getattr(settings, "FRONT_KEEP_ALIVE", "-1") + if raw_keep_alive is None or raw_keep_alive == "": + self.keep_alive = None + else: + try: + self.keep_alive = int(raw_keep_alive) + except (TypeError, ValueError): + self.keep_alive = str(raw_keep_alive) + + def run(self, user_input): + messages = self.messages.copy() + messages.append({"role": "user", "content": user_input}) + return self.chat(messages) + + def chat(self, messages): + started = time.monotonic() + logger.info( + "Front Llama chat via %s model=%s. Preview: %r", + self.url, + self.model, + messages[-1]["content"][:150], + ) + + options = { + "temperature": self.temperature, + "top_p": self.top_p, + "num_ctx": self.num_ctx, + } + if self.max_tokens: + options["num_predict"] = self.max_tokens + + payload = { + "model": self.model, + "messages": messages, + "options": options, + "stream": False, + } + if self.response_format and self.response_format.get("type") == "json_object": + payload["format"] = "json" + if self.keep_alive is not None: + payload["keep_alive"] = self.keep_alive + + headers = {} + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + + try: + resp = requests.post( + f"{self.url}/api/chat", + json=payload, + headers=headers, + timeout=self.timeout, + ) + resp.raise_for_status() + response_text = resp.json().get("message", {}).get("content") or "" + except requests.RequestException as exc: + logger.error("Front Llama HTTP error: %s", exc) + raise FrontLlamaUnavailableError( + f"Front Llama service unavailable: {exc}" + ) from exc + + elapsed = time.monotonic() - started + logger.info( + "Front Llama chat: %d chars in %.2fs", + len(response_text), + elapsed, + ) + return {"choices": [{"message": {"content": response_text}}]} diff --git a/front/tests/__init__.py b/front/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/front/tests/test_api.py b/front/tests/test_api.py new file mode 100644 index 0000000..171258e --- /dev/null +++ b/front/tests/test_api.py @@ -0,0 +1,159 @@ +import json +from unittest.mock import MagicMock + +import pytest +from django.contrib.auth import get_user_model +from rest_framework.test import APIClient + +from front.exceptions import FrontLlamaUnavailableError +from front.models import Front + +SAMPLE_MARKED = { + "titles": [{"kind": "main", "text": "Título de teste"}], + "authors": [ + { + "given_names": "Ana", + "surname": "Silva", + "affiliations": ["aff1"], + } + ], + "affiliations": [ + { + "id": "aff1", + "orgname": "Universidade Exemplo", + "city": "São Paulo", + "country": "Brasil", + "country_code": "BR", + } + ], + "abstracts": [{"kind": "main", "title": "Resumo", "text": "Texto do resumo."}], + "keywords": [{"language": "pt", "keywords": ["ciência", "dados"]}], + "article_ids": [{"pub_id_type": "doi", "value": "10.1590/example"}], +} + + +@pytest.mark.django_db +def test_api_marks_front_json(monkeypatch): + monkeypatch.setattr( + "front.data_utils.mark_front", + lambda _text: json.dumps(SAMPLE_MARKED), + ) + User = get_user_model() + user = User.objects.create_user(username="frontuser", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/front/", + data=json.dumps( + { + "front": "Título de teste\nAna Silva\nDOI: 10.1590/example", + "type": "json", + "language": "pt", + } + ), + content_type="application/json", + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["data"]["titles"][0]["text"] == "Título de teste" + assert payload["data"]["article_ids"][0]["value"] == "10.1590/example" + assert Front.objects.count() == 1 + + +@pytest.mark.django_db +def test_api_marks_front_xml(monkeypatch): + monkeypatch.setattr( + "front.data_utils.mark_front", + lambda _text: json.dumps(SAMPLE_MARKED), + ) + User = get_user_model() + user = User.objects.create_user(username="frontxml", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/front/", + data=json.dumps({"front": "Título de teste\nAna Silva", "type": "xml"}), + content_type="application/json", + ) + + assert response.status_code == 200 + xml = response.json()["data"] + assert xml.strip().startswith("" in xml + assert "Título de teste" in xml + + +@pytest.mark.django_db +def test_api_requires_front(): + User = get_user_model() + user = User.objects.create_user(username="frontempty", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/front/", + data=json.dumps({"type": "json"}), + content_type="application/json", + ) + assert response.status_code == 400 + + response = client.post( + "/api/v1/front/", + data=json.dumps({"front": " ", "type": "json"}), + content_type="application/json", + ) + assert response.status_code == 400 + + +@pytest.mark.django_db +def test_api_requires_auth(): + client = APIClient() + response = client.post( + "/api/v1/front/", + data=json.dumps({"front": "Título", "type": "json"}), + content_type="application/json", + ) + assert response.status_code in (401, 403) + + +@pytest.mark.django_db +def test_api_returns_503_without_persisting(monkeypatch): + def raise_unavailable(_text): + raise FrontLlamaUnavailableError("Front Llama service unavailable") + + monkeypatch.setattr("front.data_utils.mark_front", raise_unavailable) + User = get_user_model() + user = User.objects.create_user(username="frontdown", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + response = client.post( + "/api/v1/front/", + data=json.dumps({"front": "Título de teste", "type": "json"}), + content_type="application/json", + ) + assert response.status_code == 503 + assert "Llama model is not available" in response.json()["error"] + assert Front.objects.count() == 0 + + +@pytest.mark.django_db +def test_api_reuses_checksum_cache(monkeypatch): + calls = MagicMock(return_value=json.dumps(SAMPLE_MARKED)) + monkeypatch.setattr("front.data_utils.mark_front", calls) + User = get_user_model() + user = User.objects.create_user(username="frontcache", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + body = json.dumps({"front": "Título de teste\nAna Silva", "type": "json"}) + first = client.post("/api/v1/front/", data=body, content_type="application/json") + second = client.post("/api/v1/front/", data=body, content_type="application/json") + + assert first.status_code == 200 + assert second.status_code == 200 + assert calls.call_count == 1 + assert Front.objects.count() == 1 diff --git a/front/tests/test_docx.py b/front/tests/test_docx.py new file mode 100644 index 0000000..2c9066b --- /dev/null +++ b/front/tests/test_docx.py @@ -0,0 +1,126 @@ +import io +import json +import zipfile +from xml.sax.saxutils import escape + +import pytest +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import SimpleUploadedFile +from rest_framework.test import APIClient + +from front.utils import extract_front_section + + +def make_docx_bytes(paragraphs): + body = "".join( + f"{escape(paragraph)}" + for paragraph in paragraphs + ) + document_xml = ( + '' + '' + f"{body}" + "" + ) + content_types = ( + '' + '' + '' + '' + "" + ) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("[Content_Types].xml", content_types) + archive.writestr("word/document.xml", document_xml) + return buffer.getvalue() + + +@pytest.mark.parametrize( + "text,expected", + [ + ("", ""), + ( + "Título\nAna Silva\nIntroduction\nBody of the article", + "Título\nAna Silva", + ), + ( + "Título\nAna Silva\n1. Introdução\nCorpo", + "Título\nAna Silva", + ), + ( + "Título\nMethods\nCorpo", + "Título", + ), + ], +) +def test_extract_front_section(text, expected): + assert extract_front_section(text) == expected + + +@pytest.mark.django_db +def test_api_docx_marks_front(monkeypatch): + monkeypatch.setattr( + "front.data_utils.mark_front", + lambda text: json.dumps( + {"titles": [{"kind": "main", "text": text.split(chr(10))[0]}]} + ), + ) + User = get_user_model() + user = User.objects.create_user(username="frontdocx", password="pass") + client = APIClient() + client.force_authenticate(user=user) + + uploaded = SimpleUploadedFile( + "article.docx", + make_docx_bytes( + ["Título de teste", "Ana Silva", "Introduction", "Corpo do artigo"] + ), + content_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ) + response = client.post( + "/api/v1/front/docx/", + data={"file": uploaded, "type": "json", "language": "pt"}, + format="multipart", + ) + assert response.status_code == 200 + assert response.json()["data"]["titles"][0]["text"] == "Título de teste" + + +@pytest.mark.django_db +def test_api_docx_rejects_non_docx(): + User = get_user_model() + user = User.objects.create_user(username="frontbaddocx", password="pass") + client = APIClient() + client.force_authenticate(user=user) + uploaded = SimpleUploadedFile( + "article.txt", b"not a docx", content_type="text/plain" + ) + response = client.post( + "/api/v1/front/docx/", + data={"file": uploaded, "type": "json"}, + format="multipart", + ) + assert response.status_code == 400 + + +@pytest.mark.django_db +def test_api_docx_rejects_empty_file(): + User = get_user_model() + user = User.objects.create_user(username="frontemptydocx", password="pass") + client = APIClient() + client.force_authenticate(user=user) + uploaded = SimpleUploadedFile( + "article.docx", + b"", + content_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ) + response = client.post( + "/api/v1/front/docx/", + data={"file": uploaded, "type": "json"}, + format="multipart", + ) + assert response.status_code == 400 diff --git a/front/tests/test_provider.py b/front/tests/test_provider.py new file mode 100644 index 0000000..7406a6f --- /dev/null +++ b/front/tests/test_provider.py @@ -0,0 +1,77 @@ +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from front.exceptions import ( + FrontLlamaDisabledError, + FrontLlamaMisconfiguredError, + FrontLlamaUnavailableError, +) +from front.providers.http import Provider + + +@pytest.fixture +def llama_settings(settings): + settings.FRONT_ENABLED = True + settings.FRONT_URL = "http://llama.example:11434" + settings.FRONT_MODEL = "llama3.2:3b" + settings.FRONT_TIMEOUT = 30 + settings.FRONT_TOKEN = "" + settings.FRONT_NUM_CTX = 8192 + settings.FRONT_KEEP_ALIVE = "-1" + return settings + + +def test_http_provider_requires_url(settings): + settings.FRONT_ENABLED = True + settings.FRONT_URL = "" + + with pytest.raises(FrontLlamaMisconfiguredError): + Provider([], {"type": "json_object"}) + + +def test_http_provider_disabled(settings): + settings.FRONT_ENABLED = False + settings.FRONT_URL = "http://llama.example:11434" + + with pytest.raises(FrontLlamaDisabledError): + Provider([], {"type": "json_object"}) + + +def test_http_provider_chat_success(llama_settings): + mock_response = MagicMock() + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = { + "message": {"content": '{"titles":[]}'}, + } + + with patch( + "front.providers.http.requests.post", return_value=mock_response + ) as post: + provider = Provider( + [{"role": "system", "content": "sys"}], + {"type": "json_object"}, + ) + result = provider.run("Título de teste") + + assert result == { + "choices": [{"message": {"content": '{"titles":[]}'}}], + } + post.assert_called_once() + args, kwargs = post.call_args + assert args[0] == "http://llama.example:11434/api/chat" + assert kwargs["json"]["model"] == "llama3.2:3b" + assert kwargs["json"]["format"] == "json" + assert kwargs["json"]["keep_alive"] == -1 + assert kwargs["headers"] == {} + + +def test_http_provider_unavailable(llama_settings): + with patch( + "front.providers.http.requests.post", + side_effect=requests.ConnectionError("down"), + ): + provider = Provider([], {"type": "json_object"}) + with pytest.raises(FrontLlamaUnavailableError): + provider.run("Título") diff --git a/front/tests/test_xml.py b/front/tests/test_xml.py new file mode 100644 index 0000000..6cafad5 --- /dev/null +++ b/front/tests/test_xml.py @@ -0,0 +1,249 @@ +from lxml import etree + +from front.data_utils import apply_language_fallback, get_front_xml + +SPS_FRONT_TAGS = { + "front", + "journal-meta", + "journal-id", + "journal-title-group", + "journal-title", + "abbrev-journal-title", + "issn", + "publisher", + "publisher-name", + "article-meta", + "article-id", + "article-categories", + "subj-group", + "subject", + "title-group", + "article-title", + "trans-title-group", + "trans-title", + "contrib-group", + "contrib", + "contrib-id", + "name", + "surname", + "given-names", + "collab", + "xref", + "sup", + "role", + "aff", + "label", + "institution", + "addr-line", + "city", + "state", + "postal-code", + "country", + "email", + "author-notes", + "corresp", + "fn", + "pub-date", + "day", + "month", + "year", + "season", + "volume", + "issue", + "fpage", + "lpage", + "elocation-id", + "history", + "date", + "permissions", + "copyright-statement", + "copyright-year", + "copyright-holder", + "license", + "license-p", + "abstract", + "trans-abstract", + "title", + "p", + "sec", + "kwd-group", + "kwd", + "funding-group", + "award-group", + "funding-source", + "award-id", + "funding-statement", + "counts", + "fig-count", + "table-count", + "equation-count", + "ref-count", +} + +SAMPLE_MARKED = { + "journal": { + "journal_ids": [ + {"type": "publisher-id", "value": "bn"}, + {"type": "nlm-ta", "value": "Biota Neotropica"}, + ], + "journal_title": "Biota Neotropica", + "abbrev_journal_title": "Biota Neotrop.", + "issns": [{"pub_type": "epub", "value": "1676-0611"}], + "publisher_name": "Instituto Virtual da Biodiversidade", + }, + "article_ids": [ + { + "pub_id_type": "doi", + "value": "https://doi.org/10.1590/1676-0611-BN-2025-1870", + } + ], + "categories": [{"subject": "Article"}], + "titles": [ + {"kind": "main", "text": "Main title", "language": "en"}, + {"kind": "translated", "language": "pt", "text": "Título traduzido"}, + ], + "authors": [ + { + "contrib_type": "author", + "given_names": "Jéssica S. de", + "surname": "Lima", + "orcid": "https://orcid.org/0000-0002-3193-9315", + "affiliations": ["aff1"], + "corresp": True, + "roles": ["Writing the original draft"], + } + ], + "affiliations": [ + { + "id": "aff1", + "label": "1", + "original": "Instituto de Pesquisas Ambientais, São Paulo, SP, Brasil.", + "orgname": "Instituto de Pesquisas Ambientais", + "city": "São Paulo", + "state": "SP", + "country": "Brasil", + } + ], + "author_notes": {"corresp": "* Correspondence: jessica@example.com"}, + "pub_dates": [ + {"type": "pub", "day": "01", "month": "01", "year": "2026"}, + {"type": "collection", "year": "2026"}, + ], + "volume": "26", + "issue": "2", + "elocation_id": "e20251870", + "history": [ + {"type": "received", "day": "10", "month": "01", "year": "2025"}, + {"type": "accepted", "day": "21", "month": "04", "year": "2026"}, + ], + "permissions": { + "license_href": "https://creativecommons.org/licenses/by/4.0/", + "license_p": "This is an Open Access article.", + }, + "abstracts": [ + {"kind": "main", "title": "Abstract", "text": "This study reports bryophytes."}, + { + "kind": "translated", + "language": "pt", + "title": "Resumo", + "text": "Este estudo relata briófitas.", + }, + ], + "keywords": [ + { + "language": "en", + "title": "Keywords", + "keywords": ["Amazon flora", "Herbarium"], + }, + { + "language": "pt", + "title": "Palavras-chave", + "keywords": ["Flora amazônica", "Herbário"], + }, + ], + "funding": { + "awards": [{"funding_source": "FAPESP", "award_id": "2024/23894-1"}], + "funding_statement": "Processo FAPESP 2024/23894-1.", + }, + "counts": {"fig_count": "03", "ref_count": "52"}, +} + + +def test_get_front_xml_is_complete_front(): + xml = get_front_xml(SAMPLE_MARKED) + root = etree.fromstring(xml.encode("utf-8")) + assert root.tag == "front" + assert root.find("journal-meta") is not None + assert root.find("article-meta") is not None + + +def test_get_front_xml_only_sps_tags(): + xml = get_front_xml(SAMPLE_MARKED) + root = etree.fromstring(xml.encode("utf-8")) + found = {node.tag.split("}")[-1] for node in root.iter()} + unexpected = found - SPS_FRONT_TAGS + assert not unexpected, unexpected + + +def test_get_front_xml_article_title_and_abstract_have_no_lang(): + xml = get_front_xml(SAMPLE_MARKED) + root = etree.fromstring(xml.encode("utf-8")) + article_title = root.find(".//article-title") + abstract = root.find(".//abstract") + xml_lang = "{http://www.w3.org/XML/1998/namespace}lang" + assert xml_lang not in article_title.attrib + assert xml_lang not in abstract.attrib + trans_title_group = root.find(".//trans-title-group") + trans_abstract = root.find(".//trans-abstract") + assert trans_title_group.get(xml_lang) == "pt" + assert trans_abstract.get(xml_lang) == "pt" + + +def test_get_front_xml_normalizes_doi_orcid_and_country(): + xml = get_front_xml(SAMPLE_MARKED) + root = etree.fromstring(xml.encode("utf-8")) + doi = root.find(".//article-id[@pub-id-type='doi']") + assert doi.text == "10.1590/1676-0611-BN-2025-1870" + orcid = root.find(".//contrib-id[@contrib-id-type='orcid']") + assert orcid.text == "0000-0002-3193-9315" + country = root.find(".//aff/country") + assert country.get("country") == "BR" + assert country.text == "Brasil" + + +def test_get_front_xml_omits_missing_optional_tags(): + xml = get_front_xml( + { + "titles": [{"kind": "main", "text": "Only a title"}], + } + ) + root = etree.fromstring(xml.encode("utf-8")) + assert root.tag == "front" + assert root.find("journal-meta") is None + assert root.find("article-meta/title-group/article-title").text == "Only a title" + assert root.find(".//volume") is None + assert root.find(".//funding-group") is None + assert root.find(".//counts") is None + assert root.find(".//abstract") is None + + +def test_get_front_xml_ignores_unknown_journal_id_type(): + xml = get_front_xml( + { + "journal": { + "journal_ids": [{"type": "invented", "value": "xxx"}], + "journal_title": "Journal", + } + } + ) + root = etree.fromstring(xml.encode("utf-8")) + assert root.find(".//journal-id") is None + assert root.find(".//journal-title").text == "Journal" + + +def test_apply_language_fallback_fills_keyword_language(): + marked = apply_language_fallback( + {"keywords": [{"keywords": ["ciência"]}]}, + "pt", + ) + assert marked["keywords"][0]["language"] == "pt" diff --git a/front/utils.py b/front/utils.py new file mode 100644 index 0000000..3f4bc54 --- /dev/null +++ b/front/utils.py @@ -0,0 +1,81 @@ +import os +import re +import tempfile +import zipfile + +from lxml import etree + +from front.exceptions import FrontDocxError + +BODY_HEADING_RE = re.compile( + r"^(?:\d+[.\)]\s*)?(?:" + r"introduction|introdução|introducao|introducción|introduccion|" + r"methods?|metodologia|metodología|methodology|" + r"materials?(?:\s+and\s+methods?)?|" + r"material(?:es)?(?:\s+y\s+métodos)?|" + r"results?|resultados|" + r"discussion|discussão|discusion|" + r"conclus(?:ion|ões|iones)?" + r")\s*$", + re.IGNORECASE, +) + +FRONT_CHAR_LIMIT = 12000 + + +def normalize_front_text(value): + return re.sub(r"\s+", " ", str(value or "").strip().lower()) + + +def extract_text_from_docx(docx_path): + with zipfile.ZipFile(docx_path) as archive: + xml_bytes = archive.read("word/document.xml") + + root = etree.fromstring(xml_bytes) + nsmap = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"} + + paragraphs = [] + for paragraph in root.xpath("//w:p", namespaces=nsmap): + text = "".join( + node.text or "" for node in paragraph.xpath(".//w:t", namespaces=nsmap) + ) + text = text.strip() + if text: + paragraphs.append(text) + return "\n".join(paragraphs) + + +def extract_front_section(text): + if not text: + return "" + lines = str(text).split("\n") + cut = None + for index, line in enumerate(lines): + if BODY_HEADING_RE.match(line.strip()): + cut = index + break + selected = lines[:cut] if cut is not None else lines + result = "\n".join(line.strip() for line in selected if line.strip()) + if cut is None: + result = result[:FRONT_CHAR_LIMIT] + return result + + +def front_from_docx_upload(uploaded): + tmp_path = None + try: + with tempfile.NamedTemporaryFile(suffix=".docx", delete=False) as tmp: + for chunk in uploaded.chunks(): + tmp.write(chunk) + tmp_path = tmp.name + text = extract_text_from_docx(tmp_path) + except Exception as exc: + raise FrontDocxError("Could not read DOCX file") from exc + finally: + if tmp_path and os.path.exists(tmp_path): + os.unlink(tmp_path) + + front_text = extract_front_section(text) + if not front_text.strip(): + raise FrontDocxError("No front section found in DOCX") + return front_text