diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 479b00b..3f61b2f 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -30,7 +30,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade flake8 pytest pycodestyle pytest-cov pytest-mock + pip install --upgrade flake8 pytest pycodestyle pytest-cov pytest-mock "mypy==2.3.*" if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | @@ -38,6 +38,10 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Type check with mypy + if: matrix.python-version == '3.11' + run: | + mypy grobid_client - name: Test with pytest run: | pytest tests/ -v --cov=grobid_client --cov-report=xml --cov-report=term-missing diff --git a/MANIFEST.in b/MANIFEST.in index 4bca278..161e286 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ -include Readme.md \ No newline at end of file +include Readme.md +include grobid_client/py.typed \ No newline at end of file diff --git a/Readme.md b/Readme.md index 31f6f19..95dfbb3 100644 --- a/Readme.md +++ b/Readme.md @@ -33,6 +33,7 @@ concurrent processing capabilities for PDF documents, reference strings, and pat - **Sentence Segmentation**: Layout-aware sentence segmentation capabilities - **JSON Output**: Convert TEI XML output to structured JSON format with CORD-19-like structure - **Markdown Output**: Convert TEI XML output to clean Markdown format with structured sections +- **Type Hints**: Ships inline type annotations and a `py.typed` marker (PEP 561) for static type checking ## 📋 Prerequisites diff --git a/grobid_client/client.py b/grobid_client/client.py index 5984282..725f0db 100644 --- a/grobid_client/client.py +++ b/grobid_client/client.py @@ -1,6 +1,10 @@ """ Generic API Client """ +from __future__ import annotations + from copy import deepcopy import json +from typing import Any, Optional, Tuple + import requests try: @@ -16,12 +20,17 @@ class ApiClient(object): service methods, i.e. ``get``, ``post``, ``put`` and ``delete``. """ - accept_type = "application/xml" - api_base = None + accept_type: str = "application/xml" + api_base: Optional[str] = None def __init__( - self, base_url, username=None, api_key=None, status_endpoint=None, timeout=60 - ): + self, + base_url: str, + username: Optional[str] = None, + api_key: Optional[str] = None, + status_endpoint: Optional[str] = None, + timeout: int = 60, + ) -> None: """Initialise client. Args: @@ -37,7 +46,7 @@ def __init__( self.timeout = timeout @staticmethod - def encode(request, data): + def encode(request: Any, data: Optional[dict]) -> Any: """Add request content data to request body, set Content-type header. Should be overridden by subclasses if not using JSON encoding. @@ -58,7 +67,7 @@ def encode(request, data): return request @staticmethod - def decode(response): + def decode(response: Any) -> Any: """Decode the returned data in the response. Should be overridden by subclasses if something else than JSON is @@ -73,9 +82,9 @@ def decode(response): try: return response.json() except ValueError as e: - return e.message + return e.message # type: ignore[attr-defined] # pre-existing (Python 2 style) - def get_credentials(self): + def get_credentials(self) -> dict: """Returns parameters to be added to authenticate the request. This lives on its own to make it easier to re-implement it if needed. @@ -87,14 +96,14 @@ def get_credentials(self): def call_api( self, - method, - url, - headers=None, - params=None, - data=None, - files=None, - timeout=None, - ): + method: str, + url: str, + headers: Optional[dict] = None, + params: Optional[dict] = None, + data: Optional[dict] = None, + files: Optional[dict] = None, + timeout: Optional[int] = None, + ) -> Tuple[requests.Response, int]: """Call API. This returns object containing data, with error details if applicable. @@ -130,7 +139,7 @@ def call_api( return r, r.status_code - def get(self, url, params=None, **kwargs): + def get(self, url: str, params: Optional[dict] = None, **kwargs: Any) -> Tuple[requests.Response, int]: """Call the API with a GET request. Args: @@ -142,7 +151,7 @@ def get(self, url, params=None, **kwargs): """ return self.call_api("GET", url, params=params, **kwargs) - def delete(self, url, params=None, **kwargs): + def delete(self, url: str, params: Optional[dict] = None, **kwargs: Any) -> Tuple[requests.Response, int]: """Call the API with a DELETE request. Args: @@ -154,7 +163,14 @@ def delete(self, url, params=None, **kwargs): """ return self.call_api("DELETE", url, params=params, **kwargs) - def put(self, url, params=None, data=None, files=None, **kwargs): + def put( + self, + url: str, + params: Optional[dict] = None, + data: Optional[dict] = None, + files: Optional[dict] = None, + **kwargs: Any, + ) -> Tuple[requests.Response, int]: """Call the API with a PUT request. Args: @@ -170,7 +186,14 @@ def put(self, url, params=None, data=None, files=None, **kwargs): "PUT", url, params=params, data=data, files=files, **kwargs ) - def post(self, url, params=None, data=None, files=None, **kwargs): + def post( + self, + url: str, + params: Optional[dict] = None, + data: Optional[dict] = None, + files: Optional[dict] = None, + **kwargs: Any, + ) -> Tuple[requests.Response, int]: """Call the API with a POST request. Args: @@ -186,7 +209,7 @@ def post(self, url, params=None, data=None, files=None, **kwargs): method="POST", url=url, params=params, data=data, files=files, **kwargs ) - def service_status(self, **kwargs): + def service_status(self, **kwargs: Any) -> Tuple[requests.Response, int]: """Call the API to get the status of the service. Returns: diff --git a/grobid_client/format/TEI2LossyJSON.py b/grobid_client/format/TEI2LossyJSON.py index 9c4f5d3..5e6675f 100644 --- a/grobid_client/format/TEI2LossyJSON.py +++ b/grobid_client/format/TEI2LossyJSON.py @@ -4,6 +4,8 @@ Original version: https://github.com/howisonlab/softcite-dataset/blob/master/code/corpus/TEI2LossyJSON.py """ +from __future__ import annotations + import logging import os import uuid @@ -12,13 +14,13 @@ import html import re from pathlib import Path -from typing import Dict, Union, BinaryIO, Iterator +from typing import Any, Dict, Union, BinaryIO, Iterator import dateparser from bs4 import BeautifulSoup, Tag # Configure module-level logger -logger = logging.getLogger(__name__) +logger: logging.Logger = logging.getLogger(__name__) logger.propagate = False # Prevent propagation to avoid duplicate logs # Only configure basic logging if nothing is set up yet @@ -35,10 +37,10 @@ class TEI2LossyJSONConverter: The class also provides utilities to process a directory of TEI files in parallel and in batches. """ - def __init__(self, validate_refs: bool = True): - self.validate_refs = validate_refs + def __init__(self, validate_refs: bool = True) -> None: + self.validate_refs: bool = validate_refs - def convert_tei_file(self, tei_file: Union[Path, BinaryIO], stream: bool = False): + def convert_tei_file(self, tei_file: Union[str, Path, BinaryIO], stream: bool = False) -> Any: """Backward-compatible function. If stream=True returns a generator that yields passages (dicts). If stream=False returns the full document dict (same shape as original function). """ @@ -226,7 +228,7 @@ def convert_tei_file(self, tei_file: Union[Path, BinaryIO], stream: bool = False return document - def _extract_comprehensive_reference_data(self, bibl_struct: Tag, index: int) -> Dict: + def _extract_comprehensive_reference_data(self, bibl_struct: Tag, index: int) -> dict[str, Any] | None: """ Extract detailed bibliographic information from TEI biblStruct elements. Implements comprehensive parsing for all standard TEI bibliographic components. @@ -241,11 +243,11 @@ def _extract_comprehensive_reference_data(self, bibl_struct: Tag, index: int) -> citation_data['target'] = xml_id # Initialize containers for different types of content - contributor_list = [] - publication_metadata = {} - identifier_collection = {} - supplementary_info = [] - link_references = [] + contributor_list: list[dict[str, Any]] = [] + publication_metadata: dict[str, Any] = {} + identifier_collection: dict[str, Any] = {} + supplementary_info: list[str] = [] + link_references: list[str] = [] # 1. Process analytic level information (article/conference paper content) analytic_section = bibl_struct.find("analytic") @@ -376,9 +378,9 @@ def _extract_comprehensive_reference_data(self, bibl_struct: Tag, index: int) -> return None - def _extract_contributor_details(self, contributor_element: Tag) -> Dict: + def _extract_contributor_details(self, contributor_element: Tag) -> dict[str, Any] | None: """Extract detailed information about authors, editors, and other contributors.""" - contributor_info = {} + contributor_info: dict[str, Any] = {} # Extract name components surname_element = contributor_element.find("surname") @@ -413,7 +415,7 @@ def _extract_contributor_details(self, contributor_element: Tag) -> Dict: return contributor_info if contributor_info.get('name') else None - def _process_identifier_element(self, identifier_element: Tag, identifier_collection: Dict, level: str): + def _process_identifier_element(self, identifier_element: Tag, identifier_collection: dict[str, Any], level: str) -> None: """Process identifier elements (DOI, ISBN, ISSN, etc.) and organize by type and level.""" identifier_text = self._clean_text(identifier_element.get_text()) identifier_type = identifier_element.get("type", "").lower() @@ -430,13 +432,13 @@ def _process_identifier_element(self, identifier_element: Tag, identifier_collec else: identifier_collection[level_key]['unknown'] = identifier_text - def _process_pointer_element(self, pointer_element: Tag, link_references: list): + def _process_pointer_element(self, pointer_element: Tag, link_references: list[str]) -> None: """Process pointer elements that contain external links.""" pointer_target = pointer_element.get("target", "").strip() if pointer_target: link_references.append(pointer_target) - def _process_imprint_details(self, imprint_element: Tag, publication_metadata: Dict): + def _process_imprint_details(self, imprint_element: Tag, publication_metadata: dict[str, Any]) -> None: """Extract and process imprint information including publisher, dates, and page ranges.""" # Extract publisher information @@ -494,9 +496,9 @@ def _process_imprint_details(self, imprint_element: Tag, publication_metadata: D elif scope_unit == "chapter": publication_metadata['chapter'] = scope_text - def _compile_citation_data(self, citation_data: Dict, contributors: list, - publication_metadata: Dict, identifiers: Dict, - supplementary_info: list, links: list): + def _compile_citation_data(self, citation_data: dict[str, Any], contributors: list[dict[str, Any]], + publication_metadata: dict[str, Any], identifiers: dict[str, Any], + supplementary_info: list[str], links: list[str]) -> None: """Compile all extracted information into the final citation structure.""" # Process contributors if contributors: @@ -546,7 +548,7 @@ def _compile_citation_data(self, citation_data: Dict, contributors: list, else: citation_data['urls'] = links - def _validate_citation_content(self, citation_data: Dict) -> bool: + def _validate_citation_content(self, citation_data: dict[str, Any]) -> bool: """Validate that the citation contains meaningful information.""" # Check for essential bibliographic elements essential_elements = ['title', 'authors', 'journal', 'doi', 'isbn', 'issn', 'pmc', 'pmid'] @@ -559,13 +561,13 @@ def _validate_citation_content(self, citation_data: Dict) -> bool: return has_essential or has_fallback - def _extract_person_data(self, person_element: Tag) -> Dict: + def _extract_person_data(self, person_element: Tag) -> dict[str, Any] | None: """ Extract person data (author/editor) from TEI persName or author elements. Handles various name formats and affiliations. """ - person_data = {} + person_data: dict[str, Any] = {} # Try different name extraction methods forename = person_element.find("forename") @@ -691,7 +693,7 @@ def _iter_passages_from_soup_for_text(self, text_node: Tag, passage_level: str) head_paragraph = None - def _process_div_with_nested_content(self, div: Tag, passage_level: str, head_paragraph: str = None) -> Iterator[Dict[str, Union[str, Dict[str, str]]]]: + def _process_div_with_nested_content(self, div: Tag, passage_level: str, head_paragraph: str | None = None) -> Iterator[Dict[str, Union[str, Dict[str, str]]]]: """ Process a div and its nested content, handling various back section types. Supports nested divs for complex back sections like annex with multiple subsections. @@ -813,7 +815,7 @@ def _process_div_with_nested_content(self, div: Tag, passage_level: str, head_pa if current_head_paragraph is not None: head_paragraph = current_head_paragraph - def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.xml", parallel: bool = True, workers: int = None) -> Iterator[Dict]: + def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.xml", parallel: bool = True, workers: int | None = None) -> Iterator[Dict]: """Process a directory of TEI files and yield converted documents. When parallel=True this uses ProcessPoolExecutor to parallelize file-level conversion. Each yielded item is a dict with keys: 'path' and 'document' (document may be None on parse error). @@ -839,7 +841,7 @@ def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.x yield {"path": f, "document": doc} -def _convert_file_worker(path: str): +def _convert_file_worker(path: str) -> Any: """Worker used by ProcessPoolExecutor. Imports inside function to avoid pickling issues.""" from bs4 import BeautifulSoup # Reuse existing top-level helpers from this module by importing here @@ -850,7 +852,7 @@ def _convert_file_worker(path: str): return converter.convert_tei_file(path, stream=False) -def box_to_dict(coord_list): +def box_to_dict(coord_list: list[str]) -> dict[str, float]: """Convert coordinate list to dictionary format.""" if len(coord_list) >= 4: return { @@ -862,14 +864,14 @@ def box_to_dict(coord_list): return {} -def get_random_id(prefix=""): +def get_random_id(prefix: str = "") -> str: """Generate a random ID with optional prefix.""" return f"{prefix}{uuid.uuid4().hex[:8]}" -def get_refs_with_offsets(element): +def get_refs_with_offsets(element: Tag) -> list[dict[str, Any]]: """Extract references with their text offsets from an element.""" - refs = [] + refs: list[dict[str, Any]] = [] # Apply the same text cleaning as get_formatted_passage def _clean_text(text: str) -> str: @@ -880,7 +882,7 @@ def _clean_text(text: str) -> str: return text # Now extract references with offsets based on the cleaned text - def traverse_and_collect(node, current_pos=0): + def traverse_and_collect(node: Any, current_pos: int = 0) -> tuple[str, int]: """ Recursively traverse the DOM tree, building cleaned text content and tracking exact positions. Returns tuple: (text_content, next_position) @@ -925,7 +927,7 @@ def traverse_and_collect(node, current_pos=0): final_text = _clean_text(raw_text) # Adjust all reference offsets to match the cleaned text - final_refs = [] + final_refs: list[dict[str, Any]] = [] for ref in refs: # Find the reference text in the cleaned text to get correct offsets ref_text = ref['text'] @@ -953,7 +955,7 @@ def traverse_and_collect(node, current_pos=0): return final_refs -def get_formatted_passage(head_paragraph, head_section, paragraph_id, element): +def get_formatted_passage(head_paragraph: str | None, head_section: str | None, paragraph_id: str, element: Tag) -> dict[str, Any]: """Format a passage (paragraph or sentence) with metadata and references.""" # Import the clean_text method def _clean_text_local(text: str) -> str: @@ -984,7 +986,7 @@ def _clean_text_local(text: str) -> str: return passage -def xml_table_to_markdown(table_element): +def xml_table_to_markdown(table_element: Tag | None) -> str | None: """Convert XML table to markdown format.""" if not table_element: return None @@ -1004,7 +1006,7 @@ def xml_table_to_markdown(table_element): return "\n".join(markdown_lines) if markdown_lines else None -def xml_table_to_json(table_element): +def xml_table_to_json(table_element: Tag | None) -> dict[str, Any] | None: """Convert XML table to JSON format.""" if not table_element: return None @@ -1055,6 +1057,6 @@ def xml_table_to_json(table_element): # Backwards compatible top-level function that uses the class -def convert_tei_file(tei_file: Union[Path, BinaryIO], stream: bool = False): +def convert_tei_file(tei_file: Union[str, Path, BinaryIO], stream: bool = False) -> Any: converter = TEI2LossyJSONConverter() return converter.convert_tei_file(tei_file, stream=stream) diff --git a/grobid_client/format/TEI2LossyJSON_cli.py b/grobid_client/format/TEI2LossyJSON_cli.py index 8eab2cd..9f59875 100644 --- a/grobid_client/format/TEI2LossyJSON_cli.py +++ b/grobid_client/format/TEI2LossyJSON_cli.py @@ -5,6 +5,8 @@ This script provides a command-line interface for converting TEI XML files to JSON format using the TEI2LossyJSONConverter. """ +from __future__ import annotations + import argparse import json import logging @@ -14,7 +16,7 @@ from .TEI2LossyJSON import TEI2LossyJSONConverter -def setup_logging(verbose: bool = False): +def setup_logging(verbose: bool = False) -> None: """Setup logging configuration.""" level = logging.INFO if verbose else logging.WARNING logging.basicConfig( @@ -54,7 +56,7 @@ def convert_single_file(input_file: Path, output_file: Path, verbose: bool = Fal return False -def main(): +def main() -> None: """Main CLI entry point.""" parser = argparse.ArgumentParser( description="Convert TEI XML files to JSON format using TEI2LossyJSON converter", diff --git a/grobid_client/format/TEI2Markdown.py b/grobid_client/format/TEI2Markdown.py index 34adc4c..06caac3 100644 --- a/grobid_client/format/TEI2Markdown.py +++ b/grobid_client/format/TEI2Markdown.py @@ -11,9 +11,11 @@ - Annex - References """ +from __future__ import annotations + import re from pathlib import Path -from typing import List, Dict, Union, Optional, BinaryIO +from typing import Any, List, Dict, Union, Optional, BinaryIO from bs4 import BeautifulSoup, NavigableString, Tag import logging import dateparser @@ -28,10 +30,10 @@ class TEI2MarkdownConverter: """Converter that converts TEI XML to Markdown format.""" - def __init__(self): + def __init__(self) -> None: pass - def convert_tei_file(self, tei_file: Union[Path, BinaryIO]) -> Optional[str]: + def convert_tei_file(self, tei_file: Union[str, Path, BinaryIO]) -> Optional[str]: """Convert a TEI file to Markdown format. Args: @@ -272,7 +274,7 @@ def _extract_annex(self, soup: BeautifulSoup) -> str: return "".join(annex_sections) - def _process_div_and_nested_divs(self, div: Tag, annex_sections: list) -> None: + def _process_div_and_nested_divs(self, div: Tag, annex_sections: list[str]) -> None: """Process a div element and its nested div elements.""" # Add section header if present for this div (avoid duplicates) head = div.find("head") @@ -456,7 +458,7 @@ def _format_reference(self, bibl_struct: Tag, ref_num: int) -> str: return formatted_reference - def _extract_bibliographic_data(self, bibl_struct: Tag) -> dict: + def _extract_bibliographic_data(self, bibl_struct: Tag) -> dict[str, Any]: """ Extract comprehensive bibliographic data from TEI structure. @@ -499,7 +501,7 @@ def _extract_bibliographic_data(self, bibl_struct: Tag) -> dict: return bib_data - def _process_analytic_section(self, analytic: Tag, bib_data: dict) -> None: + def _process_analytic_section(self, analytic: Tag, bib_data: dict[str, Any]) -> None: """Process the analytic section containing article-level information.""" # Extract article title title = analytic.find("title", level="a") @@ -512,7 +514,7 @@ def _process_analytic_section(self, analytic: Tag, bib_data: dict) -> None: if author_info: bib_data['authors'].append(author_info) - def _process_monograph_section(self, monogr: Tag, bib_data: dict) -> None: + def _process_monograph_section(self, monogr: Tag, bib_data: dict[str, Any]) -> None: """Process the monograph section containing publication-level information.""" # Extract title if no analytic title was found if not bib_data['title']: @@ -537,7 +539,7 @@ def _process_monograph_section(self, monogr: Tag, bib_data: dict) -> None: if imprint: self._process_imprint_section(imprint, bib_data) - def _process_series_section(self, series: Tag, bib_data: dict) -> None: + def _process_series_section(self, series: Tag, bib_data: dict[str, Any]) -> None: """Process series information for multi-part publications.""" series_title = series.find("title", level="s") if series_title and series_title.get_text().strip(): @@ -546,7 +548,7 @@ def _process_series_section(self, series: Tag, bib_data: dict) -> None: else: bib_data['venue'] = series_title.get_text().strip() - def _process_imprint_section(self, imprint: Tag, bib_data: dict) -> None: + def _process_imprint_section(self, imprint: Tag, bib_data: dict[str, Any]) -> None: """Process the imprint section containing publication details.""" # Extract publication date date = imprint.find("date") @@ -579,7 +581,7 @@ def _process_imprint_section(self, imprint: Tag, bib_data: dict) -> None: # Plain text, no from/to attributes bib_data['pages'] = text - def _extract_author_info(self, author: Tag) -> dict: + def _extract_author_info(self, author: Tag) -> dict[str, str] | None: """Extract author information from a TEI author element.""" author_info = {} @@ -600,7 +602,7 @@ def _extract_author_info(self, author: Tag) -> dict: return author_info if author_info else None - def _extract_identifiers(self, bibl_struct: Tag, bib_data: dict) -> None: + def _extract_identifiers(self, bibl_struct: Tag, bib_data: dict[str, Any]) -> None: """Extract various identifier types from the bibliographic structure.""" identifier_sections = [bibl_struct] @@ -624,7 +626,7 @@ def _extract_identifiers(self, bibl_struct: Tag, bib_data: dict) -> None: if id_type and id_value: bib_data['identifiers'][id_type] = id_value - def _extract_urls(self, bibl_struct: Tag, bib_data: dict) -> None: + def _extract_urls(self, bibl_struct: Tag, bib_data: dict[str, Any]) -> None: """Extract URLs and external links from ptr elements.""" url_sections = [bibl_struct] @@ -656,7 +658,7 @@ def _extract_year(self, date_text: str) -> str: # Fallback to returning the original text return date_text.strip() - def _format_authors(self, authors: list) -> str: + def _format_authors(self, authors: list[dict[str, str]]) -> str: """Format author list for display.""" formatted_authors = [] @@ -678,7 +680,7 @@ def _format_authors(self, authors: list) -> str: else: return f"{formatted_authors[0]} et al." - def _build_publication_details(self, ref_data: dict) -> str: + def _build_publication_details(self, ref_data: dict[str, Any]) -> str: """Build publication details string from extracted data.""" details = [] @@ -696,7 +698,7 @@ def _build_publication_details(self, ref_data: dict) -> str: return " ".join(details) - def _build_identifiers_and_links(self, ref_data: dict) -> list: + def _build_identifiers_and_links(self, ref_data: dict[str, Any]) -> list[str]: """Build list of formatted identifiers and links.""" identifiers_and_links = [] @@ -727,7 +729,7 @@ def _build_identifiers_and_links(self, ref_data: dict) -> list: return identifiers_and_links - def _extract_raw_reference(self, bibl_struct: Tag) -> str: + def _extract_raw_reference(self, bibl_struct: Tag) -> str | None: """Extract raw reference text as fallback.""" # Look for raw reference notes raw_ref = bibl_struct.find("note", attrs={"type": "raw_reference"}) @@ -749,7 +751,7 @@ def _extract_raw_reference(self, bibl_struct: Tag) -> str: # Backwards compatible top-level function -def convert_tei_file_to_markdown(tei_file: Union[Path, BinaryIO]) -> Optional[str]: +def convert_tei_file_to_markdown(tei_file: Union[str, Path, BinaryIO]) -> Optional[str]: """Convert a TEI file to Markdown format. Args: diff --git a/grobid_client/format/TEI2Markdown_cli.py b/grobid_client/format/TEI2Markdown_cli.py index 5f840c7..5161343 100644 --- a/grobid_client/format/TEI2Markdown_cli.py +++ b/grobid_client/format/TEI2Markdown_cli.py @@ -5,6 +5,8 @@ This script provides a command-line interface for converting TEI XML files to Markdown format using the TEI2MarkdownConverter. """ +from __future__ import annotations + import argparse import logging import sys @@ -13,7 +15,7 @@ from .TEI2Markdown import TEI2MarkdownConverter -def setup_logging(verbose: bool = False): +def setup_logging(verbose: bool = False) -> None: """Setup logging configuration.""" level = logging.INFO if verbose else logging.WARNING logging.basicConfig( @@ -53,7 +55,7 @@ def convert_single_file(input_file: Path, output_file: Path, verbose: bool = Fal return False -def main(): +def main() -> None: """Main CLI entry point.""" parser = argparse.ArgumentParser( description="Convert TEI XML files to Markdown format using TEI2Markdown converter", diff --git a/grobid_client/format/__main__.py b/grobid_client/format/__main__.py index 81f5ae1..4441b04 100644 --- a/grobid_client/format/__main__.py +++ b/grobid_client/format/__main__.py @@ -3,11 +3,13 @@ This provides a menu to choose between TEI2LossyJSON and TEI2Markdown converters. """ +from __future__ import annotations + import argparse import sys -def main(): +def main() -> None: """Main entry point that provides a menu for converter selection.""" # Check if a converter was specified diff --git a/grobid_client/format/validate_json_refs.py b/grobid_client/format/validate_json_refs.py index f6c5d97..e7f86a2 100755 --- a/grobid_client/format/validate_json_refs.py +++ b/grobid_client/format/validate_json_refs.py @@ -14,6 +14,7 @@ Example: python validate_json_refs.py ./output --verbose --output validation_report.json """ +from __future__ import annotations import json import os @@ -28,7 +29,7 @@ class JSONReferenceValidator: """Validates reference offsets in JSON files.""" - def __init__(self, verbose: bool = False): + def __init__(self, verbose: bool = False) -> None: self.verbose = verbose self.results = { 'total_files': 0, @@ -297,7 +298,7 @@ def save_json_report(self, output_path: str) -> None: json.dump(report_data, f, indent=2, ensure_ascii=False) -def main(): +def main() -> None: """Main function.""" parser = argparse.ArgumentParser( description="Validate reference offsets in JSON files generated from TEI documents" diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index a525b9f..8176103 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -14,6 +14,8 @@ which is not implemented for the moment. """ +from __future__ import annotations + import os import json import argparse @@ -24,7 +26,7 @@ import requests import pathlib import logging -from typing import Tuple +from typing import Any, Optional, Tuple, Union import copy from .format.TEI2LossyJSON import TEI2LossyJSONConverter @@ -34,7 +36,7 @@ class ServerUnavailableException(Exception): """Exception raised when GROBID server is not available or not responding.""" - def __init__(self, message="GROBID server is not available"): + def __init__(self, message: str = "GROBID server is not available") -> None: super().__init__(message) self.message = message @@ -46,7 +48,7 @@ class GrobidClient(ApiClient): CONSOLIDATE_CITATIONS_MIN_TIMEOUT = 120 # Default configuration values - DEFAULT_CONFIG = { + DEFAULT_CONFIG: dict = { 'grobid_server': 'http://localhost:8070', 'batch_size': 10, 'sleep_time': 5, @@ -77,15 +79,15 @@ class GrobidClient(ApiClient): def __init__( self, - grobid_server=None, - batch_size=None, - coordinates=None, - sleep_time=None, - timeout=None, - config_path=None, - check_server=True, - verbose=False - ): + grobid_server: Optional[str] = None, + batch_size: Optional[int] = None, + coordinates: Optional[list] = None, + sleep_time: Optional[int] = None, + timeout: Optional[int] = None, + config_path: Optional[str] = None, + check_server: bool = True, + verbose: bool = False + ) -> None: # Store verbose parameter for logging configuration self.verbose = verbose @@ -112,13 +114,13 @@ def __init__( if check_server: self._test_server_connection() - def _set_config_params(self, params): + def _set_config_params(self, params: dict) -> None: """Set configuration parameters, only if they are not None.""" for key, value in params.items(): if value is not None: self.config[key] = value - def _warn_on_consolidation_timeout(self, consolidate_citations): + def _warn_on_consolidation_timeout(self, consolidate_citations: bool) -> None: """Warn when citation consolidation is enabled with a low client timeout. Consolidating citations makes GROBID query external services and can be @@ -139,23 +141,25 @@ def _warn_on_consolidation_timeout(self, consolidate_citations): f"(2-3 minutes is recommended)." ) - def _handle_server_busy_retry(self, file_path, retry_func, *args, **kwargs): + def _handle_server_busy_retry(self, file_path: str, retry_func: Any, *args: Any, **kwargs: Any) -> Any: """Handle server busy (503) retry logic.""" self.logger.warning(f"Server busy (503), retrying {file_path} after {self.config['sleep_time']} seconds") time.sleep(self.config["sleep_time"]) return retry_func(*args, **kwargs) - def _handle_request_error(self, file_path, error, error_type="Request"): + def _handle_request_error( + self, file_path: str, error: Exception, error_type: str = "Request" + ) -> Tuple[str, int, str]: """Handle request errors with consistent logging and return format.""" self.logger.error(f"{error_type} failed for {file_path}: {str(error)}") return (file_path, 500, f"{error_type} failed: {str(error)}") - def _handle_unexpected_error(self, file_path, error): + def _handle_unexpected_error(self, file_path: str, error: Exception) -> Tuple[str, int, str]: """Handle unexpected errors with consistent logging and return format.""" self.logger.error(f"Unexpected error processing {file_path}: {str(error)}") return (file_path, 500, f"Unexpected error: {str(error)}") - def _configure_logging(self): + def _configure_logging(self) -> None: """Configure logging based on the configuration settings.""" # Get logging config with defaults log_config = self.config.get('logging', {}) @@ -206,7 +210,7 @@ def _configure_logging(self): backup_count = log_config.get('backup_count', 3) from logging.handlers import RotatingFileHandler - file_handler = RotatingFileHandler( + file_handler: logging.Handler = RotatingFileHandler( log_file, maxBytes=max_bytes, backupCount=backup_count @@ -231,7 +235,7 @@ def _configure_logging(self): self.logger.info( f"Logging configured - Level: {log_level_str}, Console: {log_config.get('console', True)}, File: {log_file or 'disabled'}") - def _parse_file_size(self, size_str): + def _parse_file_size(self, size_str: Union[str, int]) -> int: """Parse file size string like '10MB', '1GB' to bytes.""" size_str = str(size_str).upper().strip() @@ -255,7 +259,7 @@ def _parse_file_size(self, size_str): return int(number * multipliers.get(unit, 1)) - def _load_config(self, path="./config.json"): + def _load_config(self, path: str = "./config.json") -> None: """ Load and merge configuration from a JSON file with default values. If the file doesn't exist, keep the default values. @@ -327,7 +331,12 @@ def _test_server_connection(self) -> Tuple[bool, int]: self.logger.error(error_msg) raise ServerUnavailableException(error_msg) from e - def _output_file_name(self, input_file, input_path, output): + def _output_file_name( + self, + input_file: str, + input_path: str, + output: Optional[str], + ) -> str: # Use pathlib for consistent cross-platform path handling input_file_path = pathlib.Path(input_file) @@ -351,23 +360,23 @@ def ping(self) -> Tuple[bool, int]: def process( self, - service, - input_path, - output=None, - n=10, - generate_ids=False, - consolidate_header=True, - consolidate_citations=False, - include_raw_citations=False, - include_raw_affiliations=False, - tei_coordinates=False, - segment_sentences=False, - force=True, - verbose=False, - flavor=None, - json_output=False, - markdown_output=False - ): + service: str, + input_path: str, + output: Optional[str] = None, + n: int = 10, + generate_ids: bool = False, + consolidate_header: bool = True, + consolidate_citations: bool = False, + include_raw_citations: bool = False, + include_raw_affiliations: bool = False, + tei_coordinates: bool = False, + segment_sentences: bool = False, + force: bool = True, + verbose: bool = False, + flavor: Optional[str] = None, + json_output: bool = False, + markdown_output: bool = False + ) -> None: start_time = time.time() batch_size_pdf = self.config["batch_size"] @@ -483,24 +492,24 @@ def process( def process_batch( self, - service, - input_files, - input_path, - output, - n, - generate_ids, - consolidate_header, - consolidate_citations, - include_raw_citations, - include_raw_affiliations, - tei_coordinates, - segment_sentences, - force, - verbose=False, - flavor=None, - json_output=False, - markdown_output=False - ): + service: str, + input_files: list, + input_path: str, + output: Optional[str], + n: int, + generate_ids: bool, + consolidate_header: bool, + consolidate_citations: bool, + include_raw_citations: bool, + include_raw_affiliations: bool, + tei_coordinates: bool, + segment_sentences: bool, + force: bool, + verbose: bool = False, + flavor: Optional[str] = None, + json_output: bool = False, + markdown_output: bool = False + ) -> Tuple[int, int, int]: batch_start_time = time.time() if verbose: self.logger.info(f"{len(input_files)} files to process in current batch") @@ -529,7 +538,7 @@ def process_batch( if not os.path.isfile(json_filename_expanded): self.logger.info(f"JSON file {json_filename} does not exist, generating JSON from existing TEI...") try: - converter = TEI2LossyJSONConverter() + converter: Any = TEI2LossyJSONConverter() json_data = converter.convert_tei_file(filename, stream=False) if json_data: @@ -564,7 +573,7 @@ def process_batch( continue - selected_process = self.process_pdf + selected_process: Any = self.process_pdf if service == 'processCitationList': selected_process = self.process_txt @@ -670,19 +679,19 @@ def process_batch( def process_pdf( self, - service, - pdf_file, - generate_ids, - consolidate_header, - consolidate_citations, - include_raw_citations, - include_raw_affiliations, - tei_coordinates, - segment_sentences, - flavor=None, - start=-1, - end=-1 - ): + service: str, + pdf_file: str, + generate_ids: bool, + consolidate_header: bool, + consolidate_citations: bool, + include_raw_citations: bool, + include_raw_affiliations: bool, + tei_coordinates: bool, + segment_sentences: bool, + flavor: Optional[str] = None, + start: int = -1, + end: int = -1 + ) -> Tuple[str, int, Optional[str]]: pdf_handle = None try: pdf_handle = open(pdf_file, "rb") @@ -760,24 +769,24 @@ def process_pdf( if pdf_handle: pdf_handle.close() - def get_server_url(self, service): + def get_server_url(self, service: str) -> str: return self.config['grobid_server'] + "/api/" + service def process_txt( self, - service, - txt_file, - generate_ids, - consolidate_header, - consolidate_citations, - include_raw_citations, - include_raw_affiliations, - tei_coordinates, - segment_sentences, - flavor=None, - start_page=-1, - end_page=-1 - ): + service: str, + txt_file: str, + generate_ids: bool, + consolidate_header: bool, + consolidate_citations: bool, + include_raw_citations: bool, + include_raw_affiliations: bool, + tei_coordinates: bool, + segment_sentences: bool, + flavor: Optional[str] = None, + start_page: int = -1, + end_page: int = -1 + ) -> Tuple[str, int, Optional[str]]: # create request based on file content try: with open(txt_file, 'r', encoding='utf-8') as f: @@ -792,7 +801,7 @@ def process_txt( the_url = self.get_server_url(service) # set the GROBID parameters - the_data = {} + the_data: dict = {} if consolidate_citations: the_data["consolidateCitations"] = "1" if include_raw_citations: @@ -826,7 +835,7 @@ def process_txt( return (txt_file, status, res.text) -def main(): +def main() -> None: # Basic logging setup for initialization only # The actual logging configuration will be done by GrobidClient based on config.json temp_logger = logging.getLogger(__name__) diff --git a/grobid_client/py.typed b/grobid_client/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index 5caa839..2f95d8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,10 +28,34 @@ dependencies = {file = ["requirements.txt"]} [tool.setuptools_scm] +[tool.mypy] +# requests/bs4/lxml/dateparser ship varying (or no) type information. +ignore_missing_imports = true + +[[tool.mypy.overrides]] +# The TEI converters rely heavily on BeautifulSoup's dynamic element API, +# whose loose typing produces many false positives under mypy. Their runtime +# behavior is covered by the conversion test-suite (tests/test_conversions.py). +module = [ + "grobid_client.format.TEI2LossyJSON", + "grobid_client.format.TEI2Markdown", + "grobid_client.format.validate_json_refs", +] +disable_error_code = [ + "attr-defined", "union-attr", "arg-type", "assignment", "operator", + "index", "var-annotated", "return-value", "misc", "func-returns-value", + "str-bytes-safe", +] + [tool.setuptools] -packages=['grobid_client'] +packages=['grobid_client', 'grobid_client.format'] license-files = [] +[tool.setuptools.package-data] +# Ship the PEP 561 marker so type checkers (e.g. mypy) pick up the inline +# type hints of this package. See issue #71. +grobid_client = ["py.typed"] + [project.urls] Homepage = "https://github.com/kermitt2/grobid_client_python" Repository = "https://github.com/kermitt2/grobid_client_python"