diff --git a/README.md b/README.md index 5775ff5..aa24030 100644 --- a/README.md +++ b/README.md @@ -394,85 +394,166 @@ Deletes an app token. - `app_token`: The app token to delete. -##### `MathpixClient.scs_file_new` +#### Files API (async document processing) -Upload a file via files-api v1 for async processing. Returns an ScsFile instance. +The Files API processes documents asynchronously — local files, or remote URIs (`s3://`, `gs://`, public `https://`, or Azure Blob HTTPS URLs) one at a time or in bulk batch calls. Non-public sources require a registered [data source](https://docs.mathpix.com/reference/files-v1-data-sources) connecting your Mathpix account to the bucket; register it once via the API following the linked guide. -Supports three upload modes (exactly one must be provided): +Submit a single document and download the result: -- `file_path`: Multipart upload from local file -- `url`: Upload from HTTP URL or S3 presigned URL -- `source_s3_uri`: Copy from S3 bucket (requires IAM role access) +```python +from mpxpy.mathpix_client import MathpixClient + +client = MathpixClient() +file = client.file_new( + source_uri="https://cdn.mathpix.com/examples/cs229-notes1.pdf", + conversion_formats={"docx": True, "md": True}, +) +file.wait_until_complete(timeout=120) +markdown = file.to_md_text() +file.to_docx_file("output.docx") +``` + +Submit a batch as a job, then collect results and failures: + +```python +job = client.file_job_new( + files=[ + {"source_uri": "s3://your-bucket/docs/contract-1.pdf", "custom_id": "contract-1"}, + {"source_uri": "https://example.com/manual.pdf", "custom_id": "manual"}, + ], + job_id="contracts-2026-07", + conversion_formats={"docx": True, "md": True}, +) +job.wait_until_complete(timeout=3600, interval=15.0) +for errored in job.files_iter(status="error"): + print("failed:", errored["custom_id"]) +file = job.file_by_custom_id("contract-1") +``` + +Batch submission is accept-and-defer: the call returns immediately and per-item failures (bad URIs, missing data sources) surface as per-file `error` statuses when you poll the job, not as request errors. -###### `MathpixClient.scs_file_new` Arguments +##### `MathpixClient.file_new` +Submit a single document for async processing, from a remote URI (`POST /files/v1/uri`) or a local file (multipart upload). Returns a `File` instance. + +###### `MathpixClient.file_new` Arguments + +- `source_uri`: Remote location of the source document (`s3://`, `gs://`, public `https://`, or Azure Blob HTTPS URL). Exactly one of `source_uri` or `file_path` is required. - `file_path`: Path to a local file to upload. -- `url`: URL of a remote file (HTTP/HTTPS or S3 presigned URL). -- `source_s3_uri`: S3 URI (s3://bucket/key) to copy from. -- `filename`: Optional filename to use (defaults to file basename). -- `scs_job_id`: Optional job ID to group files together. -- `conversion_formats`: Dict of format names to enable (e.g., `{'mmd': True, 'docx': True}`). -- `conversion_options`: Additional conversion options dict. -- `destination_s3_uri`: Optional S3 URI to write output files. -- `destination_basename`: Optional basename for output files (defaults to file_id). -- `s3_region`: Optional AWS region for S3 operations (default us-east-1). -- `image_output_mode`: Image output mode (e.g., 'local' to upload to destination_s3_uri). -- `include_page_info`: Include page info in output (default None). +- `job_id`: Optional job to associate this file with. Required whenever `custom_id` is supplied. +- `custom_id`: Optional case-sensitive customer-supplied identifier. `(job_id, custom_id)` is the idempotency key: re-submitting the same pair returns the original file. +- `idempotency_key`: Optional client-generated key sent as the `Idempotency-Key` header; makes a standalone submission safe to retry. +- `filename`: Optional display name for the file. +- `conversion_formats`: Dict of format names to enable (e.g., `{'docx': True, 'md': True}`). Mathpix Markdown (`mmd`) is always produced. +- `extra_options`: Additional request options dict merged into the request body — an escape hatch for API options this SDK version does not model yet (validated server-side). May not override the validated request fields. +- `destination_uri`: Optional destination for results; must be backed by a registered data source. When omitted, results stay in Mathpix storage and are fetched via the download helpers. +- `destination_basename`: Optional basename for output objects (defaults to the file_id). +- `s3_region`: Optional region of the `destination_uri` S3 bucket. +- `image_output_mode`: Set to `'local'` to write cropped images into `destination_uri` storage instead of the Mathpix CDN. +- `include_page_info`: Include per-page information in the output. +- `metadata`: Optional dict to attach metadata to the request. +- Plus the same OCR options as `pdf_new` (`alphabets_allowed`, `rm_spaces`, `include_smiles`, `math_inline_delimiters`, `page_ranges`, etc.). + +##### `MathpixClient.file_job_new` + +Submit a batch of documents in one call (the server enforces an items-per-call ceiling). Returns a `FileJob` instance. + +###### `MathpixClient.file_job_new` Arguments + +- `files`: List of `FileSubmission` instances or dicts. Each item takes `source_uri` (required) plus optional `custom_id`, `filename`, `destination_uri`, `s3_region`, `destination_basename`, and `page_ranges`. +- `job_id`: Optional caller-supplied job id (server-generated when omitted). Required whenever any item carries a `custom_id`. +- `idempotency_key`: Optional key making the whole batch safe to retry; honored only when no `job_id` is supplied. +- `conversion_formats`: Job-wide conversion formats, applied to every file. +- `image_output_mode`: Job-wide; `'local'` writes cropped images to each file's `destination_uri`. - `metadata`: Optional dict to attach metadata to the request. -- `alphabets_allowed`: Optional dict to list alphabets allowed in the output. -- `rm_spaces`: Remove extra white space from equations (default True). -- `rm_fonts`: Remove font commands from equations (default False). -- `idiomatic_eqn_arrays`: Use aligned/gathered/cases instead of array (default False). -- `include_equation_tags`: Include equation number tags in LaTeX (default False). -- `include_smiles`: Enable chemistry diagram OCR via SMILES (default True). -- `include_chemistry_as_image`: Return image crop for chemical diagrams (default False). -- `include_diagram_text`: Enable text extraction from diagrams (default False). -- `numbers_default_to_math`: Numbers are always math (default False). -- `math_inline_delimiters`: Tuple of (begin, end) delimiters for inline math. -- `math_display_delimiters`: Tuple of (begin, end) delimiters for display math. -- `page_ranges`: Page range string (e.g., "2,4-6" or "2--2"). -- `enable_spell_check`: Enable predictive mode for English handwriting (default False). -- `auto_number_sections`: Auto-number sections (default False). -- `remove_section_numbering`: Remove existing section numbering (default False). -- `preserve_section_numbering`: Keep existing section numbering (default True). -- `enable_tables_fallback`: Enable advanced table processing (default False). -- `fullwidth_punctuation`: Use fullwidth Unicode punctuation (default None). +- `extra_options`: Additional request options dict merged into the request body — an escape hatch for API options this SDK version does not model yet (validated server-side). May not override the validated request fields. +- Plus the same OCR options as `pdf_new`, applied to every file in the request. + +##### `MathpixClient.file_job_list` + +List submitted jobs, newest first. Returns a dict with `jobs` and `next_page_token`. + +###### `MathpixClient.file_job_list` Arguments + +- `start`: Earliest submission date to include, `yyyy-MM-dd` (UTC). +- `end`: Latest submission date to include, `yyyy-MM-dd` (UTC). +- `limit`: Maximum jobs per page (default 100). +- `paging_state`: Pagination cursor from the previous response's `next_page_token`. + +##### `MathpixClient.file_get` / `MathpixClient.file_delete` / `MathpixClient.file_job_get` -##### `MathpixClient.list_scs_files` +- `file_get(file_id)`: Fetches an existing file and returns a `File` instance seeded with its status; raises `FilesApiError` for unknown ids. +- `file_delete(file_id)`: Permanently removes a file and its results from Mathpix-owned storage. Only files in a terminal state can be deleted; repeat deletes are idempotent. +- `file_job_get(job_id)`: Fetches an existing job and returns a `FileJob` instance seeded with its `file_count`; raises `FilesApiError` for unknown ids. -List files from files-api v1. Requires exactly one filter: scs_job_id or filename. +##### `File` -###### `MathpixClient.list_scs_files` Arguments +Returned by `file_new`, `file_get`, and `FileJob.file_by_custom_id`. Methods: -- `scs_job_id`: Filter by job ID. -- `filename`: Filter by filename. -- `limit`: Maximum number of results (default 100). -- `paging_state`: Optional paging state for pagination. +- `status()`: Current status (`pending` | `split` | `completed` | `error`), progress, and per-format conversion statuses. When the status is `error`, the `error` and `error_info` attributes carry the failure details. +- `wait_until_complete(timeout)`: Poll until processing finishes. +- `wait_for_format(format, timeout)`: Poll until a specific conversion finishes. Conversions complete independently of the file status and can lag behind it; download a format only after it is ready. +- `text_result` / `bytes_result` / `json_result` / `save_file`, plus the `to_*` convenience methods (`to_md_text`, `to_docx_bytes`, `to_docx_file`, ...). +- `delete()`: Permanently remove the file and its results from Mathpix-owned storage. -Returns a dict containing 'file_ids' list and 'next_page_token' for pagination. +##### `FileJob` + +Returned by `file_job_new` and `file_job_get`. Methods: + +- `status()`: Job status and counters (`file_count`, `files_completed`, `files_errored`). +- `wait_until_complete(timeout, interval=5.0)`: Poll until every file reaches a terminal state. Per-file failures don't fail the job. +- `files(status=None, limit=None, paging_state=None)`: One page of the job's file listing, optionally filtered to `pending`, `completed`, or `error`. +- `files_iter(status=None, limit=None)`: Iterate over all files, following pagination. +- `file_by_custom_id(custom_id)`: Fetch one file by the `(job_id, custom_id)` you supplied at submission. + +##### Data sources (cloud storage setup) + +Non-public `source_uri`/`destination_uri` buckets (`s3://`, `gs://`, Azure Blob) require a registered **data source**: a pointer from your Mathpix account to a bucket you own, with an access grant. The grant model is keyless — no secrets are uploaded to Mathpix (AWS supports a legacy `access_key` fallback). Set up the cloud-side grant following the [per-provider guides](https://docs.mathpix.com/reference/files-v1-data-sources), then register and verify: + +```python +identities = client.onboarding_identities() # fetch BEFORE creating grants +external_id = identities["aws"]["external_id"] # goes in your IAM trust policy + +data_source = client.data_source_new( + provider="aws", + bucket="your-bucket", + auth_method="iam_role", + provider_specific_details={ + "iam_role_arn": "arn:aws:iam::123456789012:role/MathpixReader", + "aws_external_id": external_id, + }, + region="us-east-1", +) +probe = data_source.test() # {'result': 'ok', 'checks': {'read': True, 'write': True}, ...} +``` -##### `MathpixClient.list_scs_jobs` +##### `MathpixClient.onboarding_identities` -List SCS jobs from files-api v1. +Returns the Mathpix identities you grant access to (AWS trust account, Azure app/tenant, and — when available — the GCS impersonator service account) plus your per-group `external_id`, a stable value used in the AWS IAM trust policy and GCS bucket-control verification. Call it before setting up cloud-side grants. -###### `MathpixClient.list_scs_jobs` Arguments +##### `MathpixClient.data_source_new` -- `start`: Optional start date filter (ISO format). -- `end`: Optional end date filter (ISO format). -- `limit`: Maximum number of results (default 100). -- `paging_state`: Optional paging state for pagination. +Register a bucket as a data source. Returns a `DataSource` instance. -Returns a dict containing 'jobs' list and optionally 'paging_state' for next page. +###### `MathpixClient.data_source_new` Arguments -##### `MathpixClient.scs_job_status` +- `provider`: Storage provider, e.g. `'aws'`, `'azure'`, `'gcp'` — see the [per-provider guides](https://docs.mathpix.com/reference/files-v1-data-sources) for the supported set. +- `bucket`: Bucket / container name. +- `auth_method`: Grant type for the provider, e.g. `'iam_role'` or `'access_key'` (aws), `'azure_ad'` (azure), `'service_account'` (gcp). Invalid combinations are rejected server-side. +- `provider_specific_details`: Provider-shaped metadata (e.g. `iam_role_arn` + `aws_external_id` for aws/`iam_role`, `aws_access_key_id` for aws/`access_key`). +- `name`: Optional human-readable label. +- `region`: Bucket region (required for aws `access_key` only). +- `secret`: Only for aws `access_key` (legacy); keyless grant types reject it server-side. -Get the current status of an SCS job. +A registration that conflicts with an existing data source raises `FilesApiError` (`'conflict'`); the server message identifies the conflict. -###### `MathpixClient.scs_job_status` Arguments +For AWS and Azure, call `DataSource.test()` afterward to verify the grant end-to-end. GCS registration verifies bucket control up front, so a successful return already confirms the grant. -- `scs_job_id`: The job ID to get status for. +##### `MathpixClient.data_source_list` / `data_source_test` / `data_source_delete` -Returns JSON response containing job status information. +- `data_source_list()`: List the group's registered data sources (secrets are never returned). +- `data_source_test(data_source_id)`: Runs the read/write probe; returns `{'result', 'checks', 'message'}` and does **not** raise on a failed probe — use it to diagnose grant issues after customer-side IAM changes. +- `data_source_delete(data_source_id)`: Removes the registration; already-started work is not interrupted, and the bucket can be registered again later. Deleting the registration does not revoke access on the cloud side — remove the grant separately to revoke access. ##### `MathpixClient.query_usage` @@ -677,16 +758,17 @@ Returns a dict with 'documents' list containing conversion results. Each documen - `results`: Get the results dict mapping url_key to OCR result for each processed item. - `keys`: Get the list of URL keys in this batch. -### `ScsFile` +### `File` -#### `ScsFile` Properties +#### `File` Properties - `auth`: An Auth instance with Mathpix credentials. - `file_id`: The unique identifier for this file. -#### `ScsFile` Methods +#### `File` Methods - `status`: Get the current status of the file processing (file_id, status, num_pages, num_pages_completed, percent_done, formats). +- `delete`: Permanently remove the file and its results from Mathpix-owned storage. - `wait_until_complete`: Wait for the file processing to complete. - `wait_for_format`: Wait for a specific format conversion to complete. - `to_mmd_text`: Get the processed file result as Mathpix Markdown string. @@ -714,7 +796,6 @@ Returns a dict with 'documents' list containing conversion results. Each documen - `to_pdf_file`: Save the processed file result to a PDF file at a local path. - `to_html_file`: Save the processed file result to an HTML file at a local path. - `to_tex_zip_file`: Save the processed file result to a tex.zip file at a local path. -- `cropped_image`: Get a cropped region from a specific page as JPEG bytes. ## Error Handling diff --git a/changelog.md b/changelog.md index d5d52ff..a23fa0d 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,21 @@ # mpxpy changelog +## July 24, 2026 + +- Support the public Files API v1 + - `file_new` submits a single document, taking exactly one of `source_uri` (a remote `s3://`, `gs://`, public `https://`, or Azure Blob URI, via `POST /files/v1/uri`) or `file_path` (a local file, via multipart `POST /files/v1`); both support `Idempotency-Key` retry safety, `(job_id, custom_id)` idempotency, and the `filename`, `s3_region`, and `include_page_info` options + - `file_job_new` submits documents in bulk (with a server-enforced per-call ceiling) via `POST /files/v1/jobs`; the new `FileJob` class polls job status, lists files with a status filter and pagination iterator, and fetches files by `custom_id` + - `file_job_list`, `file_get`, `file_delete`, and `file_job_get` cover the jobs listing and file lifecycle endpoints; the `*_get` methods fetch the resource and return a seeded handle, raising `FilesApiError` for unknown ids + - New `File` class adds `delete()` and lazy status attributes, raises on failed status requests, and disambiguates download errors per the public API (`format_not_ready` vs `not_found` vs `unsupported_format`) + - New `FilesApiError` exception carries the Files API error code (`error_id`) and HTTP status; it is raised only for responses with a recognizable Files API error body, other failures raise `MathpixClientError` + - `file_new` and `file_job_new` accept an `extra_options` dict merged into the request body — an escape hatch for API options the SDK does not model yet, validated server-side; it may not override the validated request fields (`source_uri`, `files`, `job_id`, `custom_id`, `metadata`) + - The client no longer duplicates server-enforced constraints (identifier charset/length, items-per-call ceiling, status filter values); the server's `FilesApiError` is authoritative for those + - `ScsFile`, `scs_file_new`, `list_scs_files`, `list_scs_jobs`, and `scs_job_status` are deprecated. `scs_file_new` is now a thin wrapper that translates its legacy argument names and forwards to `file_new`; the listing/status methods keep their legacy endpoints and response shapes during the deprecation window. Migrate to `File`, `file_new`, `file_job_get(job_id).files()`, `file_job_list`, and `file_job_get(job_id).status()` + - Data Sources API: `onboarding_identities`, `data_source_new`, `data_source_list`, `data_source_test`, and `data_source_delete` register and manage cloud storage (S3, GCS, Azure Blob) for use as `source_uri`/`destination_uri`; new `DataSource` class with `test()` and `delete()` + - Debug logging redacts remote URIs to scheme and host, since signed URLs carry credentials in their query strings + - `files_api_url` now defaults to the resolved `api_url` as documented, so a custom `api_url` applies to Files API requests too +- All requests now send a `User-Agent: mpxpy/` header + ## June 5, 2025 - Add improve_mathpix argument to the Client, Image, and Pdf classes diff --git a/mpxpy/__init__.py b/mpxpy/__init__.py index 00ad249..53a5452 100644 --- a/mpxpy/__init__.py +++ b/mpxpy/__init__.py @@ -1,4 +1,7 @@ from mpxpy.mathpix_client import MathpixClient +from mpxpy.file import File +from mpxpy.file_job import FileJob, FileSubmission +from mpxpy.data_source import DataSource from mpxpy.scs_file import ScsFile from mpxpy.pdf import Pdf from mpxpy.image import Image @@ -10,10 +13,15 @@ ValidationError, ConversionIncompleteError, FilesystemError, + FilesApiError, ) __all__ = [ "MathpixClient", + "File", + "FileJob", + "FileSubmission", + "DataSource", "ScsFile", "Pdf", "Image", @@ -24,4 +32,5 @@ "ValidationError", "ConversionIncompleteError", "FilesystemError", + "FilesApiError", ] diff --git a/mpxpy/auth.py b/mpxpy/auth.py index c11ec87..cae5258 100644 --- a/mpxpy/auth.py +++ b/mpxpy/auth.py @@ -1,11 +1,19 @@ import os import pathlib import urllib.parse +from importlib.metadata import version, PackageNotFoundError from dotenv import load_dotenv from typing import Optional from mpxpy.logger import logger from mpxpy.errors import AuthenticationError, ValidationError +try: + MPXPY_VERSION: str = version("mpxpy") +except PackageNotFoundError: + MPXPY_VERSION = "unknown" + +USER_AGENT: str = f"mpxpy/{MPXPY_VERSION}" + class Auth: """Authentication and configuration handler for Mathpix API. @@ -53,7 +61,7 @@ def __init__( self.app_id = app_id or os.getenv('MATHPIX_APP_ID') self.app_key = app_key or os.getenv('MATHPIX_APP_KEY') raw_api_url = api_url or os.getenv('MATHPIX_URL', 'https://api.mathpix.com') - raw_files_api_url = files_api_url or os.getenv('MATHPIX_FILES_API_URL', 'https://api.mathpix.com') + raw_files_api_url = files_api_url or os.getenv('MATHPIX_FILES_API_URL') or raw_api_url if not self.app_id: logger.error("Client requires an App ID") raise AuthenticationError("Mathpix App ID is required") @@ -66,6 +74,7 @@ def __init__( self.headers = { 'app_id': self.app_id, 'app_key': self.app_key, + 'User-Agent': USER_AGENT, } def load_config(self): diff --git a/mpxpy/data_source.py b/mpxpy/data_source.py new file mode 100644 index 0000000..d812616 --- /dev/null +++ b/mpxpy/data_source.py @@ -0,0 +1,104 @@ +from typing import Optional, Dict, Any +from urllib.parse import urljoin, quote +import requests +from mpxpy.auth import Auth +from mpxpy.logger import logger +from mpxpy.request_handler import post, delete +from mpxpy.errors import ValidationError, error_from_response + + +class DataSource: + """Manages a registered Files API data source. + + A data source is a registered pointer from your Mathpix account to a bucket + or container you own, with an attached access grant. Once registered, the + bucket can be referenced via source_uri (read) or destination_uri (write) + on Files API submissions. + + Attributes: + auth: An Auth instance with Mathpix credentials. + data_source_id: The unique identifier for this data source. + """ + def __init__( + self, + auth: Auth, + data_source_id: Optional[str] = None, + request_options: Optional[Dict[str, Any]] = None, + ) -> None: + """Initialize a DataSource instance. + + Args: + auth: Auth instance containing Mathpix API credentials. + data_source_id: The unique identifier for the data source. + request_options: Optional dict of kwargs to pass to requests. + + Raises: + ValidationError: If auth is not provided or data_source_id is empty. + """ + self.auth: Auth = auth + has_auth: bool = self.auth is not None + if not has_auth: + logger.error("DataSource requires an authenticated client") + raise ValidationError("DataSource requires an authenticated client") + self.data_source_id: str = data_source_id or '' + has_data_source_id: bool = bool(self.data_source_id) + if not has_data_source_id: + logger.error("DataSource requires a Data Source ID") + raise ValidationError("DataSource requires a Data Source ID") + self.request_options: Dict[str, Any] = request_options or {} + + def test(self) -> Dict[str, Any]: + """Verify Mathpix can reach the bucket using the registered credentials. + + Performs a read probe (and, where applicable, a write probe). The API + returns HTTP 200 for both outcomes; this method returns the probe body + as-is and does NOT raise on a failed probe — the message is diagnostic + data. This is the canonical check after any customer-side IAM change. + + Returns: + dict: Response containing 'result' ('ok' or 'failed'), 'checks' + (per-probe booleans, e.g. {'read': True, 'write': False}), and + 'message' (diagnostic detail for failures). + + Raises: + FilesApiError: If the request itself fails (e.g. 'not_found' for an + unknown data source id). + """ + logger.debug(f"Testing data source {self.data_source_id}") + endpoint: str = urljoin( + self.auth.files_api_url, + f'/files/v1/data-sources/{quote(self.data_source_id, safe="")}/test' + ) + response: requests.Response = post(endpoint, headers=self.auth.headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + return response.json() + + def delete(self) -> Dict[str, Any]: + """Remove the data source registration. + + Subsequent submissions that reference the bucket return + 'data_source_not_found'; already-started work is not interrupted. The + bucket can be registered again later. Deleting the registration does + not revoke access on the cloud side — remove the grant (IAM role, + Azure role assignment, or GCS service-account binding) separately to + revoke access. + + Returns: + dict: Response containing 'data_source_id' and 'status': 'deleted'. + + Raises: + FilesApiError: If no accessible data source has this id, including + one that was already deleted ('not_found'). + """ + logger.debug(f"Deleting data source {self.data_source_id}") + endpoint: str = urljoin( + self.auth.files_api_url, + f'/files/v1/data-sources/{quote(self.data_source_id, safe="")}' + ) + response: requests.Response = delete(endpoint, headers=self.auth.headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + return response.json() diff --git a/mpxpy/errors.py b/mpxpy/errors.py index ba6b742..70990bd 100644 --- a/mpxpy/errors.py +++ b/mpxpy/errors.py @@ -1,3 +1,9 @@ +from typing import Any, Dict, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + import requests + + class MathpixClientError(Exception): """Base exception class for Mathpix client errors.""" pass @@ -21,3 +27,60 @@ class ConversionIncompleteError(MathpixClientError): """Exception raised when a conversion is not complete.""" def __init__(self, message, status_info=None): super().__init__(message) + +class FilesApiError(MathpixClientError): + """Errors returned by the Files API error envelope. + + Attributes: + error_id: Stable machine-readable error code (e.g. 'not_found', 'conflict', + 'data_source_not_found', 'quota_exceeded'). + http_status: The HTTP status code of the failed response. + """ + def __init__(self, message: str, error_id: Optional[str] = None, http_status: Optional[int] = None) -> None: + super().__init__(message) + self.error_id: Optional[str] = error_id + self.http_status: Optional[int] = http_status + + +def parse_files_api_error_envelope(response: "requests.Response") -> Optional[Dict[str, Any]]: + """Parse a Files API error envelope from a response body. + + Returns: + dict: {'error_id': str, 'message': str} when the body is a JSON object + carrying a recognizable error code ('error' string or + 'error_info.id'), otherwise None. + """ + try: + body: Any = response.json() + except ValueError: + return None + is_json_object: bool = isinstance(body, dict) + if not is_json_object: + return None + error_info: Any = body.get('error_info') + has_error_info: bool = isinstance(error_info, dict) + if not has_error_info: + error_info = {} + error_id: Any = body.get('error') or error_info.get('id') + has_error_id: bool = isinstance(error_id, str) and bool(error_id) + if not has_error_id: + return None + message: Any = error_info.get('message') or body.get('message') or error_id + return {'error_id': error_id, 'message': str(message)} + + +def error_from_response(response: "requests.Response") -> MathpixClientError: + """Build the appropriate exception for a non-2xx Files API response. + + Returns a FilesApiError when the body is a recognizable Files API error + envelope, and a plain MathpixClientError for anything else (HTML error + pages, empty bodies, non-object JSON). + """ + envelope: Optional[Dict[str, Any]] = parse_files_api_error_envelope(response) + if envelope is None: + return MathpixClientError(f"Files API request failed with HTTP {response.status_code}") + return FilesApiError( + envelope['message'], + error_id=envelope['error_id'], + http_status=response.status_code, + ) diff --git a/mpxpy/file.py b/mpxpy/file.py new file mode 100644 index 0000000..a82832b --- /dev/null +++ b/mpxpy/file.py @@ -0,0 +1,461 @@ +import os +import time +import json +from typing import Optional, Dict, Any +from urllib.parse import urljoin +import requests +from mpxpy.auth import Auth +from mpxpy.logger import logger +from mpxpy.request_handler import get, delete +from mpxpy.errors import ( + FilesystemError, + ValidationError, + ConversionIncompleteError, + MathpixClientError, + error_from_response, + parse_files_api_error_envelope, +) + + +class File: + """Manages a document submitted to the Files API (files/v1). + + This class handles operations on Files API files, including checking status, + downloading results in different formats, waiting for processing to complete, + and deleting the file from Mathpix storage. + + Attributes: + auth: An Auth instance with Mathpix credentials. + file_id: The unique identifier for this file. + """ + def __init__( + self, + auth: Auth, + file_id: Optional[str] = None, + request_options: Optional[Dict[str, Any]] = None, + status_result: Optional[Dict[str, Any]] = None, + ) -> None: + """Initialize a File instance. + + Args: + auth: Auth instance containing Mathpix API credentials. + file_id: The unique identifier for the file. + request_options: Optional dict of kwargs to pass to requests. + status_result: Optional file-status response body to seed the + lazy attributes with, avoiding an extra status request. + + Raises: + ValidationError: If auth is not provided or file_id is empty. + """ + self.auth: Auth = auth + has_auth: bool = self.auth is not None + if not has_auth: + logger.error("File requires an authenticated client") + raise ValidationError("File requires an authenticated client") + self.file_id: str = file_id or '' + has_file_id: bool = bool(self.file_id) + if not has_file_id: + logger.error("File requires a File ID") + raise ValidationError("File requires a File ID") + self.request_options: Dict[str, Any] = request_options or {} + self._last_status: Optional[Dict[str, Any]] = status_result + + def _status_field(self, field: str) -> Any: + """Return a field from the most recent status response, fetching one if needed.""" + has_cached_status: bool = self._last_status is not None + if not has_cached_status: + self.status() + return (self._last_status or {}).get(field) + + @property + def custom_id(self) -> Optional[str]: + """The customer-supplied identifier echoed by the API, or None.""" + return self._status_field('custom_id') + + @property + def filename(self) -> Optional[str]: + """The display name supplied at submit, or the API default.""" + return self._status_field('filename') + + @property + def num_pages(self) -> Optional[int]: + """Total pages detected in the document (0 until the page split runs).""" + return self._status_field('num_pages') + + @property + def percent_done(self) -> Optional[float]: + """Processing progress from 0.0 to 100.0.""" + return self._status_field('percent_done') + + @property + def destination_uri(self) -> Optional[str]: + """The result destination supplied at submit, or None.""" + return self._status_field('destination_uri') + + @property + def destination_basename(self) -> Optional[str]: + """The output basename supplied at submit, or None.""" + return self._status_field('destination_basename') + + @property + def error(self) -> Optional[str]: + """Machine-readable error code when status is 'error', else None.""" + return self._status_field('error') + + @property + def error_info(self) -> Optional[Dict[str, Any]]: + """The error id/message pair when status is 'error', else None.""" + return self._status_field('error_info') + + def wait_until_complete(self, timeout: int = 60) -> bool: + """Wait for the file processing to complete. + + Polls the file status until it's complete or the timeout is reached. + On failure the error details remain available via the `error` and + `error_info` attributes. + + Args: + timeout: Maximum number of seconds to wait. Must be a positive, non-zero integer. + + Returns: + bool: True if processing completed successfully, False if it errored or timed out. + + Raises: + ValidationError: If timeout is an invalid value + """ + is_valid_timeout: bool = isinstance(timeout, int) and timeout > 0 + if not is_valid_timeout: + raise ValidationError("Timeout must be a positive, non-zero integer") + logger.debug(f"Waiting for file {self.file_id} to complete (timeout: {timeout}s)") + attempt: int = 1 + while attempt < timeout: + logger.debug(f'Checking file status... ({attempt}/{timeout})') + file_status: Dict[str, Any] = self.status() + is_completed: bool = file_status.get('status') == 'completed' + has_errored: bool = file_status.get('status') == 'error' + if is_completed: + logger.debug(f"File {self.file_id} completed successfully") + return True + elif has_errored: + logger.error(f"File {self.file_id} processing failed") + return False + time.sleep(1) + attempt += 1 + logger.warning(f"File {self.file_id} did not complete within timeout period ({timeout}s)") + return False + + def wait_for_format(self, format: str, timeout: int = 60) -> bool: + """Wait for a specific format conversion to complete. + + Polls the file status until the format is complete or the timeout is reached. + Format conversions complete independently of the top-level file status and + can lag behind it; a requested format that has not started converting yet is + absent from the status response's `formats` map and is treated the same as + one that is not yet completed. + + Args: + format: The format to wait for (e.g., 'md', 'docx', 'latex', 'tex.zip'). + Note: Use 'latex' not 'tex' - the API uses 'latex' for status polling. + timeout: Maximum number of seconds to wait. Must be a positive, non-zero integer. + + Returns: + bool: True if format conversion completed successfully, False if it timed out or errored. + + Raises: + ValidationError: If timeout is an invalid value + """ + is_valid_timeout: bool = isinstance(timeout, int) and timeout > 0 + if not is_valid_timeout: + raise ValidationError("Timeout must be a positive, non-zero integer") + is_tex_alias: bool = format == 'tex' + if is_tex_alias: + logger.info("wait_for_format: 'tex' converted to 'latex' (API uses 'latex' for status)") + format = 'latex' + logger.debug(f"Waiting for file {self.file_id} format '{format}' to complete (timeout: {timeout}s)") + attempt: int = 1 + while attempt < timeout: + logger.debug(f'Checking format status... ({attempt}/{timeout})') + file_status: Dict[str, Any] = self.status() + format_status: Optional[str] = file_status.get('formats', {}).get(format) + is_format_completed: bool = format_status == 'completed' + has_format_errored: bool = format_status == 'error' + if is_format_completed: + logger.debug(f"File {self.file_id} format '{format}' completed successfully") + return True + elif has_format_errored: + logger.error(f"File {self.file_id} format '{format}' failed") + return False + time.sleep(1) + attempt += 1 + logger.warning(f"File {self.file_id} format '{format}' did not complete within timeout period ({timeout}s)") + return False + + def status(self) -> Dict[str, Any]: + """Get the current status of the file processing. + + Returns: + dict: JSON response containing file status information including: + - file_id: The file identifier + - status: pending|split|completed|error + - num_pages: Total number of pages + - num_pages_completed: Pages processed so far + - percent_done: Processing progress percentage + - formats: Dict of per-format conversion statuses + - filename, custom_id, destination_uri, destination_basename, format_primary + - error, error_info: present when status is 'error' + + Raises: + FilesApiError: If the file does not exist ('not_found') or belongs to + a different group ('forbidden'). + MathpixClientError: If the request fails without a Files API error body. + """ + logger.debug(f"Getting status for file {self.file_id}") + endpoint: str = urljoin(self.auth.files_api_url, f'/files/v1/{self.file_id}') + response: requests.Response = get(endpoint, headers=self.auth.headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + result: Dict[str, Any] = response.json() + self._last_status = result + return result + + def delete(self) -> Dict[str, Any]: + """Permanently remove the file and its results from Mathpix-owned storage. + + Only files in a terminal state (completed or error) can be deleted; a file + still being processed returns a conflict. Deleting is idempotent: repeating + the call on an already-deleted file returns the same success body. Results + delivered to a customer-owned bucket via destination_uri are not affected. + + Returns: + dict: Response containing 'file_id' and 'status': 'deleted'. + + Raises: + FilesApiError: If the file does not exist ('not_found'), belongs to a + different group ('forbidden'), or is still processing ('conflict'). + """ + logger.debug(f"Deleting file {self.file_id}") + endpoint: str = urljoin(self.auth.files_api_url, f'/files/v1/{self.file_id}') + response: requests.Response = delete(endpoint, headers=self.auth.headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + return response.json() + + def _check_download_response(self, response: requests.Response) -> None: + """Raise the appropriate error for a failed result download. + + Per the Files API, a format that is still converting returns 404 with an + error body of 'format_not_ready'; a plain 'not_found' 404 means the file id + does not exist; 415 'unsupported_format' means the extension was never + requested. A 409 is also treated as format-not-ready for compatibility with + older deployments. Any other failure raises FilesApiError when the body is + a Files API error envelope, and MathpixClientError otherwise. + """ + is_success: bool = response.ok + if is_success: + return + envelope: Optional[Dict[str, Any]] = parse_files_api_error_envelope(response) + error_id: Optional[str] = envelope['error_id'] if envelope is not None else None + is_unsupported_format: bool = response.status_code == 415 or error_id == 'unsupported_format' + if is_unsupported_format: + raise ValidationError(f"Format was not requested or is not supported: {error_id or response.status_code}") + is_format_not_ready: bool = response.status_code == 409 or error_id == 'format_not_ready' + if is_format_not_ready: + raise ConversionIncompleteError("Format not ready yet") + is_missing_file: bool = error_id == 'not_found' + if is_missing_file: + raise MathpixClientError(f"File not found: {self.file_id}") + raise error_from_response(response) + + def save_file(self, path: str, conversion_format: str) -> str: + """Helper function to save the processed file result to a local path. + + Args: + path: The local file path where the output will be saved + conversion_format: The format in which the output will be saved + + Returns: + output_path: The path of the saved file + + Raises: + ConversionIncompleteError: If the conversion is not complete + """ + is_directory_path: bool = path.endswith('/') or path.endswith('\\') + if is_directory_path: + filename: str = f"{self.file_id}.{conversion_format}" + path = os.path.join(path, filename) + logger.debug(f"Downloading output for file {self.file_id} in format {conversion_format} to path {path}") + endpoint: str = urljoin(self.auth.files_api_url, f'/files/v1/{self.file_id}.{conversion_format}') + response: requests.Response = get(endpoint, headers=self.auth.headers, **self.request_options) + self._check_download_response(response) + try: + directory: str = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + with open(path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + except Exception: + raise FilesystemError('Failed to save file to system') + logger.debug(f"File saved successfully to {path}") + return path + + def text_result(self, conversion_format: str) -> str: + """Helper method to download the processed file result as text. + + Args: + conversion_format: Output format extension + + Returns: + text: The result as a string (mmd, md, txt) + + Raises: + ConversionIncompleteError: If the conversion is not complete + """ + logger.debug(f"Downloading output for file {self.file_id} in format: {conversion_format}") + endpoint: str = urljoin(self.auth.files_api_url, f'/files/v1/{self.file_id}.{conversion_format}') + response: requests.Response = get(endpoint, headers=self.auth.headers, **self.request_options) + self._check_download_response(response) + return response.text + + def bytes_result(self, conversion_format: str) -> bytes: + """Helper method to download the processed file result bytes. + + Args: + conversion_format: Output format extension + + Returns: + bytes: The binary content of the result + + Raises: + ConversionIncompleteError: If the conversion is not complete + """ + logger.debug(f"Downloading output for file {self.file_id} in format: {conversion_format}") + endpoint: str = urljoin(self.auth.files_api_url, f'/files/v1/{self.file_id}.{conversion_format}') + response: requests.Response = get(endpoint, headers=self.auth.headers, **self.request_options) + self._check_download_response(response) + return response.content + + def json_result(self, conversion_format: str) -> Any: + """Helper method to download the processed file result as JSON. + + Args: + conversion_format: Output format extension (e.g., lines.json) + + Returns: + dict: The result as a dictionary + + Raises: + ConversionIncompleteError: If the conversion is not complete + """ + logger.debug(f"Downloading output for file {self.file_id} in format: {conversion_format}") + endpoint: str = urljoin(self.auth.files_api_url, f'/files/v1/{self.file_id}.{conversion_format}') + response: requests.Response = get(endpoint, headers=self.auth.headers, **self.request_options) + self._check_download_response(response) + return json.loads(response.text) + + # Text format methods + def to_mmd_text(self) -> str: + """Get the processed file result as Mathpix Markdown string.""" + return self.text_result(conversion_format='mmd') + + def to_md_text(self) -> str: + """Get the processed file result as Markdown string.""" + return self.text_result(conversion_format='md') + + def to_tex_text(self) -> str: + """Get the processed file result as LaTeX string.""" + return self.text_result(conversion_format='tex') + + # Binary format methods + def to_docx_bytes(self) -> bytes: + """Get the processed file result as DOCX bytes.""" + return self.bytes_result(conversion_format='docx') + + def to_xlsx_bytes(self) -> bytes: + """Get the processed file result as XLSX bytes.""" + return self.bytes_result(conversion_format='xlsx') + + def to_pptx_bytes(self) -> bytes: + """Get the processed file result as PPTX bytes.""" + return self.bytes_result(conversion_format='pptx') + + def to_pdf_bytes(self) -> bytes: + """Get the processed file result as PDF bytes.""" + return self.bytes_result(conversion_format='pdf') + + def to_latex_pdf_bytes(self) -> bytes: + """Get the processed file result as LaTeX-rendered PDF bytes.""" + return self.bytes_result(conversion_format='latex.pdf') + + def to_html_bytes(self) -> bytes: + """Get the processed file result as HTML bytes.""" + return self.bytes_result(conversion_format='html') + + def to_tex_zip_bytes(self) -> bytes: + """Get the processed file result as tex.zip bytes.""" + return self.bytes_result(conversion_format='tex.zip') + + def to_md_zip_bytes(self) -> bytes: + """Get the processed file result as md.zip bytes.""" + return self.bytes_result(conversion_format='md.zip') + + def to_mmd_zip_bytes(self) -> bytes: + """Get the processed file result as mmd.zip bytes.""" + return self.bytes_result(conversion_format='mmd.zip') + + def to_html_zip_bytes(self) -> bytes: + """Get the processed file result as html.zip bytes.""" + return self.bytes_result(conversion_format='html.zip') + + def to_jpg_bytes(self) -> bytes: + """Get the processed file result as JPG bytes.""" + return self.bytes_result(conversion_format='jpg') + + def to_png_bytes(self) -> bytes: + """Get the processed file result as PNG bytes.""" + return self.bytes_result(conversion_format='png') + + # JSON format methods + def to_lines_json(self) -> Any: + """Get the processed file result as lines.json.""" + return self.json_result(conversion_format='lines.json') + + def to_lines_mmd_json(self) -> Any: + """Get the processed file result as lines.mmd.json.""" + return self.json_result(conversion_format='lines.mmd.json') + + # File save methods + def to_mmd_file(self, path: str) -> str: + """Save the processed file result to a MMD file at a local path.""" + return self.save_file(path=path, conversion_format='mmd') + + def to_md_file(self, path: str) -> str: + """Save the processed file result to a Markdown file at a local path.""" + return self.save_file(path=path, conversion_format='md') + + def to_docx_file(self, path: str) -> str: + """Save the processed file result to a DOCX file at a local path.""" + return self.save_file(path=path, conversion_format='docx') + + def to_xlsx_file(self, path: str) -> str: + """Save the processed file result to an XLSX file at a local path.""" + return self.save_file(path=path, conversion_format='xlsx') + + def to_pptx_file(self, path: str) -> str: + """Save the processed file result to a PPTX file at a local path.""" + return self.save_file(path=path, conversion_format='pptx') + + def to_pdf_file(self, path: str) -> str: + """Save the processed file result to a PDF file at a local path.""" + return self.save_file(path=path, conversion_format='pdf') + + def to_html_file(self, path: str) -> str: + """Save the processed file result to an HTML file at a local path.""" + return self.save_file(path=path, conversion_format='html') + + def to_tex_zip_file(self, path: str) -> str: + """Save the processed file result to a tex.zip file at a local path.""" + return self.save_file(path=path, conversion_format='tex.zip') diff --git a/mpxpy/file_job.py b/mpxpy/file_job.py new file mode 100644 index 0000000..aea30b7 --- /dev/null +++ b/mpxpy/file_job.py @@ -0,0 +1,270 @@ +import time +from dataclasses import dataclass, asdict +from typing import Optional, Dict, Any, Iterator, Union +from urllib.parse import urljoin, quote +import requests +from mpxpy.auth import Auth +from mpxpy.file import File +from mpxpy.logger import logger +from mpxpy.request_handler import get +from mpxpy.errors import ValidationError, error_from_response + + +@dataclass +class FileSubmission: + """One document in a batch submission to the Files API jobs endpoint. + + Attributes: + source_uri: Remote location of the source document. Accepted schemes: + s3://, gs://, public https://, or an Azure Blob HTTPS URL. + custom_id: Optional case-sensitive customer-supplied identifier. + Requires the job to have an explicit job_id; (job_id, custom_id) + is the idempotency key. + filename: Optional display name for the file. + destination_uri: Optional per-file destination for results. Requires a + registered data source for the bucket. + s3_region: Optional region of the destination_uri S3 bucket. + destination_basename: Optional basename for output objects within + destination_uri. + page_ranges: Optional page range string, e.g. "1-5,8". + """ + source_uri: str + custom_id: Optional[str] = None + filename: Optional[str] = None + destination_uri: Optional[str] = None + s3_region: Optional[str] = None + destination_basename: Optional[str] = None + page_ranges: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + """Return the submission as a dict with unset fields omitted.""" + return {key: value for key, value in asdict(self).items() if value is not None} + + +class FileJob: + """Manages a Files API job: a named container of file submissions. + + Attributes: + auth: An Auth instance with Mathpix credentials. + job_id: The unique identifier for this job. + file_count: The number of submitted items, when known (set from the + submission response; not kept up to date afterwards, use status()). + """ + def __init__( + self, + auth: Auth, + job_id: Optional[str] = None, + file_count: Optional[int] = None, + request_options: Optional[Dict[str, Any]] = None, + ) -> None: + """Initialize a FileJob instance. + + Args: + auth: Auth instance containing Mathpix API credentials. + job_id: The unique identifier for the job. + file_count: Optional item count from the submission response. + request_options: Optional dict of kwargs to pass to requests. + + Raises: + ValidationError: If auth is not provided or job_id is empty. + """ + self.auth: Auth = auth + has_auth: bool = self.auth is not None + if not has_auth: + logger.error("FileJob requires an authenticated client") + raise ValidationError("FileJob requires an authenticated client") + self.job_id: str = job_id or '' + has_job_id: bool = bool(self.job_id) + if not has_job_id: + logger.error("FileJob requires a Job ID") + raise ValidationError("FileJob requires a Job ID") + self.file_count: Optional[int] = file_count + self.request_options: Dict[str, Any] = request_options or {} + + def status(self) -> Dict[str, Any]: + """Get the job's status and counters. + + Returns: + dict: JSON response containing: + - job_id: The job identifier + - status: 'processing' while any file is pending, 'completed' + when every file has reached a terminal state + - file_count: Total files accepted into the job + - files_completed: Files with final results + - files_errored: Files in terminal error state + - created_at, modified_at: ISO 8601 timestamps + + Raises: + FilesApiError: If the request fails (e.g. 'not_found'). + """ + logger.debug(f"Getting status for job {self.job_id}") + endpoint: str = urljoin(self.auth.files_api_url, f'/files/v1/jobs/{quote(self.job_id, safe="")}') + response: requests.Response = get(endpoint, headers=self.auth.headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + return response.json() + + def wait_until_complete(self, timeout: int, interval: float = 5.0) -> bool: + """Wait for the job to complete. + + Polls the job status until it is 'completed' or the timeout is reached. + Per-file failures do not fail the job; check files_errored on status() + and list them with files(status='error'). + + Args: + timeout: Maximum number of seconds to wait. Must be a positive, non-zero integer. + interval: Seconds between polls (default 5.0). Large jobs can take a + long time to complete; use a longer interval for them. + + Returns: + bool: True if the job completed within the timeout, False otherwise. + + Raises: + ValidationError: If timeout or interval is an invalid value. + """ + is_valid_timeout: bool = isinstance(timeout, int) and timeout > 0 + if not is_valid_timeout: + raise ValidationError("Timeout must be a positive, non-zero integer") + is_valid_interval: bool = interval > 0 + if not is_valid_interval: + raise ValidationError("Interval must be a positive number") + logger.debug(f"Waiting for job {self.job_id} to complete (timeout: {timeout}s)") + deadline: float = time.monotonic() + timeout + while time.monotonic() < deadline: + job_status: Dict[str, Any] = self.status() + is_completed: bool = job_status.get('status') == 'completed' + if is_completed: + logger.debug(f"Job {self.job_id} completed") + return True + time.sleep(min(interval, max(0.0, deadline - time.monotonic()))) + logger.warning(f"Job {self.job_id} did not complete within timeout period ({timeout}s)") + return False + + def files( + self, + status: Optional[str] = None, + limit: Optional[int] = None, + paging_state: Optional[str] = None, + ) -> Dict[str, Any]: + """Get one page of the job's file listing. + + Args: + status: Optional filter, one of 'pending', 'completed', 'error'. + 'pending' covers every file that has not reached a terminal state. + The per-file 'status' field is populated only when this filter is + used; it is null in the unfiltered listing. + limit: Optional maximum items per page. + paging_state: Opaque pagination cursor from the previous response's + 'next_page_token'. + + Returns: + dict: Response containing 'files' (each with file_id, filename, + status, custom_id) and 'next_page_token' (non-null when more + pages remain). + + Raises: + FilesApiError: If the request fails (e.g. an unknown status value). + """ + logger.debug(f"Listing files for job {self.job_id} (status={status})") + endpoint: str = urljoin(self.auth.files_api_url, f'/files/v1/jobs/{quote(self.job_id, safe="")}/files') + params: Dict[str, Any] = {} + if status is not None: + params['status'] = status + if limit is not None: + params['limit'] = limit + if paging_state is not None: + params['paging_state'] = paging_state + response: requests.Response = get(endpoint, headers=self.auth.headers, params=params, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + return response.json() + + def files_iter( + self, + status: Optional[str] = None, + limit: Optional[int] = None, + ) -> Iterator[Dict[str, Any]]: + """Iterate over all files in the job, following pagination. + + Args: + status: Optional filter, one of 'pending', 'completed', 'error'. + limit: Optional page size for the underlying requests. + + Yields: + dict: One file entry per iteration (file_id, filename, status, custom_id). + """ + paging_state: Optional[str] = None + while True: + page: Dict[str, Any] = self.files(status=status, limit=limit, paging_state=paging_state) + for file_entry in page.get('files', []): + yield file_entry + paging_state = page.get('next_page_token') + has_more_pages: bool = bool(paging_state) + if not has_more_pages: + return + + def file_by_custom_id(self, custom_id: str) -> File: + """Fetch a single file by the custom_id supplied at submission. + + Available only when the original submission supplied an explicit job_id + and a per-item custom_id. + + Args: + custom_id: The per-item identifier supplied at submission. + + Returns: + File: A File instance for the matched file, seeded with its status. + + Raises: + ValidationError: If custom_id is empty. + FilesApiError: If the (job_id, custom_id) pair is unknown or belongs + to another account ('not_found'; the two cases are + indistinguishable by design). + """ + has_custom_id: bool = bool(custom_id) + if not has_custom_id: + raise ValidationError("custom_id is required") + logger.debug(f"Getting file by custom_id in job {self.job_id}") + endpoint: str = urljoin( + self.auth.files_api_url, + f'/files/v1/jobs/{quote(self.job_id, safe="")}/files/{quote(custom_id, safe="")}' + ) + response: requests.Response = get(endpoint, headers=self.auth.headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + result: Dict[str, Any] = response.json() + return File( + auth=self.auth, + file_id=result['file_id'], + request_options=self.request_options, + status_result=result, + ) + + +def normalize_file_submission(item: Union[FileSubmission, Dict[str, Any]]) -> Dict[str, Any]: + """Validate and convert one batch item to its request dict. + + Args: + item: A FileSubmission or a plain dict with the same keys. + + Returns: + dict: The submission dict with unset fields omitted. + + Raises: + ValidationError: If the item is not a FileSubmission or dict, or has no source_uri. + """ + is_submission_instance: bool = isinstance(item, FileSubmission) + is_dict: bool = isinstance(item, dict) + if is_submission_instance: + submission: Dict[str, Any] = item.to_dict() + elif is_dict: + submission = {key: value for key, value in item.items() if value is not None} + else: + raise ValidationError(f"Each file must be a FileSubmission or dict, got: {type(item).__name__}") + has_source_uri: bool = bool(submission.get('source_uri')) + if not has_source_uri: + raise ValidationError("Each file submission requires a source_uri") + return submission diff --git a/mpxpy/image.py b/mpxpy/image.py index a77751f..b668e56 100644 --- a/mpxpy/image.py +++ b/mpxpy/image.py @@ -31,7 +31,7 @@ def __init__( result: Dict[str, Any], file_path: Optional[str] = None, url: Optional[str] = None, - improve_mathpix: bool = True, + improve_mathpix: Optional[bool] = True, include_line_data: Optional[bool] = False, metadata: Optional[Dict[str, Any]] = None, is_async: Optional[bool] = False, diff --git a/mpxpy/mathpix_client.py b/mpxpy/mathpix_client.py index 04c4a36..227c972 100644 --- a/mpxpy/mathpix_client.py +++ b/mpxpy/mathpix_client.py @@ -1,20 +1,124 @@ +import sys import json import requests -from typing import Dict, Any, Optional, List, Tuple +from typing import Dict, Any, Optional, List, Set, Tuple, Union +if sys.version_info >= (3, 13): + from warnings import deprecated +else: + from typing_extensions import deprecated from pathlib import Path -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse from mpxpy.pdf import Pdf from mpxpy.image import Image +from mpxpy.file import File +from mpxpy.scs_file import ScsFile from mpxpy.file_batch import FileBatch +from mpxpy.file_job import FileJob, FileSubmission, normalize_file_submission +from mpxpy.data_source import DataSource from mpxpy.conversion import Conversion -from mpxpy.scs_file import ScsFile from mpxpy.batch import Batch from mpxpy.auth import Auth from mpxpy.logger import logger, configure_logging -from mpxpy.errors import MathpixClientError, ValidationError +from mpxpy.errors import MathpixClientError, ValidationError, error_from_response from mpxpy.request_handler import post, get +def _apply_processing_options( + options: Dict[str, Any], + alphabets_allowed: Optional[Dict[str, str]] = None, + rm_spaces: Optional[bool] = True, + rm_fonts: Optional[bool] = False, + idiomatic_eqn_arrays: Optional[bool] = False, + include_equation_tags: Optional[bool] = False, + include_smiles: Optional[bool] = True, + include_chemistry_as_image: Optional[bool] = False, + include_diagram_text: Optional[bool] = False, + numbers_default_to_math: Optional[bool] = False, + math_inline_delimiters: Optional[Tuple[str, str]] = None, + math_display_delimiters: Optional[Tuple[str, str]] = None, + page_ranges: Optional[str] = None, + enable_spell_check: Optional[bool] = False, + auto_number_sections: Optional[bool] = False, + remove_section_numbering: Optional[bool] = False, + preserve_section_numbering: Optional[bool] = True, + enable_tables_fallback: Optional[bool] = False, + fullwidth_punctuation: Optional[bool] = None, +) -> None: + """Apply the OCR/conversion options shared with v3/pdf to a request options dict. + + Only non-default values are added, matching the request shape used across + the client. + """ + if alphabets_allowed is not None: + options["alphabets_allowed"] = alphabets_allowed + if not rm_spaces: + options["rm_spaces"] = rm_spaces + if rm_fonts: + options["rm_fonts"] = rm_fonts + if idiomatic_eqn_arrays: + options["idiomatic_eqn_arrays"] = idiomatic_eqn_arrays + if include_equation_tags: + options["include_equation_tags"] = True + if not include_smiles: + options["include_smiles"] = include_smiles + if include_chemistry_as_image: + options["include_chemistry_as_image"] = True + if include_diagram_text: + options["include_diagram_text"] = include_diagram_text + if numbers_default_to_math: + options["numbers_default_to_math"] = numbers_default_to_math + if math_inline_delimiters is not None: + options["math_inline_delimiters"] = math_inline_delimiters + if math_display_delimiters is not None: + options["math_display_delimiters"] = math_display_delimiters + if page_ranges is not None: + options["page_ranges"] = page_ranges + if enable_spell_check: + options["enable_spell_check"] = enable_spell_check + if auto_number_sections: + options["auto_number_sections"] = auto_number_sections + if remove_section_numbering: + options["remove_section_numbering"] = remove_section_numbering + if not preserve_section_numbering: + options["preserve_section_numbering"] = preserve_section_numbering + if enable_tables_fallback: + options["enable_tables_fallback"] = enable_tables_fallback + if fullwidth_punctuation: + options["fullwidth_punctuation"] = fullwidth_punctuation + + +def _redact_uri(uri: Optional[str]) -> str: + """Reduce a URI to its scheme and host for logging. + + Signed S3/GCS URLs and Azure SAS URLs carry bearer credentials in their + query strings, and debug logs are commonly forwarded to shared log + aggregation, so never log the full URI. + """ + parsed = urlparse(uri or '') + return f"{parsed.scheme}://{parsed.hostname or ''}" + + +def _reject_reserved_extra_options( + extra_options: Optional[Dict[str, object]], + reserved: Set[str], +) -> None: + """Reject extra_options keys that would override validated request fields. + + The extra_options pass-through is merged into the request body after the + explicit arguments, so without this check a caller could silently replace + fields the method has already validated (e.g. source_uri, files, custom_id). + """ + has_extra_options: bool = bool(extra_options) + if not has_extra_options: + return + conflicting: Set[str] = reserved.intersection(extra_options or {}) + has_conflicts: bool = bool(conflicting) + if has_conflicts: + raise ValidationError( + f"extra_options may not override validated request fields: {', '.join(sorted(conflicting))}" + ) + + class MathpixClient: """Client for interacting with the Mathpix API. @@ -87,7 +191,7 @@ def image_new( enable_tables_fallback: Optional[bool] = False, fullwidth_punctuation: Optional[bool] = None ): - """Process an image either from a local file or remote URL. + r"""Process an image either from a local file or remote URL. Args: file_path: Path to a local image file. @@ -135,7 +239,7 @@ def image_new( logger.error("Invalid parameters: Exactly one of file_path or url must be provided") raise ValidationError("Exactly one of file_path or url must be provided") endpoint = urljoin(self.auth.api_url, 'v3/text') - image_options = { + image_options: Dict[str, Any] = { "metadata": { "mpxpy": True, **(metadata or {}) @@ -285,7 +389,7 @@ def pdf_new( webhook_payload: Optional[Dict[str, Any]] = None, webhook_enabled_events: Optional[List[str]] = None, ) -> Pdf: - """Uploads a PDF, document, or ebook from a local file or remote URL and optionally requests conversions. + r"""Uploads a PDF, document, or ebook from a local file or remote URL and optionally requests conversions. Args: file_path: Path to a local PDF file. @@ -445,6 +549,7 @@ def pdf_new( raise FileNotFoundError(f"File path not found: {file_path}") with path.open("rb") as pdf_file: files = {"file": pdf_file} + response_json = None try: response = post(endpoint, data=data, files=files, headers=self.auth.headers, **self.request_options) response.raise_for_status() @@ -480,6 +585,7 @@ def pdf_new( else: logger.debug(f"Creating new PDF: url={url}") options["url"] = url + response_json = None try: response = post(endpoint, json=options, headers=self.auth.headers, **self.request_options) response.raise_for_status() @@ -634,6 +740,7 @@ def conversion_new( options["formats"]['html.zip'] = True if len(options['formats'].items()) == 0: raise ValidationError("At least one format is required.") + response_json = None try: response = post(endpoint, json=options, headers=self.auth.headers) response.raise_for_status() @@ -663,16 +770,17 @@ def conversion_new( logger.error(f"Conversion failed: {response_json}") raise MathpixClientError(f"Mathpix conversion request failed: {e}") - def scs_file_new( + def file_new( self, + source_uri: Optional[str] = None, file_path: Optional[str] = None, - url: Optional[str] = None, - source_s3_uri: Optional[str] = None, + job_id: Optional[str] = None, + custom_id: Optional[str] = None, + idempotency_key: Optional[str] = None, filename: Optional[str] = None, - scs_job_id: Optional[str] = None, conversion_formats: Optional[Dict[str, bool]] = None, - conversion_options: Optional[Dict[str, object]] = None, - destination_s3_uri: Optional[str] = None, + extra_options: Optional[Dict[str, object]] = None, + destination_uri: Optional[str] = None, destination_basename: Optional[str] = None, s3_region: Optional[str] = None, image_output_mode: Optional[str] = None, @@ -696,27 +804,49 @@ def scs_file_new( preserve_section_numbering: Optional[bool] = True, enable_tables_fallback: Optional[bool] = False, fullwidth_punctuation: Optional[bool] = None, - ): - """Upload a file via files-api v1 for async processing. + ) -> File: + """Submit a single document for async processing. - Supports three upload modes (exactly one must be provided): - - file_path: Multipart upload from local file - - url: Upload from HTTP URL or S3 presigned URL - - source_s3_uri: Copy from S3 bucket (requires IAM role access) + Exactly one of source_uri or file_path must be provided. A source_uri + submits via POST /files/v1/uri and may be an s3://, gs://, public + https://, or Azure Blob HTTPS URL; non-public sources require a + registered data source for the bucket, see + https://docs.mathpix.com/reference/files-v1-data-sources + A file_path uploads a local file via multipart POST /files/v1. Args: + source_uri: Remote location of the source document. file_path: Path to a local file to upload. - url: URL of a remote file (HTTP/HTTPS or S3 presigned URL). - source_s3_uri: S3 URI (s3://bucket/key) to copy from. - filename: Optional filename to use (defaults to file basename). - scs_job_id: Optional job ID to group files together. - conversion_formats: Dict of format names to enable (e.g., {'mmd': True, 'docx': True}). - conversion_options: Additional conversion options dict. - destination_s3_uri: Optional S3 URI to write output files. - destination_basename: Optional basename for output files (defaults to file_id). - s3_region: Optional AWS region for S3 operations (default us-east-1). - image_output_mode: Image output mode (e.g., 'local' to upload to destination_s3_uri). - include_page_info: Include page info in output (default None). + job_id: Optional job to associate this file with. Required whenever + custom_id is supplied. + custom_id: Optional case-sensitive customer-supplied identifier. + Requires job_id; (job_id, custom_id) is the idempotency key: + re-submitting the same pair returns the original file rather + than creating a new one, as long as the original file is still + live (pending, split, or completed). + idempotency_key: Optional client-generated key sent as the + Idempotency-Key header (same constraints as custom_id). Makes a + standalone submission safe to retry: re-sending the same request + returns the original file_id instead of creating a duplicate. If + both a (job_id, custom_id) pair and an idempotency_key are present, + the pair takes precedence. + filename: Optional display name for the file (defaults to + '.pdf'). + conversion_formats: Dict of format names to enable (e.g., {'docx': True, + 'md': True}). Mathpix Markdown (mmd) is always produced. + extra_options: Additional request options dict, merged into the + request body last. May not override the validated request fields + source_uri, job_id, custom_id, or metadata. + destination_uri: Optional destination for results. Same scheme rules as + source_uri; must be backed by a registered data source. When omitted, + results stay in Mathpix storage and are fetched via the download + helpers on File. + destination_basename: Optional basename for output objects within + destination_uri (defaults to the file_id). + s3_region: Optional region of the destination_uri S3 bucket. + image_output_mode: Set to 'local' to write cropped images into + destination_uri storage under images/, instead of the Mathpix CDN. + include_page_info: Include per-page information in the output. metadata: Optional dict to attach metadata to the request. alphabets_allowed: Optional dict to list alphabets allowed in the output. rm_spaces: Remove extra white space from equations (default True). @@ -729,7 +859,7 @@ def scs_file_new( numbers_default_to_math: Numbers are always math (default False). math_inline_delimiters: Tuple of (begin, end) delimiters for inline math. math_display_delimiters: Tuple of (begin, end) delimiters for display math. - page_ranges: Page range string (e.g., "2,4-6" or "2--2"). + page_ranges: Page range string (e.g., "2,4-6"). enable_spell_check: Enable predictive mode for English handwriting (default False). auto_number_sections: Auto-number sections (default False). remove_section_numbering: Remove existing section numbering (default False). @@ -737,26 +867,177 @@ def scs_file_new( enable_tables_fallback: Enable advanced table processing (default False). fullwidth_punctuation: Use fullwidth Unicode punctuation (default None). + Returns: + File: A new File instance for polling status and downloading results. + Raises: - ValidationError: If not exactly one of file_path, url, or source_s3_uri is provided. + ValidationError: If not exactly one of source_uri and file_path is + provided, custom_id is supplied without job_id, or + extra_options contains a reserved request field. FileNotFoundError: If the specified file_path does not exist. - MathpixClientError: If the API request fails. + FilesApiError: If the API rejects the submission. + MathpixClientError: If the request fails. + """ + has_exactly_one_source: bool = sum(x is not None for x in [source_uri, file_path]) == 1 + if not has_exactly_one_source: + raise ValidationError("Exactly one of source_uri or file_path must be provided") + _reject_reserved_extra_options(extra_options, {'source_uri', 'job_id', 'custom_id', 'metadata'}) + has_custom_id: bool = custom_id is not None + if has_custom_id: + has_job_id: bool = job_id is not None + if not has_job_id: + raise ValidationError("custom_id requires an explicit job_id") + has_idempotency_key: bool = idempotency_key is not None + if file_path is not None: + # The endpoint also recognizes job_id under a deprecated alias, so + # the reserved-keys guard must cover it too. + _reject_reserved_extra_options(extra_options, {'scs_job_id'}) + return self._file_new_multipart( + file_path=file_path, + job_id=job_id, + custom_id=custom_id, + idempotency_key=idempotency_key, + filename=filename, + conversion_formats=conversion_formats, + extra_options=extra_options, + destination_uri=destination_uri, + destination_basename=destination_basename, + s3_region=s3_region, + image_output_mode=image_output_mode, + include_page_info=include_page_info, + metadata=metadata, + alphabets_allowed=alphabets_allowed, + rm_spaces=rm_spaces, + rm_fonts=rm_fonts, + idiomatic_eqn_arrays=idiomatic_eqn_arrays, + include_equation_tags=include_equation_tags, + include_smiles=include_smiles, + include_chemistry_as_image=include_chemistry_as_image, + include_diagram_text=include_diagram_text, + numbers_default_to_math=numbers_default_to_math, + math_inline_delimiters=math_inline_delimiters, + math_display_delimiters=math_display_delimiters, + page_ranges=page_ranges, + enable_spell_check=enable_spell_check, + auto_number_sections=auto_number_sections, + remove_section_numbering=remove_section_numbering, + preserve_section_numbering=preserve_section_numbering, + enable_tables_fallback=enable_tables_fallback, + fullwidth_punctuation=fullwidth_punctuation, + ) + has_source_uri: bool = bool(source_uri) + if not has_source_uri: + raise ValidationError("source_uri must be a non-empty string") + options: Dict[str, object] = { + "source_uri": source_uri, + } + if metadata: + options["metadata"] = metadata + if filename: + options["filename"] = filename + if conversion_formats: + options["conversion_formats"] = conversion_formats + if destination_uri: + options["destination_uri"] = destination_uri + if destination_basename: + options["destination_basename"] = destination_basename + if s3_region: + options["s3_region"] = s3_region + if image_output_mode: + options["image_output_mode"] = image_output_mode + if include_page_info is not None: + options["include_page_info"] = include_page_info + if custom_id: + options["custom_id"] = custom_id + if job_id: + options["job_id"] = job_id + _apply_processing_options( + options, + alphabets_allowed=alphabets_allowed, + rm_spaces=rm_spaces, + rm_fonts=rm_fonts, + idiomatic_eqn_arrays=idiomatic_eqn_arrays, + include_equation_tags=include_equation_tags, + include_smiles=include_smiles, + include_chemistry_as_image=include_chemistry_as_image, + include_diagram_text=include_diagram_text, + numbers_default_to_math=numbers_default_to_math, + math_inline_delimiters=math_inline_delimiters, + math_display_delimiters=math_display_delimiters, + page_ranges=page_ranges, + enable_spell_check=enable_spell_check, + auto_number_sections=auto_number_sections, + remove_section_numbering=remove_section_numbering, + preserve_section_numbering=preserve_section_numbering, + enable_tables_fallback=enable_tables_fallback, + fullwidth_punctuation=fullwidth_punctuation, + ) + if extra_options: + options.update(extra_options) + logger.debug(f"Creating new file via Files API: source={_redact_uri(source_uri)}") + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1/uri') + headers: Dict[str, str] = dict(self.auth.headers) + if has_idempotency_key: + headers['Idempotency-Key'] = idempotency_key + try: + response: requests.Response = post(endpoint, json=options, headers=headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + response_json: Dict[str, Any] = response.json() + file_id: str = response_json['file_id'] + logger.debug(f"File from URI started, file_id: {file_id}") + return File(auth=self.auth, file_id=file_id, request_options=self.request_options) + except requests.exceptions.RequestException as e: + raise MathpixClientError(f"Mathpix Files API request failed: {e}") + + def _file_new_multipart( + self, + file_path: str, + job_id: Optional[str] = None, + custom_id: Optional[str] = None, + idempotency_key: Optional[str] = None, + filename: Optional[str] = None, + conversion_formats: Optional[Dict[str, bool]] = None, + extra_options: Optional[Dict[str, object]] = None, + destination_uri: Optional[str] = None, + destination_basename: Optional[str] = None, + s3_region: Optional[str] = None, + image_output_mode: Optional[str] = None, + include_page_info: Optional[bool] = None, + metadata: Optional[Dict[str, object]] = None, + alphabets_allowed: Optional[Dict[str, str]] = None, + rm_spaces: Optional[bool] = True, + rm_fonts: Optional[bool] = False, + idiomatic_eqn_arrays: Optional[bool] = False, + include_equation_tags: Optional[bool] = False, + include_smiles: Optional[bool] = True, + include_chemistry_as_image: Optional[bool] = False, + include_diagram_text: Optional[bool] = False, + numbers_default_to_math: Optional[bool] = False, + math_inline_delimiters: Optional[Tuple[str, str]] = None, + math_display_delimiters: Optional[Tuple[str, str]] = None, + page_ranges: Optional[str] = None, + enable_spell_check: Optional[bool] = False, + auto_number_sections: Optional[bool] = False, + remove_section_numbering: Optional[bool] = False, + preserve_section_numbering: Optional[bool] = True, + enable_tables_fallback: Optional[bool] = False, + fullwidth_punctuation: Optional[bool] = None, + ) -> File: + """Upload a local file via multipart POST /files/v1 for file_new. + + The endpoint takes job_id, custom_id, and filename as form fields + alongside the file part; the remaining options travel in the + options_json form field. """ - source_count = sum(x is not None for x in [file_path, url, source_s3_uri]) - if source_count != 1: - logger.error("Invalid parameters: Exactly one of file_path, url, or source_s3_uri must be provided") - raise ValidationError("Exactly one of file_path, url, or source_s3_uri must be provided") options: Dict[str, object] = {} - _metadata: Dict[str, object] = {"mpxpy": True} if metadata: - _metadata.update(metadata) - options["metadata"] = _metadata + options["metadata"] = metadata if conversion_formats: options["conversion_formats"] = conversion_formats - if scs_job_id: - options["scs_job_id"] = scs_job_id - if destination_s3_uri: - options["destination_s3_uri"] = destination_s3_uri + if destination_uri: + options["destination_uri"] = destination_uri if destination_basename: options["destination_basename"] = destination_basename if s3_region: @@ -765,99 +1046,645 @@ def scs_file_new( options["image_output_mode"] = image_output_mode if include_page_info is not None: options["include_page_info"] = include_page_info - if alphabets_allowed is not None: - options["alphabets_allowed"] = alphabets_allowed - if not rm_spaces: - options["rm_spaces"] = rm_spaces - if rm_fonts: - options["rm_fonts"] = rm_fonts - if idiomatic_eqn_arrays: - options["idiomatic_eqn_arrays"] = idiomatic_eqn_arrays - if include_equation_tags: - options["include_equation_tags"] = True - if not include_smiles: - options["include_smiles"] = include_smiles - if include_chemistry_as_image: - options["include_chemistry_as_image"] = True - if include_diagram_text: - options["include_diagram_text"] = include_diagram_text - if numbers_default_to_math: - options["numbers_default_to_math"] = numbers_default_to_math - if math_inline_delimiters is not None: - options["math_inline_delimiters"] = math_inline_delimiters - if math_display_delimiters is not None: - options["math_display_delimiters"] = math_display_delimiters - if page_ranges is not None: - options["page_ranges"] = page_ranges - if enable_spell_check: - options["enable_spell_check"] = enable_spell_check - if auto_number_sections: - options["auto_number_sections"] = auto_number_sections - if remove_section_numbering: - options["remove_section_numbering"] = remove_section_numbering - if not preserve_section_numbering: - options["preserve_section_numbering"] = preserve_section_numbering - if enable_tables_fallback: - options["enable_tables_fallback"] = enable_tables_fallback - if fullwidth_punctuation: - options["fullwidth_punctuation"] = fullwidth_punctuation - if conversion_options: - options.update(conversion_options) - if file_path: - logger.debug(f"Creating new file via files-api: path={file_path}") - path = Path(file_path) - if not path.is_file(): - logger.error(f"File not found: {file_path}") - raise FileNotFoundError(f"File path not found: {file_path}") - endpoint = urljoin(self.auth.files_api_url, '/files/v1') - data = {"options_json": json.dumps(options)} - # For files/v1 multipart, filename and scs_job_id are form fields (not in options_json) - if filename: - data["filename"] = filename - if scs_job_id: - data["scs_job_id"] = scs_job_id - with path.open("rb") as f: - files = {"file": f} - try: - response = post(endpoint, data=data, files=files, headers=self.auth.headers, **self.request_options) - response.raise_for_status() - response_json = response.json() - file_id = response_json['file_id'] - logger.debug(f"File upload started, file_id: {file_id}") - return ScsFile(auth=self.auth, file_id=file_id, request_options=self.request_options) - except requests.exceptions.RequestException as e: - raise MathpixClientError(f"Mathpix files-api request failed: {e}") - elif url: - logger.debug(f"Creating new file via files-api: url={url}") - endpoint = urljoin(self.auth.files_api_url, '/files/v1/url') - options["url"] = url - if filename: - options["filename"] = filename - try: - response = post(endpoint, json=options, headers=self.auth.headers, **self.request_options) - response.raise_for_status() - response_json = response.json() - file_id = response_json['file_id'] - logger.debug(f"File from URL started, file_id: {file_id}") - return ScsFile(auth=self.auth, file_id=file_id, request_options=self.request_options) - except requests.exceptions.RequestException as e: - raise MathpixClientError(f"Mathpix files-api request failed: {e}") - else: - logger.debug(f"Creating new file via files-api: source_s3_uri={source_s3_uri}") - endpoint = urljoin(self.auth.files_api_url, '/files/v1/s3') - options["source_s3_uri"] = source_s3_uri - if filename: - options["filename"] = filename + _apply_processing_options( + options, + alphabets_allowed=alphabets_allowed, + rm_spaces=rm_spaces, + rm_fonts=rm_fonts, + idiomatic_eqn_arrays=idiomatic_eqn_arrays, + include_equation_tags=include_equation_tags, + include_smiles=include_smiles, + include_chemistry_as_image=include_chemistry_as_image, + include_diagram_text=include_diagram_text, + numbers_default_to_math=numbers_default_to_math, + math_inline_delimiters=math_inline_delimiters, + math_display_delimiters=math_display_delimiters, + page_ranges=page_ranges, + enable_spell_check=enable_spell_check, + auto_number_sections=auto_number_sections, + remove_section_numbering=remove_section_numbering, + preserve_section_numbering=preserve_section_numbering, + enable_tables_fallback=enable_tables_fallback, + fullwidth_punctuation=fullwidth_punctuation, + ) + if extra_options: + options.update(extra_options) + logger.debug("Creating new file via Files API multipart upload") + path: Path = Path(file_path) + is_existing_file: bool = path.is_file() + if not is_existing_file: + raise FileNotFoundError(f"File path not found: {file_path}") + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1') + data: Dict[str, str] = {"options_json": json.dumps(options)} + if filename: + data["filename"] = filename + if job_id: + data["job_id"] = job_id + if custom_id: + data["custom_id"] = custom_id + headers: Dict[str, str] = dict(self.auth.headers) + if idempotency_key is not None: + headers['Idempotency-Key'] = idempotency_key + with path.open("rb") as f: + files: Dict[str, Any] = {"file": f} try: - response = post(endpoint, json=options, headers=self.auth.headers, **self.request_options) - response.raise_for_status() - response_json = response.json() - file_id = response_json['file_id'] - logger.debug(f"File from S3 started, file_id: {file_id}") - return ScsFile(auth=self.auth, file_id=file_id, request_options=self.request_options) + response: requests.Response = post(endpoint, data=data, files=files, headers=headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + response_json: Dict[str, Any] = response.json() + file_id: str = response_json['file_id'] + logger.debug(f"File upload started, file_id: {file_id}") + return File(auth=self.auth, file_id=file_id, request_options=self.request_options) except requests.exceptions.RequestException as e: - raise MathpixClientError(f"Mathpix files-api request failed: {e}") + raise MathpixClientError(f"Mathpix Files API multipart request failed: {e}") + + def file_job_new( + self, + files: List[Union[FileSubmission, Dict[str, Any]]], + job_id: Optional[str] = None, + idempotency_key: Optional[str] = None, + conversion_formats: Optional[Dict[str, bool]] = None, + extra_options: Optional[Dict[str, object]] = None, + image_output_mode: Optional[str] = None, + metadata: Optional[Dict[str, object]] = None, + alphabets_allowed: Optional[Dict[str, str]] = None, + rm_spaces: Optional[bool] = True, + rm_fonts: Optional[bool] = False, + idiomatic_eqn_arrays: Optional[bool] = False, + include_equation_tags: Optional[bool] = False, + include_smiles: Optional[bool] = True, + include_chemistry_as_image: Optional[bool] = False, + include_diagram_text: Optional[bool] = False, + numbers_default_to_math: Optional[bool] = False, + math_inline_delimiters: Optional[Tuple[str, str]] = None, + math_display_delimiters: Optional[Tuple[str, str]] = None, + enable_spell_check: Optional[bool] = False, + auto_number_sections: Optional[bool] = False, + remove_section_numbering: Optional[bool] = False, + preserve_section_numbering: Optional[bool] = True, + enable_tables_fallback: Optional[bool] = False, + fullwidth_punctuation: Optional[bool] = None, + ) -> FileJob: + """Submit a batch of documents for async processing in one call. + + Submits documents in bulk via POST /files/v1/jobs; the server enforces + an items-per-call ceiling. The request is + accept-and-defer: it returns immediately with a job_id and file_count, + then submits the items in the background. Per-item failures (bad or + unsupported source_uri, missing data source) are NOT reported + synchronously; each surfaces as that file's error status when you poll + the job. Poll FileJob.status() for completion and list failed items with + FileJob.files(status='error'). + + OCR and conversion options apply to every file in the submitted request. + To vary settings across subsets of a larger job, make multiple calls with + the same job_id and different options. + + Args: + files: List of FileSubmission instances or dicts with the same keys + (source_uri required; custom_id, filename, destination_uri, + s3_region, destination_basename, page_ranges optional). + job_id: Optional caller-supplied job id. If omitted the server + generates one. Required whenever any item carries a custom_id. + idempotency_key: Optional client-generated key sent as the + Idempotency-Key header, making the entire batch submission safe to + retry: re-sending the same request returns the original response + without re-enqueuing any file. Honored only when no job_id is + supplied; an explicit job_id wins and the header is ignored for + job derivation. + conversion_formats: Job-wide conversion formats, applied to every file + (e.g., {'docx': True, 'md': True}). + extra_options: Additional request options dict, merged into the + request body last. May not override the validated request fields + files, job_id, or metadata. + image_output_mode: Job-wide. Set to 'local' to write cropped images + into each file's destination_uri storage. Applies only to files + that set a destination_uri. + metadata: Optional dict to attach metadata to the request. + alphabets_allowed: Optional dict to list alphabets allowed in the output. + rm_spaces: Remove extra white space from equations (default True). + rm_fonts: Remove font commands from equations (default False). + idiomatic_eqn_arrays: Use aligned/gathered/cases instead of array (default False). + include_equation_tags: Include equation number tags in LaTeX (default False). + include_smiles: Enable chemistry diagram OCR via SMILES (default True). + include_chemistry_as_image: Return image crop for chemical diagrams (default False). + include_diagram_text: Enable text extraction from diagrams (default False). + numbers_default_to_math: Numbers are always math (default False). + math_inline_delimiters: Tuple of (begin, end) delimiters for inline math. + math_display_delimiters: Tuple of (begin, end) delimiters for display math. + enable_spell_check: Enable predictive mode for English handwriting (default False). + auto_number_sections: Auto-number sections (default False). + remove_section_numbering: Remove existing section numbering (default False). + preserve_section_numbering: Keep existing section numbering (default True). + enable_tables_fallback: Enable advanced table processing (default False). + fullwidth_punctuation: Use fullwidth Unicode punctuation (default None). + + Returns: + FileJob: A new FileJob instance seeded with the response's job_id and + file_count. + + Raises: + ValidationError: If files is empty, an item is malformed or missing + source_uri, a custom_id is duplicated within the batch, any + custom_id is supplied without an explicit job_id, or + extra_options contains a reserved request field. + FilesApiError: If the API rejects the submission (e.g. over the + items-per-call ceiling or an identifier failing the + charset/length constraint). + MathpixClientError: If the request fails. + """ + has_files: bool = bool(files) + if not has_files: + raise ValidationError("files must be a non-empty list") + _reject_reserved_extra_options(extra_options, {'files', 'job_id', 'metadata'}) + normalized: List[Dict[str, Any]] = [normalize_file_submission(item) for item in files] + has_explicit_job_id: bool = job_id is not None + seen_custom_ids: Set[str] = set() + for submission in normalized: + item_custom_id: Optional[str] = submission.get('custom_id') + has_item_custom_id: bool = item_custom_id is not None + if not has_item_custom_id: + continue + if not has_explicit_job_id: + raise ValidationError("custom_id requires an explicit job_id") + is_duplicate_custom_id: bool = item_custom_id in seen_custom_ids + if is_duplicate_custom_id: + raise ValidationError(f"Duplicate custom_id within the batch: {item_custom_id!r}") + seen_custom_ids.add(item_custom_id) + has_idempotency_key: bool = idempotency_key is not None + if has_idempotency_key and has_explicit_job_id: + logger.warning("idempotency_key is ignored for job derivation when an explicit job_id is supplied") + logger.debug(f"Submitting job with {len(normalized)} files") + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1/jobs') + body: Dict[str, Any] = { + "files": normalized, + } + if metadata: + body["metadata"] = metadata + if job_id: + body["job_id"] = job_id + if conversion_formats: + body["conversion_formats"] = conversion_formats + if image_output_mode: + body["image_output_mode"] = image_output_mode + _apply_processing_options( + body, + alphabets_allowed=alphabets_allowed, + rm_spaces=rm_spaces, + rm_fonts=rm_fonts, + idiomatic_eqn_arrays=idiomatic_eqn_arrays, + include_equation_tags=include_equation_tags, + include_smiles=include_smiles, + include_chemistry_as_image=include_chemistry_as_image, + include_diagram_text=include_diagram_text, + numbers_default_to_math=numbers_default_to_math, + math_inline_delimiters=math_inline_delimiters, + math_display_delimiters=math_display_delimiters, + enable_spell_check=enable_spell_check, + auto_number_sections=auto_number_sections, + remove_section_numbering=remove_section_numbering, + preserve_section_numbering=preserve_section_numbering, + enable_tables_fallback=enable_tables_fallback, + fullwidth_punctuation=fullwidth_punctuation, + ) + if extra_options: + body.update(extra_options) + headers: Dict[str, str] = dict(self.auth.headers) + if has_idempotency_key: + headers['Idempotency-Key'] = idempotency_key + try: + response: requests.Response = post(endpoint, json=body, headers=headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + response_json: Dict[str, Any] = response.json() + response_job_id: str = response_json['job_id'] + logger.debug(f"Job accepted, job_id: {response_job_id}") + return FileJob( + auth=self.auth, + job_id=response_job_id, + file_count=response_json.get('file_count'), + request_options=self.request_options, + ) + except requests.exceptions.RequestException as e: + raise MathpixClientError(f"Mathpix Files API job request failed: {e}") + + def file_job_list( + self, + start: Optional[str] = None, + end: Optional[str] = None, + limit: int = 100, + paging_state: Optional[str] = None, + ) -> Dict[str, Any]: + """List the jobs submitted under your account, newest first. + + Args: + start: Earliest submission date to include, yyyy-MM-dd (UTC). + Providing only one of start/end queries that single day. + end: Latest submission date to include, yyyy-MM-dd (UTC). + limit: Maximum jobs per page (default 100). + paging_state: Opaque pagination cursor from the previous response's + 'next_page_token'. + + Returns: + dict: Response containing 'jobs' (each with job_id and created_at) + and 'next_page_token' (non-null when more pages remain). + + Raises: + FilesApiError: If the request fails (e.g. 'bad_request' for a + malformed date or an out-of-range limit). + """ + logger.debug("Listing jobs from the Files API") + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1/jobs') + params: Dict[str, object] = {"limit": limit} + if start: + params["start"] = start + if end: + params["end"] = end + if paging_state: + params["paging_state"] = paging_state + try: + response: requests.Response = get(endpoint, headers=self.auth.headers, params=params, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + return response.json() + except requests.exceptions.RequestException as e: + raise MathpixClientError(f"Mathpix Files API list jobs request failed: {e}") + + def file_get(self, file_id: str) -> File: + """Fetch an existing file and return its File instance. + + Performs GET /files/v1/{file_id}; the returned File is seeded with the + response, so the lazy status attributes are populated without another + request. + + Args: + file_id: The file's identifier. + + Returns: + File: A File instance seeded with the file's current status. + + Raises: + FilesApiError: If the file does not exist ('not_found') or belongs + to a different group ('forbidden'). + MathpixClientError: If the request fails without a Files API error body. + """ + file: File = File(auth=self.auth, file_id=file_id, request_options=self.request_options) + file.status() + return file + + def file_delete(self, file_id: str) -> Dict[str, Any]: + """Permanently remove a file and its results from Mathpix-owned storage. + + See File.delete for the full semantics (terminal-state requirement, + idempotent repeats, customer-owned buckets unaffected). + + Args: + file_id: The file's identifier. + + Returns: + dict: Response containing 'file_id' and 'status': 'deleted'. + + Raises: + FilesApiError: If the file does not exist, belongs to a different + group, or is still processing. + """ + return File(auth=self.auth, file_id=file_id, request_options=self.request_options).delete() + + def file_job_get(self, job_id: str) -> FileJob: + """Fetch an existing job and return its FileJob instance. + + Performs GET /files/v1/jobs/{job_id} and seeds the returned FileJob's + file_count from the response. + + Args: + job_id: The job's identifier. + + Returns: + FileJob: A FileJob instance seeded with the job's current file_count. + + Raises: + FilesApiError: If the job does not exist ('not_found'). + MathpixClientError: If the request fails without a Files API error body. + """ + job: FileJob = FileJob(auth=self.auth, job_id=job_id, request_options=self.request_options) + job_status: Dict[str, Any] = job.status() + job.file_count = job_status.get('file_count') + return job + + def onboarding_identities(self) -> Dict[str, Any]: + """Get the Mathpix identities you grant cloud storage access to. + + Call this BEFORE setting up cloud-side grants: it returns the Mathpix + AWS trust account id, the Azure application/tenant ids, and, when + available, the GCS impersonator service-account email, plus your + per-group external_id. The external_id is a stable per-group value; it + is used in the AWS IAM trust policy and as the GCS bucket-control + verification id. The endpoint is idempotent. + + Returns: + dict: Response with 'aws' (trust_account_id, external_id) and + 'azure' (app_id, tenant_id) blocks, plus a 'gcp' + (service_account_email, external_id) block when GCS onboarding + is available. + + Raises: + FilesApiError: If the request fails. + MathpixClientError: If the request cannot be made. + """ + logger.debug("Getting data source onboarding identities") + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1/onboarding/identities') + try: + response: requests.Response = get(endpoint, headers=self.auth.headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + return response.json() + except requests.exceptions.RequestException as e: + raise MathpixClientError(f"Mathpix onboarding identities request failed: {e}") + + def data_source_new( + self, + provider: str, + bucket: str, + auth_method: str, + provider_specific_details: Dict[str, str], + name: Optional[str] = None, + region: Optional[str] = None, + secret: Optional[str] = None, + ) -> DataSource: + """Register a bucket or container as a Files API data source. + + Complete the cloud-side grant first (IAM role for AWS, RBAC assignment + for Azure, service-account impersonation binding plus the + .mathpix-verify object for GCS); see + https://docs.mathpix.com/reference/files-v1-data-sources for the + per-provider guides. Use onboarding_identities() to fetch the Mathpix + identities and your external_id before setting up grants. + + For AWS and Azure, call DataSource.test() afterward to verify the grant + end-to-end. GCS registration only succeeds once bucket-control + verification passes, so a successful return already confirms the grant. + + Args: + provider: Storage provider, e.g. 'aws', 'azure', or 'gcp'; see the + per-provider guides for the supported set. + bucket: Bucket / container name (S3 bucket, Azure container, or GCS + bucket). + auth_method: Grant type for the provider, e.g. 'iam_role' or + 'access_key' for aws, 'azure_ad' for azure, 'service_account' + for gcp. Invalid provider/auth_method combinations are rejected + by the server ('bad_request'). + provider_specific_details: Provider-shaped metadata, e.g. + {'iam_role_arn': ..., 'aws_external_id': ...} for aws/iam_role, + {'aws_access_key_id': ...} for aws/access_key, + {'azure_tenant_id': ..., 'storage_account': ...} for azure, or + {'gcp_project_id': ..., 'target_sa_email': ...} for gcp. + name: Optional human-readable label. + region: Bucket region; required for aws with 'access_key', optional + for 'iam_role' (discovered via the bucket). + secret: Only for aws with 'access_key' (legacy fallback); the + keyless grant types reject it server-side. + + Returns: + DataSource: A new DataSource instance for the registered data source. + + Raises: + ValidationError: If provider, bucket, auth_method, or + provider_specific_details is missing. + FilesApiError: If the API rejects the registration ('bad_request', + including invalid provider/auth_method combinations, a secret + for a keyless grant type, and GCS bucket-control verification + failures), the registration conflicts with an existing data + source ('conflict'; the server message identifies the + conflict), or the GCS verification probe could not reach the + bucket ('unavailable', 503; retryable). + MathpixClientError: If the request cannot be made. + """ + has_provider: bool = bool(provider) + if not has_provider: + raise ValidationError("provider is required") + has_bucket: bool = bool(bucket) + if not has_bucket: + raise ValidationError("bucket is required") + has_auth_method: bool = bool(auth_method) + if not has_auth_method: + raise ValidationError("auth_method is required") + has_details: bool = bool(provider_specific_details) + if not has_details: + raise ValidationError("provider_specific_details is required") + has_secret: bool = secret is not None + logger.debug(f"Registering data source: provider={provider}, auth_method={auth_method}") + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1/data-sources') + body: Dict[str, Any] = { + "provider": provider, + "bucket": bucket, + "auth_method": auth_method, + "provider_specific_details": provider_specific_details, + } + if name: + body["name"] = name + if region: + body["region"] = region + if has_secret: + body["secret"] = secret + try: + response: requests.Response = post(endpoint, json=body, headers=self.auth.headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + response_json: Dict[str, Any] = response.json() + data_source_id: str = response_json['data_source_id'] + logger.debug(f"Data source registered, data_source_id: {data_source_id}") + return DataSource(auth=self.auth, data_source_id=data_source_id, request_options=self.request_options) + except requests.exceptions.RequestException as e: + raise MathpixClientError(f"Mathpix data source registration failed: {e}") + + def data_source_list(self) -> Dict[str, Any]: + """List the data sources registered for your group. + + Secrets (for aws 'access_key' sources) are never returned. + + Returns: + dict: Response containing 'data_sources', each with data_source_id, + name, provider, bucket, region, auth_method, and created_at. + + Raises: + FilesApiError: If the request fails. + MathpixClientError: If the request cannot be made. + """ + logger.debug("Listing data sources") + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1/data-sources') + try: + response: requests.Response = get(endpoint, headers=self.auth.headers, **self.request_options) + has_failed: bool = not response.ok + if has_failed: + raise error_from_response(response) + return response.json() + except requests.exceptions.RequestException as e: + raise MathpixClientError(f"Mathpix data sources list request failed: {e}") + + def data_source_test(self, data_source_id: str) -> Dict[str, Any]: + """Verify Mathpix can reach a registered bucket. + + See DataSource.test for the full semantics: HTTP 200 for both outcomes; + the {'result', 'checks', 'message'} probe body is returned as-is and a + failed probe does not raise. + + Args: + data_source_id: The data source's identifier. + + Returns: + dict: The probe body ('result', 'checks', 'message'). + + Raises: + FilesApiError: If the request itself fails (e.g. 'not_found'). + """ + return DataSource(auth=self.auth, data_source_id=data_source_id, request_options=self.request_options).test() + + def data_source_delete(self, data_source_id: str) -> Dict[str, Any]: + """Remove a data source registration. + + See DataSource.delete for the full semantics (already-started work is + not interrupted; cloud-side grants must be revoked separately). + + Args: + data_source_id: The data source's identifier. + + Returns: + dict: Response containing 'data_source_id' and 'status': 'deleted'. + + Raises: + FilesApiError: If no accessible data source has this id ('not_found'). + """ + return DataSource(auth=self.auth, data_source_id=data_source_id, request_options=self.request_options).delete() + + @deprecated("scs_file_new is deprecated; use file_new(source_uri=...) instead") + def scs_file_new( + self, + file_path: Optional[str] = None, + url: Optional[str] = None, + source_s3_uri: Optional[str] = None, + filename: Optional[str] = None, + scs_job_id: Optional[str] = None, + conversion_formats: Optional[Dict[str, bool]] = None, + conversion_options: Optional[Dict[str, object]] = None, + destination_s3_uri: Optional[str] = None, + destination_basename: Optional[str] = None, + s3_region: Optional[str] = None, + image_output_mode: Optional[str] = None, + include_page_info: Optional[bool] = None, + metadata: Optional[Dict[str, object]] = None, + alphabets_allowed: Optional[Dict[str, str]] = None, + rm_spaces: Optional[bool] = True, + rm_fonts: Optional[bool] = False, + idiomatic_eqn_arrays: Optional[bool] = False, + include_equation_tags: Optional[bool] = False, + include_smiles: Optional[bool] = True, + include_chemistry_as_image: Optional[bool] = False, + include_diagram_text: Optional[bool] = False, + numbers_default_to_math: Optional[bool] = False, + math_inline_delimiters: Optional[Tuple[str, str]] = None, + math_display_delimiters: Optional[Tuple[str, str]] = None, + page_ranges: Optional[str] = None, + enable_spell_check: Optional[bool] = False, + auto_number_sections: Optional[bool] = False, + remove_section_numbering: Optional[bool] = False, + preserve_section_numbering: Optional[bool] = True, + enable_tables_fallback: Optional[bool] = False, + fullwidth_punctuation: Optional[bool] = None, + ) -> ScsFile: + """Upload a file via files-api v1 for async processing. + Deprecated: use file_new instead. This wrapper translates url and + source_s3_uri to source_uri, scs_job_id to job_id, and + destination_s3_uri to destination_uri, then forwards to file_new. + + Args: + file_path: Path to a local file to upload via multipart POST /files/v1. + url: URL of a remote file, forwarded as source_uri. + source_s3_uri: S3 URI (s3://bucket/key), forwarded as source_uri. + filename: Optional display name for the file. + scs_job_id: Forwarded as job_id. + conversion_formats: Dict of format names to enable (e.g., {'mmd': True, 'docx': True}). + conversion_options: Additional request options dict, forwarded as extra_options. + destination_s3_uri: Forwarded as destination_uri. + destination_basename: Optional basename for output files (defaults to file_id). + s3_region: Region of the destination_s3_uri bucket. + image_output_mode: Image output mode (e.g., 'local'). + include_page_info: Include per-page information in the output. + metadata: Optional dict to attach metadata to the request. + alphabets_allowed: Optional dict to list alphabets allowed in the output. + rm_spaces: Remove extra white space from equations (default True). + rm_fonts: Remove font commands from equations (default False). + idiomatic_eqn_arrays: Use aligned/gathered/cases instead of array (default False). + include_equation_tags: Include equation number tags in LaTeX (default False). + include_smiles: Enable chemistry diagram OCR via SMILES (default True). + include_chemistry_as_image: Return image crop for chemical diagrams (default False). + include_diagram_text: Enable text extraction from diagrams (default False). + numbers_default_to_math: Numbers are always math (default False). + math_inline_delimiters: Tuple of (begin, end) delimiters for inline math. + math_display_delimiters: Tuple of (begin, end) delimiters for display math. + page_ranges: Page range string (e.g., "2,4-6" or "2--2"). + enable_spell_check: Enable predictive mode for English handwriting (default False). + auto_number_sections: Auto-number sections (default False). + remove_section_numbering: Remove existing section numbering (default False). + preserve_section_numbering: Keep existing section numbering (default True). + enable_tables_fallback: Enable advanced table processing (default False). + fullwidth_punctuation: Use fullwidth Unicode punctuation (default None). + + Returns: + ScsFile: A new ScsFile instance for polling status and downloading results. + + Raises: + ValidationError: If not exactly one of file_path, url, or source_s3_uri is provided. + FileNotFoundError: If the specified file_path does not exist. + FilesApiError: If the API rejects the submission. + MathpixClientError: If the request fails. + """ + source_count: int = sum(x is not None for x in [file_path, url, source_s3_uri]) + has_exactly_one_source: bool = source_count == 1 + if not has_exactly_one_source: + logger.error("Invalid parameters: Exactly one of file_path, url, or source_s3_uri must be provided") + raise ValidationError("Exactly one of file_path, url, or source_s3_uri must be provided") + resolved_source_uri: Optional[str] = url if url is not None else source_s3_uri + submitted_file: File = self.file_new( + source_uri=resolved_source_uri, + file_path=file_path, + job_id=scs_job_id, + filename=filename, + conversion_formats=conversion_formats, + extra_options=conversion_options, + destination_uri=destination_s3_uri, + destination_basename=destination_basename, + s3_region=s3_region, + image_output_mode=image_output_mode, + include_page_info=include_page_info, + metadata=metadata, + alphabets_allowed=alphabets_allowed, + rm_spaces=rm_spaces, + rm_fonts=rm_fonts, + idiomatic_eqn_arrays=idiomatic_eqn_arrays, + include_equation_tags=include_equation_tags, + include_smiles=include_smiles, + include_chemistry_as_image=include_chemistry_as_image, + include_diagram_text=include_diagram_text, + numbers_default_to_math=numbers_default_to_math, + math_inline_delimiters=math_inline_delimiters, + math_display_delimiters=math_display_delimiters, + page_ranges=page_ranges, + enable_spell_check=enable_spell_check, + auto_number_sections=auto_number_sections, + remove_section_numbering=remove_section_numbering, + preserve_section_numbering=preserve_section_numbering, + enable_tables_fallback=enable_tables_fallback, + fullwidth_punctuation=fullwidth_punctuation, + ) + return ScsFile(auth=self.auth, file_id=submitted_file.file_id, request_options=self.request_options) + + @deprecated("list_scs_files is deprecated; use file_job_get(job_id).files() instead") def list_scs_files( self, scs_job_id: Optional[str] = None, @@ -867,7 +1694,9 @@ def list_scs_files( ): """List files from files-api v1. - Requires exactly one filter: scs_job_id or filename. + Deprecated: for listing a job's files use file_job_get(job_id).files() + instead, which targets the public GET /files/v1/jobs/{job_id}/files + endpoint and supports a status filter. Args: scs_job_id: Filter by job ID. @@ -879,7 +1708,7 @@ def list_scs_files( dict: Response containing 'file_ids' list and 'next_page_token' for pagination. """ logger.debug("Listing files from files-api") - endpoint = urljoin(self.auth.files_api_url, '/files/v1/list') + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1/list') params: Dict[str, object] = {"limit": limit} if scs_job_id: params["scs_job_id"] = scs_job_id @@ -894,6 +1723,7 @@ def list_scs_files( except requests.exceptions.RequestException as e: raise MathpixClientError(f"Mathpix files-api list request failed: {e}") + @deprecated("list_scs_jobs is deprecated; use file_job_list instead") def list_scs_jobs( self, start: Optional[str] = None, @@ -903,6 +1733,11 @@ def list_scs_jobs( ): """List SCS jobs from files-api v1. + Deprecated: use file_job_list instead, which targets the public + GET /files/v1/jobs endpoint. During the deprecation window this method + stays on the legacy GET /files/v1/scs-jobs endpoint so existing callers + keep receiving the legacy job entries and response metadata. + Args: start: Optional start date filter (ISO format). end: Optional end date filter (ISO format). @@ -913,7 +1748,7 @@ def list_scs_jobs( dict: Response containing 'jobs' list and optionally 'paging_state' for next page. """ logger.debug("Listing jobs from files-api") - endpoint = urljoin(self.auth.files_api_url, '/files/v1/scs-jobs') + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1/scs-jobs') params: Dict[str, object] = {"limit": limit} if start: params["start"] = start @@ -922,15 +1757,21 @@ def list_scs_jobs( if paging_state: params["paging_state"] = paging_state try: - response = get(endpoint, headers=self.auth.headers, params=params, **self.request_options) + response: requests.Response = get(endpoint, headers=self.auth.headers, params=params, **self.request_options) response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: raise MathpixClientError(f"Mathpix files-api list jobs request failed: {e}") + @deprecated("scs_job_status is deprecated; use file_job_get(job_id).status() instead") def scs_job_status(self, scs_job_id: str): """Get the current status of an SCS job. + Deprecated: use file_job_get(job_id).status() instead, which targets the + public GET /files/v1/jobs/{job_id} endpoint. During the deprecation + window this method stays on the legacy GET /files/v1/scs-jobs/status + endpoint so existing callers keep receiving the legacy response shape. + Args: scs_job_id: The job ID to get status for. @@ -938,10 +1779,10 @@ def scs_job_status(self, scs_job_id: str): JSON response containing job status information. """ logger.debug(f"Getting status for SCS job {scs_job_id}") - endpoint = urljoin(self.auth.files_api_url, '/files/v1/scs-jobs/status') - params = {'scs_job_id': scs_job_id} + endpoint: str = urljoin(self.auth.files_api_url, '/files/v1/scs-jobs/status') + params: Dict[str, str] = {'scs_job_id': scs_job_id} try: - response = get(endpoint, headers=self.auth.headers, params=params, **self.request_options) + response: requests.Response = get(endpoint, headers=self.auth.headers, params=params, **self.request_options) response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: diff --git a/mpxpy/scs_file.py b/mpxpy/scs_file.py index 4afd6e1..149974b 100644 --- a/mpxpy/scs_file.py +++ b/mpxpy/scs_file.py @@ -1,16 +1,26 @@ import os +import sys import time import json from typing import Optional, Dict, Any from urllib.parse import urljoin +if sys.version_info >= (3, 13): + from warnings import deprecated +else: + from typing_extensions import deprecated from mpxpy.auth import Auth from mpxpy.logger import logger from mpxpy.request_handler import get from mpxpy.errors import FilesystemError, ValidationError, ConversionIncompleteError +@deprecated("ScsFile is deprecated; use mpxpy.file.File instead") class ScsFile: - """Manages a file through the files-api v1 endpoint. + """Manages a file through the legacy files-api v1 endpoints. + + Deprecated: use mpxpy.file.File instead, which targets the public Files API + and raises FilesApiError with the API's error codes. ScsFile is kept with + its original behavior for compatibility during the deprecation window. This class handles operations on Mathpix files, including checking status, downloading results in different formats, and waiting for processing to complete. @@ -121,7 +131,7 @@ def status(self) -> Dict[str, Any]: Returns: dict: JSON response containing file status information including: - file_id: The file identifier - - status: pending|processing|completed|error + - status: pending|split|completed|error - num_pages: Total number of pages - num_pages_completed: Pages processed so far - percent_done: Processing progress percentage @@ -334,37 +344,3 @@ def to_html_file(self, path: str) -> str: def to_tex_zip_file(self, path: str) -> str: """Save the processed file result to a tex.zip file at a local path.""" return self.save_file(path=path, conversion_format='tex.zip') - - def cropped_image(self, page: int, top_left_x: int, top_left_y: int, - width: int = 0, height: int = 0) -> bytes: - """Get a cropped region from a specific page. - - Args: - page: Page number (0-indexed) - top_left_x: X coordinate of crop top-left corner - top_left_y: Y coordinate of crop top-left corner - width: Crop width (0 = original_width - top_left_x) - height: Crop height (0 = original_height - top_left_y) - - Returns: - bytes: The cropped JPEG image bytes - """ - logger.debug(f"Getting cropped image for file {self.file_id} page {page}") - endpoint = urljoin( - self.auth.files_api_url, - f'/files/v1/cropped/{self.file_id}-{page}.jpg' - ) - params = { - 'top_left_x': top_left_x, - 'top_left_y': top_left_y, - } - if width > 0: - params['width'] = width - if height > 0: - params['height'] = height - response = get(endpoint, headers=self.auth.headers, params=params, **self.request_options) - if response.status_code == 404: - raise ConversionIncompleteError("File or page not found") - if response.status_code == 400: - raise ValidationError("Invalid crop dimensions") - return response.content diff --git a/pyproject.toml b/pyproject.toml index b196b1d..cb4a930 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "mpxpy" -version = "0.0.20" +version = "0.0.21" description = "Official Mathpix client for Python" readme = "README.md" requires-python = ">=3.8" @@ -31,6 +31,7 @@ dependencies = [ "requests>=2.20.0", "python-dotenv>=0.15.0", "pydantic>=2.10.6", + "typing-extensions>=4.5.0; python_version < '3.13'", ] keywords = ["mathpix", "mpx", "mpxpy", "py", "client", "ocr", "api", "sdk", "python","package", "development"] diff --git a/tests/test_files_api.py b/tests/test_files_api.py new file mode 100644 index 0000000..d9d47d8 --- /dev/null +++ b/tests/test_files_api.py @@ -0,0 +1,229 @@ +"""Integration tests for the public Files API v1 client surface. + +These tests make real requests. Point them at a local Files API deployment via +the MATHPIX_FILES_API_URL environment variable (or the files_api_url client +argument), the same override tests/test_scs.py uses for local docker testing. +""" +import time +import uuid +from typing import Any, Dict +import pytest +from mpxpy.mathpix_client import MathpixClient +from mpxpy.file import File +from mpxpy.file_job import FileJob, FileSubmission +from mpxpy.errors import FilesApiError + +SAMPLE_PDF_URL: str = "https://mathpix-ocr-examples.s3.amazonaws.com/bitcoin-7.pdf" +BAD_PDF_URL: str = "https://mathpix-ocr-examples.s3.amazonaws.com/does-not-exist-mpxpy-test.pdf" + + +@pytest.fixture +def client() -> MathpixClient: + return MathpixClient() + + +def unique_id(prefix: str) -> str: + return f"{prefix}-{uuid.uuid4().hex[:8]}" + + +def wait_for_file_by_custom_id(job: FileJob, custom_id: str, timeout: float = 60.0) -> File: + """Poll until a submitted item is registered in the job. + + Job submission is accept-and-defer: items are enqueued in the background, + so a lookup immediately after submit can be a not_found miss. + """ + deadline: float = time.monotonic() + timeout + while True: + try: + return job.file_by_custom_id(custom_id) + except FilesApiError as error: + is_not_registered_yet: bool = error.error_id == 'not_found' + has_time_left: bool = time.monotonic() < deadline + can_retry: bool = is_not_registered_yet and has_time_left + if not can_retry: + raise + time.sleep(1.0) + + +def onboarding_identities_or_skip(client: MathpixClient) -> Dict[str, Any]: + """Fetch onboarding identities, skipping when the deployment lacks them. + + The endpoint depends on deployment configuration; deployments without it + return a 500 internal_error. + """ + try: + return client.onboarding_identities() + except FilesApiError as error: + is_unconfigured_deployment: bool = error.http_status == 500 + if is_unconfigured_deployment: + pytest.skip("onboarding identities are not configured on this deployment") + raise + + +def test_file_new_uri_lifecycle(client: MathpixClient) -> None: + """Submit a public URL, wait, download mmd, then delete (idempotently).""" + file = client.file_new( + source_uri=SAMPLE_PDF_URL, + conversion_formats={'md': True}, + ) + assert isinstance(file, File) + assert file.file_id + assert file.wait_until_complete(timeout=120) + status = file.status() + assert status['status'] == 'completed' + mmd_text = file.to_mmd_text() + assert mmd_text + # Delete, then delete again: the repeat must return the same success body. + result = file.delete() + assert result['status'] == 'deleted' + repeat = file.delete() + assert repeat['status'] == 'deleted' + + +def test_file_new_idempotency_key(client: MathpixClient) -> None: + """Re-submitting with the same Idempotency-Key returns the original file_id.""" + key = unique_id('mpxpy-idem') + first = client.file_new(source_uri=SAMPLE_PDF_URL, idempotency_key=key) + second = client.file_new(source_uri=SAMPLE_PDF_URL, idempotency_key=key) + assert first.file_id == second.file_id + + +def test_file_job_lifecycle(client: MathpixClient) -> None: + """Submit a 2-file job with one bad URI; poll; verify the error listing and + custom_id lookups.""" + job_id = unique_id('mpxpy-job') + job = client.file_job_new( + files=[ + FileSubmission(source_uri=SAMPLE_PDF_URL, custom_id='good'), + FileSubmission(source_uri=BAD_PDF_URL, custom_id='bad'), + ], + job_id=job_id, + conversion_formats={'md': True}, + ) + assert isinstance(job, FileJob) + assert job.job_id == job_id + assert job.file_count == 2 + assert job.wait_until_complete(timeout=300, interval=5.0) + status = job.status() + assert status['status'] == 'completed' + assert status['file_count'] == 2 + assert status['files_completed'] == 1 + assert status['files_errored'] == 1 + errored = list(job.files_iter(status='error')) + assert [f['custom_id'] for f in errored] == ['bad'] + good_file = job.file_by_custom_id('good') + assert isinstance(good_file, File) + assert good_file.status()['status'] == 'completed' + + +def test_file_job_custom_id_idempotency(client: MathpixClient) -> None: + """Re-submitting the same (job_id, custom_id) returns the original file_id.""" + job_id = unique_id('mpxpy-idem-job') + client.file_job_new( + files=[{'source_uri': SAMPLE_PDF_URL, 'custom_id': 'doc-1'}], + job_id=job_id, + ) + first = wait_for_file_by_custom_id(client.file_job_get(job_id), 'doc-1') + client.file_job_new( + files=[{'source_uri': SAMPLE_PDF_URL, 'custom_id': 'doc-1'}], + job_id=job_id, + ) + second = wait_for_file_by_custom_id(client.file_job_get(job_id), 'doc-1') + assert first.file_id == second.file_id + + +def test_file_job_list(client: MathpixClient) -> None: + """A submitted job appears in the jobs listing.""" + job_id = unique_id('mpxpy-list-job') + client.file_job_new( + files=[{'source_uri': SAMPLE_PDF_URL}], + job_id=job_id, + ) + found = False + paging_state = None + for _ in range(10): + result = client.file_job_list(limit=100, paging_state=paging_state) + if any(job['job_id'] == job_id for job in result['jobs']): + found = True + break + paging_state = result.get('next_page_token') + if not paging_state: + break + assert found, f"Job {job_id} not found in the jobs listing" + + +def test_file_job_status_unknown_job(client: MathpixClient) -> None: + with pytest.raises(FilesApiError): + client.file_job_get(unique_id('mpxpy-missing')).status() + + +# Data sources. +# These use a dummy AWS role ARN: registration stores metadata without probing +# the grant (only GCS verifies at registration), so no real bucket is needed. + +def test_onboarding_identities_shape(client: MathpixClient) -> None: + identities = onboarding_identities_or_skip(client) + assert 'aws' in identities + assert 'azure' in identities + # The 'gcp' block is optional + assert identities['aws']['external_id'] + # Idempotent: a second call returns the same external_id. + repeat = client.onboarding_identities() + assert repeat['aws']['external_id'] == identities['aws']['external_id'] + + +def test_data_source_lifecycle(client: MathpixClient) -> None: + """Register (dummy grant) -> conflict -> test probe -> list -> delete.""" + external_id = onboarding_identities_or_skip(client)['aws']['external_id'] + bucket = unique_id('mpxpy-test-bucket') + details = { + 'iam_role_arn': 'arn:aws:iam::123456789012:role/mpxpy-integration-dummy', + 'aws_external_id': external_id, + } + data_source = client.data_source_new( + provider='aws', + bucket=bucket, + auth_method='iam_role', + provider_specific_details=details, + name='mpxpy integration test', + region='us-east-1', + ) + assert data_source.data_source_id + try: + # Re-registering the same (provider, bucket) conflicts. + with pytest.raises(FilesApiError) as exc_info: + client.data_source_new(provider='aws', bucket=bucket, auth_method='iam_role', + provider_specific_details=details) + assert exc_info.value.error_id == 'conflict' + # The dummy role can't be assumed, so the probe reports failure without raising. + probe = data_source.test() + assert probe['result'] in ('ok', 'failed') + assert 'checks' in probe + # The source appears in the listing. + listing = client.data_source_list() + listed_ids = [entry['data_source_id'] for entry in listing['data_sources']] + assert data_source.data_source_id in listed_ids + finally: + result = data_source.delete() + assert result['status'] == 'deleted' + # Deleting again reports not_found. + with pytest.raises(FilesApiError) as exc_info: + data_source.delete() + assert exc_info.value.error_id == 'not_found' + + +def test_data_source_external_id_mismatch_rejected(client: MathpixClient) -> None: + """An aws_external_id that doesn't match the group's is a bad_request.""" + onboarding_identities_or_skip(client) # mismatch detection needs a configured deployment + bucket = unique_id('mpxpy-badid-bucket') + with pytest.raises(FilesApiError) as exc_info: + client.data_source_new( + provider='aws', + bucket=bucket, + auth_method='iam_role', + provider_specific_details={ + 'iam_role_arn': 'arn:aws:iam::123456789012:role/mpxpy-integration-dummy', + 'aws_external_id': 'not-the-group-external-id', + }, + ) + assert exc_info.value.http_status in (400, 403) diff --git a/tests/test_files_api_unit.py b/tests/test_files_api_unit.py new file mode 100644 index 0000000..14c5eac --- /dev/null +++ b/tests/test_files_api_unit.py @@ -0,0 +1,813 @@ +"""Unit tests for the public Files API v1 client surface. + +These tests mock the request layer; no network access is required. +""" +import json +import logging +from typing import Any, Dict, Iterator, Optional +from unittest.mock import patch +import pytest +from mpxpy.mathpix_client import MathpixClient +from mpxpy.file import File +from mpxpy.file_job import FileJob, FileSubmission +from mpxpy.data_source import DataSource +from mpxpy.scs_file import ScsFile +from mpxpy.errors import ( + ValidationError, + ConversionIncompleteError, + MathpixClientError, + FilesApiError, +) + + +class FakeResponse: + def __init__( + self, + status_code: int = 200, + json_body: Optional[Dict[str, Any]] = None, + content: bytes = b'', + text: str = '', + ) -> None: + self.status_code: int = status_code + self._json_body: Optional[Dict[str, Any]] = json_body + self.content: bytes = content + self.text: str = text + + @property + def ok(self) -> bool: + return self.status_code < 400 + + def json(self) -> Dict[str, Any]: + has_json_body: bool = self._json_body is not None + if not has_json_body: + raise ValueError("No JSON body") + return self._json_body or {} + + def raise_for_status(self) -> None: + has_failed: bool = not self.ok + if has_failed: + import requests + raise requests.exceptions.HTTPError(f"HTTP {self.status_code}") + + def iter_content(self, chunk_size: int = 8192) -> Iterator[bytes]: + yield self.content + + +@pytest.fixture +def client() -> MathpixClient: + return MathpixClient(app_id='test-app', app_key='test-key') + + +# Auth URL resolution + +def test_files_api_url_follows_api_url(monkeypatch) -> None: + from mpxpy.auth import Auth + monkeypatch.setattr(Auth, 'load_config', lambda self: False) + monkeypatch.delenv('MATHPIX_URL', raising=False) + monkeypatch.delenv('MATHPIX_FILES_API_URL', raising=False) + # A custom api_url carries over to the Files API endpoints + assert Auth(app_id='x', app_key='y', api_url='https://example.test').files_api_url == 'https://example.test' + # Default with nothing set + assert Auth(app_id='x', app_key='y').files_api_url == 'https://api.mathpix.com' + # An explicit files_api_url wins + explicit = Auth(app_id='x', app_key='y', api_url='https://example.test', files_api_url='https://files.example.test') + assert explicit.files_api_url == 'https://files.example.test' + # The env var wins over the api_url fallback + monkeypatch.setenv('MATHPIX_FILES_API_URL', 'https://env-files.example.test') + assert Auth(app_id='x', app_key='y', api_url='https://example.test').files_api_url == 'https://env-files.example.test' + + +# file_new + +def test_file_new_uri_request_shape(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'abc-123'}) + file = client.file_new( + source_uri='s3://bucket/doc.pdf', + job_id='job-1', + custom_id='doc-1', + conversion_formats={'docx': True}, + destination_uri='s3://bucket/outputs/doc-1/', + page_ranges='1-5', + ) + assert isinstance(file, File) + assert file.file_id == 'abc-123' + args, kwargs = mock_post.call_args + assert args[0].endswith('/files/v1/uri') + body = kwargs['json'] + assert body['source_uri'] == 's3://bucket/doc.pdf' + assert body['job_id'] == 'job-1' + assert body['custom_id'] == 'doc-1' + assert body['conversion_formats'] == {'docx': True} + assert body['destination_uri'] == 's3://bucket/outputs/doc-1/' + assert body['page_ranges'] == '1-5' + assert 'metadata' not in body + + +def test_requests_carry_mpxpy_user_agent(client: MathpixClient) -> None: + assert client.auth.headers['User-Agent'].startswith('mpxpy/') + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'abc-123'}) + client.file_new(source_uri='https://example.com/doc.pdf') + _, kwargs = mock_post.call_args + assert kwargs['headers']['User-Agent'].startswith('mpxpy/') + + +def test_file_new_sends_idempotency_key_header(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'abc-123'}) + client.file_new(source_uri='https://example.com/doc.pdf', idempotency_key='retry-key-1') + _, kwargs = mock_post.call_args + assert kwargs['headers']['Idempotency-Key'] == 'retry-key-1' + assert kwargs['headers']['app_key'] == 'test-key' + + +def test_file_new_requires_source_uri(client: MathpixClient) -> None: + with pytest.raises(ValidationError): + client.file_new(source_uri='') + + +def test_file_new_custom_id_requires_job_id(client: MathpixClient) -> None: + with pytest.raises(ValidationError): + client.file_new(source_uri='s3://b/k.pdf', custom_id='doc-1') + + +def test_file_new_requires_exactly_one_source(client: MathpixClient) -> None: + with pytest.raises(ValidationError): + client.file_new() + with pytest.raises(ValidationError): + client.file_new(source_uri='s3://b/k.pdf', file_path='/tmp/doc.pdf') + + +def test_file_new_local_upload_multipart(client: MathpixClient, tmp_path) -> None: + doc = tmp_path / 'doc.pdf' + doc.write_bytes(b'%PDF-1.4 test') + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'f-local'}) + file = client.file_new( + file_path=str(doc), + job_id='job-1', + filename='doc.pdf', + destination_uri='s3://bucket/out/', + s3_region='us-east-1', + include_page_info=True, + ) + assert isinstance(file, File) + assert file.file_id == 'f-local' + args, kwargs = mock_post.call_args + assert args[0].endswith('/files/v1') + assert not args[0].endswith('/files/v1/uri') + # job_id and filename travel as form fields; the rest in options_json + assert kwargs['data']['job_id'] == 'job-1' + assert kwargs['data']['filename'] == 'doc.pdf' + options = json.loads(kwargs['data']['options_json']) + assert options['destination_uri'] == 's3://bucket/out/' + assert options['s3_region'] == 'us-east-1' + assert options['include_page_info'] is True + assert 'scs_job_id' not in options + + +def test_file_new_local_upload_forwards_custom_id_and_idempotency_key(client: MathpixClient, tmp_path) -> None: + # The multipart endpoint accepts custom_id as a form field and + # Idempotency-Key as a header, same as the URI transport. + doc = tmp_path / 'doc.pdf' + doc.write_bytes(b'%PDF-1.4 test') + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'f-local'}) + client.file_new( + file_path=str(doc), + job_id='job-1', + custom_id='doc-1', + idempotency_key='retry-key-1', + ) + _, kwargs = mock_post.call_args + assert kwargs['data']['custom_id'] == 'doc-1' + assert kwargs['data']['job_id'] == 'job-1' + assert kwargs['headers']['Idempotency-Key'] == 'retry-key-1' + + +def test_file_new_local_upload_custom_id_requires_job_id(client: MathpixClient, tmp_path) -> None: + doc = tmp_path / 'doc.pdf' + doc.write_bytes(b'%PDF-1.4 test') + with pytest.raises(ValidationError): + client.file_new(file_path=str(doc), custom_id='doc-1') + + +def test_file_new_local_upload_rejects_legacy_reserved_alias(client: MathpixClient, tmp_path) -> None: + # The multipart body carries job_id under its legacy field name; the + # reserved-keys guard must cover the alias too. + doc = tmp_path / 'doc.pdf' + doc.write_bytes(b'%PDF-1.4 test') + with pytest.raises(ValidationError): + client.file_new(file_path=str(doc), job_id='job-1', extra_options={'scs_job_id': 'other-job'}) + + +def test_file_new_raises_files_api_error(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse( + status_code=404, + json_body={'error': 'data_source_not_found', + 'error_info': {'id': 'data_source_not_found', + 'message': 'No data source registered for source'}}, + ) + with pytest.raises(FilesApiError) as exc_info: + client.file_new(source_uri='s3://unregistered/doc.pdf') + assert exc_info.value.error_id == 'data_source_not_found' + assert exc_info.value.http_status == 404 + + +def test_file_new_does_not_log_signed_uri_credentials(client: MathpixClient, caplog) -> None: + # Signed URLs carry bearer credentials in their query strings; logs must + # only ever contain the redacted scheme/host. + signed_uri = 'https://bucket.s3.amazonaws.com/doc.pdf?X-Amz-Credential=AKIAEXAMPLE&X-Amz-Signature=sigvalue' + with caplog.at_level(logging.DEBUG, logger='mathpix'): + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'abc-123'}) + client.file_new(source_uri=signed_uri) + assert 'X-Amz-Signature' not in caplog.text + assert 'sigvalue' not in caplog.text + assert 'AKIAEXAMPLE' not in caplog.text + + +def test_file_new_non_envelope_error_is_client_error(client: MathpixClient) -> None: + # A failure without a Files API error body (e.g. an HTML 502 from a proxy) + # must not be dressed up as a FilesApiError. + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(status_code=502, text='Bad Gateway') + with pytest.raises(MathpixClientError) as exc_info: + client.file_new(source_uri='s3://bucket/doc.pdf') + assert not isinstance(exc_info.value, FilesApiError) + + +def test_file_new_metadata_and_optional_fields_forwarded(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'abc-123'}) + client.file_new( + source_uri='s3://bucket/doc.pdf', + filename='doc.pdf', + s3_region='us-east-1', + include_page_info=True, + metadata={'batch': 'july'}, + ) + _, kwargs = mock_post.call_args + body = kwargs['json'] + assert body['filename'] == 'doc.pdf' + assert body['s3_region'] == 'us-east-1' + assert body['include_page_info'] is True + assert body['metadata'] == {'batch': 'july'} + + +def test_file_new_rejects_reserved_extra_options(client: MathpixClient) -> None: + for reserved_key in ('source_uri', 'job_id', 'custom_id', 'metadata'): + with pytest.raises(ValidationError): + client.file_new( + source_uri='s3://bucket/doc.pdf', + extra_options={reserved_key: 'injected'}, + ) + + +# file_job_new + +def test_file_job_new_request_shape(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'job_id': 'job-1', 'file_count': 2}) + job = client.file_job_new( + files=[ + FileSubmission(source_uri='s3://bucket/a.pdf', custom_id='a'), + {'source_uri': 'https://example.com/b.pdf', 'custom_id': 'b'}, + ], + job_id='job-1', + conversion_formats={'md': True}, + ) + assert isinstance(job, FileJob) + assert job.job_id == 'job-1' + assert job.file_count == 2 + args, kwargs = mock_post.call_args + assert args[0].endswith('/files/v1/jobs') + body = kwargs['json'] + assert body['job_id'] == 'job-1' + assert body['conversion_formats'] == {'md': True} + assert body['files'] == [ + {'source_uri': 's3://bucket/a.pdf', 'custom_id': 'a'}, + {'source_uri': 'https://example.com/b.pdf', 'custom_id': 'b'}, + ] + + +def test_file_job_new_validation(client: MathpixClient) -> None: + with pytest.raises(ValidationError): + client.file_job_new(files=[]) + # custom_id without job_id + with pytest.raises(ValidationError): + client.file_job_new(files=[{'source_uri': 's3://b/k', 'custom_id': 'a'}]) + # duplicate custom_id within batch + with pytest.raises(ValidationError): + client.file_job_new( + files=[{'source_uri': 's1', 'custom_id': 'a'}, {'source_uri': 's2', 'custom_id': 'a'}], + job_id='j', + ) + # missing source_uri + with pytest.raises(ValidationError): + client.file_job_new(files=[{'filename': 'x.pdf'}]) + + +def test_file_job_new_rejects_reserved_extra_options(client: MathpixClient) -> None: + for reserved_key in ('files', 'job_id', 'metadata'): + with pytest.raises(ValidationError): + client.file_job_new( + files=[{'source_uri': 's3://bucket/a.pdf'}], + extra_options={reserved_key: 'injected'}, + ) + + +def test_file_job_new_idempotency_key_header(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'job_id': 'derived-1', 'file_count': 1}) + job = client.file_job_new( + files=[{'source_uri': 'https://example.com/a.pdf'}], + idempotency_key='batch-key-1', + ) + assert job.job_id == 'derived-1' + _, kwargs = mock_post.call_args + assert kwargs['headers']['Idempotency-Key'] == 'batch-key-1' + + +# file_job_list + +def test_file_job_list_params(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.get') as mock_get: + mock_get.return_value = FakeResponse(json_body={'jobs': [], 'next_page_token': None}) + result = client.file_job_list(start='2026-07-01', end='2026-07-31', limit=50, paging_state='cursor') + assert result == {'jobs': [], 'next_page_token': None} + args, kwargs = mock_get.call_args + assert args[0].endswith('/files/v1/jobs') + assert kwargs['params'] == {'limit': 50, 'start': '2026-07-01', 'end': '2026-07-31', 'paging_state': 'cursor'} + + +# file_get / file_job_get fetch semantics + +def test_file_get_fetches_and_seeds_status(client: MathpixClient) -> None: + status_body = {'file_id': 'f-1', 'status': 'completed', 'custom_id': 'doc-1', 'num_pages': 4} + with patch('mpxpy.file.get') as mock_get: + mock_get.return_value = FakeResponse(json_body=status_body) + file = client.file_get('f-1') + assert isinstance(file, File) + args, _ = mock_get.call_args + assert args[0].endswith('/files/v1/f-1') + # Seeded from the fetch; no extra status request needed + assert file.custom_id == 'doc-1' + assert file.num_pages == 4 + assert mock_get.call_count == 1 + + +def test_file_get_unknown_id_raises(client: MathpixClient) -> None: + with patch('mpxpy.file.get') as mock_get: + mock_get.return_value = FakeResponse(status_code=404, json_body={'error': 'not_found'}) + with pytest.raises(FilesApiError) as exc_info: + client.file_get('f-missing') + assert exc_info.value.error_id == 'not_found' + + +def test_file_job_get_fetches_and_seeds_file_count(client: MathpixClient) -> None: + with patch('mpxpy.file_job.get') as mock_get: + mock_get.return_value = FakeResponse( + json_body={'job_id': 'job-1', 'status': 'completed', 'file_count': 7}, + ) + job = client.file_job_get('job-1') + assert isinstance(job, FileJob) + assert job.file_count == 7 + args, _ = mock_get.call_args + assert args[0].endswith('/files/v1/jobs/job-1') + + +def test_file_job_get_unknown_id_raises(client: MathpixClient) -> None: + with patch('mpxpy.file_job.get') as mock_get: + mock_get.return_value = FakeResponse(status_code=404, json_body={'error': 'not_found'}) + with pytest.raises(FilesApiError) as exc_info: + client.file_job_get('job-missing') + assert exc_info.value.error_id == 'not_found' + + +# FileJob + +def test_file_job_status_endpoint(client: MathpixClient) -> None: + job = FileJob(auth=client.auth, job_id='job-1') + with patch('mpxpy.file_job.get') as mock_get: + mock_get.return_value = FakeResponse(json_body={'job_id': 'job-1', 'status': 'completed'}) + status = job.status() + assert status['status'] == 'completed' + args, _ = mock_get.call_args + assert args[0].endswith('/files/v1/jobs/job-1') + + +def test_file_job_files_status_filter_passthrough(client: MathpixClient) -> None: + # Filter values are the server's contract; the client passes them through + job = FileJob(auth=client.auth, job_id='job-1') + with patch('mpxpy.file_job.get') as mock_get: + mock_get.return_value = FakeResponse(json_body={'files': [], 'next_page_token': None}) + job.files(status='error') + _, kwargs = mock_get.call_args + assert kwargs['params'] == {'status': 'error'} + + +def test_file_job_files_iter_pagination(client: MathpixClient) -> None: + job = FileJob(auth=client.auth, job_id='job-1') + pages = [ + FakeResponse(json_body={'files': [{'file_id': 'f1'}, {'file_id': 'f2'}], 'next_page_token': 'p2'}), + FakeResponse(json_body={'files': [{'file_id': 'f3'}], 'next_page_token': None}), + ] + with patch('mpxpy.file_job.get') as mock_get: + mock_get.side_effect = pages + files = list(job.files_iter(status='error')) + assert [f['file_id'] for f in files] == ['f1', 'f2', 'f3'] + assert mock_get.call_count == 2 + second_params = mock_get.call_args_list[1][1]['params'] + assert second_params['paging_state'] == 'p2' + assert second_params['status'] == 'error' + + +def test_file_job_file_by_custom_id(client: MathpixClient) -> None: + job = FileJob(auth=client.auth, job_id='job-1') + status_body = {'file_id': 'f-9', 'status': 'completed', 'custom_id': 'doc-9', 'num_pages': 3} + with patch('mpxpy.file_job.get') as mock_get: + mock_get.return_value = FakeResponse(json_body=status_body) + file = job.file_by_custom_id('doc-9') + assert isinstance(file, File) + assert file.file_id == 'f-9' + # Seeded from the response; no extra status request needed + assert file.custom_id == 'doc-9' + assert file.num_pages == 3 + args, _ = mock_get.call_args + assert args[0].endswith('/files/v1/jobs/job-1/files/doc-9') + + +def test_file_job_file_by_custom_id_not_found(client: MathpixClient) -> None: + job = FileJob(auth=client.auth, job_id='job-1') + with patch('mpxpy.file_job.get') as mock_get: + mock_get.return_value = FakeResponse(status_code=404, json_body={'error': 'not_found'}) + with pytest.raises(FilesApiError) as exc_info: + job.file_by_custom_id('unknown') + assert exc_info.value.error_id == 'not_found' + + +# File.status + +def test_file_status_raises_on_error_response(client: MathpixClient) -> None: + file = File(auth=client.auth, file_id='f-missing') + with patch('mpxpy.file.get') as mock_get: + mock_get.return_value = FakeResponse(status_code=404, json_body={'error': 'not_found'}) + with pytest.raises(FilesApiError) as exc_info: + file.status() + assert exc_info.value.error_id == 'not_found' + + +def test_file_status_non_envelope_error_is_client_error(client: MathpixClient) -> None: + file = File(auth=client.auth, file_id='f-1') + with patch('mpxpy.file.get') as mock_get: + mock_get.return_value = FakeResponse(status_code=500, text='oops') + with pytest.raises(MathpixClientError) as exc_info: + file.status() + assert not isinstance(exc_info.value, FilesApiError) + + +# File downloads: error disambiguation + +def test_download_format_not_ready_is_conversion_incomplete(client: MathpixClient) -> None: + file = File(auth=client.auth, file_id='f-1') + with patch('mpxpy.file.get') as mock_get: + mock_get.return_value = FakeResponse( + status_code=404, + json_body={'error': 'format_not_ready', 'status': 'split'}, + ) + with pytest.raises(ConversionIncompleteError): + file.text_result('docx') + + +def test_download_not_found_is_client_error(client: MathpixClient) -> None: + file = File(auth=client.auth, file_id='f-1') + with patch('mpxpy.file.get') as mock_get: + mock_get.return_value = FakeResponse(status_code=404, json_body={'error': 'not_found'}) + with pytest.raises(MathpixClientError) as exc_info: + file.text_result('mmd') + assert not isinstance(exc_info.value, ConversionIncompleteError) + + +def test_download_unsupported_format_is_validation_error(client: MathpixClient) -> None: + file = File(auth=client.auth, file_id='f-1') + with patch('mpxpy.file.get') as mock_get: + mock_get.return_value = FakeResponse(status_code=415, json_body={'error': 'unsupported_format'}) + with pytest.raises(ValidationError): + file.bytes_result('docx') + + +def test_download_legacy_409_is_conversion_incomplete(client: MathpixClient) -> None: + file = File(auth=client.auth, file_id='f-1') + with patch('mpxpy.file.get') as mock_get: + mock_get.return_value = FakeResponse(status_code=409) + with pytest.raises(ConversionIncompleteError): + file.bytes_result('docx') + + +# File.delete + +def test_file_delete_success(client: MathpixClient) -> None: + file = File(auth=client.auth, file_id='f-1') + with patch('mpxpy.file.delete') as mock_delete: + mock_delete.return_value = FakeResponse(json_body={'file_id': 'f-1', 'status': 'deleted'}) + result = file.delete() + assert result == {'file_id': 'f-1', 'status': 'deleted'} + args, _ = mock_delete.call_args + assert args[0].endswith('/files/v1/f-1') + + +def test_file_delete_conflict_while_processing(client: MathpixClient) -> None: + file = File(auth=client.auth, file_id='f-1') + with patch('mpxpy.file.delete') as mock_delete: + mock_delete.return_value = FakeResponse( + status_code=409, + json_body={'error': 'conflict', 'error_info': {'id': 'conflict', 'message': 'File is still processing'}}, + ) + with pytest.raises(FilesApiError) as exc_info: + file.delete() + assert exc_info.value.error_id == 'conflict' + assert exc_info.value.http_status == 409 + + +# Data sources + +def test_onboarding_identities(client: MathpixClient) -> None: + identities = { + 'aws': {'trust_account_id': '123456789012', 'external_id': 'group-uuid'}, + 'azure': {'app_id': 'app-uuid', 'tenant_id': 'tenant-uuid'}, + 'gcp': {'service_account_email': 'ingest@example.iam.gserviceaccount.com', 'external_id': 'group-uuid'}, + } + with patch('mpxpy.mathpix_client.get') as mock_get: + mock_get.return_value = FakeResponse(json_body=identities) + result = client.onboarding_identities() + assert result == identities + args, _ = mock_get.call_args + assert args[0].endswith('/files/v1/onboarding/identities') + + +def test_onboarding_identities_gcp_block_is_optional(client: MathpixClient) -> None: + identities = { + 'aws': {'trust_account_id': '123456789012', 'external_id': 'group-uuid'}, + 'azure': {'app_id': 'app-uuid', 'tenant_id': 'tenant-uuid'}, + } + with patch('mpxpy.mathpix_client.get') as mock_get: + mock_get.return_value = FakeResponse(json_body=identities) + result = client.onboarding_identities() + assert result == identities + assert 'gcp' not in result + + +def test_data_source_new_request_shape(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'data_source_id': 'ds-1'}) + data_source = client.data_source_new( + provider='aws', + bucket='my-bucket', + auth_method='iam_role', + provider_specific_details={ + 'iam_role_arn': 'arn:aws:iam::123456789012:role/MathpixReader', + 'aws_external_id': 'group-uuid', + }, + name='prod-source', + region='us-east-1', + ) + assert isinstance(data_source, DataSource) + assert data_source.data_source_id == 'ds-1' + args, kwargs = mock_post.call_args + assert args[0].endswith('/files/v1/data-sources') + body = kwargs['json'] + assert body['provider'] == 'aws' + assert body['bucket'] == 'my-bucket' + assert body['auth_method'] == 'iam_role' + assert body['name'] == 'prod-source' + assert body['region'] == 'us-east-1' + assert 'secret' not in body + + +def test_data_source_new_requires_shape_fields(client: MathpixClient) -> None: + # Only required-value checks are client-side; provider/auth_method + # combinations are the server's contract. + details = {'iam_role_arn': 'arn', 'aws_external_id': 'x'} + with pytest.raises(ValidationError): + client.data_source_new(provider='', bucket='b', auth_method='iam_role', + provider_specific_details=details) + with pytest.raises(ValidationError): + client.data_source_new(provider='aws', bucket='', auth_method='iam_role', + provider_specific_details=details) + with pytest.raises(ValidationError): + client.data_source_new(provider='aws', bucket='b', auth_method='', + provider_specific_details=details) + with pytest.raises(ValidationError): + client.data_source_new(provider='aws', bucket='b', auth_method='iam_role', + provider_specific_details={}) + + +def test_data_source_new_bucket_conflict_raises(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse( + status_code=409, + json_body={'error': 'conflict', + 'error_info': {'id': 'conflict', + 'message': 'A data source for this bucket is already registered.'}}, + ) + with pytest.raises(FilesApiError) as exc_info: + client.data_source_new(provider='aws', bucket='b', auth_method='iam_role', + provider_specific_details={'iam_role_arn': 'arn', 'aws_external_id': 'x'}) + assert exc_info.value.error_id == 'conflict' + assert exc_info.value.http_status == 409 + assert 'bucket is already registered' in str(exc_info.value) + + +def test_data_source_new_duplicate_name_conflict_raises(client: MathpixClient) -> None: + # The same 409 'conflict' also covers a duplicate name; the server message + # is preserved so the two cases are distinguishable. + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse( + status_code=409, + json_body={'error': 'conflict', + 'error_info': {'id': 'conflict', + 'message': 'A data source with this name is already registered.'}}, + ) + with pytest.raises(FilesApiError) as exc_info: + client.data_source_new(provider='aws', bucket='other-bucket', auth_method='iam_role', + provider_specific_details={'iam_role_arn': 'arn', 'aws_external_id': 'x'}, + name='taken-name') + assert exc_info.value.error_id == 'conflict' + assert 'name is already registered' in str(exc_info.value) + + +def test_data_source_new_azure_request_shape(client: MathpixClient) -> None: + details = {'azure_tenant_id': 'tenant-uuid', 'storage_account': 'mystorageacct'} + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'data_source_id': 'ds-az'}) + client.data_source_new(provider='azure', bucket='my-container', auth_method='azure_ad', + provider_specific_details=details) + _, kwargs = mock_post.call_args + body = kwargs['json'] + assert body['provider'] == 'azure' + assert body['bucket'] == 'my-container' + assert body['auth_method'] == 'azure_ad' + assert body['provider_specific_details'] == details + assert 'secret' not in body + + +def test_data_source_new_gcp_request_shape(client: MathpixClient) -> None: + details = {'gcp_project_id': 'my-project', 'target_sa_email': 'ingest@my-project.iam.gserviceaccount.com'} + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'data_source_id': 'ds-gcp'}) + client.data_source_new(provider='gcp', bucket='my-bucket', auth_method='service_account', + provider_specific_details=details) + _, kwargs = mock_post.call_args + body = kwargs['json'] + assert body['provider'] == 'gcp' + assert body['auth_method'] == 'service_account' + assert body['provider_specific_details'] == details + assert 'secret' not in body + + +def test_data_source_new_aws_access_key_request_shape(client: MathpixClient) -> None: + details = {'aws_access_key_id': 'AKIAEXAMPLE'} + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'data_source_id': 'ds-key'}) + client.data_source_new(provider='aws', bucket='my-bucket', auth_method='access_key', + provider_specific_details=details, secret='key-material', + region='us-east-1') + _, kwargs = mock_post.call_args + body = kwargs['json'] + assert body['provider'] == 'aws' + assert body['bucket'] == 'my-bucket' + assert body['auth_method'] == 'access_key' + assert body['provider_specific_details'] == {'aws_access_key_id': 'AKIAEXAMPLE'} + assert body['secret'] == 'key-material' + assert body['region'] == 'us-east-1' + + +def test_data_source_test_returns_failed_probe_without_raising(client: MathpixClient) -> None: + probe = {'result': 'failed', 'checks': {'read': True, 'write': False}, + 'message': 'Write probe failed: 403 AccessDenied (missing s3:PutObject)'} + with patch('mpxpy.data_source.post') as mock_post: + mock_post.return_value = FakeResponse(json_body=probe) + result = client.data_source_test('ds-1') + assert result == probe + args, _ = mock_post.call_args + assert args[0].endswith('/files/v1/data-sources/ds-1/test') + + +def test_data_source_delete(client: MathpixClient) -> None: + with patch('mpxpy.data_source.delete') as mock_delete: + mock_delete.return_value = FakeResponse(json_body={'data_source_id': 'ds-1', 'status': 'deleted'}) + result = client.data_source_delete('ds-1') + assert result == {'data_source_id': 'ds-1', 'status': 'deleted'} + args, _ = mock_delete.call_args + assert args[0].endswith('/files/v1/data-sources/ds-1') + + +def test_data_source_delete_not_found(client: MathpixClient) -> None: + with patch('mpxpy.data_source.delete') as mock_delete: + mock_delete.return_value = FakeResponse(status_code=404, json_body={'error': 'not_found'}) + with pytest.raises(FilesApiError) as exc_info: + client.data_source_delete('ds-missing') + assert exc_info.value.error_id == 'not_found' + + +def test_data_source_list(client: MathpixClient) -> None: + listing = {'data_sources': [{'data_source_id': 'ds-1', 'provider': 'aws', 'bucket': 'b'}]} + with patch('mpxpy.mathpix_client.get') as mock_get: + mock_get.return_value = FakeResponse(json_body=listing) + result = client.data_source_list() + assert result == listing + args, _ = mock_get.call_args + assert args[0].endswith('/files/v1/data-sources') + + +# Deprecated wrappers + +def test_scs_file_new_url_delegates_to_file_new(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'f-1'}) + with pytest.warns(DeprecationWarning): + file = client.scs_file_new( + url='https://example.com/doc.pdf', + scs_job_id='job-1', + filename='doc.pdf', + destination_s3_uri='s3://bucket/out/', + s3_region='us-east-1', + include_page_info=True, + ) + assert isinstance(file, ScsFile) + args, kwargs = mock_post.call_args + assert args[0].endswith('/files/v1/uri') + body = kwargs['json'] + assert body['source_uri'] == 'https://example.com/doc.pdf' + assert body['job_id'] == 'job-1' + assert body['filename'] == 'doc.pdf' + assert body['destination_uri'] == 's3://bucket/out/' + assert body['s3_region'] == 'us-east-1' + assert body['include_page_info'] is True + + +def test_scs_file_new_s3_delegates_to_file_new(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'f-1'}) + with pytest.warns(DeprecationWarning): + client.scs_file_new(source_s3_uri='s3://bucket/doc.pdf') + args, kwargs = mock_post.call_args + assert args[0].endswith('/files/v1/uri') + assert kwargs['json']['source_uri'] == 's3://bucket/doc.pdf' + + +def test_list_scs_jobs_stays_on_legacy_endpoint(client: MathpixClient) -> None: + legacy_body = {'jobs': [{'scs_job_id': 'job-1', 'group_id': 'g-1', 'app_id': 'a-1'}], 'paging_state': None} + with patch('mpxpy.mathpix_client.get') as mock_get: + mock_get.return_value = FakeResponse(json_body=legacy_body) + with pytest.warns(DeprecationWarning): + result = client.list_scs_jobs(limit=10) + assert result == legacy_body + args, kwargs = mock_get.call_args + assert args[0].endswith('/files/v1/scs-jobs') + assert kwargs['params'] == {'limit': 10} + + +def test_scs_job_status_stays_on_legacy_endpoint(client: MathpixClient) -> None: + legacy_body = {'scs_job_id': 'job-1', 'group_id': 'g-1', 'app_id': 'a-1', 'num_pages_completed': 5} + with patch('mpxpy.mathpix_client.get') as mock_get: + mock_get.return_value = FakeResponse(json_body=legacy_body) + with pytest.warns(DeprecationWarning): + status = client.scs_job_status('job-1') + assert status == legacy_body + args, kwargs = mock_get.call_args + assert args[0].endswith('/files/v1/scs-jobs/status') + assert kwargs['params'] == {'scs_job_id': 'job-1'} + + +def test_scs_file_direct_use_warns(client: MathpixClient) -> None: + with pytest.warns(DeprecationWarning): + file = ScsFile(auth=client.auth, file_id='f-1') + assert file.file_id == 'f-1' + + +def test_scs_file_new_file_path_multipart(client: MathpixClient, tmp_path) -> None: + doc = tmp_path / 'doc.pdf' + doc.write_bytes(b'%PDF-1.4 test') + with patch('mpxpy.mathpix_client.post') as mock_post: + mock_post.return_value = FakeResponse(json_body={'file_id': 'f-1'}) + with pytest.warns(DeprecationWarning): + file = client.scs_file_new(file_path=str(doc), scs_job_id='job-1') + assert isinstance(file, ScsFile) + args, kwargs = mock_post.call_args + assert args[0].endswith('/files/v1') + assert not args[0].endswith('/files/v1/uri') + # The wrapper translates scs_job_id to the job_id form field + assert kwargs['data']['job_id'] == 'job-1' + + +def test_list_scs_files_deprecated(client: MathpixClient) -> None: + with patch('mpxpy.mathpix_client.get') as mock_get: + mock_get.return_value = FakeResponse(json_body={'file_ids': [], 'next_page_token': None}) + with pytest.warns(DeprecationWarning): + client.list_scs_files(scs_job_id='job-1') + args, _ = mock_get.call_args + assert args[0].endswith('/files/v1/list') diff --git a/tests/test_image.py b/tests/test_image.py index 4b70fd0..923365b 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -69,7 +69,7 @@ def test_invalid_image_arguments(client): def test_async_image(client): image_file_url = "https://mathpix-ocr-examples.s3.amazonaws.com/cases_hw.jpg" image = client.image_new(url=image_file_url, is_async=True) - assert image.wait_until_complete(timeout=20), f"Async image request did not complete within the timeout period" + assert image.wait_until_complete(timeout=20), "Async image request did not complete within the timeout period" result = image.results() assert result is not None, "Async image result is None" assert 'image_id' in result, "Async image result has no image_id" diff --git a/tests/test_query_converter_results.py b/tests/test_query_converter_results.py index 8c45ef9..4377749 100644 --- a/tests/test_query_converter_results.py +++ b/tests/test_query_converter_results.py @@ -1,5 +1,4 @@ import time -import uuid from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Callable, Any diff --git a/tests/test_scs.py b/tests/test_scs.py index a891eb4..572be7f 100644 --- a/tests/test_scs.py +++ b/tests/test_scs.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta, timezone import os from pathlib import Path -import re from typing import List import pytest from mpxpy.mathpix_client import MathpixClient @@ -101,7 +100,7 @@ def test_file_status(client: MathpixClient): status = file.status() assert 'file_id' in status assert 'status' in status - assert status['status'] in ['pending', 'processing', 'completed', 'error'] + assert status['status'] in ['pending', 'split', 'completed', 'error'] def test_file_download_mmd(client: MathpixClient): @@ -617,43 +616,3 @@ def test_file_save_to_directory(client: MathpixClient, tmp_path: Path): assert saved_path.endswith('.mmd') assert os.path.getsize(saved_path) > 0 - -def test_cropped_image(client: MathpixClient): - """Test getting a cropped image from a processed PDF. - - Uses multiple_images_1.pdf which contains images, then extracts crop - coordinates from the mmd output and verifies the cropped_image endpoint. - """ - pdf_path = os.path.join(current_dir, 'files', 'pdfs', 'multiple_images_1.pdf') - file = client.scs_file_new( - file_path=pdf_path, - conversion_formats={'mmd': True}, - ) - assert file.wait_until_complete(timeout=120) - # Get mmd output and extract a cropped image URL - mmd_text = file.to_mmd_text() - # Pattern: https://cdn.mathpix.com/cropped/{file_id}-{page}.jpg?height=X&width=Y&top_left_y=Z&top_left_x=W - pattern = re.compile( - r'https://cdn\.mathpix\.com/cropped/[^?]+\?' - r'height=(\d+)&width=(\d+)&top_left_y=(\d+)&top_left_x=(\d+)' - ) - match = pattern.search(mmd_text) - assert match, "No cropped image URL found in mmd output" - height, width, top_left_y, top_left_x = map(int, match.groups()) - # Extract page number from the URL (format: {file_id}-{page}.jpg) - page_pattern = re.compile(r'/cropped/.+-(\d+)\.jpg') - page_match = page_pattern.search(match.group(0)) - assert page_match, f"Could not extract page number from cropped URL: {match.group(0)}" - page = int(page_match.group(1)) - # Call cropped_image endpoint - cropped_bytes = file.cropped_image( - page=page, - top_left_x=top_left_x, - top_left_y=top_left_y, - width=width, - height=height, - ) - assert cropped_bytes is not None - assert len(cropped_bytes) > 0 - # Verify JPEG magic bytes - assert cropped_bytes[:2] == b'\xff\xd8', "Response is not a valid JPEG"