Pass **kwargs through Form functions so rendering has access to a CharacterEncoding option set, e.g. in ToString

mathics/builtin/atomic/strings.py

@@ -866,17 +866,32 @@ class ToString(Builtin):
 
     >> ToString[2]
      = 2
+
+    Notice how the output changes when we switch formatting from 'StandardForm', the default, to \
+    'InputForm':
+
     >> ToString[2] // InputForm
      = "2"
+
+    'ToString' can act a translator of expressions to one expression format to another:
+    >> ToString[Integrate[f[x],x], TeXForm]
+     = \\int f(x) \\, dx
+
+    'ToString' also handles character encoding changes, when passed the 'CharacterEncoding' option:
+    >> ToString[a >= b, CharacterEncoding-> "UTF-8"]
+     = a ≥ b
+
+    ## Reinstate when we've worked out how to handle MS-Windows
+    ## >> ToString[a ≥ b, CharacterEncoding-> "ASCII"]
+    ## = a >= b
+
     >> ToString[a+b]
      = a + b
     >> "U" <> 2
      : String expected.
      = U <> 2
     >> "U" <> ToString[2]
      = U2
-    >> ToString[Integrate[f[x],x], TeXForm]
-     = \\int f(x) \\, dx
 
     """
 

mathics/format/box/formatvalues.py

@@ -5,7 +5,6 @@
 formatting rules.
 """
 
-
 from typing import Any, Callable, Dict, List, Optional, Type
 
 from mathics.core.atoms import Complex, Integer, Rational, String, SymbolI
@@ -43,22 +42,28 @@
 
 _element_formatters: Dict[
     Type[BaseElement],
-    Callable[[BaseElement, Evaluation, Symbol], Optional[BaseElement]],
+    Callable[[BaseElement, Evaluation, Symbol, Optional[str]], Optional[BaseElement]],
 ] = {}
 
 
 def do_format(
-    element: BaseElement, evaluation: Evaluation, form: Symbol
+    element: BaseElement,
+    evaluation: Evaluation,
+    form: Symbol,
+    encoding: Optional[str] = None,
 ) -> BaseElement:
     do_format_method = _element_formatters.get(type(element), do_format_element)
-    result = do_format_method(element, evaluation, form)
+    result = do_format_method(element, evaluation, form, encoding)
     if result is None:
         return element
     return result
 
 
 def do_format_element(
-    element: BaseElement, evaluation: Evaluation, form: Symbol
+    element: BaseElement,
+    evaluation: Evaluation,
+    form: Symbol,
+    encoding: Optional[str] = None,
 ) -> Optional[BaseElement]:
     """
     Applies formats associated to the expression and removes
@@ -135,7 +140,7 @@ def format_expr(expr):
         formatted = format_expr(expr) if isinstance(expr, EvalMixin) else None
         if formatted is not None:
             do_format_fn = _element_formatters.get(type(formatted), do_format_element)
-            result = do_format_fn(formatted, evaluation, form)
+            result = do_format_fn(formatted, evaluation, form, encoding)
             if include_form and result is not None:
                 result = Expression(form, result)
             return result
@@ -153,7 +158,7 @@ def format_expr(expr):
             if len(expr.get_elements()) != 1:
                 return expr
             do_format_fn = _element_formatters.get(type(element), do_format_element)
-            result = do_format_fn(expr, evaluation, form)
+            result = do_format_fn(expr, evaluation, form, encoding)
             if isinstance(result, Expression):
                 expr = result
 
@@ -165,14 +170,14 @@ def format_expr(expr):
             new_elements = tuple(
                 (
                     _element_formatters.get(type(element), do_format_element)(
-                        element, evaluation, form
+                        element, evaluation, form, encoding
                     )
                     for element in expr.elements
                 )
             )
             expr_head = expr.head
             do_format = _element_formatters.get(type(expr_head), do_format_element)
-            head = do_format(expr_head, evaluation, form) or expr_head
+            head = do_format(expr_head, evaluation, form, encoding) or expr_head
             expr = to_expression_with_specialization(head, *new_elements)
 
         if include_form:
@@ -183,7 +188,10 @@ def format_expr(expr):
 
 
 def do_format_rational(
-    element: BaseElement, evaluation: Evaluation, form: Symbol
+    element: BaseElement,
+    evaluation: Evaluation,
+    form: Symbol,
+    encoding: Optional[str] = None,
 ) -> Optional[BaseElement]:
     if not isinstance(element, Rational):
         return None
@@ -197,12 +205,15 @@ def do_format_rational(
     if minus:
         result = Expression(SymbolMinus, result)
     result = Expression(SymbolHoldForm, result)
-    result = do_format_expression(result, evaluation, form) or result
+    result = do_format_expression(result, evaluation, form, encoding) or result
     return result
 
 
 def do_format_complex(
-    element: BaseElement, evaluation: Evaluation, form: Symbol
+    element: BaseElement,
+    evaluation: Evaluation,
+    form: Symbol,
+    encoding: Optional[str] = None,
 ) -> Optional[BaseElement]:
     if not isinstance(element, Complex):
         return None
@@ -220,27 +231,18 @@ def do_format_complex(
     else:
         result = Expression(SymbolPlus, *parts)
 
-    return do_format_expression(Expression(SymbolHoldForm, result), evaluation, form)
+    return do_format_expression(
+        Expression(SymbolHoldForm, result), evaluation, form, encoding
+    )
 
 
 def do_format_expression(
-    element: BaseElement, evaluation: Evaluation, form: Symbol
+    element: BaseElement,
+    evaluation: Evaluation,
+    form: Symbol,
+    encoding: Optional[str] = None,
 ) -> BaseElement:
-    # # not sure how much useful is this format_cache
-    # if element._format_cache is None:
-    #    element._format_cache = {}
-
-    # last_evaluated_time, expr = element._format_cache.get(form, (None, None))
-    # if last_evaluated_time is not None and expr is not None:
-    # if True
-    #    symbolname = expr.get_name()
-    #    if symbolname != "":
-    #        if not evaluation.definitions.is_uncertain_final_value(
-    #            last_evaluated_time, set((symbolname,))
-    #        ):
-    #            return expr
-    expr = do_format_element(element, evaluation, form) or element
-    # element._format_cache[form] = (evaluation.definitions.now, expr)
+    expr = do_format_element(element, evaluation, form, encoding) or element
     return expr
 
 

mathics/format/box/makeboxes.py

@@ -76,7 +76,10 @@ def eval_makeboxes_traditional_form(expr, evaluation):
 
 
 def apply_makeboxes_rules(
-    expr: BaseElement, evaluation: Evaluation, form: Symbol = SymbolStandardForm
+    expr: BaseElement,
+    evaluation: Evaluation,
+    form: Symbol = SymbolStandardForm,
+    **kwargs,
 ) -> BoxElementMixin:
     """
     This function takes the definitions provided by the evaluation
@@ -124,7 +127,7 @@ def yield_rules():
     if parent_form is not form:
         return apply_makeboxes_rules(expr, evaluation, parent_form)
 
-    return eval_generic_makeboxes(expr, form, evaluation)
+    return eval_generic_makeboxes(expr, form, evaluation, **kwargs)
 
 
 # TODO: evaluation is needed because `atom_to_boxes` uses it. Can we remove this
@@ -205,7 +208,7 @@ def eval_makeboxes_fullform_recursive(
     return RowBox(*result_elements)
 
 
-def eval_generic_makeboxes(expr, f, evaluation):
+def eval_generic_makeboxes(expr, f, evaluation, **kwargs):
     """MakeBoxes[expr_,
     f:TraditionalForm|StandardForm]"""
     from mathics.builtin.box.layout import RowBox
@@ -224,7 +227,7 @@ def eval_generic_makeboxes(expr, f, evaluation):
 
         printform_callback = PRINT_FORMS_CALLBACK.get(head.get_name(), None)
         if printform_callback is not None:
-            return printform_callback(elements[0], evaluation)
+            return printform_callback(elements[0], evaluation, **kwargs)
 
         f_name = f.get_name()
         if f_name == "System`TraditionalForm":
@@ -292,11 +295,11 @@ def format_element(
     Applies formats associated to the expression, and then calls Makeboxes
     """
     evaluation.is_boxing = True
-    formatted_expr = do_format(element, evaluation, form)
+    formatted_expr = do_format(element, evaluation, form, **kwargs)
     if form not in evaluation.definitions.boxforms:
         formatted_expr = Expression(form, formatted_expr)
         form = SymbolStandardForm
-    result_box = apply_makeboxes_rules(formatted_expr, evaluation, form)
+    result_box = apply_makeboxes_rules(formatted_expr, evaluation, form, **kwargs)
     if isinstance(result_box, BoxElementMixin):
         return result_box
     return eval_makeboxes_fullform_recursive(element, evaluation)

mathics/format/box/outputforms.py

@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 
 from mathics.core.atoms import Integer, String
 from mathics.core.element import BaseElement, BoxElementMixin
@@ -24,7 +25,9 @@
 
 
 @is_print_form_callback("System`MathMLForm")
-def eval_mathmlform(expr: BaseElement, evaluation: Evaluation) -> BoxElementMixin:
+def eval_mathmlform(
+    expr: BaseElement, evaluation: Evaluation, encoding: Optional[str] = None
+) -> BoxElementMixin:
     "MakeBoxes[MathMLForm[expr_], form_]"
     from mathics.builtin.box.layout import InterpretationBox
 
@@ -63,7 +66,12 @@ def eval_mathmlform(expr: BaseElement, evaluation: Evaluation) -> BoxElementMixi
 
 
 def eval_tableform(
-    self, table: BaseElement, f: Symbol, evaluation: Evaluation, options
+    self,
+    table: BaseElement,
+    f: Symbol,
+    evaluation: Evaluation,
+    options,
+    encoding: Optional[str] = None,
 ):
     """MakeBoxes[TableForm[table_], f_]"""
     from mathics.builtin.box.layout import GridBox
@@ -124,7 +132,9 @@ def transform_item(item):
 
 
 @is_print_form_callback("System`TeXForm")
-def eval_texform(expr: BaseElement, evaluation: Evaluation) -> BoxElementMixin:
+def eval_texform(
+    expr: BaseElement, evaluation: Evaluation, encoding: Optional[str] = None
+) -> BoxElementMixin:
     from mathics.builtin.box.layout import InterpretationBox
 
     boxes = format_element(expr, evaluation, SymbolTraditionalForm)

test/builtin/test_comparison.py

@@ -113,7 +113,7 @@ def test_compare_many_members(
 ):
     #    if str_expr is None:
     #        reset_session()
-    result = session.evaluate(f"ToString[{str_expr}]").value
+    result = session.evaluate(f'ToString[{str_expr}, CharacterEncoding->"ASCII"]').value
     print("result:", result)
     if assert_fail_message:
         assert result == str_expected, assert_fail_message

test/helper.py

@@ -110,7 +110,7 @@ def check_evaluation(
         str_expected = "Null"
 
     if to_string_expr:
-        str_expr = f"ToString[{str_expr}]"
+        str_expr = f'ToString[{str_expr}, CharacterEncoding->"ASCII"]'
         result = evaluate_value(str_expr)
     elif to_string_expr is None:
         result = str_expr
@@ -123,7 +123,7 @@ def check_evaluation(
         if hold_expected:
             expected = str_expected
         else:
-            str_expected = f"ToString[{str_expected}]"
+            str_expected = f'ToString[{str_expected}, CharacterEncoding->"ASCII"]'
             expected = evaluate_value(str_expected)
     elif to_string_expected is None:
         expected = str_expected