Break equal-start uprating ties the way auto-carry-over does

changelog.d/fix-uprating-equal-start.fixed.md

@@ -0,0 +1 @@
+When two earlier inputs in a variable's own unit start on the same day (such as `2012` and `year:2012:2` for a yearly variable), uprating now starts from the one that ends last, the input auto-carry-over would carry, so the uprated value no longer depends on which input was stored first or on whether it is in memory or on disk.

policyengine_core/simulations/simulation.py

@@ -135,6 +135,28 @@ def _end_order(period: Period) -> float:
     return days_before_year + _DAYS_BEFORE_MONTH[month - 1] + (leap and month > 2) + day
 
 
+def _latest_input_key(period: Period, definition_period: str) -> tuple:
+    """Sort key for the stored input that auto-carry-over or uprating reads.
+
+    The input at the variable's own definition-period unit beats one in
+    another unit; then the one that starts last wins; on a tie, the one that
+    ends last (``_end_order``), then the larger unit, then the period's
+    string form. Distinct periods never tie, so the input chosen depends only
+    on which periods are stored, never on the order they were stored in (or
+    on whether a value is in memory or on disk). Compare ``period.start``
+    (temporal order): sorting Period tuples lexicographically puts "year"
+    before "month" alphabetically, so a known "2023" annual value would win
+    over a later "2024-06" monthly value (bug H1).
+    """
+    return (
+        period.unit == definition_period,
+        period.start,
+        _end_order(period),
+        periods.unit_weight(period.unit),
+        str(period),
+    )
+
+
 if TYPE_CHECKING:
     from policyengine_core.taxbenefitsystems import TaxBenefitSystem
 
@@ -957,9 +979,15 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike:
                     # Take the latest period from the filtered list itself.
                     # Indexing ``known_periods`` with a position in the
                     # filtered list picked the wrong period whenever a later
-                    # one was stored first.
+                    # one was stored first. Two inputs can start on the same
+                    # day: a yearly variable stores a ``year:2012:2`` input as
+                    # given, beside one for ``2012``. Break that tie as
+                    # auto-carry-over does (the one that ends last), so the
+                    # source never depends on which was stored first. The
+                    # factor below runs from the source's start either way.
                     latest_known_period = max(
-                        earlier_input_periods, key=lambda p: p.start
+                        earlier_input_periods,
+                        key=lambda p: _latest_input_key(p, variable.definition_period),
                     )
                     try:
                         uprating_parameter = get_parameter(
@@ -996,16 +1024,11 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike:
                 ):
                     # Carry over the latest input: of the stored periods that
                     # start no later than ``period``, the one that starts last
-                    # (on a tie, the one that ends last, then the larger unit,
-                    # so the choice never depends on the order inputs were
-                    # stored), preferring periods at
-                    # the variable's own definition-period unit and using
-                    # another unit only when there is none, as for an input to
-                    # a variable with no ``set_input`` helper. Compare
-                    # period.start (temporal order): sorting Period tuples
-                    # lexicographically puts "year" before "month"
-                    # alphabetically, so a known "2023" annual value would win
-                    # over a later "2024-06" monthly value (bug H1).
+                    # (on a tie, the one that ends last, then the larger unit;
+                    # see ``_latest_input_key``), preferring periods at the
+                    # variable's own definition-period unit and using another
+                    # unit only when there is none, as for an input to a
+                    # variable with no ``set_input`` helper.
                     #
                     # Only inputs carry. Every value this simulation
                     # calculated is marked derived when cached (formula
@@ -1025,13 +1048,7 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike:
                             )
                             if input_period.start <= period.start
                         ),
-                        key=lambda p: (
-                            p.unit == variable.definition_period,
-                            p.start,
-                            _end_order(p),
-                            periods.unit_weight(p.unit),
-                            str(p),
-                        ),
+                        key=lambda p: _latest_input_key(p, variable.definition_period),
                         default=None,
                     )
                     if last_known_period is not None:

pyproject.toml

@@ -43,7 +43,7 @@ dev = [
     "ruff>=0.9.0",
     "coverage",
     "furo",
-    "hypothesis>=6,<7",
+    "hypothesis>=6.62,<7",
     "jupyter-book<1",
     "markupsafe==2.0.1",
     "mypy==0.991",

tests/core/test_uprating_order.py

@@ -27,6 +27,12 @@
 Each regression here compares with a fresh simulation that calculates only
 the period in question, byte for byte.
 
+Two earlier inputs can also start on the same day: a yearly variable stores a
+``year:2012:2`` input as given, beside one for ``2012`` (a ``set_input``
+helper is only called for an input in another unit). The uprating source was
+then whichever was stored first, or the one in memory over the one on disk.
+It is now the one that ends last, the input auto-carry-over would carry.
+
 A yearly input for a monthly variable is stored month by month by its
 ``set_input`` helper. The helpers took a month the simulation had calculated
 for an input and kept it, so the uprating rule above skipped it. They now
@@ -50,7 +56,9 @@
     alone,
     assert_bitwise_equal,
     build_system,
+    can_store_on_disk,
     index,
+    reference,
     simulation,
 )
 
@@ -293,6 +301,214 @@ def test_uprated_value_is_cached_as_derived(system):
     assert not holder.is_derived(periods.period("2012"))
 
 
+# Inputs that start on the same day.
+
+TIED = {"year:2012:2": [1.0, 2.0], "2012": [5.0, 6.0]}
+
+
+def _in_order(values, longer_first):
+    order = list(values) if longer_first else list(values)[::-1]
+    return {key: values[key] for key in order}
+
+
+@pytest.mark.parametrize("variable", ["uprated_any_unit", "uprated"])
+@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"])
+def test_inputs_starting_together_uprate_the_one_ending_last(
+    system, variable, longer_first
+):
+    """``uprated`` has a ``set_input`` helper and ``uprated_any_unit`` none:
+    both store ``year:2012:2`` as given."""
+    inputs = {variable: _in_order(TIED, longer_first)}
+    result = alone(system, inputs, variable, "2015")
+    # Before the fix: [5, 6] uprated whenever 2012 was stored first.
+    np.testing.assert_allclose(
+        result, np.array([1, 2]) * index(2015) / index(2012), rtol=1e-6
+    )
+    assert_bitwise_equal(result, reference(system, inputs, variable, "2015"))
+
+
+@pytest.mark.parametrize("reverse", [False, True], ids=["forward", "reverse"])
+def test_inputs_starting_together_uprate_the_longest_not_the_first_by_name(
+    system, reverse
+):
+    """``year:2012:10`` ends last; as a string it sorts between ``2012`` and
+    ``year:2012:2``."""
+    values = {"2012": [5.0, 6.0], "year:2012:10": [1.0, 2.0], "year:2012:2": [3, 4]}
+    order = list(values)[::-1] if reverse else list(values)
+    inputs = {"uprated_any_unit": {key: values[key] for key in order}}
+    result = alone(system, inputs, "uprated_any_unit", "2015")
+    np.testing.assert_allclose(
+        result, np.array([1, 2]) * index(2015) / index(2012), rtol=1e-6
+    )
+    assert_bitwise_equal(result, reference(system, inputs, "uprated_any_unit", "2015"))
+
+
+@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"])
+def test_monthly_inputs_starting_together_uprate_the_one_ending_last(
+    system, longer_first
+):
+    values = {"month:2012-01:3": [1.0, 2.0], "2012-01": [5.0, 6.0]}
+    inputs = {"uprated_monthly": _in_order(values, longer_first)}
+    result = alone(system, inputs, "uprated_monthly", "2014-03")
+    np.testing.assert_allclose(
+        result, np.array([1, 2]) * index(2014) / index(2012), rtol=1e-6
+    )
+    assert_bitwise_equal(
+        result, reference(system, inputs, "uprated_monthly", "2014-03")
+    )
+
+
+@pytest.mark.filterwarnings(
+    "ignore::policyengine_core.warnings.memory_config_warning.MemoryConfigWarning"
+)
+@pytest.mark.parametrize(
+    "longer_on_disk",
+    [
+        pytest.param(
+            True,
+            id="longer",
+            marks=pytest.mark.skipif(
+                not can_store_on_disk("year:2012:2"),
+                reason="OnDiskStorage cannot store year:2012:2 here (Windows; #526)",
+            ),
+        ),
+        pytest.param(False, id="shorter"),
+    ],
+)
+def test_inputs_starting_together_uprate_the_same_one_from_memory_or_disk(
+    system, longer_on_disk
+):
+    """``get_known_periods`` lists the periods in memory before those on
+    disk, whatever order they were stored in."""
+    built = simulation(system, {})
+    built.memory_config = MemoryConfig(max_memory_occupation=0)
+    holder = built.get_holder("uprated_any_unit")
+    holder._disk_storage = holder.create_disk_storage()
+    on_disk, in_memory = ("year:2012:2", "2012")[:: 1 if longer_on_disk else -1]
+    holder._on_disk_storable = False
+    built.set_input("uprated_any_unit", in_memory, np.array(TIED[in_memory]))
+    holder._on_disk_storable = True
+    built.set_input("uprated_any_unit", on_disk, np.array(TIED[on_disk]))
+    assert holder._disk_storage.has(periods.period(on_disk))
+    assert holder._memory_storage.has(periods.period(in_memory))
+    result = built.calculate("uprated_any_unit", "2015")
+    expected = alone(system, {"uprated_any_unit": TIED}, "uprated_any_unit", "2015")
+    assert_bitwise_equal(result, expected)
+    np.testing.assert_allclose(
+        result, np.array([1, 2]) * index(2015) / index(2012), rtol=1e-6
+    )
+
+
+@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"])
+def test_branch_input_starting_with_a_parent_input_is_uprated_by_the_same_rule(
+    system, longer_first
+):
+    first, second = ("year:2012:2", "2012")[:: 1 if longer_first else -1]
+    built = simulation(system, {"uprated": {first: TIED[first]}})
+    branch = built.get_branch("reform")
+    branch.set_input("uprated", second, np.array(TIED[second]))
+    np.testing.assert_allclose(
+        branch.calculate("uprated", "2015"),
+        np.array([1, 2]) * index(2015) / index(2012),
+        rtol=1e-6,
+    )
+
+
+@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"])
+def test_flat_uprating_reads_the_input_auto_carry_over_reads(longer_first):
+    """Uprating by an index that never moves returns what auto-carry-over
+    returns for the same inputs, ties included, as long as no input in the
+    variable's own unit starts on the period's first day (see the next
+    test)."""
+    system = SYSTEMS[True]
+    values = {**TIED, "2012-06": [7.0, 8.0], "2014-02": [9.0, 10.0]}
+    ordered = _in_order(values, longer_first)
+    built = simulation(system, {"uprated_flat": ordered, "carried_any_unit": ordered})
+    for year in ("2013", "2014", "2015"):
+        assert_bitwise_equal(
+            built.calculate("uprated_flat", year),
+            built.calculate("carried_any_unit", year),
+            year,
+        )
+    np.testing.assert_array_equal(built.calculate("uprated_flat", "2015"), [1, 2])
+
+
+def test_longer_input_starting_on_the_period_is_not_an_uprating_source(system):
+    """The rule this change keeps: only inputs that start before the period
+    are uprated from, so ``year:2012:2`` is not a source for 2012 itself.
+    2012 gets 2011's input uprated; auto-carry-over would carry
+    ``year:2012:2``. Without an earlier input, 2012 gets what auto-carry-over
+    gives it. Whether that should change is PolicyEngine/policyengine-core#583."""
+    inputs = {"uprated_any_unit": {"2011": [1.0, 2.0], "year:2012:2": [5.0, 6.0]}}
+    np.testing.assert_allclose(
+        alone(system, inputs, "uprated_any_unit", "2012"),
+        np.array([1, 2]) * index(2012) / index(2011),
+        rtol=1e-6,
+    )
+    inputs = {"uprated_any_unit": {"year:2012:2": [5.0, 6.0]}}
+    expected = [5, 6] if system.auto_carry_over_input_variables else [0, 0]
+    np.testing.assert_array_equal(
+        alone(system, inputs, "uprated_any_unit", "2012"), expected
+    )
+
+
+@pytest.mark.parametrize(
+    "stored,asked",
+    [("day:9999-12-30:3", "9999-12-31"), ("day:2012-01-01:4000000", "2013-01-01")],
+)
+def test_an_input_ending_after_the_last_date_is_uprated(system, stored, asked):
+    """A period that ends after 9999-12-31 has no ``stop`` (it raises
+    ``OverflowError``); it is still an uprating source."""
+    inputs = {"uprated_daily_flat": {stored: [10.0, 20.0]}}
+    np.testing.assert_array_equal(
+        alone(system, inputs, "uprated_daily_flat", asked), [10, 20]
+    )
+    assert_bitwise_equal(
+        alone(system, inputs, "uprated_daily_flat", asked),
+        reference(system, inputs, "uprated_daily_flat", asked),
+    )
+
+
+@pytest.mark.parametrize(
+    "values,asked",
+    [
+        (
+            {"day:2012-01-01:4000000": [10.0, 20.0], "day:2012-01-01:2": [1.0, 2.0]},
+            "2013-01-01",
+        ),
+        # Both end after 9999-12-31. As strings, ":10" sorts before ":3".
+        (
+            {"day:9999-12-30:10": [10.0, 20.0], "day:9999-12-30:3": [1.0, 2.0]},
+            "9999-12-31",
+        ),
+    ],
+    ids=["one_after", "both_after"],
+)
+@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"])
+def test_an_input_ending_after_the_last_date_ends_last(
+    system, values, asked, longer_first
+):
+    inputs = {"uprated_daily_flat": _in_order(values, longer_first)}
+    result = alone(system, inputs, "uprated_daily_flat", asked)
+    np.testing.assert_array_equal(result, [10, 20])
+    assert_bitwise_equal(result, reference(system, inputs, "uprated_daily_flat", asked))
+
+
+@pytest.mark.parametrize("longer_first", [True, False], ids=["longer", "shorter"])
+def test_inputs_ending_after_the_last_date_carry_the_one_ending_last(
+    system, longer_first
+):
+    """``stop`` raises ``OverflowError`` for days that end after 9999-12-31,
+    but not for months: ``month:9999-01:13`` ends on 10000-01-31, after
+    ``day:9999-01-01:370`` (10000-01-05), so it is the one carried."""
+    values = {"month:9999-01:13": [1.0, 2.0], "day:9999-01-01:370": [3.0, 4.0]}
+    inputs = {"carried_any_unit": _in_order(values, longer_first)}
+    result = alone(system, inputs, "carried_any_unit", "9999")
+    expected = [1, 2] if system.auto_carry_over_input_variables else [0, 0]
+    np.testing.assert_array_equal(result, expected)
+    assert_bitwise_equal(result, reference(system, inputs, "carried_any_unit", "9999"))
+
+
 # A yearly input for a monthly variable goes through its ``set_input`` helper,
 # which stores it in each month of the year: copied to each month for a stock
 # (``set_input_dispatch_by_period``), divided between them for a flow