Skip to content

Migrate RareLabelEncoder to narwhals, add polars support #1030

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
solegalli wants to merge 2 commits into
base: narwhals-migration
Choose a base branch
from
Open

Migrate RareLabelEncoder to narwhals, add polars support #1030

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d506d3a
Migrate RareLabelEncoder to narwhals, add polars support
solegalli Aug 26, 2026
b6eaf5c
Adapt RareLabelEncoder to narwhals-returning check_X
solegalli Aug 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 36 additions & 2 deletions docs/user_guide/encoding/RareLabelEncoder.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,11 +179,12 @@ In the following output, we see the number of observations per category:

.. code:: python

var_A
A 10
B 10
C 2
D 1
Name: var_A, dtype: int64
Name: count, dtype: int64

Now, we group categories only for variables with more than 3 unique categories:

Expand Down Expand Up @@ -216,10 +217,43 @@ a new category called `Rare`:

.. code:: python

var_A
A 10
B 10
Rare 3
Name: var_A, dtype: int64
Name: count, dtype: int64

With polars
-----------

:class:`RareLabelEncoder()` works the same way with polars dataframes:

.. code:: python

import polars as pl
from feature_engine.encoding import RareLabelEncoder

data = {'var_A': ['A'] * 10 + ['B'] * 10 + ['C'] * 2 + ['D'] * 1}
data = pl.DataFrame(data)

rare_encoder = RareLabelEncoder(tol=0.05, n_categories=3, max_n_categories=2)
Xt = rare_encoder.fit_transform(data)
Xt['var_A'].value_counts().sort('var_A')

We see the same grouping as with the pandas dataframe:

.. code:: text

shape: (3, 2)
┌───────┬───────┐
│ var_A ┆ count │
│ --- ┆ --- │
│ str ┆ u32 │
╞═══════╪═══════╡
│ A ┆ 10 │
│ B ┆ 10 │
│ Rare ┆ 3 │
└───────┴───────┘

Considerations
--------------
Expand Down
118 changes: 86 additions & 32 deletions feature_engine/encoding/rare_label.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import warnings
from typing import List, Optional, Union

import numpy as np
import pandas as pd
import narwhals as nw
from narwhals.typing import IntoDataFrame, IntoSeries

from feature_engine._check_init_parameters.check_init_input_params import (
_check_return_empty_is_bool,
Expand Down Expand Up @@ -137,6 +137,28 @@ class RareLabelEncoder(CategoricalMethodsMixin, CategoricalInitMixinNA):
3 4 b
4 5 b
5 6 Rare

With polars

>>> import polars as pl
>>> from feature_engine.encoding import RareLabelEncoder
>>> X = pl.DataFrame(dict(x1 = [1,2,3,4,5,6], x2 = ["b", "b", "b", "b", "b", "a"]))
>>> rle = RareLabelEncoder(n_categories = 1, tol=0.2)
>>> rle.fit(X)
>>> rle.transform(X)
shape: (6, 2)
┌─────┬──────┐
│ x1 ┆ x2 │
│ --- ┆ --- │
│ i64 ┆ str │
╞═════╪══════╡
│ 1 ┆ b │
│ 2 ┆ b │
│ 3 ┆ b │
│ 4 ┆ b │
│ 5 ┆ b │
│ 6 ┆ Rare │
└─────┴──────┘
"""

def __init__(
Expand Down Expand Up @@ -186,40 +208,52 @@ def __init__(
self.replace_with = replace_with
self.return_empty = return_empty

def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None):
"""
Learn the frequent categories for each variable.

Parameters
----------
X: pandas dataframe of shape = [n_samples, n_features]
X: dataframe of shape = [n_samples, n_features]
The training input samples. Can be the entire dataframe, not just selected
variables

y: None
y is not required. You can pass y or None.
"""

X = check_X(X)
nw_X = check_X(X)
variables_ = self._check_or_select_variables(X)
self._check_na(X, variables_)

self.encoder_dict_ = {}

# n_unique() counts a null as its own category, matching pandas'
# plain unique() (used for the cardinality check below), unlike
# pandas' nunique() which drops nulls by default.
for var in variables_:
if len(X[var].unique()) > self.n_categories:
col = nw_X.get_column(var)

if col.n_unique() > self.n_categories:

# if the variable has more than the indicated number of categories
# the encoder will learn the most frequent categories
t = X[var].value_counts(normalize=True)
# the encoder will learn the most frequent categories.
# drop_nulls() mirrors pandas' value_counts(dropna=True)
# default, which narwhals' value_counts() doesn't apply on
# its own. sort=True matches pandas' own value_counts()
# default order (descending by count).
counts = col.drop_nulls().value_counts(sort=True, normalize=True)
cat_col, freq_col = counts.columns

# non-rare labels:
freq_idx = t[t >= self.tol].index
freq_idx = counts.filter(
counts.get_column(freq_col) >= self.tol
).get_column(cat_col).to_list()

if self.max_n_categories:
self.encoder_dict_[var] = list(freq_idx[: self.max_n_categories])
self.encoder_dict_[var] = freq_idx[: self.max_n_categories]
else:
self.encoder_dict_[var] = list(freq_idx)
self.encoder_dict_[var] = freq_idx

else:
# if the total number of categories is smaller than the indicated
Expand All @@ -229,56 +263,76 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
"indicated in n_categories. Thus, all categories will be "
"considered frequent".format(var)
)
self.encoder_dict_[var] = list(X[var].unique())
self.encoder_dict_[var] = col.unique(maintain_order=True).to_list()

self.variables_ = variables_
self._get_feature_names_in(X)
return self

def transform(self, X: pd.DataFrame) -> pd.DataFrame:
def transform(self, X: IntoDataFrame) -> IntoDataFrame:
"""
Group infrequent categories. Replace infrequent categories by the string 'Rare'
or any other name provided by the user.

Parameters
----------
X: pandas dataframe of shape = [n_samples, n_features]
X: dataframe of shape = [n_samples, n_features]
The input samples.

Returns
-------
X: pandas dataframe of shape = [n_samples, n_features]
X: dataframe of shape = [n_samples, n_features]
The dataframe where rare categories have been grouped.
"""

X = self._check_transform_input_and_state(X)
nw_X = self._check_transform_input_and_state(X)

# check if dataset contains na
if self.missing_values == "raise":
_check_contains_na(X, self.variables_, error_msg="optional")
with_nan = []
else:
with_nan = [np.nan]

# a pandas Categorical column rejects an unseen label on assignment
# until the label is added to its categories; narwhals has no
# cross-backend equivalent (polars has no comparable dtype
# restriction here), so this stays a native pandas step. Operate on a
# copy so the user's dataframe is not mutated.
if nw_X.implementation.is_pandas():
native_X = nw_X.to_native().copy()
for feature in self.variables_:
if native_X[feature].dtype == "category":
native_X[feature] = native_X[feature].cat.add_categories(
self.replace_with
)
nw_X = nw.from_native(native_X, eager_only=True)

# nw.when(<Series>).then(<Series>).otherwise(nw.lit(...)) lets each
# column keep its own dtype where frequent, and take the (possibly
# differently-typed) replace_with value elsewhere - narwhals
# resolves the common dtype per backend, e.g. object in pandas,
# cast-to-string in polars, so no manual dtype fixup is needed
# before it, unlike the old pandas-only .astype("O"). Passing
# Series (from get_column(), not nw.col()) into when/then/otherwise
# keeps this working for pandas integer column names, and nw.lit()
# broadcasts replace_with natively instead of materialising a
# same-length replacement array (benchmarked ~1.7x faster than the
# zip_with(col, new_series(...)) equivalent at 100k rows).
new_columns = []
for feature in self.variables_:
# Setting an item of incompatible dtype is deprecated
# and will raise an error in a future version of pandas
if self.ignore_format is True and isinstance(self.replace_with, str):
num_vars = list(
X[self.variables_].select_dtypes(include="number").columns
col = nw_X.get_column(feature)
keep = col.is_in(self.encoder_dict_[feature])
if self.missing_values == "ignore":
keep = keep | col.is_null()
new_columns.append(
nw.when(keep).then(col).otherwise(nw.lit(self.replace_with)).alias(
feature
)
X[num_vars] = X[num_vars].astype("O")

if X[feature].dtype == "category":
X[feature] = X[feature].cat.add_categories(self.replace_with)

X.loc[~X[feature].isin(self.encoder_dict_[feature] + with_nan), feature] = (
self.replace_with
)

X = nw_X.with_columns(*new_columns).to_native()

return X

def inverse_transform(self, X: pd.DataFrame):
def inverse_transform(self, X: IntoDataFrame):
"""inverse_transform is not implemented for this transformer."""
raise NotImplementedError(
"inverse_transform is not implemented for this transformer."
Expand Down
Loading