TableTensor#

Bases: Tensor

A torch.Tensor for tensorized, lossless table data.

A TableTensor stores column blocks separately per semantic type, while exposing a single tensor-shaped table interface. The last dimension represents named columns.

import torch
from sdm import CategoricalTensor, StringTensor, Stype, TableTensor

table = TableTensor(
    columns={
        "numerical": ["age", "income"],
        "categorical": ["country", "segment"],
    },
    numerical=torch.randn(10, 2),
    categorical=CategoricalTensor(
        code=torch.randint(0, 2, size=(10, 2)),
        categories=(
            StringTensor.from_list(["USA", "Germany"]),
            StringTensor.from_list(["enterprise", "startup"]),
        ),
    ),
)

print(table)
# TableTensor(
#   size=(10, 4),
#   blocks={
#     numerical (2): ['age', 'income'],
#     categorical (2): ['country', 'segment'],
#   },
# )

# DataFrame-like column selection, but still tensor-native:
features = table[["age", "country"]]
assert features.size() == (10, 2)

# Normal PyTorch indexing still works on row/batch dimensions:
batch = table[[1, 0, 2], ["income", "segment"]]
assert batch.size() == (3, 2)

# Semantic blocks stay separate for model input:
x_num = table.numerical
x_cat = table.categorical

# Tensor ops preserve the table container:
stacked = torch.stack([table, table], dim=0)
assert stacked.size() == (2, 10, 4)

# Column-wise cat extends the schema:
wide = torch.cat([table[["age"]], table[["country"]]], dim=-1)

Parameters:

size (Sequence[int] | None) – The shape of the tensor [..., C].
columns (Mapping[StypeLike, Sequence[str]] | None) – Column names grouped by semantic type.
numerical (Tensor | None) – The numerical column block of shape [..., C_num].
categorical (CategoricalTensor | None) – The categorical column block of shape [..., C_cat].
datetime (Tensor | None) – The datetime64[us] column block of shape [..., C_dt].
text (StringTensor | None) – The text column block of shape [..., C_text].
id (ColumnarTensor | None) – The identifier column block of shape [..., C_id].
device (torch.device | str | None) – The device.

Return type:

Self

classmethod from_arrow(table: Table, stypes: Mapping[str, Stype | str], *, device: device | str | None = None) → Self#

Create a tensor from a pyarrow.Table.

import pyarrow as pa
from sdm import TableTensor

arrow_table = pa.table({
    "age": pa.array([25, 31, 42], type=pa.int64()),
    "city": pa.array(["SF", "NYC", "SF"], type=pa.string()),
})
tensor = TableTensor.from_arrow(
    table=arrow_table,
    stypes={"age": "numerical", "city": "categorical"},
)

Parameters:

table (Table) – The table.
stypes (Mapping[str, Stype | str]) – The semantic type for each column. Columns that are present in table but not included in stypes will be ignored.
device (device | str | None) – The device.

Return type:

Self

to_arrow() → Table#

Convert this tensor to a pyarrow.Table.

Return type:: Table

classmethod from_pandas(df: pd.DataFrame, stypes: Mapping[str, StypeLike], *, device: torch.device | str | None = None) → Self#

Create a tensor from a pandas.DataFrame.

Parameters:

df (pd.DataFrame) – The dataframe.
stypes (Mapping[str, StypeLike]) – The semantic type for each column. Columns that are present in df but not included in stypes will be ignored.
device (torch.device | str | None) – The device.

Return type:

Self

classmethod from_columns(data: Mapping[str, Sequence[Any]], stypes: Mapping[str, Stype | str], *, device: device | str | None = None) → Self#

Create a tensor from column data.

Parameters:

data (Mapping[str, Sequence[Any]]) – Column data keyed by column name.
stypes (Mapping[str, Stype | str]) – The semantic type for each column. Columns that are present in data but not included in stypes will be ignored.
device (device | str | None) – The device.

Return type:

Self

to_pandas() → pd.DataFrame#

Convert this tensor to a pandas.DataFrame.

Return type:: pd.DataFrame

classmethod from_tensor(tensor: Tensor, columns: Sequence[str] | None = None) → Self#

Create a table from a torch.Tensor.

Parameters:

tensor (Tensor) –
The input tensor with shape [..., C], interpreted as:
- A floating-point torch.Tensor becomes numerical columns.
- An integer torch.Tensor becomes categorical columns.
- A StringTensor becomes text columns.
columns (Sequence[str] | None) – The C column names.

Return type:

Self

classmethod from_cudf(df: cudf.DataFrame, stypes: Mapping[str, StypeLike], *, device: torch.device | str | None = None) → Self#

Create a tensor from a cudf.DataFrame.

Parameters:

df (cudf.DataFrame) – The dataframe.
stypes (Mapping[str, StypeLike]) – The semantic type for each column. Columns that are present in df but not included in stypes will be ignored.
device (torch.device | str | None) – The device.

Return type:

Self

to_cudf() → cudf.DataFrame#

Convert this tensor to a cudf.DataFrame.

Return type:: cudf.DataFrame

property columns: Mapping[Stype, tuple[str, ...]]#: Return column names grouped by semantic type.

property column_names: frozenset[str]#: The column names of this tensor.

property stypes: Mapping[str, Stype]#: Return the semantic type for each column.

stype(column: str) → Stype#

Return the semantic type for a column.

Parameters:: column (str) – The column name.
Return type:: Stype

property active_stypes: frozenset[Stype]#: Semantic types with at least one column.

property numerical: Tensor#: Return the numerical column block.

property categorical: CategoricalTensor#: Return the categorical column block.

property datetime: Tensor#: Return the datetime column block.

property text: StringTensor#: Return the text column block.

property id: ColumnarTensor#: Return the identifier column block.

items() → Iterator[tuple[Stype, Tensor]]#

Yield (stype, block) pairs for typed column blocks.

Return type:: Iterator[tuple[Stype, Tensor]]

property blocks: Mapping[Stype, Tensor]#: Return typed column blocks per semantic type.

property schema: TableSchema#: The schema of this table.

is_same_schema(other: TableTensor) → bool#

Whether other has the same schema layout.

Parameters:: other (TableTensor) – The object to compare against.
Return type:: bool

Return a table with one or more semantic blocks replaced.

Provided blocks replace the corresponding semantic type while omitted blocks are reused from this table. The returned table preserves the current column schema and is validated by the TableTensor constructor.

Parameters:

numerical (Tensor | None) – Replacement numerical block with shape [..., C_num].
categorical (CategoricalTensor | None) – Replacement categorical block with shape [..., C_cat].
datetime (Tensor | None) – Replacement datetime block with shape [..., C_dt].
text (StringTensor | None) – Replacement text block with shape [..., C_text].
id (ColumnarTensor | None) – Replacement identifier block with shape [..., C_id].

Return type:

Self

select_stypes(stypes: Stype | str | Iterable[Stype | str]) → Self#

Return a table containing only stypes columns.

Parameters:: stypes (Stype | str | Iterable[Stype | str]) – The semantic type or semantic types to select.
Return type:: Self

drop_stypes(stypes: Stype | str | Iterable[Stype | str]) → Self#

Return a table with stypes columns removed.

assert table.columns[Stype.categorical] == ("country", "segment")
table = table.drop_stypes("categorical")
assert table.columns[Stype.categorical] == ()
assert table.columns[Stype.numerical] == ("age", "income")

Parameters:: stypes (Stype | str | Iterable[Stype | str]) – The semantic type or semantic types to drop.
Return type:: Self

select_columns(columns: str | Iterable[str]) → Self#

Return a table containing only columns.

assert table.size() == (10, 4)
table = table.select_columns(["age", "country"])
assert table.size() == (10, 2)

Parameters:: columns (str | Iterable[str]) – The columns to select.
Return type:: Self

drop_columns(columns: str | Iterable[str]) → Self#

Return a table with columns removed.

assert table.size() == (10, 4)
table = table.drop_columns(["age", "country"])
assert table.size() == (10, 2)

Parameters:: columns (str | Iterable[str]) – The columns to drop.
Return type:: Self