[LAML-4] Add SAINT

lightautoml/automl/presets/tabular_presets.py

@@ -609,6 +609,7 @@ def create_automl(self, **fit_args):
                 "autoint",
                 "tabnet",
                 "fttransformer",
+                "saint",
             ]
             available_nn_models = available_nn_models + [x + "_tuned" for x in available_nn_models]
             nn_models = [

lightautoml/dataset/base.py

@@ -365,6 +365,19 @@ def shape(self) -> Tuple[Optional[int], Optional[int]]:
         return rows, cols
 
     # static methods - how to make 1d slice, 2s slice, concat of feature matrix etc ...
+    @staticmethod
+    def _vstack(datasets: Sequence[Any]) -> Any:
+        """Abstract method - define horizontal stack of feature arrays.
+
+        Args:
+            datasets: Sequence of feature arrays.
+
+        Returns:  # noqa DAR202
+            Single feature array.
+
+        """
+        raise NotImplementedError("Horizontal Stack not implemented.")
+
     @staticmethod
     def _hstack(datasets: Sequence[Any]) -> Any:
         """Abstract method - define horizontal stack of feature arrays.
@@ -472,7 +485,42 @@ def concat(cls, datasets: Sequence["LAMLDataset"]) -> "LAMLDataset":
         dataset.set_data(data, features, roles)
 
         return dataset
+    @classmethod
+    def vconcat(cls, datasets: Sequence["LAMLDataset"]) -> "LAMLDataset":
+        """Concat multiple dataset.
+
+        Default behavior - takes empty dataset from datasets[0]
+        and concat all features from others.
+
+        Args:
+            datasets: Sequence of datasets.
+
+        Returns:
+            Concated dataset.
+
+        """
+        for check in cls._concat_checks:
+            check(datasets)
+
+        dataset = datasets[0].empty()
+        data = []
+        features = [*datasets[0].features]
+        roles = {**datasets[0].roles}
+        atrs = set(dataset._array_like_attrs)
 
+        for ds in datasets:
+            data.append(ds.data)
+            for atr in ds._array_like_attrs:
+                if atr not in atrs:
+                    dataset._array_like_attrs.append(atr)
+                    dataset.__dict__[atr] = ds.__dict__[atr]
+                    atrs.update({atr})
+
+        data = cls._vstack(data)
+        dataset.set_data(data, features, roles)
+
+        return dataset
+
     def drop_features(self, droplist: Sequence[str]):
         """Inplace drop columns from dataset.
 

lightautoml/dataset/np_pd_dataset.py

@@ -212,6 +212,18 @@ def _hstack(datasets: Sequence[np.ndarray]) -> np.ndarray:
 
         """
         return np.hstack(datasets)
+    @staticmethod
+    def _vstack(datasets: Sequence[np.ndarray]) -> np.ndarray:
+        """Concatenate function for numpy arrays.
+
+        Args:
+            datasets: Sequence of np.ndarray.
+
+        Returns:
+            Stacked features array.
+
+        """
+        return np.vstack(datasets)
 
     @staticmethod
     def _get_rows(data: np.ndarray, k: IntIdx) -> np.ndarray:
@@ -400,6 +412,17 @@ def _hstack(datasets: Sequence[Union[sparse.csr_matrix, np.ndarray]]) -> sparse.
 
         """
         return sparse.hstack(datasets, format="csr")
+    def _vstack(datasets: Sequence[Union[sparse.csr_matrix, np.ndarray]]) -> sparse.csr_matrix:
+        """Concatenate function for sparse and numpy arrays.
+
+        Args:
+            datasets: Sequence of csr_matrix or np.ndarray.
+
+        Returns:
+            Sparse matrix.
+
+        """
+        return sparse.vstack(datasets, format="csr")
 
     def __init__(
         self,
@@ -609,6 +632,19 @@ def _hstack(datasets: Sequence[DataFrame]) -> DataFrame:
 
         """
         return pd.concat(datasets, axis=1)
+
+    @staticmethod
+    def _vstack(datasets: Sequence[DataFrame]) -> DataFrame:
+        """Define how to concat features arrays.
+
+        Args:
+            datasets: Sequence of tables.
+
+        Returns:
+            concatenated table.
+
+        """
+        return pd.concat(datasets, axis=0)
 
     @staticmethod
     def _get_rows(data: DataFrame, k: IntIdx) -> FrameOrSeries:

lightautoml/dataset/utils.py

@@ -158,3 +158,118 @@ def concatenate(datasets: Sequence[LAMLDataset]) -> LAMLDataset:
         datasets = [datasets[n]] + [x for (y, x) in enumerate(datasets) if n != y]
 
     return conc(datasets)
+
+
+
+def get_common_vconcat(
+    datasets: Sequence[LAMLDataset],
+) -> Tuple[Callable, Optional[type]]:
+    """Get concatenation function for datasets of different types.
+
+    Takes multiple datasets as input and check,
+    if is's ok to concatenate it and return function.
+
+    Args:
+        datasets: Sequence of datasets.
+
+    Returns:
+        Function, that is able to concatenate datasets.
+
+    """
+    # TODO: Add pandas + numpy via transforming to numpy?
+    dataset_types = set([type(x) for x in datasets])
+
+    # general - if single type, concatenation for that type
+    if len(dataset_types) == 1:
+        klass = list(dataset_types)[0]
+        return klass.vconcat, None
+
+    # np and sparse goes to sparse
+    elif dataset_types == {NumpyDataset, CSRSparseDataset}:
+        return CSRSparseDataset.vconcat, CSRSparseDataset
+
+    elif dataset_types == {NumpyDataset, PandasDataset}:
+        return numpy_and_pandas_vconcat, None
+
+    elif (dataset_types == {NumpyDataset, SeqNumpyPandasDataset}) or (
+        dataset_types == {PandasDataset, SeqNumpyPandasDataset}
+    ):
+        return numpy_or_pandas_and_seq_vconcat, None
+
+    raise TypeError("Unable to concatenate dataset types {0}".format(list(dataset_types)))
+
+
+def numpy_and_pandas_vconcat(datasets: Sequence[Union[NumpyDataset, PandasDataset]]) -> PandasDataset:
+    """Concat of numpy and pandas dataset.
+
+    Args:
+        datasets: Sequence of datasets to concatenate.
+
+    Returns:
+        Concatenated dataset.
+
+    """
+    datasets = [x.to_pandas() for x in datasets]
+
+    return PandasDataset.vconcat(datasets)
+
+
+def numpy_or_pandas_and_seq_vconcat(
+    datasets: Sequence[Union[NumpyDataset, PandasDataset, SeqNumpyPandasDataset]]
+) -> Union[NumpyDataset, PandasDataset]:
+    """Concat plain and sequential dataset.
+
+    If both datasets have same size then concat them as plain, otherwise include seq dataset inside plain one.
+
+    Args:
+        datasets: one plain and one seq dataset.
+
+    Returns:
+        Concatenated dataset.
+
+    """
+    assert len(datasets) == 2, "should be 1 sequential and 1 plain dataset"
+    # get 1 numpy / pandas dataset
+    for n, dataset in enumerate(datasets):
+        if type(dataset) == SeqNumpyPandasDataset:
+            seq_dataset = dataset
+        else:
+            plain_dataset = dataset
+
+    if len(seq_dataset.data) == len(plain_dataset):
+        return SeqNumpyPandasDataset.vconcat([seq_dataset, plain_dataset.to_pandas()])
+    else:
+        if hasattr(plain_dataset, "seq_data"):
+            plain_dataset.seq_data[seq_dataset.name] = seq_dataset
+        else:
+            plain_dataset.seq_data = {seq_dataset.name: seq_dataset}
+
+        return plain_dataset
+
+
+def vconcatenate(datasets: Sequence[LAMLDataset]) -> LAMLDataset:
+    """Dataset concatenation function.
+
+    Check if datasets have common concat function and then apply.
+    Assume to take target/folds/weights etc from first one.
+
+    Args:
+        datasets: Sequence of datasets.
+
+    Returns:
+        Dataset with concatenated features.
+
+    """
+    conc, klass = get_common_vconcat([ds for ds in datasets if ds is not None])
+
+    # this part is made to avoid setting first dataset of required type
+    if klass is not None:
+
+        n = 0
+        for n, ds in enumerate(datasets):
+            if type(ds) is klass:
+                break
+
+        datasets = [datasets[n]] + [x for (y, x) in enumerate(datasets) if n != y]
+
+    return conc(datasets)

lightautoml/ml_algo/base.py

@@ -16,7 +16,7 @@
 
 import numpy as np
 
-from lightautoml.validation.base import TrainValidIterator
+from lightautoml.validation.base import HoldoutIterator, TrainValidIterator
 
 from ..dataset.base import LAMLDataset
 from ..dataset.np_pd_dataset import CSRSparseDataset
@@ -271,8 +271,8 @@ def fit_predict(self, train_valid_iterator: TrainValidIterator) -> NumpyDataset:
                     "===== Start working with \x1b[1mfold {}\x1b[0m for \x1b[1m{}\x1b[0m =====".format(n, self._name)
                 )
             self.timer.set_control_point()
-
-            model, pred = self.fit_predict_single_fold(train, valid)
+            self.params['is_holdout'] = isinstance(train_valid_iterator,HoldoutIterator)
+            model, pred = self.fit_predict_single_fold(train, valid, 0)
             self.models.append(model)
             preds_arr[idx] += pred.reshape((pred.shape[0], -1))
             counter_arr[idx] += 1