This notebook shows that, when using a clustered splitter, if the clusters explain a part of the variance, using a non-clustered analysis will lead to higher false positive rate than expected.
+In particular, we use a clustered splitter and:
+from datetime import date
+
+import numpy as np
+from cluster_experiments import PowerAnalysis, ConstantPerturbator, BalancedClusteredSplitter, ExperimentAnalysis, ClusteredOLSAnalysis
+import pandas as pd
+import statsmodels.api as sm
+
+
+
+# Create fake data
+N = 10_000
+clusters = [f"Cluster {i}" for i in range(10)]
+dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(1, 15)]
+df = pd.DataFrame(
+ {
+ "cluster": np.random.choice(clusters, size=N),
+ "date": np.random.choice(dates, size=N),
+ }
+).assign(
+ # Target is a linear combination of cluster and day of week, plus some noise
+ cluster_id=lambda df: df["cluster"].astype("category").cat.codes,
+ day_of_week=lambda df: pd.to_datetime(df["date"]).dt.dayofweek,
+ target=lambda df: df["cluster_id"] + df["day_of_week"] + np.random.normal(size=N),
+)
+df.head()
+| + | cluster | +date | +cluster_id | +day_of_week | +target | +
|---|---|---|---|---|---|
| 0 | +Cluster 3 | +2022-01-08 | +3 | +5 | +7.534487 | +
| 1 | +Cluster 2 | +2022-01-06 | +2 | +3 | +5.039041 | +
| 2 | +Cluster 1 | +2022-01-14 | +1 | +4 | +5.341845 | +
| 3 | +Cluster 7 | +2022-01-12 | +7 | +2 | +9.468617 | +
| 4 | +Cluster 0 | +2022-01-10 | +0 | +0 | +-0.644678 | +
Some clusters have a higher average outcome than others
+ +df.groupby("cluster").agg({"target": ["mean", "std"]})
+| + | target | +|
|---|---|---|
| + | mean | +std | +
| cluster | ++ | + |
| Cluster 0 | +3.027335 | +2.223308 | +
| Cluster 1 | +3.907833 | +2.211297 | +
| Cluster 2 | +4.895215 | +2.270596 | +
| Cluster 3 | +6.045043 | +2.269786 | +
| Cluster 4 | +6.902209 | +2.224554 | +
| Cluster 5 | +8.028794 | +2.313159 | +
| Cluster 6 | +9.046213 | +2.253462 | +
| Cluster 7 | +10.055748 | +2.226720 | +
| Cluster 8 | +11.048716 | +2.273583 | +
| Cluster 9 | +11.939075 | +2.216478 | +
# Simple ols to run the analysis
+class NonClusteredOLS(ExperimentAnalysis):
+ def analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = sm.OLS.from_formula("target ~ treatment", data=df).fit()
+ return results_ols.pvalues[self.treatment_col]
+cluster_cols = ["cluster", "date"]
+
+splitter = BalancedClusteredSplitter(
+ cluster_cols=cluster_cols,
+)
+
+perturbator = ConstantPerturbator()
+
+alpha = 0.05
+n_simulations = 100
+
+# Right power analysis, we use clustered splitter and ols clustered analysis
+pw_right = PowerAnalysis(
+ splitter=splitter,
+ perturbator=perturbator,
+ alpha=alpha,
+ n_simulations=n_simulations,
+ analysis=ClusteredOLSAnalysis(
+ cluster_cols=cluster_cols,
+ ),
+)
+
+# Wrong power analysis, we use clustered splitter and regular ols
+pw_wrong = PowerAnalysis(
+ splitter=splitter,
+ perturbator=perturbator,
+ alpha=alpha,
+ n_simulations=n_simulations,
+ analysis=NonClusteredOLS(
+ # We pass cluster_cols here, but we don't use them!!!
+ cluster_cols=cluster_cols,
+ ),
+)
+Right way of doing it: in the AA test we get a power similar to the type I error of the test
+ +pw_right.power_analysis(df, average_effect=0.0)
+0.06+
Wrong way of doing it: the AA test fails, we have too much power
+ +pw_wrong.power_analysis(df, average_effect=0.0)
+0.79+
The goal of this notebook is to understand how different hypotheses change the power of an experiment. We start from some theory, moving to a pratical perspective using simulations and the implementations from this package.
+In hypothesis testing, various hypotheses can be examined, but the most common are:
+In most cases the one-sided (less or greater) p-value is half the two-sided p-value. So, if the two-sided p-value is 5%, the one-sided p-value is 2.5%. However, if the actual difference (effect) went opposite to the predicted direction, in this case the one-sided p-value equals one minus half the two-sided value. So if the two-sided p-value is 2.5%, the one-tailed p-value is 97.5%.
+ +
from datetime import date
+import numpy as np
+import pandas as pd
+from cluster_experiments import PowerAnalysis
+import matplotlib.pyplot as plt
+import warnings
+from plotnine import ggplot,theme_classic, facet_wrap, geom_density, labs, geom_vline, aes, geom_line, geom_histogram, geom_text,scale_y_continuous
+
+warnings.filterwarnings('ignore')
+np.random.seed(42)
+
+N = 1000
+clusters = [f"Cluster {i}" for i in range(50)]
+dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(1, 32)]
+df = pd.DataFrame(
+ {
+ "cluster": np.random.choice(clusters, size=N),
+ "target": np.random.normal(0, 1, size=N),
+ "date": np.random.choice(dates, size=N),
+ }
+)
+Let's start with a simple OLS model, not clustered, to understand the difference in power. We will try 2 different splitters: constant and normal
+ +splitters = ['constant', 'normal']
+results = []
+
+for hypothesis in ["two-sided", "less", "greater"]:
+ for splitter in splitters:
+ config = {
+ "analysis": 'ols',
+ "perturbator": splitter,
+ "splitter": "non_clustered",
+ "n_simulations": 50,
+ "hypothesis": hypothesis,
+ "seed":41
+ }
+ pw = PowerAnalysis.from_dict(config)
+
+ power_dict = pw.power_line(df, average_effects=list(np.linspace(0.000001, 0.5, 15)))
+ power_df = pd.DataFrame(list(power_dict.items()), columns=['average_effect', 'power'])
+
+ power_df['hypothesis'] = hypothesis
+ power_df['splitter'] = splitter
+
+ results.append(power_df)
+
+
+final_df = pd.concat(results, ignore_index=True)
+final_df.head()
+| + | average_effect | +power | +hypothesis | +splitter | +
|---|---|---|---|---|
| 0 | +0.000001 | +0.04 | +two-sided | +constant | +
| 1 | +0.035715 | +0.06 | +two-sided | +constant | +
| 2 | +0.071429 | +0.22 | +two-sided | +constant | +
| 3 | +0.107144 | +0.40 | +two-sided | +constant | +
| 4 | +0.142858 | +0.64 | +two-sided | +constant | +
def plot(breakdown:str, ncol_facetting:int):
+ p = (ggplot(final_df, aes(x='average_effect', y = 'power', color='hypothesis'))
+ + geom_line()
+ + theme_classic()
+ + facet_wrap(breakdown, ncol = 1))
+
+ print(p)
+We can clearly see that using the correct side (higher, as the effect is positive) increase the power of the experiment. It's also great to see such a low power in case of hypothesis 'less'.
+ +plot(breakdown = 'splitter', ncol_facetting = 1)
+
+++
Now we will quantify this difference in power between greater and two-sided
+ +pivot_df = (
+ final_df
+ .pivot(index = [ 'splitter', 'average_effect'], columns = 'hypothesis', values = 'power')
+ .reset_index()
+ .assign(diff = lambda x: x['greater'] - x['two-sided'])
+)
+mean_diff = pivot_df['diff'].mean()
+max_count = np.max(pivot_df['diff'].value_counts())
+Below we plot the distribution of the difference between 2-sided and 'greater' hypotheses. The mean equals to 4% is the estimated increase in power when using a one-sided experiment. From the 30 iterations, we see that 12 times it didn't change the power, in 2 cases it actually decrease it and in 2 cases the incease in power was 15pp.
+ +(ggplot(pivot_df, aes(x = 'diff'))
+ + geom_histogram( alpha=0.5, bins = 10, position="identity")
+ + geom_density()
+ + geom_vline(xintercept=mean_diff, color="red", linetype="dashed", size=1)
+ + geom_text(x=mean_diff, y=np.max(pivot_df['diff'].value_counts()), label=f'Mean: {mean_diff:.2f}', va='bottom', ha='left', color="red")
+ + scale_y_continuous(breaks=range(0, int(max_count) + 1))
+ + labs(title = 'Histogram of power difference between greater and two sided hypothesis', x='Difference in power')
+
+)
+
+<Figure Size: (640 x 480)>+
Now let's move to clustered methods. To keep the notebook tidy, we will just run the constant perturbator.
+ +results = []
+
+for hypothesis in ["two-sided", "less", "greater"]:
+ for analysis in ["ols_clustered", "gee", 'ttest_clustered']:
+ config = {
+ "analysis": analysis,
+ "perturbator": "constant",
+ "splitter": "clustered",
+ "n_simulations": 50,
+ "hypothesis": hypothesis,
+ "cluster_cols": ['cluster', 'date'],
+ "seed":41
+ }
+ pw = PowerAnalysis.from_dict(config)
+
+ power_dict = pw.power_line(df, average_effects=list(np.linspace(0.000001, 0.5, 15)))
+ power_df = pd.DataFrame(list(power_dict.items()), columns=['average_effect', 'power'])
+
+ power_df['hypothesis'] = hypothesis
+ power_df['analysis'] = analysis
+
+ results.append(power_df)
+
+
+final_df = pd.concat(results, ignore_index=True)
+Here, again, we see an increase in power using the correct one-sided hypothesis compared to two-sided.
+ +plot(breakdown = 'analysis', ncol_facetting=1)
+
+++
from cluster_experiments.inference.analysis_plan import *¶
+AnalysisPlan
+
+
+
+¶A class used to represent an Analysis Plan with a list of hypothesis tests and a list of variants. +All the hypothesis tests in the same analysis plan will be analysed with the same dataframe, which will need to be passed in the analyze() method.
+tests : List[HypothesisTest] + A list of HypothesisTest instances +variants : List[Variant] + A list of Variant instances +variant_col : str + name of the column with the experiment groups +alpha : float + significance level used to construct confidence intervals
+ +cluster_experiments/inference/analysis_plan.pyclass AnalysisPlan:
+ """
+ A class used to represent an Analysis Plan with a list of hypothesis tests and a list of variants.
+ All the hypothesis tests in the same analysis plan will be analysed with the same dataframe, which will need to be passed in the analyze() method.
+
+ Attributes
+ ----------
+ tests : List[HypothesisTest]
+ A list of HypothesisTest instances
+ variants : List[Variant]
+ A list of Variant instances
+ variant_col : str
+ name of the column with the experiment groups
+ alpha : float
+ significance level used to construct confidence intervals
+ """
+
+ def __init__(
+ self,
+ tests: List[HypothesisTest],
+ variants: List[Variant],
+ variant_col: str = "treatment",
+ alpha: float = 0.05,
+ ):
+ """
+ Parameters
+ ----------
+ tests : List[HypothesisTest]
+ A list of HypothesisTest instances
+ variants : List[Variant]
+ A list of Variant instances
+ variant_col : str
+ The name of the column containing the variant names.
+ alpha : float
+ significance level used to construct confidence intervals
+ """
+
+ self.tests = tests
+ self.variants = variants
+ self.variant_col = variant_col
+ self.alpha = alpha
+
+ self._validate_inputs()
+
+ def _validate_inputs(self):
+ """
+ Validates the inputs for the AnalysisPlan class.
+
+ Raises
+ ------
+ TypeError
+ If tests is not a list of HypothesisTest instances or if variants is not a list of Variant instances.
+ ValueError
+ If tests or variants are empty lists.
+ """
+ if not isinstance(self.tests, list) or not all(
+ isinstance(test, HypothesisTest) for test in self.tests
+ ):
+ raise TypeError("Tests must be a list of HypothesisTest instances")
+ if not isinstance(self.variants, list) or not all(
+ isinstance(variant, Variant) for variant in self.variants
+ ):
+ raise TypeError("Variants must be a list of Variant instances")
+ if not isinstance(self.variant_col, str):
+ raise TypeError("Variant_col must be a string")
+ if not self.tests:
+ raise ValueError("Tests list cannot be empty")
+ if not self.variants:
+ raise ValueError("Variants list cannot be empty")
+
+ def analyze(
+ self,
+ exp_data: pd.DataFrame,
+ pre_exp_data: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ ) -> AnalysisPlanResults:
+ """
+ Method to run the experiment analysis.
+ """
+
+ # Validate input data at the beginning
+ self._validate_data(exp_data, pre_exp_data)
+
+ analysis_results = AnalysisPlanResults()
+
+ for test in self.tests:
+ exp_data = test.add_covariates(exp_data, pre_exp_data)
+
+ for treatment_variant in self.treatment_variants:
+ for dimension in test.dimensions:
+ for dimension_value in dimension.iterate_dimension_values():
+
+ if verbose:
+ logger.info(
+ f"Metric: {test.metric.alias}, "
+ f"Treatment: {treatment_variant.name}, "
+ f"Dimension: {dimension.name}, "
+ f"Value: {dimension_value}"
+ )
+
+ test_results = test.get_test_results(
+ exp_data=exp_data,
+ control_variant=self.control_variant,
+ treatment_variant=treatment_variant,
+ variant_col=self.variant_col,
+ dimension=dimension,
+ dimension_value=dimension_value,
+ alpha=self.alpha,
+ )
+
+ analysis_results = analysis_results + test_results
+
+ return analysis_results
+
+ def _validate_data(
+ self, exp_data: pd.DataFrame, pre_exp_data: Optional[pd.DataFrame] = None
+ ):
+ """
+ Validates the input dataframes for the analyze method.
+
+ Parameters
+ ----------
+ exp_data : pd.DataFrame
+ The experimental data
+ pre_exp_data : Optional[pd.DataFrame]
+ The pre-experimental data (optional)
+
+ Raises
+ ------
+ ValueError
+ If exp_data is not a DataFrame or is empty
+ If pre_exp_data is provided and is not a DataFrame or is empty
+ """
+ if not isinstance(exp_data, pd.DataFrame):
+ raise ValueError("exp_data must be a pandas DataFrame")
+ if exp_data.empty:
+ raise ValueError("exp_data cannot be empty")
+ if pre_exp_data is not None:
+ if not isinstance(pre_exp_data, pd.DataFrame):
+ raise ValueError("pre_exp_data must be a pandas DataFrame if provided")
+ if pre_exp_data.empty:
+ raise ValueError("pre_exp_data cannot be empty if provided")
+
+ @property
+ def control_variant(self) -> Variant:
+ """
+ Returns the control variant from the list of variants. Raises an error if no control variant is found.
+
+ Returns
+ -------
+ Variant
+ The control variant
+
+ Raises
+ ------
+ ValueError
+ If no control variant is found
+ """
+ for variant in self.variants:
+ if variant.is_control:
+ return variant
+ raise ValueError("No control variant found")
+
+ @property
+ def treatment_variants(self) -> List[Variant]:
+ """
+ Returns the treatment variants from the list of variants. Raises an error if no treatment variants are found.
+
+ Returns
+ -------
+ List[Variant]
+ A list of treatment variants
+
+ Raises
+ ------
+ ValueError
+ If no treatment variants are found
+ """
+ treatments = [variant for variant in self.variants if not variant.is_control]
+ if not treatments:
+ raise ValueError("No treatment variants found")
+ return treatments
+
+ @classmethod
+ def from_metrics(
+ cls,
+ metrics: List[Metric],
+ variants: List[Variant],
+ variant_col: str = "treatment",
+ alpha: float = 0.05,
+ dimensions: Optional[List[Dimension]] = None,
+ analysis_type: str = "default",
+ analysis_config: Optional[Dict[str, Any]] = None,
+ custom_analysis_type_mapper: Optional[Dict[str, ExperimentAnalysis]] = None,
+ ) -> "AnalysisPlan":
+ """
+ Creates a simplified AnalysisPlan instance from a list of metrics. It will create HypothesisTest objects under the hood.
+ This shortcut does not support cupac, and uses the same dimensions, analysis type and analysis config for all metrics.
+
+ Parameters
+ ----------
+ metrics : List[Metric]
+ A list of Metric instances
+ variants : List[Variant]
+ A list of Variant instances
+ variant_col : str
+ The name of the column containing the variant names.
+ alpha : float
+ Significance level used to construct confidence intervals
+ dimensions : Optional[List[Dimension]]
+ A list of Dimension instances (optional)
+ analysis_type : str
+ The type of analysis to be conducted (default: "default")
+ analysis_config : Optional[Dict[str, Any]]
+ A dictionary containing analysis configuration options (optional)
+ custom_analysis_type_mapper : Optional[Dict[str, ExperimentAnalysis]]
+ An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes
+
+ Returns
+ -------
+ AnalysisPlan
+ An instance of AnalysisPlan
+ """
+ tests = [
+ HypothesisTest(
+ metric=metric,
+ dimensions=dimensions or [],
+ analysis_type=analysis_type,
+ analysis_config=analysis_config or {},
+ custom_analysis_type_mapper=custom_analysis_type_mapper or {},
+ )
+ for metric in metrics
+ ]
+
+ return cls(
+ tests=tests,
+ variants=variants,
+ variant_col=variant_col,
+ alpha=alpha,
+ )
+control_variant: Variant
+
+
+ property
+ readonly
+
+
+¶treatment_variants: List[cluster_experiments.inference.variant.Variant]
+
+
+ property
+ readonly
+
+
+¶__init__(self, tests, variants, variant_col='treatment', alpha=0.05)
+
+
+ special
+
+
+¶tests : List[HypothesisTest] + A list of HypothesisTest instances +variants : List[Variant] + A list of Variant instances +variant_col : str + The name of the column containing the variant names. +alpha : float + significance level used to construct confidence intervals
+ +cluster_experiments/inference/analysis_plan.pydef __init__(
+ self,
+ tests: List[HypothesisTest],
+ variants: List[Variant],
+ variant_col: str = "treatment",
+ alpha: float = 0.05,
+):
+ """
+ Parameters
+ ----------
+ tests : List[HypothesisTest]
+ A list of HypothesisTest instances
+ variants : List[Variant]
+ A list of Variant instances
+ variant_col : str
+ The name of the column containing the variant names.
+ alpha : float
+ significance level used to construct confidence intervals
+ """
+
+ self.tests = tests
+ self.variants = variants
+ self.variant_col = variant_col
+ self.alpha = alpha
+
+ self._validate_inputs()
+analyze(self, exp_data, pre_exp_data=None, verbose=False)
+
+
+¶Method to run the experiment analysis.
+ +cluster_experiments/inference/analysis_plan.pydef analyze(
+ self,
+ exp_data: pd.DataFrame,
+ pre_exp_data: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+) -> AnalysisPlanResults:
+ """
+ Method to run the experiment analysis.
+ """
+
+ # Validate input data at the beginning
+ self._validate_data(exp_data, pre_exp_data)
+
+ analysis_results = AnalysisPlanResults()
+
+ for test in self.tests:
+ exp_data = test.add_covariates(exp_data, pre_exp_data)
+
+ for treatment_variant in self.treatment_variants:
+ for dimension in test.dimensions:
+ for dimension_value in dimension.iterate_dimension_values():
+
+ if verbose:
+ logger.info(
+ f"Metric: {test.metric.alias}, "
+ f"Treatment: {treatment_variant.name}, "
+ f"Dimension: {dimension.name}, "
+ f"Value: {dimension_value}"
+ )
+
+ test_results = test.get_test_results(
+ exp_data=exp_data,
+ control_variant=self.control_variant,
+ treatment_variant=treatment_variant,
+ variant_col=self.variant_col,
+ dimension=dimension,
+ dimension_value=dimension_value,
+ alpha=self.alpha,
+ )
+
+ analysis_results = analysis_results + test_results
+
+ return analysis_results
+from_metrics(metrics, variants, variant_col='treatment', alpha=0.05, dimensions=None, analysis_type='default', analysis_config=None, custom_analysis_type_mapper=None)
+
+
+ classmethod
+
+
+¶Creates a simplified AnalysisPlan instance from a list of metrics. It will create HypothesisTest objects under the hood. +This shortcut does not support cupac, and uses the same dimensions, analysis type and analysis config for all metrics.
+metrics : List[Metric] + A list of Metric instances +variants : List[Variant] + A list of Variant instances +variant_col : str + The name of the column containing the variant names. +alpha : float + Significance level used to construct confidence intervals +dimensions : Optional[List[Dimension]] + A list of Dimension instances (optional) +analysis_type : str + The type of analysis to be conducted (default: "default") +analysis_config : Optional[Dict[str, Any]] + A dictionary containing analysis configuration options (optional) +custom_analysis_type_mapper : Optional[Dict[str, ExperimentAnalysis]] + An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes
+AnalysisPlan + An instance of AnalysisPlan
+ +cluster_experiments/inference/analysis_plan.py@classmethod
+def from_metrics(
+ cls,
+ metrics: List[Metric],
+ variants: List[Variant],
+ variant_col: str = "treatment",
+ alpha: float = 0.05,
+ dimensions: Optional[List[Dimension]] = None,
+ analysis_type: str = "default",
+ analysis_config: Optional[Dict[str, Any]] = None,
+ custom_analysis_type_mapper: Optional[Dict[str, ExperimentAnalysis]] = None,
+) -> "AnalysisPlan":
+ """
+ Creates a simplified AnalysisPlan instance from a list of metrics. It will create HypothesisTest objects under the hood.
+ This shortcut does not support cupac, and uses the same dimensions, analysis type and analysis config for all metrics.
+
+ Parameters
+ ----------
+ metrics : List[Metric]
+ A list of Metric instances
+ variants : List[Variant]
+ A list of Variant instances
+ variant_col : str
+ The name of the column containing the variant names.
+ alpha : float
+ Significance level used to construct confidence intervals
+ dimensions : Optional[List[Dimension]]
+ A list of Dimension instances (optional)
+ analysis_type : str
+ The type of analysis to be conducted (default: "default")
+ analysis_config : Optional[Dict[str, Any]]
+ A dictionary containing analysis configuration options (optional)
+ custom_analysis_type_mapper : Optional[Dict[str, ExperimentAnalysis]]
+ An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes
+
+ Returns
+ -------
+ AnalysisPlan
+ An instance of AnalysisPlan
+ """
+ tests = [
+ HypothesisTest(
+ metric=metric,
+ dimensions=dimensions or [],
+ analysis_type=analysis_type,
+ analysis_config=analysis_config or {},
+ custom_analysis_type_mapper=custom_analysis_type_mapper or {},
+ )
+ for metric in metrics
+ ]
+
+ return cls(
+ tests=tests,
+ variants=variants,
+ variant_col=variant_col,
+ alpha=alpha,
+ )
+from cluster_experiments.inference.analysis_results import *¶
+AnalysisPlanResults
+
+
+
+ dataclass
+
+
+¶A dataclass used to represent the results of the experiment analysis.
+metric_alias : List[str] + The alias of the metric used in the test +control_variant_name : List[str] + The name of the control variant +treatment_variant_name : List[str] + The name of the treatment variant +control_variant_mean : List[float] + The mean value of the control variant +treatment_variant_mean : List[float] + The mean value of the treatment variant +analysis_type : List[str] + The type of analysis performed +ate : List[float] + The average treatment effect +ate_ci_lower : List[float] + The lower bound of the confidence interval for the ATE +ate_ci_upper : List[float] + The upper bound of the confidence interval for the ATE +p_value : List[float] + The p-value of the test +std_error : List[float] + The standard error of the test +dimension_name : List[str] + The name of the dimension +dimension_value : List[str] + The value of the dimension +!!! alpha "List[float]" + The significance level of the test
+ +cluster_experiments/inference/analysis_results.pyclass AnalysisPlanResults:
+ """
+ A dataclass used to represent the results of the experiment analysis.
+
+ Attributes
+ ----------
+ metric_alias : List[str]
+ The alias of the metric used in the test
+ control_variant_name : List[str]
+ The name of the control variant
+ treatment_variant_name : List[str]
+ The name of the treatment variant
+ control_variant_mean : List[float]
+ The mean value of the control variant
+ treatment_variant_mean : List[float]
+ The mean value of the treatment variant
+ analysis_type : List[str]
+ The type of analysis performed
+ ate : List[float]
+ The average treatment effect
+ ate_ci_lower : List[float]
+ The lower bound of the confidence interval for the ATE
+ ate_ci_upper : List[float]
+ The upper bound of the confidence interval for the ATE
+ p_value : List[float]
+ The p-value of the test
+ std_error : List[float]
+ The standard error of the test
+ dimension_name : List[str]
+ The name of the dimension
+ dimension_value : List[str]
+ The value of the dimension
+ alpha: List[float]
+ The significance level of the test
+ """
+
+ metric_alias: List[str] = field(default_factory=lambda: [])
+ control_variant_name: List[str] = field(default_factory=lambda: [])
+ treatment_variant_name: List[str] = field(default_factory=lambda: [])
+ control_variant_mean: List[float] = field(default_factory=lambda: [])
+ treatment_variant_mean: List[float] = field(default_factory=lambda: [])
+ analysis_type: List[str] = field(default_factory=lambda: [])
+ ate: List[float] = field(default_factory=lambda: [])
+ ate_ci_lower: List[float] = field(default_factory=lambda: [])
+ ate_ci_upper: List[float] = field(default_factory=lambda: [])
+ p_value: List[float] = field(default_factory=lambda: [])
+ std_error: List[float] = field(default_factory=lambda: [])
+ dimension_name: List[str] = field(default_factory=lambda: [])
+ dimension_value: List[str] = field(default_factory=lambda: [])
+ alpha: List[float] = field(default_factory=lambda: [])
+
+ def __add__(self, other):
+ if not isinstance(other, AnalysisPlanResults):
+ return NotImplemented
+
+ return AnalysisPlanResults(
+ metric_alias=self.metric_alias + other.metric_alias,
+ control_variant_name=self.control_variant_name + other.control_variant_name,
+ treatment_variant_name=self.treatment_variant_name
+ + other.treatment_variant_name,
+ control_variant_mean=self.control_variant_mean + other.control_variant_mean,
+ treatment_variant_mean=self.treatment_variant_mean
+ + other.treatment_variant_mean,
+ analysis_type=self.analysis_type + other.analysis_type,
+ ate=self.ate + other.ate,
+ ate_ci_lower=self.ate_ci_lower + other.ate_ci_lower,
+ ate_ci_upper=self.ate_ci_upper + other.ate_ci_upper,
+ p_value=self.p_value + other.p_value,
+ std_error=self.std_error + other.std_error,
+ dimension_name=self.dimension_name + other.dimension_name,
+ dimension_value=self.dimension_value + other.dimension_value,
+ alpha=self.alpha + other.alpha,
+ )
+
+ def to_dataframe(self):
+ return pd.DataFrame(asdict(self))
+from cluster_experiments.cupac import *¶
+CupacHandler
+
+
+
+¶CupacHandler class. It handles operations related to the cupac model.
+Its main goal is to call the add_covariates method, where it will add the ouptut from the cupac model, +and this should be used as covariates in the regression method for the hypothesis test.
+ +cluster_experiments/cupac.pyclass CupacHandler:
+ """
+ CupacHandler class. It handles operations related to the cupac model.
+
+ Its main goal is to call the add_covariates method, where it will add the ouptut from the cupac model,
+ and this should be used as covariates in the regression method for the hypothesis test.
+ """
+
+ def __init__(
+ self,
+ cupac_model: Optional[BaseEstimator] = None,
+ target_col: str = "target",
+ features_cupac_model: Optional[List[str]] = None,
+ cache_fit: bool = True,
+ ):
+ self.cupac_model: BaseEstimator = cupac_model or EmptyRegressor()
+ self.target_col = target_col
+ self.cupac_outcome_name = f"estimate_{target_col}"
+ self.features_cupac_model: List[str] = features_cupac_model or []
+ self.is_cupac = not isinstance(self.cupac_model, EmptyRegressor)
+ self.cache_fit = cache_fit
+
+ def _prep_data_cupac(
+ self, df: pd.DataFrame, pre_experiment_df: pd.DataFrame
+ ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.Series]:
+ """Prepares data for training and prediction"""
+ df = df.copy()
+ pre_experiment_df = pre_experiment_df.copy()
+ df_predict = df.drop(columns=[self.target_col])
+ # Split data into X and y
+ pre_experiment_x = pre_experiment_df.drop(columns=[self.target_col])
+ pre_experiment_y = pre_experiment_df[self.target_col]
+
+ # Keep only cupac features
+ if self.features_cupac_model:
+ pre_experiment_x = pre_experiment_x[self.features_cupac_model]
+ df_predict = df_predict[self.features_cupac_model]
+
+ return df_predict, pre_experiment_x, pre_experiment_y
+
+ def add_covariates(
+ self, df: pd.DataFrame, pre_experiment_df: Optional[pd.DataFrame] = None
+ ) -> pd.DataFrame:
+ """
+ Train model to predict outcome variable (based on pre-experiment data)
+ and add the prediction to the experiment dataframe. Only do this if
+ we use cupac
+ Args:
+ pre_experiment_df: Dataframe with pre-experiment data.
+ df: Dataframe with outcome and treatment variables.
+ """
+ self.check_cupac_inputs(pre_experiment_df)
+
+ # Early return if no need to add covariates
+ if not self.need_covariates(pre_experiment_df):
+ return df
+
+ df = df.copy()
+ pre_experiment_df = pre_experiment_df.copy()
+ df_predict, pre_experiment_x, pre_experiment_y = self._prep_data_cupac(
+ df=df, pre_experiment_df=pre_experiment_df
+ )
+
+ # Fit model if it has not been fitted before
+ self._fit_cupac_model(pre_experiment_x, pre_experiment_y)
+
+ # Predict
+ estimated_target = self._predict_cupac_model(df_predict)
+
+ # Add cupac outcome name to df
+ df[self.cupac_outcome_name] = estimated_target
+ return df
+
+ def _fit_cupac_model(
+ self, pre_experiment_x: pd.DataFrame, pre_experiment_y: pd.Series
+ ):
+ """Fits the cupac model.
+ Caches the fitted model in the object, so we only fit it once.
+ We can disable this by setting cache_fit to False.
+ """
+ if not self.cache_fit:
+ self.cupac_model.fit(pre_experiment_x, pre_experiment_y)
+ return
+
+ try:
+ check_is_fitted(self.cupac_model)
+ except NotFittedError:
+ self.cupac_model.fit(pre_experiment_x, pre_experiment_y)
+
+ def _predict_cupac_model(self, df_predict: pd.DataFrame) -> ArrayLike:
+ """Predicts the cupac model"""
+ if hasattr(self.cupac_model, "predict_proba"):
+ return self.cupac_model.predict_proba(df_predict)[:, 1]
+ if hasattr(self.cupac_model, "predict"):
+ return self.cupac_model.predict(df_predict)
+ raise ValueError("cupac_model should have predict or predict_proba method.")
+
+ def need_covariates(self, pre_experiment_df: Optional[pd.DataFrame] = None) -> bool:
+ return pre_experiment_df is not None and self.is_cupac
+
+ def check_cupac_inputs(self, pre_experiment_df: Optional[pd.DataFrame] = None):
+ if self.is_cupac and pre_experiment_df is None:
+ raise ValueError("If cupac is used, pre_experiment_df should be provided.")
+
+ if not self.is_cupac and pre_experiment_df is not None:
+ raise ValueError(
+ "If cupac is not used, pre_experiment_df should not be provided - "
+ "remove pre_experiment_df argument or set cupac_model to not None."
+ )
+add_covariates(self, df, pre_experiment_df=None)
+
+
+¶Train model to predict outcome variable (based on pre-experiment data) +and add the prediction to the experiment dataframe. Only do this if +we use cupac
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
cluster_experiments/cupac.pydef add_covariates(
+ self, df: pd.DataFrame, pre_experiment_df: Optional[pd.DataFrame] = None
+) -> pd.DataFrame:
+ """
+ Train model to predict outcome variable (based on pre-experiment data)
+ and add the prediction to the experiment dataframe. Only do this if
+ we use cupac
+ Args:
+ pre_experiment_df: Dataframe with pre-experiment data.
+ df: Dataframe with outcome and treatment variables.
+ """
+ self.check_cupac_inputs(pre_experiment_df)
+
+ # Early return if no need to add covariates
+ if not self.need_covariates(pre_experiment_df):
+ return df
+
+ df = df.copy()
+ pre_experiment_df = pre_experiment_df.copy()
+ df_predict, pre_experiment_x, pre_experiment_y = self._prep_data_cupac(
+ df=df, pre_experiment_df=pre_experiment_df
+ )
+
+ # Fit model if it has not been fitted before
+ self._fit_cupac_model(pre_experiment_x, pre_experiment_y)
+
+ # Predict
+ estimated_target = self._predict_cupac_model(df_predict)
+
+ # Add cupac outcome name to df
+ df[self.cupac_outcome_name] = estimated_target
+ return df
+
+EmptyRegressor (BaseEstimator)
+
+
+
+
+¶Empty regressor class. It does not do anything, used to glue the code of other estimators and PowerAnalysis
+Each Regressor should have: +- fit method: Uses pre experiment data to fit some kind of model to be used as a covariate and reduce variance. +- predict method: Uses the fitted model to add the covariate on the experiment data.
+It can add aggregates of the target in older data as a covariate, or a model (cupac) to predict the target.
+ +cluster_experiments/cupac.pyclass EmptyRegressor(BaseEstimator):
+ """
+ Empty regressor class. It does not do anything, used to glue the code of other estimators and PowerAnalysis
+
+ Each Regressor should have:
+ - fit method: Uses pre experiment data to fit some kind of model to be used as a covariate and reduce variance.
+ - predict method: Uses the fitted model to add the covariate on the experiment data.
+
+ It can add aggregates of the target in older data as a covariate, or a model (cupac) to predict the target.
+ """
+
+ @classmethod
+ def from_config(cls, config):
+ return cls()
+
+TargetAggregation (BaseEstimator)
+
+
+
+
+¶Adds average of target using pre-experiment data
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
agg_col |
+ str |
+ Column to group by to aggregate target |
+ required | +
target_col |
+ str |
+ Column to aggregate |
+ 'target' |
+
smoothing_factor |
+ int |
+ Smoothing factor for the smoothed mean |
+ 20 |
+
Usage: +
import pandas as pd
+from cluster_experiments.cupac import TargetAggregation
+
+df = pd.DataFrame({"agg_col": ["a", "a", "b", "b", "c", "c"], "target_col": [1, 2, 3, 4, 5, 6]})
+new_df = pd.DataFrame({"agg_col": ["a", "a", "b", "b", "c", "c"]})
+target_agg = TargetAggregation("agg_col", "target_col")
+target_agg.fit(df.drop(columns="target_col"), df["target_col"])
+df_with_target_agg = target_agg.predict(new_df)
+print(df_with_target_agg)
+cluster_experiments/cupac.pyclass TargetAggregation(BaseEstimator):
+ """
+ Adds average of target using pre-experiment data
+
+ Args:
+ agg_col: Column to group by to aggregate target
+ target_col: Column to aggregate
+ smoothing_factor: Smoothing factor for the smoothed mean
+ Usage:
+ ```python
+ import pandas as pd
+ from cluster_experiments.cupac import TargetAggregation
+
+ df = pd.DataFrame({"agg_col": ["a", "a", "b", "b", "c", "c"], "target_col": [1, 2, 3, 4, 5, 6]})
+ new_df = pd.DataFrame({"agg_col": ["a", "a", "b", "b", "c", "c"]})
+ target_agg = TargetAggregation("agg_col", "target_col")
+ target_agg.fit(df.drop(columns="target_col"), df["target_col"])
+ df_with_target_agg = target_agg.predict(new_df)
+ print(df_with_target_agg)
+ ```
+ """
+
+ def __init__(
+ self,
+ agg_col: str,
+ target_col: str = "target",
+ smoothing_factor: int = 20,
+ ):
+ self.agg_col = agg_col
+ self.target_col = target_col
+ self.smoothing_factor = smoothing_factor
+ self.is_empty = False
+ self.mean_target_col = f"{self.target_col}_mean"
+ self.smooth_mean_target_col = f"{self.target_col}_smooth_mean"
+ self.pre_experiment_agg_df = pd.DataFrame()
+
+ def _get_pre_experiment_mean(self, pre_experiment_df: pd.DataFrame) -> float:
+ return pre_experiment_df[self.target_col].mean()
+
+ def fit(self, X: pd.DataFrame, y: pd.Series) -> "TargetAggregation":
+ """Fits "target encoder" model to pre-experiment data"""
+ pre_experiment_df = X.copy()
+ pre_experiment_df[self.target_col] = y
+
+ self.pre_experiment_mean = self._get_pre_experiment_mean(pre_experiment_df)
+ self.pre_experiment_agg_df = (
+ pre_experiment_df.assign(count=1)
+ .groupby(self.agg_col, as_index=False)
+ .agg({self.target_col: "sum", "count": "sum"})
+ .assign(
+ **{
+ self.mean_target_col: lambda x: x[self.target_col] / x["count"],
+ self.smooth_mean_target_col: lambda x: (
+ x[self.target_col]
+ + self.smoothing_factor * self.pre_experiment_mean
+ )
+ / (x["count"] + self.smoothing_factor),
+ }
+ )
+ .drop(columns=["count", self.target_col])
+ )
+ return self
+
+ def predict(self, X: pd.DataFrame) -> ArrayLike:
+ """Adds average target of pre-experiment data to experiment data"""
+ return (
+ X.merge(self.pre_experiment_agg_df, how="left", on=self.agg_col)[
+ self.smooth_mean_target_col
+ ]
+ .fillna(self.pre_experiment_mean)
+ .values
+ )
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates TargetAggregation from PowerConfig"""
+ return cls(
+ agg_col=config.agg_col,
+ target_col=config.target_col,
+ smoothing_factor=config.smoothing_factor,
+ )
+fit(self, X, y)
+
+
+¶Fits "target encoder" model to pre-experiment data
+ +cluster_experiments/cupac.pydef fit(self, X: pd.DataFrame, y: pd.Series) -> "TargetAggregation":
+ """Fits "target encoder" model to pre-experiment data"""
+ pre_experiment_df = X.copy()
+ pre_experiment_df[self.target_col] = y
+
+ self.pre_experiment_mean = self._get_pre_experiment_mean(pre_experiment_df)
+ self.pre_experiment_agg_df = (
+ pre_experiment_df.assign(count=1)
+ .groupby(self.agg_col, as_index=False)
+ .agg({self.target_col: "sum", "count": "sum"})
+ .assign(
+ **{
+ self.mean_target_col: lambda x: x[self.target_col] / x["count"],
+ self.smooth_mean_target_col: lambda x: (
+ x[self.target_col]
+ + self.smoothing_factor * self.pre_experiment_mean
+ )
+ / (x["count"] + self.smoothing_factor),
+ }
+ )
+ .drop(columns=["count", self.target_col])
+ )
+ return self
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates TargetAggregation from PowerConfig
+ +cluster_experiments/cupac.py@classmethod
+def from_config(cls, config):
+ """Creates TargetAggregation from PowerConfig"""
+ return cls(
+ agg_col=config.agg_col,
+ target_col=config.target_col,
+ smoothing_factor=config.smoothing_factor,
+ )
+predict(self, X)
+
+
+¶Adds average target of pre-experiment data to experiment data
+ +cluster_experiments/cupac.pydef predict(self, X: pd.DataFrame) -> ArrayLike:
+ """Adds average target of pre-experiment data to experiment data"""
+ return (
+ X.merge(self.pre_experiment_agg_df, how="left", on=self.agg_col)[
+ self.smooth_mean_target_col
+ ]
+ .fillna(self.pre_experiment_mean)
+ .values
+ )
+from cluster_experiments.inference.dimension import *¶
+DefaultDimension (Dimension)
+
+
+
+
+ dataclass
+
+
+¶A class used to represent a Dimension with a default value representing total, i.e. no slicing.
+ +cluster_experiments/inference/dimension.pyclass DefaultDimension(Dimension):
+ """
+ A class used to represent a Dimension with a default value representing total, i.e. no slicing.
+ """
+
+ def __init__(self):
+ super().__init__(name="__total_dimension", values=["total"])
+__init__(self)
+
+
+ special
+
+
+¶Initialize self. See help(type(self)) for accurate signature.
+ +cluster_experiments/inference/dimension.pydef __init__(self):
+ super().__init__(name="__total_dimension", values=["total"])
+
+Dimension
+
+
+
+ dataclass
+
+
+¶A class used to represent a Dimension with a name and values.
+name : str + The name of the dimension +values : List[str] + A list of strings representing the possible values of the dimension
+ +cluster_experiments/inference/dimension.pyclass Dimension:
+ """
+ A class used to represent a Dimension with a name and values.
+
+ Attributes
+ ----------
+ name : str
+ The name of the dimension
+ values : List[str]
+ A list of strings representing the possible values of the dimension
+ """
+
+ name: str
+ values: List[str]
+
+ def __post_init__(self):
+ """
+ Validates the inputs after initialization.
+ """
+ self._validate_inputs()
+
+ def _validate_inputs(self):
+ """
+ Validates the inputs for the Dimension class.
+
+ Raises
+ ------
+ TypeError
+ If the name is not a string or if values is not a list of strings.
+ """
+ if not isinstance(self.name, str):
+ raise TypeError("Dimension name must be a string")
+ if not isinstance(self.values, list) or not all(
+ isinstance(val, str) for val in self.values
+ ):
+ raise TypeError("Dimension values must be a list of strings")
+
+ def iterate_dimension_values(self):
+ """
+ A generator method to yield name and values from the dimension.
+
+ Yields
+ ------
+ Any
+ A unique value from the dimension.
+ """
+ seen = set()
+ for value in self.values:
+ if value not in seen:
+ seen.add(value)
+ yield value
+__post_init__(self)
+
+
+ special
+
+
+¶Validates the inputs after initialization.
+ +cluster_experiments/inference/dimension.pydef __post_init__(self):
+ """
+ Validates the inputs after initialization.
+ """
+ self._validate_inputs()
+iterate_dimension_values(self)
+
+
+¶A generator method to yield name and values from the dimension.
+Any + A unique value from the dimension.
+ +cluster_experiments/inference/dimension.pydef iterate_dimension_values(self):
+ """
+ A generator method to yield name and values from the dimension.
+
+ Yields
+ ------
+ Any
+ A unique value from the dimension.
+ """
+ seen = set()
+ for value in self.values:
+ if value not in seen:
+ seen.add(value)
+ yield value
+from cluster_experiments.experiment_analysis import *¶
+ClusteredOLSAnalysis (ExperimentAnalysis)
+
+
+
+
+¶Class to run OLS clustered analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_cols |
+ List[str] |
+ list of columns to use as clusters |
+ required | +
target_col |
+ str |
+ name of the column containing the variable to measure |
+ 'target' |
+
treatment_col |
+ str |
+ name of the column containing the treatment variable |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group |
+ 'B' |
+
covariates |
+ Optional[List[str]] |
+ list of columns to use as covariates |
+ None |
+
hypothesis |
+ str |
+ one of "two-sided", "less", "greater" indicating the alternative hypothesis |
+ 'two-sided' |
+
Usage:
+from cluster_experiments.experiment_analysis import ClusteredOLSAnalysis
+import pandas as pd
+
+df = pd.DataFrame({
+ 'x': [1, 2, 3, 0, 0, 1, 2, 0],
+ 'treatment': ["A"] * 2 + ["B"] * 2 + ["A"] * 2 + ["B"] * 2,
+ 'cluster': [1, 1, 2, 2, 3, 3, 4, 4],
+})
+
+ClusteredOLSAnalysis(
+ cluster_cols=['cluster'],
+ target_col='x',
+).get_pvalue(df)
+cluster_experiments/experiment_analysis.pyclass ClusteredOLSAnalysis(ExperimentAnalysis):
+ """
+ Class to run OLS clustered analysis
+
+ Arguments:
+ cluster_cols: list of columns to use as clusters
+ target_col: name of the column containing the variable to measure
+ treatment_col: name of the column containing the treatment variable
+ treatment: name of the treatment to use as the treated group
+ covariates: list of columns to use as covariates
+ hypothesis: one of "two-sided", "less", "greater" indicating the alternative hypothesis
+
+ Usage:
+
+ ```python
+ from cluster_experiments.experiment_analysis import ClusteredOLSAnalysis
+ import pandas as pd
+
+ df = pd.DataFrame({
+ 'x': [1, 2, 3, 0, 0, 1, 2, 0],
+ 'treatment': ["A"] * 2 + ["B"] * 2 + ["A"] * 2 + ["B"] * 2,
+ 'cluster': [1, 1, 2, 2, 3, 3, 4, 4],
+ })
+
+ ClusteredOLSAnalysis(
+ cluster_cols=['cluster'],
+ target_col='x',
+ ).get_pvalue(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ cluster_cols: List[str],
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ covariates: Optional[List[str]] = None,
+ hypothesis: str = "two-sided",
+ ):
+ super().__init__(
+ target_col=target_col,
+ treatment_col=treatment_col,
+ cluster_cols=cluster_cols,
+ treatment=treatment,
+ covariates=covariates,
+ hypothesis=hypothesis,
+ )
+ self.regressors = [self.treatment_col] + self.covariates
+ self.formula = f"{self.target_col} ~ {' + '.join(self.regressors)}"
+ self.cov_type = "cluster"
+
+ def fit_ols_clustered(self, df: pd.DataFrame):
+ """Returns the fitted OLS model"""
+ return sm.OLS.from_formula(self.formula, data=df,).fit(
+ cov_type=self.cov_type,
+ cov_kwds={"groups": self._get_cluster_column(df)},
+ )
+
+ def analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+ if verbose:
+ print(results_ols.summary())
+
+ p_value = self.pvalue_based_on_hypothesis(results_ols)
+ return p_value
+
+ def analysis_point_estimate(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the point estimate of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+ return results_ols.params[self.treatment_col]
+
+ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the standard error of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+ return results_ols.bse[self.treatment_col]
+
+ def analysis_confidence_interval(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+ ) -> ConfidenceInterval:
+ """Returns the confidence interval of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_ols.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_ols.summary())
+
+ # Return the confidence interval
+ return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha)
+
+ def analysis_inference_results(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+ ) -> InferenceResults:
+ """Returns the inference results of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+
+ std_error = results_ols.bse[self.treatment_col]
+ ate = results_ols.params[self.treatment_col]
+ p_value = self.pvalue_based_on_hypothesis(results_ols)
+
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_ols.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_ols.summary())
+
+ # Return the confidence interval
+ return InferenceResults(
+ ate=ate,
+ p_value=p_value,
+ std_error=std_error,
+ conf_int=ConfidenceInterval(
+ lower=lower_bound, upper=upper_bound, alpha=alpha
+ ),
+ )
+analysis_confidence_interval(self, df, alpha, verbose=False)
+
+
+¶Returns the confidence interval of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_confidence_interval(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+) -> ConfidenceInterval:
+ """Returns the confidence interval of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_ols.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_ols.summary())
+
+ # Return the confidence interval
+ return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha)
+analysis_inference_results(self, df, alpha, verbose=False)
+
+
+¶Returns the inference results of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_inference_results(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+) -> InferenceResults:
+ """Returns the inference results of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+
+ std_error = results_ols.bse[self.treatment_col]
+ ate = results_ols.params[self.treatment_col]
+ p_value = self.pvalue_based_on_hypothesis(results_ols)
+
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_ols.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_ols.summary())
+
+ # Return the confidence interval
+ return InferenceResults(
+ ate=ate,
+ p_value=p_value,
+ std_error=std_error,
+ conf_int=ConfidenceInterval(
+ lower=lower_bound, upper=upper_bound, alpha=alpha
+ ),
+ )
+analysis_point_estimate(self, df, verbose=False)
+
+
+¶Returns the point estimate of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_point_estimate(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the point estimate of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+ return results_ols.params[self.treatment_col]
+analysis_pvalue(self, df, verbose=False)
+
+
+¶Returns the p-value of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+ if verbose:
+ print(results_ols.summary())
+
+ p_value = self.pvalue_based_on_hypothesis(results_ols)
+ return p_value
+analysis_standard_error(self, df, verbose=False)
+
+
+¶Returns the standard error of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the standard error of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols_clustered(df)
+ return results_ols.bse[self.treatment_col]
+fit_ols_clustered(self, df)
+
+
+¶Returns the fitted OLS model
+ +cluster_experiments/experiment_analysis.pydef fit_ols_clustered(self, df: pd.DataFrame):
+ """Returns the fitted OLS model"""
+ return sm.OLS.from_formula(self.formula, data=df,).fit(
+ cov_type=self.cov_type,
+ cov_kwds={"groups": self._get_cluster_column(df)},
+ )
+
+ConfidenceInterval
+
+
+
+ dataclass
+
+
+¶Class to define the structure of a confidence interval.
+ +cluster_experiments/experiment_analysis.pyclass ConfidenceInterval:
+ """
+ Class to define the structure of a confidence interval.
+ """
+
+ lower: float
+ upper: float
+ alpha: float
+
+ExperimentAnalysis (ABC)
+
+
+
+
+¶Abstract class to run the analysis of a given experiment
+In order to create your own ExperimentAnalysis, +you should create a derived class that implements the analysis_pvalue method.
+It can also be used as a component of the PowerAnalysis class.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_cols |
+ List[str] |
+ list of columns to use as clusters |
+ required | +
target_col |
+ str |
+ name of the column containing the variable to measure |
+ 'target' |
+
treatment_col |
+ str |
+ name of the column containing the treatment variable |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group |
+ 'B' |
+
covariates |
+ Optional[List[str]] |
+ list of columns to use as covariates |
+ None |
+
hypothesis |
+ str |
+ one of "two-sided", "less", "greater" indicating the alternative hypothesis |
+ 'two-sided' |
+
cluster_experiments/experiment_analysis.pyclass ExperimentAnalysis(ABC):
+ """
+ Abstract class to run the analysis of a given experiment
+
+ In order to create your own ExperimentAnalysis,
+ you should create a derived class that implements the analysis_pvalue method.
+
+ It can also be used as a component of the PowerAnalysis class.
+
+ Arguments:
+ cluster_cols: list of columns to use as clusters
+ target_col: name of the column containing the variable to measure
+ treatment_col: name of the column containing the treatment variable
+ treatment: name of the treatment to use as the treated group
+ covariates: list of columns to use as covariates
+ hypothesis: one of "two-sided", "less", "greater" indicating the alternative hypothesis
+
+ """
+
+ def __init__(
+ self,
+ cluster_cols: List[str],
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ covariates: Optional[List[str]] = None,
+ hypothesis: str = "two-sided",
+ ):
+ self.target_col = target_col
+ self.treatment = treatment
+ self.treatment_col = treatment_col
+ self.cluster_cols = cluster_cols
+ self.covariates = covariates or []
+ self.hypothesis = hypothesis
+
+ def _get_cluster_column(self, df: pd.DataFrame) -> pd.Series:
+ """Paste all strings of cluster_cols in one single column"""
+ df = df.copy()
+ return df[self.cluster_cols].astype(str).sum(axis=1)
+
+ def _create_binary_treatment(self, df: pd.DataFrame) -> pd.DataFrame:
+ """Transforms treatment column into 0 - 1 column"""
+ df = df.copy()
+ df[self.treatment_col] = (df[self.treatment_col] == self.treatment).astype(int)
+ return df
+
+ @abstractmethod
+ def analysis_pvalue(
+ self,
+ df: pd.DataFrame,
+ verbose: bool = False,
+ ) -> float:
+ """
+ Returns the p-value of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+
+ def analysis_point_estimate(
+ self,
+ df: pd.DataFrame,
+ verbose: bool = False,
+ ) -> float:
+ """
+ Returns the point estimate of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ raise NotImplementedError("Point estimate not implemented for this analysis")
+
+ def analysis_standard_error(
+ self,
+ df: pd.DataFrame,
+ verbose: bool = False,
+ ) -> float:
+ """
+ Returns the standard error of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ raise NotImplementedError("Standard error not implemented for this analysis")
+
+ def analysis_confidence_interval(
+ self,
+ df: pd.DataFrame,
+ alpha: float,
+ verbose: bool = False,
+ ) -> ConfidenceInterval:
+ """
+ Returns the confidence interval of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ raise NotImplementedError(
+ "Confidence Interval not implemented for this analysis"
+ )
+
+ def analysis_inference_results(
+ self,
+ df: pd.DataFrame,
+ alpha: float,
+ verbose: bool = False,
+ ) -> InferenceResults:
+ """
+ Returns the InferenceResults object of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ raise NotImplementedError(
+ "Inference results are not implemented for this analysis"
+ )
+
+ def _data_checks(self, df: pd.DataFrame) -> None:
+ """Checks that the data is correct"""
+ if df[self.target_col].isnull().any():
+ raise ValueError(
+ f"There are null values in outcome column {self.treatment_col}"
+ )
+
+ if not is_numeric_dtype(df[self.target_col]):
+ raise ValueError(
+ f"Outcome column {self.target_col} should be numeric and not {df[self.target_col].dtype}"
+ )
+
+ def get_pvalue(self, df: pd.DataFrame) -> float:
+ """Returns the p-value of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_pvalue(df)
+
+ def get_point_estimate(self, df: pd.DataFrame) -> float:
+ """Returns the point estimate of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_point_estimate(df)
+
+ def get_standard_error(self, df: pd.DataFrame) -> float:
+ """Returns the standard error of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_standard_error(df)
+
+ def get_confidence_interval(
+ self, df: pd.DataFrame, alpha: float
+ ) -> ConfidenceInterval:
+ """Returns the confidence interval of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_confidence_interval(df, alpha)
+
+ def get_inference_results(self, df: pd.DataFrame, alpha: float) -> InferenceResults:
+ """Returns the inference results of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_inference_results(df, alpha)
+
+ def pvalue_based_on_hypothesis(
+ self, model_result
+ ) -> float: # todo add typehint statsmodels result
+ """Returns the p-value of the analysis
+ Arguments:
+ model_result: statsmodels result object
+ verbose (Optional): bool, prints the regression summary if True
+
+ """
+ treatment_effect = model_result.params[self.treatment_col]
+ p_value = model_result.pvalues[self.treatment_col]
+
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.LESS:
+ return p_value / 2 if treatment_effect <= 0 else 1 - p_value / 2
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.GREATER:
+ return p_value / 2 if treatment_effect >= 0 else 1 - p_value / 2
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.TWO_SIDED:
+ return p_value
+ raise ValueError(f"{self.hypothesis} is not a valid HypothesisEntries")
+
+ def _split_pre_experiment_df(self, df: pd.DataFrame):
+ raise NotImplementedError(
+ "This method should be implemented in the child class"
+ )
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates an ExperimentAnalysis object from a PowerConfig object"""
+ return cls(
+ cluster_cols=config.cluster_cols,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ covariates=config.covariates,
+ hypothesis=config.hypothesis,
+ )
+analysis_confidence_interval(self, df, alpha, verbose=False)
+
+
+¶Returns the confidence interval of the analysis. Expects treatment to be 0-1 variable
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_confidence_interval(
+ self,
+ df: pd.DataFrame,
+ alpha: float,
+ verbose: bool = False,
+) -> ConfidenceInterval:
+ """
+ Returns the confidence interval of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ raise NotImplementedError(
+ "Confidence Interval not implemented for this analysis"
+ )
+analysis_inference_results(self, df, alpha, verbose=False)
+
+
+¶Returns the InferenceResults object of the analysis. Expects treatment to be 0-1 variable
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_inference_results(
+ self,
+ df: pd.DataFrame,
+ alpha: float,
+ verbose: bool = False,
+) -> InferenceResults:
+ """
+ Returns the InferenceResults object of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ raise NotImplementedError(
+ "Inference results are not implemented for this analysis"
+ )
+analysis_point_estimate(self, df, verbose=False)
+
+
+¶Returns the point estimate of the analysis. Expects treatment to be 0-1 variable
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_point_estimate(
+ self,
+ df: pd.DataFrame,
+ verbose: bool = False,
+) -> float:
+ """
+ Returns the point estimate of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ raise NotImplementedError("Point estimate not implemented for this analysis")
+analysis_pvalue(self, df, verbose=False)
+
+
+¶Returns the p-value of the analysis. Expects treatment to be 0-1 variable
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.py@abstractmethod
+def analysis_pvalue(
+ self,
+ df: pd.DataFrame,
+ verbose: bool = False,
+) -> float:
+ """
+ Returns the p-value of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+analysis_standard_error(self, df, verbose=False)
+
+
+¶Returns the standard error of the analysis. Expects treatment to be 0-1 variable
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_standard_error(
+ self,
+ df: pd.DataFrame,
+ verbose: bool = False,
+) -> float:
+ """
+ Returns the standard error of the analysis. Expects treatment to be 0-1 variable
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ raise NotImplementedError("Standard error not implemented for this analysis")
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates an ExperimentAnalysis object from a PowerConfig object
+ +cluster_experiments/experiment_analysis.py@classmethod
+def from_config(cls, config):
+ """Creates an ExperimentAnalysis object from a PowerConfig object"""
+ return cls(
+ cluster_cols=config.cluster_cols,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ covariates=config.covariates,
+ hypothesis=config.hypothesis,
+ )
+get_confidence_interval(self, df, alpha)
+
+
+¶Returns the confidence interval of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
cluster_experiments/experiment_analysis.pydef get_confidence_interval(
+ self, df: pd.DataFrame, alpha: float
+) -> ConfidenceInterval:
+ """Returns the confidence interval of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_confidence_interval(df, alpha)
+get_inference_results(self, df, alpha)
+
+
+¶Returns the inference results of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
cluster_experiments/experiment_analysis.pydef get_inference_results(self, df: pd.DataFrame, alpha: float) -> InferenceResults:
+ """Returns the inference results of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_inference_results(df, alpha)
+get_point_estimate(self, df)
+
+
+¶Returns the point estimate of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
cluster_experiments/experiment_analysis.pydef get_point_estimate(self, df: pd.DataFrame) -> float:
+ """Returns the point estimate of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_point_estimate(df)
+get_pvalue(self, df)
+
+
+¶Returns the p-value of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
cluster_experiments/experiment_analysis.pydef get_pvalue(self, df: pd.DataFrame) -> float:
+ """Returns the p-value of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_pvalue(df)
+get_standard_error(self, df)
+
+
+¶Returns the standard error of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
cluster_experiments/experiment_analysis.pydef get_standard_error(self, df: pd.DataFrame) -> float:
+ """Returns the standard error of the analysis
+
+ Arguments:
+ df: dataframe containing the data to analyze
+ """
+ df = df.copy()
+ df = self._create_binary_treatment(df)
+ self._data_checks(df=df)
+ return self.analysis_standard_error(df)
+pvalue_based_on_hypothesis(self, model_result)
+
+
+¶Returns the p-value of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
model_result |
+ + | statsmodels result object |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ required | +
cluster_experiments/experiment_analysis.pydef pvalue_based_on_hypothesis(
+ self, model_result
+) -> float: # todo add typehint statsmodels result
+ """Returns the p-value of the analysis
+ Arguments:
+ model_result: statsmodels result object
+ verbose (Optional): bool, prints the regression summary if True
+
+ """
+ treatment_effect = model_result.params[self.treatment_col]
+ p_value = model_result.pvalues[self.treatment_col]
+
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.LESS:
+ return p_value / 2 if treatment_effect <= 0 else 1 - p_value / 2
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.GREATER:
+ return p_value / 2 if treatment_effect >= 0 else 1 - p_value / 2
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.TWO_SIDED:
+ return p_value
+ raise ValueError(f"{self.hypothesis} is not a valid HypothesisEntries")
+
+GeeExperimentAnalysis (ExperimentAnalysis)
+
+
+
+
+¶Class to run GEE clustered analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_cols |
+ List[str] |
+ list of columns to use as clusters |
+ required | +
target_col |
+ str |
+ name of the column containing the variable to measure |
+ 'target' |
+
treatment_col |
+ str |
+ name of the column containing the treatment variable |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group |
+ 'B' |
+
covariates |
+ Optional[List[str]] |
+ list of columns to use as covariates |
+ None |
+
hypothesis |
+ str |
+ one of "two-sided", "less", "greater" indicating the alternative hypothesis |
+ 'two-sided' |
+
Usage:
+from cluster_experiments.experiment_analysis import GeeExperimentAnalysis
+import pandas as pd
+
+df = pd.DataFrame({
+ 'x': [1, 2, 3, 0, 0, 1],
+ 'treatment': ["A"] * 3 + ["B"] * 3,
+ 'cluster': [1] * 6,
+})
+
+GeeExperimentAnalysis(
+ cluster_cols=['cluster'],
+ target_col='x',
+).get_pvalue(df)
+cluster_experiments/experiment_analysis.pyclass GeeExperimentAnalysis(ExperimentAnalysis):
+ """
+ Class to run GEE clustered analysis
+
+ Arguments:
+ cluster_cols: list of columns to use as clusters
+ target_col: name of the column containing the variable to measure
+ treatment_col: name of the column containing the treatment variable
+ treatment: name of the treatment to use as the treated group
+ covariates: list of columns to use as covariates
+ hypothesis: one of "two-sided", "less", "greater" indicating the alternative hypothesis
+
+ Usage:
+
+ ```python
+ from cluster_experiments.experiment_analysis import GeeExperimentAnalysis
+ import pandas as pd
+
+ df = pd.DataFrame({
+ 'x': [1, 2, 3, 0, 0, 1],
+ 'treatment': ["A"] * 3 + ["B"] * 3,
+ 'cluster': [1] * 6,
+ })
+
+ GeeExperimentAnalysis(
+ cluster_cols=['cluster'],
+ target_col='x',
+ ).get_pvalue(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ cluster_cols: List[str],
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ covariates: Optional[List[str]] = None,
+ hypothesis: str = "two-sided",
+ ):
+ super().__init__(
+ target_col=target_col,
+ treatment_col=treatment_col,
+ cluster_cols=cluster_cols,
+ treatment=treatment,
+ covariates=covariates,
+ hypothesis=hypothesis,
+ )
+ self.regressors = [self.treatment_col] + self.covariates
+ self.formula = f"{self.target_col} ~ {' + '.join(self.regressors)}"
+ self.fam = sm.families.Gaussian()
+ self.va = sm.cov_struct.Exchangeable()
+
+ def fit_gee(self, df: pd.DataFrame) -> sm.GEE:
+ """Returns the fitted GEE model"""
+ return sm.GEE.from_formula(
+ self.formula,
+ data=df,
+ groups=self._get_cluster_column(df),
+ family=self.fam,
+ cov_struct=self.va,
+ ).fit()
+
+ def analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+ if verbose:
+ print(results_gee.summary())
+
+ p_value = self.pvalue_based_on_hypothesis(results_gee)
+ return p_value
+
+ def analysis_point_estimate(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the point estimate of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+ return results_gee.params[self.treatment_col]
+
+ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the standard error of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+ return results_gee.bse[self.treatment_col]
+
+ def analysis_confidence_interval(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+ ) -> ConfidenceInterval:
+ """Returns the confidence interval of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_gee.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_gee.summary())
+
+ # Return the confidence interval
+ return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha)
+
+ def analysis_inference_results(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+ ) -> InferenceResults:
+ """Returns the inference results of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+
+ std_error = results_gee.bse[self.treatment_col]
+ ate = results_gee.params[self.treatment_col]
+ p_value = self.pvalue_based_on_hypothesis(results_gee)
+
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_gee.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_gee.summary())
+
+ # Return the confidence interval
+ return InferenceResults(
+ ate=ate,
+ p_value=p_value,
+ std_error=std_error,
+ conf_int=ConfidenceInterval(
+ lower=lower_bound, upper=upper_bound, alpha=alpha
+ ),
+ )
+analysis_confidence_interval(self, df, alpha, verbose=False)
+
+
+¶Returns the confidence interval of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_confidence_interval(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+) -> ConfidenceInterval:
+ """Returns the confidence interval of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_gee.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_gee.summary())
+
+ # Return the confidence interval
+ return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha)
+analysis_inference_results(self, df, alpha, verbose=False)
+
+
+¶Returns the inference results of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_inference_results(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+) -> InferenceResults:
+ """Returns the inference results of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+
+ std_error = results_gee.bse[self.treatment_col]
+ ate = results_gee.params[self.treatment_col]
+ p_value = self.pvalue_based_on_hypothesis(results_gee)
+
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_gee.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_gee.summary())
+
+ # Return the confidence interval
+ return InferenceResults(
+ ate=ate,
+ p_value=p_value,
+ std_error=std_error,
+ conf_int=ConfidenceInterval(
+ lower=lower_bound, upper=upper_bound, alpha=alpha
+ ),
+ )
+analysis_point_estimate(self, df, verbose=False)
+
+
+¶Returns the point estimate of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_point_estimate(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the point estimate of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+ return results_gee.params[self.treatment_col]
+analysis_pvalue(self, df, verbose=False)
+
+
+¶Returns the p-value of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+ if verbose:
+ print(results_gee.summary())
+
+ p_value = self.pvalue_based_on_hypothesis(results_gee)
+ return p_value
+analysis_standard_error(self, df, verbose=False)
+
+
+¶Returns the standard error of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the standard error of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_gee = self.fit_gee(df)
+ return results_gee.bse[self.treatment_col]
+fit_gee(self, df)
+
+
+¶Returns the fitted GEE model
+ +cluster_experiments/experiment_analysis.pydef fit_gee(self, df: pd.DataFrame) -> sm.GEE:
+ """Returns the fitted GEE model"""
+ return sm.GEE.from_formula(
+ self.formula,
+ data=df,
+ groups=self._get_cluster_column(df),
+ family=self.fam,
+ cov_struct=self.va,
+ ).fit()
+
+InferenceResults
+
+
+
+ dataclass
+
+
+¶Class to define the structure of complete statistical analysis results.
+ +cluster_experiments/experiment_analysis.pyclass InferenceResults:
+ """
+ Class to define the structure of complete statistical analysis results.
+ """
+
+ ate: float
+ p_value: float
+ std_error: float
+ conf_int: ConfidenceInterval
+
+MLMExperimentAnalysis (ExperimentAnalysis)
+
+
+
+
+¶Class to run Mixed Linear Models clustered analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_cols |
+ List[str] |
+ list of columns to use as clusters |
+ required | +
target_col |
+ str |
+ name of the column containing the variable to measure |
+ 'target' |
+
treatment_col |
+ str |
+ name of the column containing the treatment variable |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group |
+ 'B' |
+
covariates |
+ Optional[List[str]] |
+ list of columns to use as covariates |
+ None |
+
hypothesis |
+ str |
+ one of "two-sided", "less", "greater" indicating the alternative hypothesis |
+ 'two-sided' |
+
Usage:
+from cluster_experiments.experiment_analysis import MLMExperimentAnalysis
+import pandas as pd
+
+df = pd.DataFrame({
+ 'x': [1, 2, 3, 0, 0, 1],
+ 'treatment': ["A"] * 3 + ["B"] * 3,
+ 'cluster': [1] * 6,
+})
+
+MLMExperimentAnalysis(
+ cluster_cols=['cluster'],
+ target_col='x',
+).get_pvalue(df)
+cluster_experiments/experiment_analysis.pyclass MLMExperimentAnalysis(ExperimentAnalysis):
+ """
+ Class to run Mixed Linear Models clustered analysis
+
+ Arguments:
+ cluster_cols: list of columns to use as clusters
+ target_col: name of the column containing the variable to measure
+ treatment_col: name of the column containing the treatment variable
+ treatment: name of the treatment to use as the treated group
+ covariates: list of columns to use as covariates
+ hypothesis: one of "two-sided", "less", "greater" indicating the alternative hypothesis
+
+ Usage:
+
+ ```python
+ from cluster_experiments.experiment_analysis import MLMExperimentAnalysis
+ import pandas as pd
+
+ df = pd.DataFrame({
+ 'x': [1, 2, 3, 0, 0, 1],
+ 'treatment': ["A"] * 3 + ["B"] * 3,
+ 'cluster': [1] * 6,
+ })
+
+ MLMExperimentAnalysis(
+ cluster_cols=['cluster'],
+ target_col='x',
+ ).get_pvalue(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ cluster_cols: List[str],
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ covariates: Optional[List[str]] = None,
+ hypothesis: str = "two-sided",
+ ):
+ super().__init__(
+ target_col=target_col,
+ treatment_col=treatment_col,
+ cluster_cols=cluster_cols,
+ treatment=treatment,
+ covariates=covariates,
+ hypothesis=hypothesis,
+ )
+ self.regressors = [self.treatment_col] + self.covariates
+ self.formula = f"{self.target_col} ~ {' + '.join(self.regressors)}"
+
+ self.re_formula = None
+ self.vc_formula = None
+
+ def fit_mlm(self, df: pd.DataFrame) -> sm.MixedLM:
+ """Returns the fitted MLM model"""
+ return sm.MixedLM.from_formula(
+ formula=self.formula,
+ data=df,
+ groups=self._get_cluster_column(df),
+ re_formula=self.re_formula,
+ vc_formula=self.vc_formula,
+ ).fit()
+
+ def analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_mlm = self.fit_mlm(df)
+ if verbose:
+ print(results_mlm.summary())
+
+ p_value = self.pvalue_based_on_hypothesis(results_mlm)
+ return p_value
+
+ def analysis_point_estimate(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the point estimate of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_mlm = self.fit_mlm(df)
+ return results_mlm.params[self.treatment_col]
+
+ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the standard error of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_mlm = self.fit_mlm(df)
+ return results_mlm.bse[self.treatment_col]
+analysis_point_estimate(self, df, verbose=False)
+
+
+¶Returns the point estimate of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_point_estimate(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the point estimate of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_mlm = self.fit_mlm(df)
+ return results_mlm.params[self.treatment_col]
+analysis_pvalue(self, df, verbose=False)
+
+
+¶Returns the p-value of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_mlm = self.fit_mlm(df)
+ if verbose:
+ print(results_mlm.summary())
+
+ p_value = self.pvalue_based_on_hypothesis(results_mlm)
+ return p_value
+analysis_standard_error(self, df, verbose=False)
+
+
+¶Returns the standard error of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the standard error of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_mlm = self.fit_mlm(df)
+ return results_mlm.bse[self.treatment_col]
+fit_mlm(self, df)
+
+
+¶Returns the fitted MLM model
+ +cluster_experiments/experiment_analysis.pydef fit_mlm(self, df: pd.DataFrame) -> sm.MixedLM:
+ """Returns the fitted MLM model"""
+ return sm.MixedLM.from_formula(
+ formula=self.formula,
+ data=df,
+ groups=self._get_cluster_column(df),
+ re_formula=self.re_formula,
+ vc_formula=self.vc_formula,
+ ).fit()
+
+OLSAnalysis (ExperimentAnalysis)
+
+
+
+
+¶Class to run OLS analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
target_col |
+ str |
+ name of the column containing the variable to measure |
+ 'target' |
+
treatment_col |
+ str |
+ name of the column containing the treatment variable |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group |
+ 'B' |
+
covariates |
+ Optional[List[str]] |
+ list of columns to use as covariates |
+ None |
+
hypothesis |
+ str |
+ one of "two-sided", "less", "greater" indicating the alternative hypothesis |
+ 'two-sided' |
+
Usage:
+from cluster_experiments.experiment_analysis import OLSAnalysis
+import pandas as pd
+
+df = pd.DataFrame({
+ 'x': [1, 2, 3, 0, 0, 1],
+ 'treatment': ["A"] * 3 + ["B"] * 3,
+})
+
+OLSAnalysis(
+ target_col='x',
+).get_pvalue(df)
+cluster_experiments/experiment_analysis.pyclass OLSAnalysis(ExperimentAnalysis):
+ """
+ Class to run OLS analysis
+
+ Arguments:
+ target_col: name of the column containing the variable to measure
+ treatment_col: name of the column containing the treatment variable
+ treatment: name of the treatment to use as the treated group
+ covariates: list of columns to use as covariates
+ hypothesis: one of "two-sided", "less", "greater" indicating the alternative hypothesis
+
+ Usage:
+
+ ```python
+ from cluster_experiments.experiment_analysis import OLSAnalysis
+ import pandas as pd
+
+ df = pd.DataFrame({
+ 'x': [1, 2, 3, 0, 0, 1],
+ 'treatment': ["A"] * 3 + ["B"] * 3,
+ })
+
+ OLSAnalysis(
+ target_col='x',
+ ).get_pvalue(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ covariates: Optional[List[str]] = None,
+ hypothesis: str = "two-sided",
+ ):
+ self.target_col = target_col
+ self.treatment = treatment
+ self.treatment_col = treatment_col
+ self.covariates = covariates or []
+ self.regressors = [self.treatment_col] + self.covariates
+ self.formula = f"{self.target_col} ~ {' + '.join(self.regressors)}"
+ self.hypothesis = hypothesis
+
+ def fit_ols(self, df: pd.DataFrame):
+ """Returns the fitted OLS model"""
+ return sm.OLS.from_formula(self.formula, data=df).fit()
+
+ def analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df=df)
+ if verbose:
+ print(results_ols.summary())
+
+ p_value = self.pvalue_based_on_hypothesis(results_ols)
+ return p_value
+
+ def analysis_point_estimate(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the point estimate of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df=df)
+ return results_ols.params[self.treatment_col]
+
+ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the standard error of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df=df)
+ return results_ols.bse[self.treatment_col]
+
+ def analysis_confidence_interval(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+ ) -> ConfidenceInterval:
+ """Returns the confidence interval of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df)
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_ols.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_ols.summary())
+
+ # Return the confidence interval
+ return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha)
+
+ def analysis_inference_results(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+ ) -> InferenceResults:
+ """Returns the inference results of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df)
+
+ std_error = results_ols.bse[self.treatment_col]
+ ate = results_ols.params[self.treatment_col]
+ p_value = self.pvalue_based_on_hypothesis(results_ols)
+
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_ols.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_ols.summary())
+
+ # Return the confidence interval
+ return InferenceResults(
+ ate=ate,
+ p_value=p_value,
+ std_error=std_error,
+ conf_int=ConfidenceInterval(
+ lower=lower_bound, upper=upper_bound, alpha=alpha
+ ),
+ )
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates an OLSAnalysis object from a PowerConfig object"""
+ return cls(
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ covariates=config.covariates,
+ hypothesis=config.hypothesis,
+ )
+analysis_confidence_interval(self, df, alpha, verbose=False)
+
+
+¶Returns the confidence interval of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_confidence_interval(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+) -> ConfidenceInterval:
+ """Returns the confidence interval of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df)
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_ols.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_ols.summary())
+
+ # Return the confidence interval
+ return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha)
+analysis_inference_results(self, df, alpha, verbose=False)
+
+
+¶Returns the inference results of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
alpha |
+ float |
+ significance level |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_inference_results(
+ self, df: pd.DataFrame, alpha: float, verbose: bool = False
+) -> InferenceResults:
+ """Returns the inference results of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ alpha: significance level
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df)
+
+ std_error = results_ols.bse[self.treatment_col]
+ ate = results_ols.params[self.treatment_col]
+ p_value = self.pvalue_based_on_hypothesis(results_ols)
+
+ # Extract the confidence interval for the treatment column
+ conf_int_df = results_ols.conf_int(alpha=alpha)
+ lower_bound, upper_bound = conf_int_df.loc[self.treatment_col]
+
+ if verbose:
+ print(results_ols.summary())
+
+ # Return the confidence interval
+ return InferenceResults(
+ ate=ate,
+ p_value=p_value,
+ std_error=std_error,
+ conf_int=ConfidenceInterval(
+ lower=lower_bound, upper=upper_bound, alpha=alpha
+ ),
+ )
+analysis_point_estimate(self, df, verbose=False)
+
+
+¶Returns the point estimate of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_point_estimate(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the point estimate of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df=df)
+ return results_ols.params[self.treatment_col]
+analysis_pvalue(self, df, verbose=False)
+
+
+¶Returns the p-value of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df=df)
+ if verbose:
+ print(results_ols.summary())
+
+ p_value = self.pvalue_based_on_hypothesis(results_ols)
+ return p_value
+analysis_standard_error(self, df, verbose=False)
+
+
+¶Returns the standard error of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the standard error of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+ results_ols = self.fit_ols(df=df)
+ return results_ols.bse[self.treatment_col]
+fit_ols(self, df)
+
+
+¶Returns the fitted OLS model
+ +cluster_experiments/experiment_analysis.pydef fit_ols(self, df: pd.DataFrame):
+ """Returns the fitted OLS model"""
+ return sm.OLS.from_formula(self.formula, data=df).fit()
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates an OLSAnalysis object from a PowerConfig object
+ +cluster_experiments/experiment_analysis.py@classmethod
+def from_config(cls, config):
+ """Creates an OLSAnalysis object from a PowerConfig object"""
+ return cls(
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ covariates=config.covariates,
+ hypothesis=config.hypothesis,
+ )
+
+PairedTTestClusteredAnalysis (ExperimentAnalysis)
+
+
+
+
+¶Class to run paired T-test analysis on aggregated data
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_cols |
+ List[str] |
+ list of columns to use as clusters |
+ required | +
target_col |
+ str |
+ name of the column containing the variable to measure |
+ 'target' |
+
treatment_col |
+ str |
+ name of the column containing the treatment variable |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group |
+ 'B' |
+
strata_cols |
+ List[str] |
+ list of index columns for paired t test. Should be a subset or equal to cluster_cols |
+ required | +
hypothesis |
+ str |
+ one of "two-sided", "less", "greater" indicating the alternative hypothesis |
+ 'two-sided' |
+
Usage:
+from cluster_experiments.experiment_analysis import PairedTTestClusteredAnalysis
+import pandas as pd
+
+df = pd.DataFrame({
+ 'x': [1, 2, 3, 4, 0, 0, 1, 1],
+ 'treatment': ["A", "B", "A", "B"] * 2,
+ 'cluster': [1, 2, 3, 4, 1, 2, 3, 4],
+})
+
+PairedTTestClusteredAnalysis(
+ cluster_cols=['cluster'],
+ strata_cols=['cluster'],
+ target_col='x',
+).get_pvalue(df)
+cluster_experiments/experiment_analysis.pyclass PairedTTestClusteredAnalysis(ExperimentAnalysis):
+ """
+ Class to run paired T-test analysis on aggregated data
+
+ Arguments:
+ cluster_cols: list of columns to use as clusters
+ target_col: name of the column containing the variable to measure
+ treatment_col: name of the column containing the treatment variable
+ treatment: name of the treatment to use as the treated group
+ strata_cols: list of index columns for paired t test. Should be a subset or equal to cluster_cols
+ hypothesis: one of "two-sided", "less", "greater" indicating the alternative hypothesis
+
+ Usage:
+
+ ```python
+ from cluster_experiments.experiment_analysis import PairedTTestClusteredAnalysis
+ import pandas as pd
+
+ df = pd.DataFrame({
+ 'x': [1, 2, 3, 4, 0, 0, 1, 1],
+ 'treatment': ["A", "B", "A", "B"] * 2,
+ 'cluster': [1, 2, 3, 4, 1, 2, 3, 4],
+ })
+
+ PairedTTestClusteredAnalysis(
+ cluster_cols=['cluster'],
+ strata_cols=['cluster'],
+ target_col='x',
+ ).get_pvalue(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ cluster_cols: List[str],
+ strata_cols: List[str],
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ hypothesis: str = "two-sided",
+ ):
+ self.strata_cols = strata_cols
+ self.target_col = target_col
+ self.treatment = treatment
+ self.treatment_col = treatment_col
+ self.cluster_cols = cluster_cols
+ self.hypothesis = hypothesis
+
+ def _preprocessing(self, df: pd.DataFrame, verbose: bool = False) -> pd.DataFrame:
+ df_grouped = df.groupby(
+ self.cluster_cols + [self.treatment_col], as_index=False
+ )[self.target_col].mean()
+
+ n_control = df_grouped[self.treatment_col].value_counts()[0]
+ n_treatment = df_grouped[self.treatment_col].value_counts()[1]
+
+ if n_control != n_treatment:
+ logging.warning(
+ f"groups don't have same number of observations, {n_treatment =} and {n_control =}"
+ )
+
+ assert all(
+ [x in self.cluster_cols for x in self.strata_cols]
+ ), f"strata should be a subset or equal to cluster_cols ({self.cluster_cols = }, {self.strata_cols = })"
+
+ df_pivot = df_grouped.pivot_table(
+ columns=self.treatment_col,
+ index=self.strata_cols,
+ values=self.target_col,
+ )
+
+ if df_pivot.isna().sum().sum() > 0:
+ logging.warning(
+ f"There are missing pairs for some clusters, removing the lonely ones: {df_pivot[df_pivot.isna().any(axis=1)].to_dict()}"
+ )
+
+ if verbose:
+ print(f"performing paired t test in this data \n {df_pivot} \n")
+
+ df_pivot = df_pivot.dropna()
+
+ return df_pivot
+
+ def analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the extra info if True
+ """
+ assert (
+ type(self.cluster_cols) is list
+ ), "cluster_cols needs to be a list of strings (even with one element)"
+ assert (
+ type(self.strata_cols) is list
+ ), "strata_cols needs to be a list of strings (even with one element)"
+
+ df_pivot = self._preprocessing(df=df)
+
+ t_test_results = ttest_rel(
+ df_pivot.iloc[:, 0], df_pivot.iloc[:, 1], alternative=self.hypothesis
+ )
+
+ if verbose:
+ print(f"paired t test results: \n {t_test_results} \n")
+
+ return t_test_results.pvalue
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a PairedTTestClusteredAnalysis object from a PowerConfig object"""
+ return cls(
+ cluster_cols=config.cluster_cols,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ strata_cols=config.strata_cols,
+ hypothesis=config.hypothesis,
+ )
+analysis_pvalue(self, df, verbose=False)
+
+
+¶Returns the p-value of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the extra info if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the extra info if True
+ """
+ assert (
+ type(self.cluster_cols) is list
+ ), "cluster_cols needs to be a list of strings (even with one element)"
+ assert (
+ type(self.strata_cols) is list
+ ), "strata_cols needs to be a list of strings (even with one element)"
+
+ df_pivot = self._preprocessing(df=df)
+
+ t_test_results = ttest_rel(
+ df_pivot.iloc[:, 0], df_pivot.iloc[:, 1], alternative=self.hypothesis
+ )
+
+ if verbose:
+ print(f"paired t test results: \n {t_test_results} \n")
+
+ return t_test_results.pvalue
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a PairedTTestClusteredAnalysis object from a PowerConfig object
+ +cluster_experiments/experiment_analysis.py@classmethod
+def from_config(cls, config):
+ """Creates a PairedTTestClusteredAnalysis object from a PowerConfig object"""
+ return cls(
+ cluster_cols=config.cluster_cols,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ strata_cols=config.strata_cols,
+ hypothesis=config.hypothesis,
+ )
+
+SyntheticControlAnalysis (ExperimentAnalysis)
+
+
+
+
+¶Class to run Synthetic control analysis. It expects only one treatment cluster.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
target_col |
+ str |
+ The name of the column containing the variable to measure. |
+ 'target' |
+
treatment_col |
+ str |
+ The name of the column containing the treatment variable. |
+ 'treatment' |
+
treatment |
+ str |
+ The name of the treatment to use as the treated group. |
+ 'B' |
+
cluster_cols |
+ list |
+ A list of columns to use as clusters. |
+ required | +
hypothesis |
+ str |
+ One of "two-sided", "less", "greater" indicating the hypothesis. |
+ 'two-sided' |
+
time_col |
+ str |
+ The name of the column containing the time data. |
+ 'date' |
+
intervention_date |
+ str |
+ The date when the intervention occurred. |
+ required | +
Usage:
+from cluster_experiments.experiment_analysis import SyntheticControlAnalysis
+import pandas as pd
+import numpy as np
+from itertools import product
+
+dates = pd.date_range("2022-01-01", "2022-01-31", freq="d")
+
+users = [f"User {i}" for i in range(10)]
+
+# Create a combination of each date with each user
+combinations = list(product(users, dates))
+
+target_values = np.random.normal(0, 1, size=len(combinations))
+
+df = pd.DataFrame(combinations, columns=["user", "date"])
+df["target"] = target_values
+
+df["treatment"] = "A"
+df.loc[(df["user"] == "User 5"), "treatment"] = "B"
+
+SyntheticControlAnalysis(
+ cluster_cols=["user"], time_col="date", intervention_date="2022-01-15"
+).get_pvalue(df)
+cluster_experiments/experiment_analysis.pyclass SyntheticControlAnalysis(ExperimentAnalysis):
+ """
+ Class to run Synthetic control analysis. It expects only one treatment cluster.
+
+ Arguments:
+
+ target_col (str): The name of the column containing the variable to measure.
+ treatment_col (str): The name of the column containing the treatment variable.
+ treatment (str): The name of the treatment to use as the treated group.
+ cluster_cols (list): A list of columns to use as clusters.
+ hypothesis (str): One of "two-sided", "less", "greater" indicating the hypothesis.
+ time_col (str): The name of the column containing the time data.
+ intervention_date (str): The date when the intervention occurred.
+ Usage:
+
+ ```python
+ from cluster_experiments.experiment_analysis import SyntheticControlAnalysis
+ import pandas as pd
+ import numpy as np
+ from itertools import product
+
+ dates = pd.date_range("2022-01-01", "2022-01-31", freq="d")
+
+ users = [f"User {i}" for i in range(10)]
+
+ # Create a combination of each date with each user
+ combinations = list(product(users, dates))
+
+ target_values = np.random.normal(0, 1, size=len(combinations))
+
+ df = pd.DataFrame(combinations, columns=["user", "date"])
+ df["target"] = target_values
+
+ df["treatment"] = "A"
+ df.loc[(df["user"] == "User 5"), "treatment"] = "B"
+
+ SyntheticControlAnalysis(
+ cluster_cols=["user"], time_col="date", intervention_date="2022-01-15"
+ ).get_pvalue(df)
+
+ ```
+ """
+
+ def __init__(
+ self,
+ intervention_date: str,
+ cluster_cols: List[str],
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ hypothesis: str = "two-sided",
+ time_col: str = "date",
+ ):
+ super().__init__(
+ treatment=treatment,
+ target_col=target_col,
+ treatment_col=treatment_col,
+ hypothesis=hypothesis,
+ cluster_cols=cluster_cols,
+ )
+
+ self.time_col = time_col
+ self.intervention_date = intervention_date
+
+ if time_col in cluster_cols:
+ raise ValueError("time columns should not be in cluster columns")
+
+ def _fit(self, pre_experiment_df: pd.DataFrame, verbose: bool) -> np.ndarray:
+ """Returns the weight of each donor"""
+
+ if not any(pre_experiment_df[self.treatment_col] == 1):
+ raise ValueError("No treatment unit found in the data.")
+
+ X = (
+ pre_experiment_df.query(f"{self.treatment_col} == 0")
+ .pivot(index=self.cluster_cols, columns=self.time_col)[self.target_col]
+ .T
+ )
+
+ y = (
+ pre_experiment_df.query(f"{self.treatment_col} == 1")
+ .pivot(index=self.cluster_cols, columns=self.time_col)[self.target_col]
+ .T.iloc[:, 0]
+ )
+
+ weights = get_w(X, y, verbose)
+
+ return weights
+
+ def _predict(
+ self, df: pd.DataFrame, weights: np.ndarray, treatment_cluster: str
+ ) -> pd.DataFrame:
+ """
+ This method adds a column with the synthetic results and filter only the treatment unit.
+
+ First, it calculates the weights of each donor in the control group using the `fit_synthetic` method.
+ It then uses these weights to create a synthetic control group that closely matches the treatment unit before the intervention.
+ The synthetic control group is added to the treatment unit in the dataframe.
+ """
+ synthetic = (
+ df[self._get_cluster_column(df) != treatment_cluster]
+ .pivot(index=self.time_col, columns=self.cluster_cols)[self.target_col]
+ .values.dot(weights)
+ )
+
+ # add synthetic to treatment cluster
+ return df[self._get_cluster_column(df) == treatment_cluster].assign(
+ synthetic=synthetic
+ )
+
+ def fit_predict_synthetic(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: pd.DataFrame,
+ treatment_cluster: str,
+ verbose: bool = False,
+ ) -> pd.DataFrame:
+ """
+ Fit the synthetic control model and predict the results for the treatment cluster.
+ Args:
+ df: The dataframe containing the data after the intervention.
+ pre_experiment_df: The dataframe containing the data before the intervention.
+ treatment_cluster: The name of the treatment cluster.
+ verbose: If True, print the status of the optimization of weights.
+
+ Returns:
+ The dataframe with the synthetic results added to the treatment cluster.
+ """
+ weights = self._fit(pre_experiment_df=pre_experiment_df, verbose=verbose)
+
+ prediction = self._predict(
+ df=df, weights=weights, treatment_cluster=treatment_cluster
+ )
+ return prediction
+
+ def pvalue_based_on_hypothesis(
+ self, ate: np.float64, avg_effects: Dict[str, float]
+ ) -> float:
+ """
+ Returns the p-value of the analysis.
+ 1. Count how many times the average effect is greater than the real treatment unit
+ 2. Average it with the number of units. The result is the p-value using Fisher permutation exact test.
+ """
+
+ avg_effects = list(avg_effects.values())
+
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.LESS:
+ return np.mean(avg_effects < ate)
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.GREATER:
+ return np.mean(avg_effects > ate)
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.TWO_SIDED:
+ avg_effects = np.abs(avg_effects)
+ return np.mean(avg_effects > ate)
+
+ raise ValueError(f"{self.hypothesis} is not a valid HypothesisEntries")
+
+ def _get_treatment_cluster(self, df: pd.DataFrame) -> str:
+ """Returns the first treatment cluster. The current implementation of Synthetic Control only accepts one treatment cluster.
+ This will be left inside Synthetic class because it doesn't apply for other analyses"""
+ treatment_df = df[df[self.treatment_col] == 1]
+ treatment_cluster = self._get_cluster_column(treatment_df).unique()[0]
+ return treatment_cluster
+
+ def analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """
+ Returns the p-value of the analysis.
+ 1. Calculate the average effect after intervention for each unit.
+ 2. Count how many times the average effect is greater than the real treatment unit
+ 3. Average it with the number of units. The result is the p-value using Fisher permutation test
+ """
+
+ clusters = self._get_cluster_column(df).unique()
+ treatment_cluster = self._get_treatment_cluster(df)
+
+ synthetic_donors = {
+ cluster: self.analysis_point_estimate(
+ treatment_cluster=cluster,
+ df=df,
+ verbose=verbose,
+ )
+ for cluster in clusters
+ }
+
+ ate = synthetic_donors[treatment_cluster]
+ synthetic_donors.pop(treatment_cluster)
+
+ return self.pvalue_based_on_hypothesis(ate=ate, avg_effects=synthetic_donors)
+
+ def analysis_point_estimate(
+ self,
+ df: pd.DataFrame,
+ treatment_cluster: Optional[str] = None,
+ verbose: bool = False,
+ ):
+ """
+ Calculate the point estimate for the treatment effect for a specified cluster by averaging across the time windows.
+ """
+ df, pre_experiment_df = self._split_pre_experiment_df(df)
+
+ if treatment_cluster is None:
+ treatment_cluster = self._get_treatment_cluster(df)
+
+ df = self.fit_predict_synthetic(
+ df, pre_experiment_df, treatment_cluster, verbose=verbose
+ )
+
+ df["effect"] = df[self.target_col] - df["synthetic"]
+ avg_effect = df["effect"].mean()
+ return avg_effect
+
+ def _split_pre_experiment_df(self, df: pd.DataFrame):
+ """Split the dataframe into pre-experiment and experiment dataframes"""
+ pre_experiment_df = df[(df[self.time_col] <= self.intervention_date)]
+ df = df[(df[self.time_col] > self.intervention_date)]
+ return df, pre_experiment_df
+analysis_point_estimate(self, df, treatment_cluster=None, verbose=False)
+
+
+¶Calculate the point estimate for the treatment effect for a specified cluster by averaging across the time windows.
+ +cluster_experiments/experiment_analysis.pydef analysis_point_estimate(
+ self,
+ df: pd.DataFrame,
+ treatment_cluster: Optional[str] = None,
+ verbose: bool = False,
+):
+ """
+ Calculate the point estimate for the treatment effect for a specified cluster by averaging across the time windows.
+ """
+ df, pre_experiment_df = self._split_pre_experiment_df(df)
+
+ if treatment_cluster is None:
+ treatment_cluster = self._get_treatment_cluster(df)
+
+ df = self.fit_predict_synthetic(
+ df, pre_experiment_df, treatment_cluster, verbose=verbose
+ )
+
+ df["effect"] = df[self.target_col] - df["synthetic"]
+ avg_effect = df["effect"].mean()
+ return avg_effect
+analysis_pvalue(self, df, verbose=False)
+
+
+¶Returns the p-value of the analysis. +1. Calculate the average effect after intervention for each unit. +2. Count how many times the average effect is greater than the real treatment unit +3. Average it with the number of units. The result is the p-value using Fisher permutation test
+ +cluster_experiments/experiment_analysis.pydef analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """
+ Returns the p-value of the analysis.
+ 1. Calculate the average effect after intervention for each unit.
+ 2. Count how many times the average effect is greater than the real treatment unit
+ 3. Average it with the number of units. The result is the p-value using Fisher permutation test
+ """
+
+ clusters = self._get_cluster_column(df).unique()
+ treatment_cluster = self._get_treatment_cluster(df)
+
+ synthetic_donors = {
+ cluster: self.analysis_point_estimate(
+ treatment_cluster=cluster,
+ df=df,
+ verbose=verbose,
+ )
+ for cluster in clusters
+ }
+
+ ate = synthetic_donors[treatment_cluster]
+ synthetic_donors.pop(treatment_cluster)
+
+ return self.pvalue_based_on_hypothesis(ate=ate, avg_effects=synthetic_donors)
+fit_predict_synthetic(self, df, pre_experiment_df, treatment_cluster, verbose=False)
+
+
+¶Fit the synthetic control model and predict the results for the treatment cluster.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ The dataframe containing the data after the intervention. |
+ required | +
pre_experiment_df |
+ DataFrame |
+ The dataframe containing the data before the intervention. |
+ required | +
treatment_cluster |
+ str |
+ The name of the treatment cluster. |
+ required | +
verbose |
+ bool |
+ If True, print the status of the optimization of weights. |
+ False |
+
Returns:
+| Type | +Description | +
|---|---|
DataFrame |
+ The dataframe with the synthetic results added to the treatment cluster. |
+
cluster_experiments/experiment_analysis.pydef fit_predict_synthetic(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: pd.DataFrame,
+ treatment_cluster: str,
+ verbose: bool = False,
+) -> pd.DataFrame:
+ """
+ Fit the synthetic control model and predict the results for the treatment cluster.
+ Args:
+ df: The dataframe containing the data after the intervention.
+ pre_experiment_df: The dataframe containing the data before the intervention.
+ treatment_cluster: The name of the treatment cluster.
+ verbose: If True, print the status of the optimization of weights.
+
+ Returns:
+ The dataframe with the synthetic results added to the treatment cluster.
+ """
+ weights = self._fit(pre_experiment_df=pre_experiment_df, verbose=verbose)
+
+ prediction = self._predict(
+ df=df, weights=weights, treatment_cluster=treatment_cluster
+ )
+ return prediction
+pvalue_based_on_hypothesis(self, ate, avg_effects)
+
+
+¶Returns the p-value of the analysis. +1. Count how many times the average effect is greater than the real treatment unit +2. Average it with the number of units. The result is the p-value using Fisher permutation exact test.
+ +cluster_experiments/experiment_analysis.pydef pvalue_based_on_hypothesis(
+ self, ate: np.float64, avg_effects: Dict[str, float]
+) -> float:
+ """
+ Returns the p-value of the analysis.
+ 1. Count how many times the average effect is greater than the real treatment unit
+ 2. Average it with the number of units. The result is the p-value using Fisher permutation exact test.
+ """
+
+ avg_effects = list(avg_effects.values())
+
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.LESS:
+ return np.mean(avg_effects < ate)
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.GREATER:
+ return np.mean(avg_effects > ate)
+ if HypothesisEntries(self.hypothesis) == HypothesisEntries.TWO_SIDED:
+ avg_effects = np.abs(avg_effects)
+ return np.mean(avg_effects > ate)
+
+ raise ValueError(f"{self.hypothesis} is not a valid HypothesisEntries")
+
+TTestClusteredAnalysis (ExperimentAnalysis)
+
+
+
+
+¶Class to run T-test analysis on aggregated data
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_cols |
+ List[str] |
+ list of columns to use as clusters |
+ required | +
target_col |
+ str |
+ name of the column containing the variable to measure |
+ 'target' |
+
treatment_col |
+ str |
+ name of the column containing the treatment variable |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group |
+ 'B' |
+
hypothesis |
+ str |
+ one of "two-sided", "less", "greater" indicating the alternative hypothesis |
+ 'two-sided' |
+
Usage:
+from cluster_experiments.experiment_analysis import TTestClusteredAnalysis
+import pandas as pd
+
+df = pd.DataFrame({
+ 'x': [1, 2, 3, 4, 0, 0, 1, 1],
+ 'treatment': ["A", "B", "A", "B"] * 2,
+ 'cluster': [1, 2, 3, 4, 1, 2, 3, 4],
+})
+
+TTestClusteredAnalysis(
+ cluster_cols=['cluster'],
+ target_col='x',
+).get_pvalue(df)
+cluster_experiments/experiment_analysis.pyclass TTestClusteredAnalysis(ExperimentAnalysis):
+ """
+ Class to run T-test analysis on aggregated data
+
+ Arguments:
+ cluster_cols: list of columns to use as clusters
+ target_col: name of the column containing the variable to measure
+ treatment_col: name of the column containing the treatment variable
+ treatment: name of the treatment to use as the treated group
+ hypothesis: one of "two-sided", "less", "greater" indicating the alternative hypothesis
+
+ Usage:
+
+ ```python
+ from cluster_experiments.experiment_analysis import TTestClusteredAnalysis
+ import pandas as pd
+
+ df = pd.DataFrame({
+ 'x': [1, 2, 3, 4, 0, 0, 1, 1],
+ 'treatment': ["A", "B", "A", "B"] * 2,
+ 'cluster': [1, 2, 3, 4, 1, 2, 3, 4],
+ })
+
+ TTestClusteredAnalysis(
+ cluster_cols=['cluster'],
+ target_col='x',
+ ).get_pvalue(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ cluster_cols: List[str],
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ hypothesis: str = "two-sided",
+ ):
+ self.target_col = target_col
+ self.treatment = treatment
+ self.treatment_col = treatment_col
+ self.cluster_cols = cluster_cols
+ self.hypothesis = hypothesis
+
+ def analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+
+ df_grouped = df.groupby(
+ self.cluster_cols + [self.treatment_col], as_index=False
+ )[self.target_col].mean()
+
+ treatment_data = df_grouped.query(f"{self.treatment_col} == 1")[self.target_col]
+ control_data = df_grouped.query(f"{self.treatment_col} == 0")[self.target_col]
+ assert len(treatment_data), "treatment data should have more than 1 cluster"
+ assert len(control_data), "control data should have more than 1 cluster"
+ t_test_results = ttest_ind(
+ treatment_data, control_data, equal_var=False, alternative=self.hypothesis
+ )
+ return t_test_results.pvalue
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a TTestClusteredAnalysis object from a PowerConfig object"""
+ return cls(
+ cluster_cols=config.cluster_cols,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ hypothesis=config.hypothesis,
+ )
+analysis_pvalue(self, df, verbose=False)
+
+
+¶Returns the p-value of the analysis
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe containing the data to analyze |
+ required | +
verbose |
+ Optional |
+ bool, prints the regression summary if True |
+ False |
+
cluster_experiments/experiment_analysis.pydef analysis_pvalue(self, df: pd.DataFrame, verbose: bool = False) -> float:
+ """Returns the p-value of the analysis
+ Arguments:
+ df: dataframe containing the data to analyze
+ verbose (Optional): bool, prints the regression summary if True
+ """
+
+ df_grouped = df.groupby(
+ self.cluster_cols + [self.treatment_col], as_index=False
+ )[self.target_col].mean()
+
+ treatment_data = df_grouped.query(f"{self.treatment_col} == 1")[self.target_col]
+ control_data = df_grouped.query(f"{self.treatment_col} == 0")[self.target_col]
+ assert len(treatment_data), "treatment data should have more than 1 cluster"
+ assert len(control_data), "control data should have more than 1 cluster"
+ t_test_results = ttest_ind(
+ treatment_data, control_data, equal_var=False, alternative=self.hypothesis
+ )
+ return t_test_results.pvalue
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a TTestClusteredAnalysis object from a PowerConfig object
+ +cluster_experiments/experiment_analysis.py@classmethod
+def from_config(cls, config):
+ """Creates a TTestClusteredAnalysis object from a PowerConfig object"""
+ return cls(
+ cluster_cols=config.cluster_cols,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ hypothesis=config.hypothesis,
+ )
+from cluster_experiments.inference.hypothesis_test import *¶
+HypothesisTest
+
+
+
+¶A class used to represent a Hypothesis Test with a metric, analysis, optional analysis configuration, and optional dimensions.
+metric : Metric + An instance of the Metric class +analysis_type : str + string mapping to an ExperimentAnalysis class. Must be either in the built-in analysis_mapping or in the custom_analysis_type_mapper if provided. +analysis_config : Optional[dict] + An optional dictionary representing the configuration for the analysis +dimensions : Optional[List[Dimension]] + An optional list of Dimension instances +cupac_config : Optional[dict] + An optional dictionary representing the configuration for the cupac model +custom_analysis_type_mapper : Optional[Dict[str, ExperimentAnalysis]] + An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes
+ +cluster_experiments/inference/hypothesis_test.pyclass HypothesisTest:
+ """
+ A class used to represent a Hypothesis Test with a metric, analysis, optional analysis configuration, and optional dimensions.
+
+ Attributes
+ ----------
+ metric : Metric
+ An instance of the Metric class
+ analysis_type : str
+ string mapping to an ExperimentAnalysis class. Must be either in the built-in analysis_mapping or in the custom_analysis_type_mapper if provided.
+ analysis_config : Optional[dict]
+ An optional dictionary representing the configuration for the analysis
+ dimensions : Optional[List[Dimension]]
+ An optional list of Dimension instances
+ cupac_config : Optional[dict]
+ An optional dictionary representing the configuration for the cupac model
+ custom_analysis_type_mapper : Optional[Dict[str, ExperimentAnalysis]]
+ An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes
+ """
+
+ def __init__(
+ self,
+ metric: Metric,
+ analysis_type: str,
+ analysis_config: Optional[dict] = None,
+ dimensions: Optional[List[Dimension]] = None,
+ cupac_config: Optional[dict] = None,
+ custom_analysis_type_mapper: Optional[Dict[str, ExperimentAnalysis]] = None,
+ ):
+ """
+ Parameters
+ ----------
+ metric : Metric
+ An instance of the Metric class
+ analysis_type : str
+ string mapping to an ExperimentAnalysis class. Must be either in the built-in analysis_mapping or in the custom_analysis_type_mapper if provided.
+ analysis_config : Optional[dict]
+ An optional dictionary representing the configuration for the analysis
+ dimensions : Optional[List[Dimension]]
+ An optional list of Dimension instances
+ cupac_config : Optional[dict]
+ An optional dictionary representing the configuration for the cupac model
+ custom_analysis_type_mapper : Optional[Dict[str, ExperimentAnalysis]]
+ An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes
+ """
+ self._validate_inputs(
+ metric,
+ analysis_type,
+ analysis_config,
+ dimensions,
+ cupac_config,
+ custom_analysis_type_mapper,
+ )
+ self.metric = metric
+ self.analysis_type = analysis_type
+ self.analysis_config = analysis_config or {}
+ self.dimensions = [DefaultDimension()] + (dimensions or [])
+ self.cupac_config = cupac_config or {}
+ self.custom_analysis_type_mapper = custom_analysis_type_mapper or {}
+
+ self.analysis_type_mapper = self.custom_analysis_type_mapper or analysis_mapping
+ self.analysis_class = self.analysis_type_mapper[self.analysis_type]
+ self.is_cupac = bool(cupac_config)
+ self.cupac_handler = (
+ CupacHandler(**self.cupac_config) if self.is_cupac else None
+ )
+ self.cupac_covariate_col = (
+ self.cupac_handler.cupac_outcome_name if self.is_cupac else None
+ )
+
+ self.new_analysis_config = None
+ self.experiment_analysis = None
+
+ @staticmethod
+ def _validate_inputs(
+ metric: Metric,
+ analysis_type: str,
+ analysis_config: Optional[dict],
+ dimensions: Optional[List[Dimension]],
+ cupac_config: Optional[dict] = None,
+ custom_analysis_type_mapper: Optional[Dict[str, ExperimentAnalysis]] = None,
+ ):
+ """
+ Validates the inputs for the HypothesisTest class.
+
+ Parameters
+ ----------
+ metric : Metric
+ An instance of the Metric class
+ analysis_type : str
+ string mapper to an ExperimentAnalysis
+ analysis_config : Optional[dict]
+ An optional dictionary representing the configuration for the analysis
+ dimensions : Optional[List[Dimension]]
+ An optional list of Dimension instances
+ cupac_config : Optional[dict]
+ An optional dictionary representing the configuration for the cupac model
+ custom_analysis_type_mapper : Optional[dict[str, ExperimentAnalysis]]
+ An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes
+ """
+ # Check if metric is a valid Metric instance
+ if not isinstance(metric, Metric):
+ raise TypeError("Metric must be an instance of Metric")
+
+ # Check if analysis_type is a string
+ if not isinstance(analysis_type, str):
+ raise TypeError("Analysis must be a string")
+
+ # Check if analysis_config is a dictionary when provided
+ if analysis_config is not None and not isinstance(analysis_config, dict):
+ raise TypeError("analysis_config must be a dictionary if provided")
+
+ # Check if cupac_config is a dictionary when provided
+ if cupac_config is not None and not isinstance(cupac_config, dict):
+ raise TypeError("cupac_config must be a dictionary if provided")
+
+ # Check if dimensions is a list of Dimension instances when provided
+ if dimensions is not None and (
+ not isinstance(dimensions, list)
+ or not all(isinstance(dim, Dimension) for dim in dimensions)
+ ):
+ raise TypeError(
+ "Dimensions must be a list of Dimension instances if provided"
+ )
+
+ # Validate custom_analysis_type_mapper if provided
+ if custom_analysis_type_mapper:
+ # Ensure it's a dictionary
+ if not isinstance(custom_analysis_type_mapper, dict):
+ raise TypeError(
+ "custom_analysis_type_mapper must be a dictionary if provided"
+ )
+
+ # Ensure all keys are strings and values are ExperimentAnalysis classes
+ for key, value in custom_analysis_type_mapper.items():
+ if not isinstance(key, str):
+ raise TypeError(
+ f"Key '{key}' in custom_analysis_type_mapper must be a string"
+ )
+ if not issubclass(value, ExperimentAnalysis):
+ raise TypeError(
+ f"Value '{value}' for key '{key}' in custom_analysis_type_mapper must be a subclass of ExperimentAnalysis"
+ )
+
+ # Ensure the analysis_type is in the custom mapper if a custom mapper is provided
+ if analysis_type not in custom_analysis_type_mapper:
+ raise ValueError(
+ f"Analysis type '{analysis_type}' not found in the provided custom_analysis_type_mapper"
+ )
+
+ # If no custom_analysis_type_mapper, check if analysis_type exists in the default mapping
+ elif analysis_type not in analysis_mapping:
+ raise ValueError(
+ f"Analysis type '{analysis_type}' not found in analysis_mapping"
+ )
+
+ def get_inference_results(self, df: pd.DataFrame, alpha: float) -> InferenceResults:
+ """
+ Performs inference analysis on the provided DataFrame using the analysis class.
+
+ Parameters
+ ----------
+ df : pd.DataFrame
+ The dataframe containing the data for analysis.
+ alpha : float
+ The significance level to be used in the inference analysis.
+
+ Returns
+ -------
+ InferenceResults
+ The results containing the statistics of the inference procedure.
+ """
+
+ self.experiment_analysis = self.analysis_class(**self.new_analysis_config)
+ inference_results = self.experiment_analysis.get_inference_results(
+ df=df, alpha=alpha
+ )
+
+ return inference_results
+
+ def _prepare_analysis_config(
+ self, target_col: str, treatment_col: str, treatment: str
+ ) -> None:
+ """
+ Extends the analysis_config provided by the user, by adding or overriding the following keys:
+ - target_col
+ - treatment_col
+ - treatment
+
+ Also handles cupac covariate.
+
+ Returns
+ -------
+ dict
+ The prepared analysis configuration, ready to be ingested by the experiment analysis class
+ """
+ new_analysis_config = copy.deepcopy(self.analysis_config)
+
+ new_analysis_config["target_col"] = target_col
+ new_analysis_config["treatment_col"] = treatment_col
+ new_analysis_config["treatment"] = treatment
+
+ covariates = new_analysis_config.get("covariates", [])
+
+ if self.cupac_covariate_col and self.cupac_covariate_col not in covariates:
+ raise ValueError(
+ f"You provided a cupac configuration but did not provide the cupac covariate called {self.cupac_covariate_col} in the analysis_config"
+ )
+
+ self.new_analysis_config = new_analysis_config
+
+ @staticmethod
+ def prepare_data(
+ data: pd.DataFrame,
+ variant_col: str,
+ treatment_variant: Variant,
+ control_variant: Variant,
+ dimension_name: str,
+ dimension_value: str,
+ ) -> pd.DataFrame:
+ """
+ Prepares the data for the experiment analysis pipeline
+ """
+ prepared_df = data.copy()
+
+ prepared_df = prepared_df.assign(__total_dimension="total")
+
+ prepared_df = prepared_df.query(
+ f"{variant_col}.isin(['{treatment_variant.name}','{control_variant.name}'])"
+ ).query(f"{dimension_name} == '{dimension_value}'")
+
+ return prepared_df
+
+ def add_covariates(
+ self, exp_data: pd.DataFrame, pre_exp_data: pd.DataFrame
+ ) -> pd.DataFrame:
+ """
+ If the test is a cupac test, adds the covariates to the experimental data.
+ """
+ if self.is_cupac:
+ exp_data = self.cupac_handler.add_covariates(
+ df=exp_data, pre_experiment_df=pre_exp_data
+ )
+
+ return exp_data
+
+ def get_test_results(
+ self,
+ control_variant: Variant,
+ treatment_variant: Variant,
+ variant_col: str,
+ exp_data: pd.DataFrame,
+ dimension: Dimension,
+ dimension_value: str,
+ alpha: float,
+ ) -> AnalysisPlanResults:
+ """
+ Performs the hypothesis test on the provided data, for the given dimension value.
+
+ Parameters
+ ----------
+ control_variant : Variant
+ The control variant
+ treatment_variant : Variant
+ The treatment variant
+ variant_col : str
+ The column name representing the variant
+ exp_data : pd.DataFrame
+ The dataframe containing the data for analysis.
+ dimension : Dimension
+ The dimension instance
+ dimension_value : str
+ The value of the dimension
+ alpha : float
+ The significance level to be used in the inference analysis.
+
+ Returns
+ -------
+ AnalysisPlanResults
+ The results of the hypothesis test
+ """
+ self._prepare_analysis_config(
+ target_col=self.metric.target_column,
+ treatment_col=variant_col,
+ treatment=treatment_variant.name,
+ )
+
+ prepared_df = self.prepare_data(
+ data=exp_data,
+ variant_col=variant_col,
+ treatment_variant=treatment_variant,
+ control_variant=control_variant,
+ dimension_name=dimension.name,
+ dimension_value=dimension_value,
+ )
+
+ inference_results = self.get_inference_results(df=prepared_df, alpha=alpha)
+
+ control_variant_mean = self.metric.get_mean(
+ prepared_df.query(f"{variant_col}=='{control_variant.name}'")
+ )
+ treatment_variant_mean = self.metric.get_mean(
+ prepared_df.query(f"{variant_col}=='{treatment_variant.name}'")
+ )
+
+ test_results = AnalysisPlanResults(
+ metric_alias=[self.metric.alias],
+ control_variant_name=[control_variant.name],
+ treatment_variant_name=[treatment_variant.name],
+ control_variant_mean=[control_variant_mean],
+ treatment_variant_mean=[treatment_variant_mean],
+ analysis_type=[self.analysis_type],
+ ate=[inference_results.ate],
+ ate_ci_lower=[inference_results.conf_int.lower],
+ ate_ci_upper=[inference_results.conf_int.upper],
+ p_value=[inference_results.p_value],
+ std_error=[inference_results.std_error],
+ dimension_name=[dimension.name],
+ dimension_value=[dimension_value],
+ alpha=[alpha],
+ )
+
+ return test_results
+__init__(self, metric, analysis_type, analysis_config=None, dimensions=None, cupac_config=None, custom_analysis_type_mapper=None)
+
+
+ special
+
+
+¶metric : Metric + An instance of the Metric class +analysis_type : str + string mapping to an ExperimentAnalysis class. Must be either in the built-in analysis_mapping or in the custom_analysis_type_mapper if provided. +analysis_config : Optional[dict] + An optional dictionary representing the configuration for the analysis +dimensions : Optional[List[Dimension]] + An optional list of Dimension instances +cupac_config : Optional[dict] + An optional dictionary representing the configuration for the cupac model +custom_analysis_type_mapper : Optional[Dict[str, ExperimentAnalysis]] + An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes
+ +cluster_experiments/inference/hypothesis_test.pydef __init__(
+ self,
+ metric: Metric,
+ analysis_type: str,
+ analysis_config: Optional[dict] = None,
+ dimensions: Optional[List[Dimension]] = None,
+ cupac_config: Optional[dict] = None,
+ custom_analysis_type_mapper: Optional[Dict[str, ExperimentAnalysis]] = None,
+):
+ """
+ Parameters
+ ----------
+ metric : Metric
+ An instance of the Metric class
+ analysis_type : str
+ string mapping to an ExperimentAnalysis class. Must be either in the built-in analysis_mapping or in the custom_analysis_type_mapper if provided.
+ analysis_config : Optional[dict]
+ An optional dictionary representing the configuration for the analysis
+ dimensions : Optional[List[Dimension]]
+ An optional list of Dimension instances
+ cupac_config : Optional[dict]
+ An optional dictionary representing the configuration for the cupac model
+ custom_analysis_type_mapper : Optional[Dict[str, ExperimentAnalysis]]
+ An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes
+ """
+ self._validate_inputs(
+ metric,
+ analysis_type,
+ analysis_config,
+ dimensions,
+ cupac_config,
+ custom_analysis_type_mapper,
+ )
+ self.metric = metric
+ self.analysis_type = analysis_type
+ self.analysis_config = analysis_config or {}
+ self.dimensions = [DefaultDimension()] + (dimensions or [])
+ self.cupac_config = cupac_config or {}
+ self.custom_analysis_type_mapper = custom_analysis_type_mapper or {}
+
+ self.analysis_type_mapper = self.custom_analysis_type_mapper or analysis_mapping
+ self.analysis_class = self.analysis_type_mapper[self.analysis_type]
+ self.is_cupac = bool(cupac_config)
+ self.cupac_handler = (
+ CupacHandler(**self.cupac_config) if self.is_cupac else None
+ )
+ self.cupac_covariate_col = (
+ self.cupac_handler.cupac_outcome_name if self.is_cupac else None
+ )
+
+ self.new_analysis_config = None
+ self.experiment_analysis = None
+add_covariates(self, exp_data, pre_exp_data)
+
+
+¶If the test is a cupac test, adds the covariates to the experimental data.
+ +cluster_experiments/inference/hypothesis_test.pydef add_covariates(
+ self, exp_data: pd.DataFrame, pre_exp_data: pd.DataFrame
+) -> pd.DataFrame:
+ """
+ If the test is a cupac test, adds the covariates to the experimental data.
+ """
+ if self.is_cupac:
+ exp_data = self.cupac_handler.add_covariates(
+ df=exp_data, pre_experiment_df=pre_exp_data
+ )
+
+ return exp_data
+get_inference_results(self, df, alpha)
+
+
+¶Performs inference analysis on the provided DataFrame using the analysis class.
+df : pd.DataFrame + The dataframe containing the data for analysis. +alpha : float + The significance level to be used in the inference analysis.
+InferenceResults + The results containing the statistics of the inference procedure.
+ +cluster_experiments/inference/hypothesis_test.pydef get_inference_results(self, df: pd.DataFrame, alpha: float) -> InferenceResults:
+ """
+ Performs inference analysis on the provided DataFrame using the analysis class.
+
+ Parameters
+ ----------
+ df : pd.DataFrame
+ The dataframe containing the data for analysis.
+ alpha : float
+ The significance level to be used in the inference analysis.
+
+ Returns
+ -------
+ InferenceResults
+ The results containing the statistics of the inference procedure.
+ """
+
+ self.experiment_analysis = self.analysis_class(**self.new_analysis_config)
+ inference_results = self.experiment_analysis.get_inference_results(
+ df=df, alpha=alpha
+ )
+
+ return inference_results
+get_test_results(self, control_variant, treatment_variant, variant_col, exp_data, dimension, dimension_value, alpha)
+
+
+¶Performs the hypothesis test on the provided data, for the given dimension value.
+control_variant : Variant + The control variant +treatment_variant : Variant + The treatment variant +variant_col : str + The column name representing the variant +exp_data : pd.DataFrame + The dataframe containing the data for analysis. +dimension : Dimension + The dimension instance +dimension_value : str + The value of the dimension +alpha : float + The significance level to be used in the inference analysis.
+AnalysisPlanResults + The results of the hypothesis test
+ +cluster_experiments/inference/hypothesis_test.pydef get_test_results(
+ self,
+ control_variant: Variant,
+ treatment_variant: Variant,
+ variant_col: str,
+ exp_data: pd.DataFrame,
+ dimension: Dimension,
+ dimension_value: str,
+ alpha: float,
+) -> AnalysisPlanResults:
+ """
+ Performs the hypothesis test on the provided data, for the given dimension value.
+
+ Parameters
+ ----------
+ control_variant : Variant
+ The control variant
+ treatment_variant : Variant
+ The treatment variant
+ variant_col : str
+ The column name representing the variant
+ exp_data : pd.DataFrame
+ The dataframe containing the data for analysis.
+ dimension : Dimension
+ The dimension instance
+ dimension_value : str
+ The value of the dimension
+ alpha : float
+ The significance level to be used in the inference analysis.
+
+ Returns
+ -------
+ AnalysisPlanResults
+ The results of the hypothesis test
+ """
+ self._prepare_analysis_config(
+ target_col=self.metric.target_column,
+ treatment_col=variant_col,
+ treatment=treatment_variant.name,
+ )
+
+ prepared_df = self.prepare_data(
+ data=exp_data,
+ variant_col=variant_col,
+ treatment_variant=treatment_variant,
+ control_variant=control_variant,
+ dimension_name=dimension.name,
+ dimension_value=dimension_value,
+ )
+
+ inference_results = self.get_inference_results(df=prepared_df, alpha=alpha)
+
+ control_variant_mean = self.metric.get_mean(
+ prepared_df.query(f"{variant_col}=='{control_variant.name}'")
+ )
+ treatment_variant_mean = self.metric.get_mean(
+ prepared_df.query(f"{variant_col}=='{treatment_variant.name}'")
+ )
+
+ test_results = AnalysisPlanResults(
+ metric_alias=[self.metric.alias],
+ control_variant_name=[control_variant.name],
+ treatment_variant_name=[treatment_variant.name],
+ control_variant_mean=[control_variant_mean],
+ treatment_variant_mean=[treatment_variant_mean],
+ analysis_type=[self.analysis_type],
+ ate=[inference_results.ate],
+ ate_ci_lower=[inference_results.conf_int.lower],
+ ate_ci_upper=[inference_results.conf_int.upper],
+ p_value=[inference_results.p_value],
+ std_error=[inference_results.std_error],
+ dimension_name=[dimension.name],
+ dimension_value=[dimension_value],
+ alpha=[alpha],
+ )
+
+ return test_results
+prepare_data(data, variant_col, treatment_variant, control_variant, dimension_name, dimension_value)
+
+
+ staticmethod
+
+
+¶Prepares the data for the experiment analysis pipeline
+ +cluster_experiments/inference/hypothesis_test.py@staticmethod
+def prepare_data(
+ data: pd.DataFrame,
+ variant_col: str,
+ treatment_variant: Variant,
+ control_variant: Variant,
+ dimension_name: str,
+ dimension_value: str,
+) -> pd.DataFrame:
+ """
+ Prepares the data for the experiment analysis pipeline
+ """
+ prepared_df = data.copy()
+
+ prepared_df = prepared_df.assign(__total_dimension="total")
+
+ prepared_df = prepared_df.query(
+ f"{variant_col}.isin(['{treatment_variant.name}','{control_variant.name}'])"
+ ).query(f"{dimension_name} == '{dimension_value}'")
+
+ return prepared_df
+from cluster_experiments.inference.metric import *¶
+Metric (ABC)
+
+
+
+
+¶An abstract base class used to represent a Metric with an alias.
+alias : str + A string representing the alias of the metric
+ +cluster_experiments/inference/metric.pyclass Metric(ABC):
+ """
+ An abstract base class used to represent a Metric with an alias.
+
+ Attributes
+ ----------
+ alias : str
+ A string representing the alias of the metric
+ """
+
+ def __init__(self, alias: str):
+ """
+ Parameters
+ ----------
+ alias : str
+ The alias of the metric
+ """
+ self.alias = alias
+ self._validate_alias()
+
+ def _validate_alias(self):
+ """
+ Validates the alias input for the Metric class.
+
+ Raises
+ ------
+ TypeError
+ If the alias is not a string
+ """
+ if not isinstance(self.alias, str):
+ raise TypeError("Metric alias must be a string")
+
+ @property
+ @abstractmethod
+ def target_column(self) -> str:
+ """
+ Abstract property to return the target column to feed the experiment analysis class, from the metric definition.
+
+ Returns
+ -------
+ str
+ The target column name
+ """
+ pass
+
+ @abstractmethod
+ def get_mean(self, df: pd.DataFrame) -> float:
+ """
+ Abstract method to return the mean value of the metric, given a dataframe.
+
+ Returns
+ -------
+ float
+ The mean value of the metric
+ """
+ pass
+target_column: str
+
+
+ property
+ readonly
+
+
+¶Abstract property to return the target column to feed the experiment analysis class, from the metric definition.
+str + The target column name
+__init__(self, alias)
+
+
+ special
+
+
+¶alias : str + The alias of the metric
+ +cluster_experiments/inference/metric.pydef __init__(self, alias: str):
+ """
+ Parameters
+ ----------
+ alias : str
+ The alias of the metric
+ """
+ self.alias = alias
+ self._validate_alias()
+get_mean(self, df)
+
+
+¶Abstract method to return the mean value of the metric, given a dataframe.
+float + The mean value of the metric
+ +cluster_experiments/inference/metric.py@abstractmethod
+def get_mean(self, df: pd.DataFrame) -> float:
+ """
+ Abstract method to return the mean value of the metric, given a dataframe.
+
+ Returns
+ -------
+ float
+ The mean value of the metric
+ """
+ pass
+
+RatioMetric (Metric)
+
+
+
+
+¶A class used to represent a Ratio Metric with an alias, a numerator name, and a denominator name. +To be used when the metric is defined at a lower level than the data used for the analysis.
+In a clustered experiment the participants were randomised based on their country of residence. +The metric of interest is the salary of each participant. If the dataset fed into the analysis is at country-level, +then a RatioMetric must be used: the numerator would be the sum of all salaries in the country, +the denominator would be the number of participants in the country.
+alias : str + A string representing the alias of the metric +numerator_name : str + A string representing the numerator name of the metric +denominator_name : str + A string representing the denominator name of the metric
+ +cluster_experiments/inference/metric.pyclass RatioMetric(Metric):
+ """
+ A class used to represent a Ratio Metric with an alias, a numerator name, and a denominator name.
+ To be used when the metric is defined at a lower level than the data used for the analysis.
+
+ Example
+ ----------
+ In a clustered experiment the participants were randomised based on their country of residence.
+ The metric of interest is the salary of each participant. If the dataset fed into the analysis is at country-level,
+ then a RatioMetric must be used: the numerator would be the sum of all salaries in the country,
+ the denominator would be the number of participants in the country.
+
+ Attributes
+ ----------
+ alias : str
+ A string representing the alias of the metric
+ numerator_name : str
+ A string representing the numerator name of the metric
+ denominator_name : str
+ A string representing the denominator name of the metric
+ """
+
+ def __init__(self, alias: str, numerator_name: str, denominator_name: str):
+ """
+ Parameters
+ ----------
+ alias : str
+ The alias of the metric
+ numerator_name : str
+ The numerator name of the metric
+ denominator_name : str
+ The denominator name of the metric
+ """
+ super().__init__(alias)
+ self.numerator_name = numerator_name
+ self.denominator_name = denominator_name
+ self._validate_names()
+
+ def _validate_names(self):
+ """
+ Validates the numerator and denominator names input for the RatioMetric class.
+
+ Raises
+ ------
+ TypeError
+ If the numerator or denominator names are not strings
+ """
+ if not isinstance(self.numerator_name, str) or not isinstance(
+ self.denominator_name, str
+ ):
+ raise TypeError("RatioMetric names must be strings")
+
+ @property
+ def target_column(self) -> str:
+ """
+ Returns the target column for the RatioMetric.
+
+ Returns
+ -------
+ str
+ The numerator name of the metric
+ """
+ return self.numerator_name
+
+ def get_mean(self, df: pd.DataFrame) -> float:
+ """
+ Returns the mean value of the metric, given a dataframe.
+
+ Returns
+ -------
+ float
+ The mean value of the metric
+ """
+ return df[self.numerator_name].mean() / df[self.denominator_name].mean()
+target_column: str
+
+
+ property
+ readonly
+
+
+¶Returns the target column for the RatioMetric.
+str + The numerator name of the metric
+__init__(self, alias, numerator_name, denominator_name)
+
+
+ special
+
+
+¶alias : str + The alias of the metric +numerator_name : str + The numerator name of the metric +denominator_name : str + The denominator name of the metric
+ +cluster_experiments/inference/metric.pydef __init__(self, alias: str, numerator_name: str, denominator_name: str):
+ """
+ Parameters
+ ----------
+ alias : str
+ The alias of the metric
+ numerator_name : str
+ The numerator name of the metric
+ denominator_name : str
+ The denominator name of the metric
+ """
+ super().__init__(alias)
+ self.numerator_name = numerator_name
+ self.denominator_name = denominator_name
+ self._validate_names()
+get_mean(self, df)
+
+
+¶Returns the mean value of the metric, given a dataframe.
+float + The mean value of the metric
+ +cluster_experiments/inference/metric.pydef get_mean(self, df: pd.DataFrame) -> float:
+ """
+ Returns the mean value of the metric, given a dataframe.
+
+ Returns
+ -------
+ float
+ The mean value of the metric
+ """
+ return df[self.numerator_name].mean() / df[self.denominator_name].mean()
+
+SimpleMetric (Metric)
+
+
+
+
+¶A class used to represent a Simple Metric with an alias and a name. +To be used when the metric is defined at the same level of the data used for the analysis.
+In a clustered experiment the participants were randomised based on their country of residence. +The metric of interest is the salary of each participant. If the dataset fed into the analysis is at participant-level, +then a SimpleMetric must be used. However, if the dataset fed into the analysis is at country-level, then a RatioMetric must be used.
+alias : str + A string representing the alias of the metric +name : str + A string representing the name of the metric
+ +cluster_experiments/inference/metric.pyclass SimpleMetric(Metric):
+ """
+ A class used to represent a Simple Metric with an alias and a name.
+ To be used when the metric is defined at the same level of the data used for the analysis.
+
+ Example
+ ----------
+ In a clustered experiment the participants were randomised based on their country of residence.
+ The metric of interest is the salary of each participant. If the dataset fed into the analysis is at participant-level,
+ then a SimpleMetric must be used. However, if the dataset fed into the analysis is at country-level, then a RatioMetric must be used.
+
+ Attributes
+ ----------
+ alias : str
+ A string representing the alias of the metric
+ name : str
+ A string representing the name of the metric
+ """
+
+ def __init__(self, alias: str, name: str):
+ """
+ Parameters
+ ----------
+ alias : str
+ The alias of the metric
+ name : str
+ The name of the metric
+ """
+ super().__init__(alias)
+ self.name = name
+ self._validate_name()
+
+ def _validate_name(self):
+ """
+ Validates the name input for the SimpleMetric class.
+
+ Raises
+ ------
+ TypeError
+ If the name is not a string
+ """
+ if not isinstance(self.name, str):
+ raise TypeError("SimpleMetric name must be a string")
+
+ @property
+ def target_column(self) -> str:
+ """
+ Returns the target column for the SimpleMetric.
+
+ Returns
+ -------
+ str
+ The name of the metric
+ """
+ return self.name
+
+ def get_mean(self, df: pd.DataFrame) -> float:
+ """
+ Returns the mean value of the metric, given a dataframe.
+
+ Returns
+ -------
+ float
+ The mean value of the metric
+ """
+ return df[self.name].mean()
+target_column: str
+
+
+ property
+ readonly
+
+
+¶__init__(self, alias, name)
+
+
+ special
+
+
+¶alias : str + The alias of the metric +name : str + The name of the metric
+ +cluster_experiments/inference/metric.pydef __init__(self, alias: str, name: str):
+ """
+ Parameters
+ ----------
+ alias : str
+ The alias of the metric
+ name : str
+ The name of the metric
+ """
+ super().__init__(alias)
+ self.name = name
+ self._validate_name()
+get_mean(self, df)
+
+
+¶Returns the mean value of the metric, given a dataframe.
+float + The mean value of the metric
+ +cluster_experiments/inference/metric.pydef get_mean(self, df: pd.DataFrame) -> float:
+ """
+ Returns the mean value of the metric, given a dataframe.
+
+ Returns
+ -------
+ float
+ The mean value of the metric
+ """
+ return df[self.name].mean()
+from cluster_experiments.perturbator import *¶
+BetaRelativePerturbator (NormalPerturbator, RelativePositivePerturbator)
+
+
+
+
+¶A stochastic Perturbator for continuous targets that applies a sampled +effect from a scaled Beta distribution. It applies the effect multiplicatively.
+The sampled effect is defined for values in the specified range +(range_min, range_max). It's recommended to set -1<range_min<0 and +range_max>0 in a "symmetric way" around 0, such that +log(1 + range_min) = -log(1 + range_max). +This ensures to have an "symmetric range" of perturbations that relatively +decrease the target as perturbations that relatively increase the target. +By "symmetry" of relative effects we mean that for an effect c > 0, an +increase of the target t via t*(1 + c) is "symmetric" to a decrease of t +via t/(1 + c). For example, an increase of 5x (i.e. by +400%, corresponding +to c_inc=4) is "symmetric" to a decrease of 5x (i.e. a decrease of -80%, +corresponding to c_dec = -0.8). In this case, 1 + c_dec = 1/(1 + c_inc), so +the relative effects c_inc and c_dec are "symmetric" in the sense that they +are inverse to each other.
+The number of samples with 0 as target remains unchanged.
+The stochastic effect is sampled from a beta distribution with parameters +mean and variance, which is linearly scaled to the range +(range_min, range_max). +If variance is not provided, the variance is abs(mean).
+target -> target * (1 + effect), where effect ~ Beta(a, b)
+The common beta parameters are derived from the mean and scale parameters, +combined with linear transformations to ensure the support in the given +range. The resulting beta parameters are scaled by abs(mu) to narrow the +beta distribution around the mean.
+mu_transformed <- (mu - range_min) / (range_max - range_min)
+scale_transformed <- (scale - range_min) / (range_max - range_min)
+a <- mu_transformed / (scale_transformed * scale_transformed)
+b <- (1-mu_transformed) / (scale_transformed * scale_transformed)
+effect_transformed ~ beta(a/abs(mu), b/abs(mu))
+effect = effect_transformed * (range_max - range_min) + range_min
+Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
average_effect |
+ Optional[float] |
+ the average effect of the treatment. Defaults to None. |
+ None |
+
target_col |
+ str |
+ name of the target_col to use as the outcome. Defaults to "target". |
+ 'target' |
+
treatment_col |
+ str |
+ the name of the column that contains the treatment. Defaults to "treatment". |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group. Defaults to "B". |
+ 'B' |
+
scale |
+ Optional[float] |
+ the scale of the effect distribution. Defaults to None. +If not provided, the variance of the beta distribution is abs(mean). |
+ None |
+
range_min |
+ float |
+ the minimum value of the target range, must be >-1. +Defaults to -0.8, which allows for up to 5x decreases of the target. |
+ None |
+
range_max |
+ float |
+ the maximum value of the target range. +Defaults to 4, which allows for up to 5x increases of the target. |
+ None |
+
reduce_variance |
+ Optional[bool] |
+ if True and if abs(average_effect)<1, we reduce +the variance of the beta distribution by multiplying the beta parameters by 1/abs(average_effect). +Defaults to None, which is equivalent to True. |
+ None |
+
cluster_experiments/perturbator.pyclass BetaRelativePerturbator(NormalPerturbator, RelativePositivePerturbator):
+ """
+ A stochastic Perturbator for continuous targets that applies a sampled
+ effect from a scaled Beta distribution. It applies the effect multiplicatively.
+
+ The sampled effect is defined for values in the specified range
+ (range_min, range_max). It's recommended to set -1<range_min<0 and
+ range_max>0 in a "symmetric way" around 0, such that
+ log(1 + range_min) = -log(1 + range_max).
+ This ensures to have an "symmetric range" of perturbations that relatively
+ decrease the target as perturbations that relatively increase the target.
+ By "symmetry" of relative effects we mean that for an effect c > 0, an
+ increase of the target t via t*(1 + c) is "symmetric" to a decrease of t
+ via t/(1 + c). For example, an increase of 5x (i.e. by +400%, corresponding
+ to c_inc=4) is "symmetric" to a decrease of 5x (i.e. a decrease of -80%,
+ corresponding to c_dec = -0.8). In this case, 1 + c_dec = 1/(1 + c_inc), so
+ the relative effects c_inc and c_dec are "symmetric" in the sense that they
+ are inverse to each other.
+
+ The number of samples with 0 as target remains unchanged.
+
+ The stochastic effect is sampled from a beta distribution with parameters
+ mean and variance, which is linearly scaled to the range
+ (range_min, range_max).
+ If variance is not provided, the variance is abs(mean).
+
+ ```
+ target -> target * (1 + effect), where effect ~ Beta(a, b)
+ ```
+
+ The common beta parameters are derived from the mean and scale parameters,
+ combined with linear transformations to ensure the support in the given
+ range. The resulting beta parameters are scaled by abs(mu) to narrow the
+ beta distribution around the mean.
+
+ ```
+ mu_transformed <- (mu - range_min) / (range_max - range_min)
+ scale_transformed <- (scale - range_min) / (range_max - range_min)
+ a <- mu_transformed / (scale_transformed * scale_transformed)
+ b <- (1-mu_transformed) / (scale_transformed * scale_transformed)
+ effect_transformed ~ beta(a/abs(mu), b/abs(mu))
+ effect = effect_transformed * (range_max - range_min) + range_min
+ ```
+
+ Arguments:
+ average_effect (Optional[float], optional): the average effect of the treatment. Defaults to None.
+ target_col (str, optional): name of the target_col to use as the outcome. Defaults to "target".
+ treatment_col (str, optional): the name of the column that contains the treatment. Defaults to "treatment".
+ treatment (str, optional): name of the treatment to use as the treated group. Defaults to "B".
+ scale (Optional[float], optional): the scale of the effect distribution. Defaults to None.
+ If not provided, the variance of the beta distribution is abs(mean).
+ range_min (float, optional): the minimum value of the target range, must be >-1.
+ Defaults to -0.8, which allows for up to 5x decreases of the target.
+ range_max (float, optional): the maximum value of the target range.
+ Defaults to 4, which allows for up to 5x increases of the target.
+ reduce_variance (Optional[bool], optional): if True and if abs(average_effect)<1, we reduce
+ the variance of the beta distribution by multiplying the beta parameters by 1/abs(average_effect).
+ Defaults to None, which is equivalent to True.
+ """
+
+ def __init__(
+ self,
+ average_effect: Optional[float] = None,
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ scale: Optional[float] = None,
+ range_min: Optional[float] = None,
+ range_max: Optional[float] = None,
+ reduce_variance: Optional[bool] = None,
+ ):
+ self._check_range(range_min, range_max)
+ super().__init__(average_effect, target_col, treatment_col, treatment, scale)
+ self._range_min = range_min or -0.8
+ self._range_max = range_max or 4
+ self._reduce_variance = reduce_variance or True
+
+ def perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+ ) -> pd.DataFrame:
+ """
+ Usage:
+ ```python
+ from cluster_experiments.perturbator import BetaRelativePerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = BetaRelativePerturbator(range_min = -0.5, range_max = 2)
+ # Increase target metric by 20% on average
+ perturbator.perturbate(df, average_effect=0.2)
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ self.check_relative_effect_bounds(average_effect)
+ scale = self.get_scale(average_effect)
+ self.check_relative_effect_bounds(scale)
+ n = self.get_number_of_treated(df)
+ sampled_effect = self._sample_scaled_beta_effect(average_effect, scale, n)
+ df = self.apply_multiplicative_effect(df, sampled_effect)
+ return df
+
+ @staticmethod
+ def _check_range(range_min: float, range_max: float):
+ if range_min < -1:
+ raise ValueError(f"range_min needs to be greater than -1, got {range_min}")
+ if range_min >= range_max:
+ raise ValueError(
+ f"range_min needs to be smaller than range_max, got "
+ f"{range_min = } and {range_max = }"
+ )
+
+ def check_relative_effect_bounds(self, average_effect: float) -> None:
+ self.check_average_effect_greater_than(average_effect, x=self._range_min)
+ self.check_average_effect_smaller_than(average_effect, x=self._range_max)
+
+ def check_average_effect_greater_than(
+ self, average_effect: float, x: float
+ ) -> Optional[NoReturn]:
+ if average_effect <= x:
+ raise ValueError(
+ f"Simulated effect needs to be greater than range_min={x}, got {average_effect}"
+ )
+
+ def check_average_effect_smaller_than(
+ self, average_effect: float, x: float
+ ) -> Optional[NoReturn]:
+ if average_effect >= x:
+ raise ValueError(
+ f"Simulated effect needs to be smaller than range_max={x}, got {average_effect}"
+ )
+
+ def _reduce_variance_beta_params(
+ self, average_effect: float, a: float, b: float
+ ) -> Tuple[float, float]:
+ """
+ Multiplying the parameters of the beta distribution with a factor >1
+ reduces variance
+ """
+ if abs(average_effect) < 1:
+ a *= 1 / abs(average_effect)
+ b *= 1 / abs(average_effect)
+ return a, b
+
+ def _sample_scaled_beta_effect(
+ self, average_effect: float, scale: float, n: int
+ ) -> np.ndarray:
+ average_effect_inv_transf = self._inv_transform_to_range(average_effect)
+ scale_inv_transf = self._inv_transform_to_range(scale)
+ a = average_effect_inv_transf / (scale_inv_transf * scale_inv_transf)
+ b = (1 - average_effect_inv_transf) / (scale_inv_transf * scale_inv_transf)
+
+ if self._reduce_variance:
+ a, b = self._reduce_variance_beta_params(average_effect, a, b)
+ beta = np.random.beta(a, b, n)
+
+ return self._transform_to_range(beta)
+
+ def _transform_to_range(self, x: Union[float, np.ndarray]):
+ return x * (self._range_max - self._range_min) + self._range_min
+
+ def _inv_transform_to_range(self, x: Union[float, np.ndarray]):
+ return (x - self._range_min) / (self._range_max - self._range_min)
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a Perturbator object from a PowerConfig object"""
+ return cls(
+ average_effect=config.average_effect,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ scale=config.scale,
+ range_min=config.range_min,
+ range_max=config.range_max,
+ )
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a Perturbator object from a PowerConfig object
+ +cluster_experiments/perturbator.py@classmethod
+def from_config(cls, config):
+ """Creates a Perturbator object from a PowerConfig object"""
+ return cls(
+ average_effect=config.average_effect,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ scale=config.scale,
+ range_min=config.range_min,
+ range_max=config.range_max,
+ )
+perturbate(self, df, average_effect=None)
+
+
+¶Usage: +
from cluster_experiments.perturbator import BetaRelativePerturbator
+import pandas as pd
+df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+perturbator = BetaRelativePerturbator(range_min = -0.5, range_max = 2)
+# Increase target metric by 20% on average
+perturbator.perturbate(df, average_effect=0.2)
+cluster_experiments/perturbator.pydef perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+) -> pd.DataFrame:
+ """
+ Usage:
+ ```python
+ from cluster_experiments.perturbator import BetaRelativePerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = BetaRelativePerturbator(range_min = -0.5, range_max = 2)
+ # Increase target metric by 20% on average
+ perturbator.perturbate(df, average_effect=0.2)
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ self.check_relative_effect_bounds(average_effect)
+ scale = self.get_scale(average_effect)
+ self.check_relative_effect_bounds(scale)
+ n = self.get_number_of_treated(df)
+ sampled_effect = self._sample_scaled_beta_effect(average_effect, scale, n)
+ df = self.apply_multiplicative_effect(df, sampled_effect)
+ return df
+
+BetaRelativePositivePerturbator (NormalPerturbator, RelativePositivePerturbator)
+
+
+
+
+¶A stochastic Perturbator for continuous, positively-defined targets that applies a +sampled effect from the Beta distribution. It applies the effect multiplicatively.
+WARNING: the average effect is only defined for values between 0 and 1 (not +included). Therefore, it only increments the target for the treated samples.
+The number of samples with 0 as target remains unchanged.
+The stochastic effect is sampled from a beta distribution with parameters mean and +variance. If variance is not provided, the variance is abs(mean). Hence, the effect +is bounded by 0 and 1.
+target -> target * (1 + effect), where effect ~ Beta(a, b); a, b > 0
+ and target > 0 for all samples
+The common beta parameters are derived from the mean and scale parameters (see +how below). That's why the average effect is only defined for values between 0 +and 1, otherwise one of the beta parameters would be negative or zero:
+a <- mu / (scale * scale)
+b <- (1-mu) / (scale * scale)
+effect ~ beta(a, b)
+Example: a mean = 0.2 and variance = 0.1, give a = 20 and b = 80 +Plot: https://www.wolframalpha.com/input?i=plot+distribution+of+beta%2820%2C+80%29
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
average_effect |
+ Optional[float] |
+ the average effect of the treatment. Defaults to None. |
+ required | +
target_col |
+ str |
+ name of the target_col to use as the outcome. Defaults to "target". |
+ required | +
treatment_col |
+ str |
+ the name of the column that contains the treatment. Defaults to "treatment". |
+ required | +
treatment |
+ str |
+ name of the treatment to use as the treated group. Defaults to "B". |
+ required | +
scale |
+ Optional[float] |
+ the scale of the effect distribution. Defaults to None. |
+ required | +
cluster_experiments/perturbator.pyclass BetaRelativePositivePerturbator(NormalPerturbator, RelativePositivePerturbator):
+ """
+ A stochastic Perturbator for continuous, positively-defined targets that applies a
+ sampled effect from the Beta distribution. It applies the effect multiplicatively.
+
+ *WARNING*: the average effect is only defined for values between 0 and 1 (not
+ included). Therefore, it only increments the target for the treated samples.
+
+ The number of samples with 0 as target remains unchanged.
+
+ The stochastic effect is sampled from a beta distribution with parameters mean and
+ variance. If variance is not provided, the variance is abs(mean). Hence, the effect
+ is bounded by 0 and 1.
+
+ ```
+ target -> target * (1 + effect), where effect ~ Beta(a, b); a, b > 0
+ and target > 0 for all samples
+ ```
+
+ The common beta parameters are derived from the mean and scale parameters (see
+ how below). That's why the average effect is only defined for values between 0
+ and 1, otherwise one of the beta parameters would be negative or zero:
+
+ ```
+ a <- mu / (scale * scale)
+ b <- (1-mu) / (scale * scale)
+ effect ~ beta(a, b)
+ ```
+ source: https://stackoverflow.com/a/51143208
+
+ Example: a mean = 0.2 and variance = 0.1, give a = 20 and b = 80
+ Plot: https://www.wolframalpha.com/input?i=plot+distribution+of+beta%2820%2C+80%29
+
+ Arguments:
+ average_effect (Optional[float], optional): the average effect of the treatment. Defaults to None.
+ target_col (str, optional): name of the target_col to use as the outcome. Defaults to "target".
+ treatment_col (str, optional): the name of the column that contains the treatment. Defaults to "treatment".
+ treatment (str, optional): name of the treatment to use as the treated group. Defaults to "B".
+ scale (Optional[float], optional): the scale of the effect distribution. Defaults to None.
+ """
+
+ def perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+ ) -> pd.DataFrame:
+ """
+ Usage:
+ ```python
+ from cluster_experiments.perturbator import BetaRelativePositivePerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = BetaRelativePositivePerturbator()
+ # Increase target metric by 20%
+ perturbator.perturbate(df, average_effect=0.2)
+ # returns pd.DataFrame({"target": [1, 3, 3], "treatment": ["A", "B", "A"]})
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ self.check_beta_positive_effect(df, average_effect)
+ scale = self.get_scale(average_effect)
+ n = self.get_number_of_treated(df)
+ sampled_effect = self._sample_beta_effect(average_effect, scale, n)
+ df = self.apply_multiplicative_effect(df, sampled_effect)
+ return df
+
+ def check_beta_positive_effect(self, df, average_effect):
+ self.check_average_effect_greater_than(average_effect, x=0)
+ self.check_average_effect_less_than(average_effect, x=1)
+ self.check_target_is_not_negative(df)
+ self.check_target_is_not_constant_zero(df, average_effect)
+
+ def check_average_effect_less_than(
+ self, average_effect: float, x: float
+ ) -> Optional[NoReturn]:
+ if average_effect >= x:
+ raise ValueError(
+ f"Simulated effect needs to be less than {x*100:.0f}%, got "
+ f"{average_effect*100:.1f}%"
+ )
+
+ def _sample_beta_effect(
+ self, average_effect: float, scale: float, n: int
+ ) -> np.ndarray:
+ a = average_effect / (scale * scale)
+ b = (1 - average_effect) / (scale * scale)
+ return np.random.beta(a, b, n)
+perturbate(self, df, average_effect=None)
+
+
+¶Usage: +
from cluster_experiments.perturbator import BetaRelativePositivePerturbator
+import pandas as pd
+df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+perturbator = BetaRelativePositivePerturbator()
+# Increase target metric by 20%
+perturbator.perturbate(df, average_effect=0.2)
+# returns pd.DataFrame({"target": [1, 3, 3], "treatment": ["A", "B", "A"]})
+cluster_experiments/perturbator.pydef perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+) -> pd.DataFrame:
+ """
+ Usage:
+ ```python
+ from cluster_experiments.perturbator import BetaRelativePositivePerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = BetaRelativePositivePerturbator()
+ # Increase target metric by 20%
+ perturbator.perturbate(df, average_effect=0.2)
+ # returns pd.DataFrame({"target": [1, 3, 3], "treatment": ["A", "B", "A"]})
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ self.check_beta_positive_effect(df, average_effect)
+ scale = self.get_scale(average_effect)
+ n = self.get_number_of_treated(df)
+ sampled_effect = self._sample_beta_effect(average_effect, scale, n)
+ df = self.apply_multiplicative_effect(df, sampled_effect)
+ return df
+
+BinaryPerturbator (Perturbator)
+
+
+
+
+¶BinaryPerturbator is a Perturbator that adds is used to deal with binary outcome variables. +It randomly selects some treated instances and flips their outcome from 0 to 1 or 1 to 0, depending on the effect being positive or negative
+ +cluster_experiments/perturbator.pyclass BinaryPerturbator(Perturbator):
+ """
+ BinaryPerturbator is a Perturbator that adds is used to deal with binary outcome variables.
+ It randomly selects some treated instances and flips their outcome from 0 to 1 or 1 to 0, depending on the effect being positive or negative
+ """
+
+ def _sample_max(self, df: pd.DataFrame, n: int) -> pd.DataFrame:
+ """Like sample without replacement,
+ but if you are to sample more than 100% of the data,
+ it just returns the whole dataframe."""
+ if n >= len(df):
+ return df
+ return df.sample(n=n)
+
+ def _data_checks(self, df: pd.DataFrame, average_effect: float) -> None:
+ """Check that outcome is indeed binary, and average effect is in (-1, 1)"""
+
+ if set(df[self.target_col].unique()) - {0, 1}:
+ raise ValueError(
+ f"Target column must be binary, found {set(df[self.target_col].unique())}"
+ )
+
+ if average_effect > 1 or average_effect < -1:
+ raise ValueError(
+ f"Average effect must be in (-1, 1), found {average_effect}"
+ )
+
+ def perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+ ) -> pd.DataFrame:
+ """
+ Usage:
+
+ ```python
+ from cluster_experiments.perturbator import BinaryPerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 0, 1], "treatment": ["A", "B", "A"]})
+ perturbator = BinaryPerturbator()
+ perturbator.perturbate(df, average_effect=0.1)
+ ```
+ """
+
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+
+ self._data_checks(df, average_effect)
+
+ from_target, to_target = 1, 0
+ if average_effect > 0:
+ from_target, to_target = 0, 1
+
+ n_transformed = abs(int(average_effect * len(df.query(self.treated_query))))
+ idx = list(
+ # Sample of negative cases in group B
+ df.query(f"{self.target_col} == {from_target} & {self.treated_query}")
+ .pipe(self._sample_max, n=n_transformed)
+ .index.drop_duplicates()
+ )
+ df.loc[idx, self.target_col] = to_target
+ return df
+perturbate(self, df, average_effect=None)
+
+
+¶Usage:
+from cluster_experiments.perturbator import BinaryPerturbator
+import pandas as pd
+df = pd.DataFrame({"target": [1, 0, 1], "treatment": ["A", "B", "A"]})
+perturbator = BinaryPerturbator()
+perturbator.perturbate(df, average_effect=0.1)
+cluster_experiments/perturbator.pydef perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+) -> pd.DataFrame:
+ """
+ Usage:
+
+ ```python
+ from cluster_experiments.perturbator import BinaryPerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 0, 1], "treatment": ["A", "B", "A"]})
+ perturbator = BinaryPerturbator()
+ perturbator.perturbate(df, average_effect=0.1)
+ ```
+ """
+
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+
+ self._data_checks(df, average_effect)
+
+ from_target, to_target = 1, 0
+ if average_effect > 0:
+ from_target, to_target = 0, 1
+
+ n_transformed = abs(int(average_effect * len(df.query(self.treated_query))))
+ idx = list(
+ # Sample of negative cases in group B
+ df.query(f"{self.target_col} == {from_target} & {self.treated_query}")
+ .pipe(self._sample_max, n=n_transformed)
+ .index.drop_duplicates()
+ )
+ df.loc[idx, self.target_col] = to_target
+ return df
+
+ConstantPerturbator (Perturbator)
+
+
+
+
+¶ConstantPerturbator is a Perturbator that adds a constant effect to the target column of the treated instances.
+ +cluster_experiments/perturbator.pyclass ConstantPerturbator(Perturbator):
+ """
+ ConstantPerturbator is a Perturbator that adds a constant effect to the target column of the treated instances.
+ """
+
+ def perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+ ) -> pd.DataFrame:
+ """
+ Usage:
+
+ ```python
+ from cluster_experiments.perturbator import ConstantPerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = ConstantPerturbator()
+ perturbator.perturbate(df, average_effect=1)
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ df = self.apply_additive_effect(df, average_effect)
+ return df
+
+ def apply_additive_effect(
+ self, df: pd.DataFrame, effect: Union[float, np.ndarray]
+ ) -> pd.DataFrame:
+ df.loc[df[self.treatment_col] == self.treatment, self.target_col] += effect
+ return df
+perturbate(self, df, average_effect=None)
+
+
+¶Usage:
+from cluster_experiments.perturbator import ConstantPerturbator
+import pandas as pd
+df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+perturbator = ConstantPerturbator()
+perturbator.perturbate(df, average_effect=1)
+cluster_experiments/perturbator.pydef perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+) -> pd.DataFrame:
+ """
+ Usage:
+
+ ```python
+ from cluster_experiments.perturbator import ConstantPerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = ConstantPerturbator()
+ perturbator.perturbate(df, average_effect=1)
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ df = self.apply_additive_effect(df, average_effect)
+ return df
+
+NormalPerturbator (ConstantPerturbator)
+
+
+
+
+¶The NormalPerturbator class implements a perturbator that adds a stochastic effect +to the target column of the treated instances. The stochastic effect is sampled from a +normal distribution with mean average_effect and variance scale. If scale is not +provided, the variance is abs(average_effect). If scale is provided, a +value not much bigger than the average_effect is suggested.
+target -> target + effect, where effect ~ Normal(average_effect, scale)
+Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
average_effect |
+ Optional[float] |
+ the average effect of the treatment. Defaults to None. |
+ None |
+
target_col |
+ str |
+ name of the target_col to use as the outcome. Defaults to "target". |
+ 'target' |
+
treatment_col |
+ str |
+ the name of the column that contains the treatment. Defaults to "treatment". |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group. Defaults to "B". |
+ 'B' |
+
scale |
+ Optional[float] |
+ the scale of the effect distribution. Defaults to None. |
+ None |
+
cluster_experiments/perturbator.pyclass NormalPerturbator(ConstantPerturbator):
+ """The NormalPerturbator class implements a perturbator that adds a stochastic effect
+ to the target column of the treated instances. The stochastic effect is sampled from a
+ normal distribution with mean average_effect and variance scale. If scale is not
+ provided, the variance is abs(average_effect). If scale is provided, a
+ value not much bigger than the average_effect is suggested.
+
+ ```
+ target -> target + effect, where effect ~ Normal(average_effect, scale)
+ ```
+
+ Arguments:
+ average_effect (Optional[float], optional): the average effect of the treatment. Defaults to None.
+ target_col (str, optional): name of the target_col to use as the outcome. Defaults to "target".
+ treatment_col (str, optional): the name of the column that contains the treatment. Defaults to "treatment".
+ treatment (str, optional): name of the treatment to use as the treated group. Defaults to "B".
+ scale (Optional[float], optional): the scale of the effect distribution. Defaults to None.
+ """
+
+ def __init__(
+ self,
+ average_effect: Optional[float] = None,
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ scale: Optional[float] = None,
+ ):
+ super().__init__(average_effect, target_col, treatment_col, treatment)
+ self._scale = scale
+
+ def perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+ ) -> pd.DataFrame:
+ """Perturbate with a normal effect with mean average_effect and
+ std abs(average_effect).
+
+ Arguments:
+ df (pd.DataFrame): the dataframe to perturbate.
+ average_effect (Optional[float], optional): the average effect. Defaults to None.
+
+ Returns:
+ pd.DataFrame: the perturbated dataframe.
+
+ Usage:
+
+ ```python
+ from cluster_experiments.perturbator import NormalPerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = NormalPerturbator()
+ perturbator.perturbate(df, average_effect=1)
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ scale = self.get_scale(average_effect)
+ n = self.get_number_of_treated(df)
+ sampled_effect = self._sample_normal_effect(average_effect, scale, n)
+ df = self.apply_additive_effect(df, sampled_effect)
+ return df
+
+ def get_scale(self, average_effect: float) -> float:
+ """Get the scale of the normal distribution. If scale is not provided, the
+ variance is abs(average_effect). Raises a ValueError if scale is not positive.
+ """
+ scale = abs(average_effect) if self._scale is None else self._scale
+ if scale <= 0:
+ raise ValueError(f"scale must be positive, got {scale}")
+ return scale
+
+ def get_number_of_treated(self, df: pd.DataFrame) -> int:
+ """Get the number of treated instances in the dataframe"""
+ return (df[self.treatment_col] == self.treatment).sum()
+
+ def _sample_normal_effect(
+ self, average_effect: float, scale: float, n: int
+ ) -> np.ndarray:
+ return np.random.normal(average_effect, scale, n)
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a Perturbator object from a PowerConfig object"""
+ return cls(
+ average_effect=config.average_effect,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ scale=config.scale,
+ )
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a Perturbator object from a PowerConfig object
+ +cluster_experiments/perturbator.py@classmethod
+def from_config(cls, config):
+ """Creates a Perturbator object from a PowerConfig object"""
+ return cls(
+ average_effect=config.average_effect,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ scale=config.scale,
+ )
+get_number_of_treated(self, df)
+
+
+¶Get the number of treated instances in the dataframe
+ +cluster_experiments/perturbator.pydef get_number_of_treated(self, df: pd.DataFrame) -> int:
+ """Get the number of treated instances in the dataframe"""
+ return (df[self.treatment_col] == self.treatment).sum()
+get_scale(self, average_effect)
+
+
+¶Get the scale of the normal distribution. If scale is not provided, the +variance is abs(average_effect). Raises a ValueError if scale is not positive.
+ +cluster_experiments/perturbator.pydef get_scale(self, average_effect: float) -> float:
+ """Get the scale of the normal distribution. If scale is not provided, the
+ variance is abs(average_effect). Raises a ValueError if scale is not positive.
+ """
+ scale = abs(average_effect) if self._scale is None else self._scale
+ if scale <= 0:
+ raise ValueError(f"scale must be positive, got {scale}")
+ return scale
+perturbate(self, df, average_effect=None)
+
+
+¶Perturbate with a normal effect with mean average_effect and +std abs(average_effect).
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ pd.DataFrame |
+ the dataframe to perturbate. |
+ required | +
average_effect |
+ Optional[float] |
+ the average effect. Defaults to None. |
+ None |
+
Returns:
+| Type | +Description | +
|---|---|
pd.DataFrame |
+ the perturbated dataframe. |
+
Usage:
+from cluster_experiments.perturbator import NormalPerturbator
+import pandas as pd
+df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+perturbator = NormalPerturbator()
+perturbator.perturbate(df, average_effect=1)
+cluster_experiments/perturbator.pydef perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+) -> pd.DataFrame:
+ """Perturbate with a normal effect with mean average_effect and
+ std abs(average_effect).
+
+ Arguments:
+ df (pd.DataFrame): the dataframe to perturbate.
+ average_effect (Optional[float], optional): the average effect. Defaults to None.
+
+ Returns:
+ pd.DataFrame: the perturbated dataframe.
+
+ Usage:
+
+ ```python
+ from cluster_experiments.perturbator import NormalPerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = NormalPerturbator()
+ perturbator.perturbate(df, average_effect=1)
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ scale = self.get_scale(average_effect)
+ n = self.get_number_of_treated(df)
+ sampled_effect = self._sample_normal_effect(average_effect, scale, n)
+ df = self.apply_additive_effect(df, sampled_effect)
+ return df
+
+Perturbator (ABC)
+
+
+
+
+¶Abstract perturbator. Perturbators are used to simulate a fictitious effect when running a power analysis.
+The idea is that, when running a power analysis, we split our instances according to a RandomSplitter, and the +instances that got the treatment, are perturbated with a fictional effect via the Perturbator.
+In order to create your own perturbator, you should create a derived class that implements the perturbate method.
+The perturbate method should add the average effect in the desired way and return the dataframe with the extra average effect,
+without affecting the initial dataframe. Keep in mind to use df = df.copy() in the first line of the perturbate method.
Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
average_effect |
+ Optional[float] |
+ The average effect of the treatment |
+ None |
+
target_col |
+ str |
+ name of the target_col to use as the outcome |
+ 'target' |
+
treatment_col |
+ str |
+ The name of the column that contains the treatment |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group |
+ 'B' |
+
cluster_experiments/perturbator.pyclass Perturbator(ABC):
+ """
+ Abstract perturbator. Perturbators are used to simulate a fictitious effect when running a power analysis.
+
+ The idea is that, when running a power analysis, we split our instances according to a RandomSplitter, and the
+ instances that got the treatment, are perturbated with a fictional effect via the Perturbator.
+
+ In order to create your own perturbator, you should create a derived class that implements the perturbate method.
+ The perturbate method should add the average effect in the desired way and return the dataframe with the extra average effect,
+ without affecting the initial dataframe. Keep in mind to use `df = df.copy()` in the first line of the perturbate method.
+
+ Arguments:
+ average_effect: The average effect of the treatment
+ target_col: name of the target_col to use as the outcome
+ treatment_col: The name of the column that contains the treatment
+ treatment: name of the treatment to use as the treated group
+
+ """
+
+ def __init__(
+ self,
+ average_effect: Optional[float] = None,
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ ):
+ self.average_effect = average_effect
+ self.target_col = target_col
+ self.treatment_col = treatment_col
+ self.treatment = treatment
+ self.treated_query = f"{self.treatment_col} == '{self.treatment}'"
+
+ def get_average_effect(self, average_effect: Optional[float] = None) -> float:
+ average_effect = (
+ average_effect if average_effect is not None else self.average_effect
+ )
+ if average_effect is None:
+ raise ValueError(
+ "average_effect must be provided, either in the constructor or in the method call"
+ )
+ return average_effect
+
+ @abstractmethod
+ def perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+ ) -> pd.DataFrame:
+ """Method to perturbate a dataframe"""
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a Perturbator object from a PowerConfig object"""
+ return cls(
+ average_effect=config.average_effect,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ )
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a Perturbator object from a PowerConfig object
+ +cluster_experiments/perturbator.py@classmethod
+def from_config(cls, config):
+ """Creates a Perturbator object from a PowerConfig object"""
+ return cls(
+ average_effect=config.average_effect,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ )
+perturbate(self, df, average_effect=None)
+
+
+¶Method to perturbate a dataframe
+ +cluster_experiments/perturbator.py@abstractmethod
+def perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+) -> pd.DataFrame:
+ """Method to perturbate a dataframe"""
+
+RelativePositivePerturbator (Perturbator)
+
+
+
+
+¶A Perturbator for continuous, positively-defined targets +applies a simulated effect multiplicatively for the treated samples, ie. +proportional to the target value for each sample. The number of samples with 0 +as target remains unchanged.
+target -> target * (1 + average_effect), where -1 < average_effect < inf
+ and target > 0 for all samples
+cluster_experiments/perturbator.pyclass RelativePositivePerturbator(Perturbator):
+ """
+ A Perturbator for continuous, positively-defined targets
+ applies a simulated effect multiplicatively for the treated samples, ie.
+ proportional to the target value for each sample. The number of samples with 0
+ as target remains unchanged.
+
+ ```
+ target -> target * (1 + average_effect), where -1 < average_effect < inf
+ and target > 0 for all samples
+ ```
+ """
+
+ def perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+ ) -> pd.DataFrame:
+ """
+ Usage:
+ ```python
+ from cluster_experiments.perturbator import RelativePositivePerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = RelativePositivePerturbator()
+ # Increase target metric by 50%
+ perturbator.perturbate(df, average_effect=0.5)
+ # returns pd.DataFrame({"target": [1, 3, 3], "treatment": ["A", "B", "A"]})
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ self.check_relative_positive_effect(df, average_effect)
+ df = self.apply_multiplicative_effect(df, average_effect)
+ return df
+
+ def check_relative_positive_effect(
+ self, df: pd.DataFrame, average_effect: float
+ ) -> None:
+ self.check_average_effect_greater_than(average_effect, x=-1)
+ self.check_target_is_not_negative(df)
+ self.check_target_is_not_constant_zero(df, average_effect)
+
+ def check_target_is_not_constant_zero(
+ self, df: pd.DataFrame, average_effect: float
+ ) -> Optional[NoReturn]:
+ treatment_zeros = (
+ (df[self.treatment_col] != self.treatment) | (df[self.target_col] == 0)
+ ).mean()
+ if 1.0 == treatment_zeros:
+ raise ValueError(
+ f"All treatment samples have {self.target_col} = 0, relative effect "
+ f"{average_effect} will have no effect"
+ )
+
+ def check_target_is_not_negative(self, df: pd.DataFrame) -> Optional[NoReturn]:
+ if any(df[self.target_col] < 0):
+ raise ValueError(
+ f"All {self.target_col} values need to be positive or 0, "
+ f"got {df[self.target_col].min()}"
+ )
+
+ def check_average_effect_greater_than(
+ self, average_effect: float, x: float
+ ) -> Optional[NoReturn]:
+ if average_effect <= x:
+ raise ValueError(
+ f"Simulated effect needs to be greater than {x*100:.0f}%, got "
+ f"{average_effect*100:.1f}%"
+ )
+
+ def apply_multiplicative_effect(
+ self, df: pd.DataFrame, effect: Union[float, np.ndarray]
+ ) -> pd.DataFrame:
+ df.loc[df[self.treatment_col] == self.treatment, self.target_col] *= 1 + effect
+ return df
+perturbate(self, df, average_effect=None)
+
+
+¶Usage: +
from cluster_experiments.perturbator import RelativePositivePerturbator
+import pandas as pd
+df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+perturbator = RelativePositivePerturbator()
+# Increase target metric by 50%
+perturbator.perturbate(df, average_effect=0.5)
+# returns pd.DataFrame({"target": [1, 3, 3], "treatment": ["A", "B", "A"]})
+cluster_experiments/perturbator.pydef perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+) -> pd.DataFrame:
+ """
+ Usage:
+ ```python
+ from cluster_experiments.perturbator import RelativePositivePerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = RelativePositivePerturbator()
+ # Increase target metric by 50%
+ perturbator.perturbate(df, average_effect=0.5)
+ # returns pd.DataFrame({"target": [1, 3, 3], "treatment": ["A", "B", "A"]})
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ self.check_relative_positive_effect(df, average_effect)
+ df = self.apply_multiplicative_effect(df, average_effect)
+ return df
+
+SegmentedBetaRelativePerturbator (BetaRelativePositivePerturbator)
+
+
+
+
+¶A stochastic Perturbator for continuous targets that applies a sampled +effect from the Beta distribution. It applies the effect multiplicatively +and based on given segments. +For each segment, the average segment effect is sampled from a beta +distribution with support in (0, 1). Within each segment, the individual +effects are sampled from a beta distribution with mean equal to the segment +average effect and support in (range_min, range_max).
+The number of samples with 0 as target remains unchanged.
+For additional details and recommendations on the parameters, see the
+documentation for the BetaRelativePerturbator class.
Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
average_effect |
+ Optional[float] |
+ the average effect of the treatment. Defaults to None. |
+ None |
+
target_col |
+ str |
+ name of the target_col to use as the outcome. Defaults to "target". |
+ 'target' |
+
treatment_col |
+ str |
+ the name of the column that contains the treatment. Defaults to "treatment". |
+ 'treatment' |
+
treatment |
+ str |
+ name of the treatment to use as the treated group. Defaults to "B". |
+ 'B' |
+
scale |
+ Optional[float] |
+ the scale of the effect distribution. Defaults to None. |
+ None |
+
range_min |
+ float |
+ the minimum value of the target range, must be >-1. +Defaults to -0.8, which allows for up to 5x decreases of the target. |
+ None |
+
range_max |
+ float |
+ the maximum value of the target range. +Defaults to 4, which allows for up to 5x increases of the target. |
+ None |
+
segment_cols |
+ Optional[List[str]] |
+ the columns to use for segmenting. Defaults to None. |
+ required | +
cluster_experiments/perturbator.pyclass SegmentedBetaRelativePerturbator(BetaRelativePositivePerturbator):
+ """
+ A stochastic Perturbator for continuous targets that applies a sampled
+ effect from the Beta distribution. It applies the effect multiplicatively
+ and based on given segments.
+ For each segment, the average segment effect is sampled from a beta
+ distribution with support in (0, 1). Within each segment, the individual
+ effects are sampled from a beta distribution with mean equal to the segment
+ average effect and support in (range_min, range_max).
+
+ The number of samples with 0 as target remains unchanged.
+
+ For additional details and recommendations on the parameters, see the
+ documentation for the `BetaRelativePerturbator` class.
+
+ Arguments:
+ average_effect (Optional[float], optional): the average effect of the treatment. Defaults to None.
+ target_col (str, optional): name of the target_col to use as the outcome. Defaults to "target".
+ treatment_col (str, optional): the name of the column that contains the treatment. Defaults to "treatment".
+ treatment (str, optional): name of the treatment to use as the treated group. Defaults to "B".
+ scale (Optional[float], optional): the scale of the effect distribution. Defaults to None.
+ range_min (float, optional): the minimum value of the target range, must be >-1.
+ Defaults to -0.8, which allows for up to 5x decreases of the target.
+ range_max (float, optional): the maximum value of the target range.
+ Defaults to 4, which allows for up to 5x increases of the target.
+ segment_cols (Optional[List[str]], optional): the columns to use for segmenting. Defaults to None.
+ """
+
+ def __init__(
+ self,
+ segment_cols: List[str],
+ average_effect: Optional[float] = None,
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ scale: Optional[float] = None,
+ range_min: Optional[float] = None,
+ range_max: Optional[float] = None,
+ ):
+ super().__init__(average_effect, target_col, treatment_col, treatment, scale)
+ self._range_min = range_min or -0.8
+ self._range_max = range_max or 4
+ self._segment_cols = segment_cols
+ self.segment_col = self._get_segment_col_name(segment_cols)
+
+ @staticmethod
+ def _get_segment_col_name(segment_cols: List[str]):
+ if not isinstance(segment_cols, list):
+ raise ValueError(
+ f"segment_cols must be of type List[str], got type {type(segment_cols)}"
+ )
+ return "_cluster_" + "_".join(segment_cols)
+
+ def _set_segment_col_values(self, df: pd.DataFrame):
+ if self.segment_col in df.columns:
+ raise ValueError(
+ f"Cannot use {self.segment_col=} as perturbator clustering "
+ f"column, as it already exists in the input dataframe!"
+ )
+ return df.copy().assign(
+ **{self.segment_col: df[self._segment_cols].astype(str).sum(axis=1)}
+ )
+
+ def get_cluster_perturbator_fixed_params(
+ self, average_effect: Optional[float] = None
+ ) -> Dict[str, Any]:
+ average_effect = self.get_average_effect(average_effect)
+ self.check_average_effect_greater_than(average_effect, x=0)
+ self.check_average_effect_less_than(average_effect, x=1)
+ scale = self.get_scale(average_effect)
+ return {
+ "average_effect": average_effect,
+ "scale": scale,
+ }
+
+ def get_cluster_perturbator(self, **kwargs) -> Perturbator:
+ sampled_effect = self._sample_beta_effect(
+ kwargs["average_effect"], kwargs["scale"], 1
+ )
+ cluster_perturbator = BetaRelativePerturbator(
+ average_effect=sampled_effect,
+ target_col=self.target_col,
+ treatment_col=self.treatment_col,
+ treatment=self.treatment,
+ range_min=self._range_min,
+ range_max=self._range_max,
+ )
+ return cluster_perturbator
+
+ def perturbate(
+ self,
+ df: pd.DataFrame,
+ average_effect: Optional[float] = None,
+ ) -> pd.DataFrame:
+ df = df.copy().reset_index(drop=True)
+ df = self._set_segment_col_values(df)
+
+ cluster_perturbator_params = self.get_cluster_perturbator_fixed_params(
+ average_effect
+ )
+ df_perturbed = pd.concat(
+ [
+ self.get_cluster_perturbator(**cluster_perturbator_params).perturbate(
+ df=df[df[self.segment_col] == cluster].copy()
+ )
+ for cluster in df[self.segment_col].unique()
+ ]
+ )
+ return df_perturbed.drop(columns=self.segment_col).reset_index(drop=True)
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a Perturbator object from a PowerConfig object"""
+ return cls(
+ average_effect=config.average_effect,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ scale=config.scale,
+ range_min=config.range_min,
+ range_max=config.range_max,
+ segment_cols=config.segment_cols,
+ )
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a Perturbator object from a PowerConfig object
+ +cluster_experiments/perturbator.py@classmethod
+def from_config(cls, config):
+ """Creates a Perturbator object from a PowerConfig object"""
+ return cls(
+ average_effect=config.average_effect,
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ scale=config.scale,
+ range_min=config.range_min,
+ range_max=config.range_max,
+ segment_cols=config.segment_cols,
+ )
+perturbate(self, df, average_effect=None)
+
+
+¶Usage: +
from cluster_experiments.perturbator import BetaRelativePositivePerturbator
+import pandas as pd
+df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+perturbator = BetaRelativePositivePerturbator()
+# Increase target metric by 20%
+perturbator.perturbate(df, average_effect=0.2)
+# returns pd.DataFrame({"target": [1, 3, 3], "treatment": ["A", "B", "A"]})
+cluster_experiments/perturbator.pydef perturbate(
+ self,
+ df: pd.DataFrame,
+ average_effect: Optional[float] = None,
+) -> pd.DataFrame:
+ df = df.copy().reset_index(drop=True)
+ df = self._set_segment_col_values(df)
+
+ cluster_perturbator_params = self.get_cluster_perturbator_fixed_params(
+ average_effect
+ )
+ df_perturbed = pd.concat(
+ [
+ self.get_cluster_perturbator(**cluster_perturbator_params).perturbate(
+ df=df[df[self.segment_col] == cluster].copy()
+ )
+ for cluster in df[self.segment_col].unique()
+ ]
+ )
+ return df_perturbed.drop(columns=self.segment_col).reset_index(drop=True)
+
+UniformPerturbator (Perturbator)
+
+
+
+
+¶UniformPerturbator is a Perturbator that adds a constant effect to the target column of the treated instances.
+ +cluster_experiments/perturbator.pyclass UniformPerturbator(Perturbator):
+ """
+ UniformPerturbator is a Perturbator that adds a constant effect to the target column of the treated instances.
+ """
+
+ def __init__(
+ self,
+ average_effect: Optional[float] = None,
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ ):
+ super().__init__(average_effect, target_col, treatment_col, treatment)
+ logging.warning(
+ "UniformPerturbator is deprecated and will be removed in future versions. "
+ "Use ConstantPerturbator instead."
+ )
+
+ def perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+ ) -> pd.DataFrame:
+ """
+ Usage:
+
+ ```python
+ from cluster_experiments.perturbator import UniformPerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = UniformPerturbator()
+ perturbator.perturbate(df, average_effect=1)
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ df = self.apply_additive_effect(df, average_effect)
+ return df
+
+ def apply_additive_effect(
+ self, df: pd.DataFrame, effect: Union[float, np.ndarray]
+ ) -> pd.DataFrame:
+ df.loc[df[self.treatment_col] == self.treatment, self.target_col] += effect
+ return df
+perturbate(self, df, average_effect=None)
+
+
+¶Usage:
+from cluster_experiments.perturbator import UniformPerturbator
+import pandas as pd
+df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+perturbator = UniformPerturbator()
+perturbator.perturbate(df, average_effect=1)
+cluster_experiments/perturbator.pydef perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float] = None
+) -> pd.DataFrame:
+ """
+ Usage:
+
+ ```python
+ from cluster_experiments.perturbator import UniformPerturbator
+ import pandas as pd
+ df = pd.DataFrame({"target": [1, 2, 3], "treatment": ["A", "B", "A"]})
+ perturbator = UniformPerturbator()
+ perturbator.perturbate(df, average_effect=1)
+ ```
+ """
+ df = df.copy().reset_index(drop=True)
+ average_effect = self.get_average_effect(average_effect)
+ df = self.apply_additive_effect(df, average_effect)
+ return df
+from cluster_experiments.power_analysis import *¶
+NormalPowerAnalysis
+
+
+
+¶Class used to run Power analysis, using the central limit theorem to estimate power based on standard errors of the estimator, +and the fact that the coefficients of a regression are normally distributed. +It does so by running simulations. In each simulation: +1. Assign treatment to dataframe randomly +2. Add pre-experiment data if needed +3. Get standard error from analysis
+Finally it returns the power of the analysis by counting how many times the effect was detected.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
splitter |
+ RandomSplitter |
+ RandomSplitter class to randomly assign treatment to dataframe. |
+ required | +
analysis |
+ ExperimentAnalysis |
+ ExperimentAnalysis class to use for analysis. |
+ required | +
cupac_model |
+ Optional[sklearn.base.BaseEstimator] |
+ Sklearn estimator class to add pre-experiment data to dataframe. If None, no pre-experiment data will be added. |
+ None |
+
target_col |
+ str |
+ Name of the column with the outcome variable. |
+ 'target' |
+
treatment_col |
+ str |
+ Name of the column with the treatment variable. |
+ 'treatment' |
+
treatment |
+ str |
+ value of treatment_col considered to be treatment (not control) |
+ 'B' |
+
control |
+ str |
+ value of treatment_col considered to be control (not treatment) |
+ 'A' |
+
n_simulations |
+ int |
+ Number of simulations to run. |
+ 100 |
+
alpha |
+ float |
+ Significance level. |
+ 0.05 |
+
features_cupac_model |
+ Optional[List[str]] |
+ Covariates to be used in cupac model |
+ None |
+
seed |
+ Optional[int] |
+ Optional. Seed to use for the splitter. |
+ None |
+
Usage: +
from datetime import date
+
+import numpy as np
+import pandas as pd
+from cluster_experiments.experiment_analysis import GeeExperimentAnalysis
+from cluster_experiments.power_analysis import NormalPowerAnalysis
+from cluster_experiments.random_splitter import ClusteredSplitter
+
+N = 1_000
+users = [f"User {i}" for i in range(1000)]
+clusters = [f"Cluster {i}" for i in range(100)]
+dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(1, 32)]
+df = pd.DataFrame(
+ {
+ "cluster": np.random.choice(clusters, size=N),
+ "target": np.random.normal(0, 1, size=N),
+ "user": np.random.choice(users, size=N),
+ "date": np.random.choice(dates, size=N),
+ }
+)
+
+experiment_dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(15, 32)]
+sw = ClusteredSplitter(
+ cluster_cols=["cluster", "date"],
+)
+
+analysis = GeeExperimentAnalysis(
+ cluster_cols=["cluster", "date"],
+)
+
+pw = NormalPowerAnalysis(
+ splitter=sw, analysis=analysis, n_simulations=50
+)
+
+power = pw.power_analysis(df, average_effect=0.1)
+print(f"{power = }")
+cluster_experiments/power_analysis.pyclass NormalPowerAnalysis:
+ """
+ Class used to run Power analysis, using the central limit theorem to estimate power based on standard errors of the estimator,
+ and the fact that the coefficients of a regression are normally distributed.
+ It does so by running simulations. In each simulation:
+ 1. Assign treatment to dataframe randomly
+ 2. Add pre-experiment data if needed
+ 3. Get standard error from analysis
+
+ Finally it returns the power of the analysis by counting how many times the effect was detected.
+
+ Args:
+ splitter: RandomSplitter class to randomly assign treatment to dataframe.
+ analysis: ExperimentAnalysis class to use for analysis.
+ cupac_model: Sklearn estimator class to add pre-experiment data to dataframe. If None, no pre-experiment data will be added.
+ target_col: Name of the column with the outcome variable.
+ treatment_col: Name of the column with the treatment variable.
+ treatment: value of treatment_col considered to be treatment (not control)
+ control: value of treatment_col considered to be control (not treatment)
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ features_cupac_model: Covariates to be used in cupac model
+ seed: Optional. Seed to use for the splitter.
+
+ Usage:
+ ```python
+ from datetime import date
+
+ import numpy as np
+ import pandas as pd
+ from cluster_experiments.experiment_analysis import GeeExperimentAnalysis
+ from cluster_experiments.power_analysis import NormalPowerAnalysis
+ from cluster_experiments.random_splitter import ClusteredSplitter
+
+ N = 1_000
+ users = [f"User {i}" for i in range(1000)]
+ clusters = [f"Cluster {i}" for i in range(100)]
+ dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(1, 32)]
+ df = pd.DataFrame(
+ {
+ "cluster": np.random.choice(clusters, size=N),
+ "target": np.random.normal(0, 1, size=N),
+ "user": np.random.choice(users, size=N),
+ "date": np.random.choice(dates, size=N),
+ }
+ )
+
+ experiment_dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(15, 32)]
+ sw = ClusteredSplitter(
+ cluster_cols=["cluster", "date"],
+ )
+
+ analysis = GeeExperimentAnalysis(
+ cluster_cols=["cluster", "date"],
+ )
+
+ pw = NormalPowerAnalysis(
+ splitter=sw, analysis=analysis, n_simulations=50
+ )
+
+ power = pw.power_analysis(df, average_effect=0.1)
+ print(f"{power = }")
+ ```
+ """
+
+ def __init__(
+ self,
+ splitter: RandomSplitter,
+ analysis: ExperimentAnalysis,
+ cupac_model: Optional[BaseEstimator] = None,
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ control: str = "A",
+ n_simulations: int = 100,
+ alpha: float = 0.05,
+ features_cupac_model: Optional[List[str]] = None,
+ seed: Optional[int] = None,
+ hypothesis: str = "two-sided",
+ time_col: Optional[str] = None,
+ ):
+ self.splitter = splitter
+ self.analysis = analysis
+ self.n_simulations = n_simulations
+ self.target_col = target_col
+ self.treatment = treatment
+ self.control = control
+ self.treatment_col = treatment_col
+ self.alpha = alpha
+ self.hypothesis = hypothesis
+ self.time_col = time_col
+
+ self.cupac_handler = CupacHandler(
+ cupac_model=cupac_model,
+ target_col=target_col,
+ features_cupac_model=features_cupac_model,
+ )
+ if seed is not None:
+ random.seed(seed) # seed for splitter
+ np.random.seed(seed) # numpy seed
+ # may need to seed other stochasticity sources if added
+
+ self.check_inputs()
+
+ def _split(self, df: pd.DataFrame) -> pd.DataFrame:
+ """
+ Split dataframe.
+ Args:
+ df: Dataframe with outcome variable
+ """
+ treatment_df = self.splitter.assign_treatment_df(df)
+ self.log_nulls(treatment_df)
+ treatment_df = treatment_df.query(
+ f"{self.treatment_col}.notnull()", engine="python"
+ ).query(
+ f"{self.treatment_col}.isin(['{self.treatment}', '{self.control}'])",
+ engine="python",
+ )
+ return treatment_df
+
+ def _get_standard_error(
+ self,
+ df: pd.DataFrame,
+ n_simulations: int,
+ verbose: bool,
+ ) -> Generator[float, None, None]:
+ for _ in tqdm(range(n_simulations), disable=not verbose):
+ split_df = self._split(df)
+ yield self.analysis.get_standard_error(split_df)
+
+ def _normal_power_calculation(
+ self, alpha: float, std_error: float, average_effect: float
+ ) -> float:
+ """Returns the power of the analysis using the normal distribution.
+ Arguments:
+ alpha: significance level
+ std_error: standard error of the analysis
+ average_effect: effect size of the analysis
+ """
+ if HypothesisEntries(self.analysis.hypothesis) == HypothesisEntries.LESS:
+ z_alpha = norm.ppf(alpha)
+ return float(norm.cdf(z_alpha - average_effect / std_error))
+
+ if HypothesisEntries(self.analysis.hypothesis) == HypothesisEntries.GREATER:
+ z_alpha = norm.ppf(1 - alpha)
+ return 1 - float(norm.cdf(z_alpha - average_effect / std_error))
+
+ if HypothesisEntries(self.analysis.hypothesis) == HypothesisEntries.TWO_SIDED:
+ z_alpha = norm.ppf(1 - alpha / 2)
+ norm_cdf_right = norm.cdf(z_alpha - average_effect / std_error)
+ norm_cdf_left = norm.cdf(-z_alpha - average_effect / std_error)
+ return float(norm_cdf_left + (1 - norm_cdf_right))
+
+ raise ValueError(f"{self.analysis.hypothesis} is not a valid HypothesisEntries")
+
+ def _normal_mde_calculation(
+ self, alpha: float, std_error: float, power: float
+ ) -> float:
+ """
+ Returns the minimum detectable effect of the analysis using the normal distribution.
+ Args:
+ alpha: Significance level.
+ std_error: Standard error of the analysis.
+ power: Power of the analysis.
+ """
+ if HypothesisEntries(self.analysis.hypothesis) == HypothesisEntries.LESS:
+ z_alpha = norm.ppf(alpha)
+ z_beta = norm.ppf(1 - power)
+ elif HypothesisEntries(self.analysis.hypothesis) == HypothesisEntries.GREATER:
+ z_alpha = norm.ppf(1 - alpha)
+ z_beta = norm.ppf(power)
+ elif HypothesisEntries(self.analysis.hypothesis) == HypothesisEntries.TWO_SIDED:
+ # we are neglecting norm_cdf_left
+ z_alpha = norm.ppf(1 - alpha / 2)
+ z_beta = norm.ppf(power)
+ else:
+ raise ValueError(
+ f"{self.analysis.hypothesis} is not a valid HypothesisEntries"
+ )
+
+ return float(z_alpha + z_beta) * std_error
+
+ def mde_power_line(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ powers: Iterable[float] = (),
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ ) -> Dict[float, float]:
+ """
+ Returns the minimum detectable effect of the analysis.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ power: Power of the analysis.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ alpha = self.alpha if alpha is None else alpha
+ std_error = self._get_average_standard_error(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ n_simulations=n_simulations,
+ )
+ return {
+ power: self._normal_mde_calculation(
+ alpha=alpha, std_error=std_error, power=power
+ )
+ for power in powers
+ }
+
+ def mde(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ power: float = 0.8,
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ ) -> float:
+ """
+ Returns the minimum detectable effect of the analysis.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ power: Power of the analysis.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ return self.mde_power_line(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ powers=[power],
+ n_simulations=n_simulations,
+ alpha=alpha,
+ )[power]
+
+ def _get_average_standard_error(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ n_simulations: Optional[int] = None,
+ ) -> float:
+ """
+ Gets standard error to be used in normal power calculation.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effects: Average effects to test.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ n_simulations = self.n_simulations if n_simulations is None else n_simulations
+
+ df = df.copy()
+ df = self.cupac_handler.add_covariates(df, pre_experiment_df)
+
+ std_errors = list(self._get_standard_error(df, n_simulations, verbose))
+ std_error_mean = float(np.mean(std_errors))
+
+ return std_error_mean
+
+ def run_average_standard_error(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ n_simulations: Optional[int] = None,
+ experiment_length: Iterable[int] = (),
+ ) -> Generator[Tuple[float, int], None, None]:
+ """
+ Run power analysis by simulation, using standard errors from the analysis.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ n_simulations: Number of simulations to run.
+ experiment_length: Length of the experiment in days.
+ """
+ n_simulations = self.n_simulations if n_simulations is None else n_simulations
+
+ for n_days in experiment_length:
+ df_time = df.copy()
+ experiment_start = df_time[self.time_col].min()
+ df_time = df_time.loc[
+ df_time[self.time_col] < experiment_start + pd.Timedelta(days=n_days)
+ ]
+ std_error_mean = self._get_average_standard_error(
+ df=df_time,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ n_simulations=n_simulations,
+ )
+ yield std_error_mean, n_days
+
+ def power_time_line(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effects: Iterable[float] = (),
+ experiment_length: Iterable[int] = (),
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ ) -> List[Dict]:
+ """
+ Run power analysis by simulation, using standard errors from the analysis.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effects: Average effects to test.
+ experiment_length: Length of the experiment in days.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ alpha = self.alpha if alpha is None else alpha
+
+ results = []
+ for std_error_mean, n_days in self.run_average_standard_error(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ n_simulations=n_simulations,
+ experiment_length=experiment_length,
+ ):
+ for effect in average_effects:
+ power = self._normal_power_calculation(
+ alpha=alpha, std_error=std_error_mean, average_effect=effect
+ )
+ results.append(
+ {"effect": effect, "power": power, "experiment_length": n_days}
+ )
+
+ return results
+
+ def mde_time_line(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ powers: Iterable[float] = (),
+ experiment_length: Iterable[int] = (),
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ ) -> List[Dict]:
+ alpha = self.alpha if alpha is None else alpha
+
+ results = []
+ for std_error_mean, n_days in self.run_average_standard_error(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ n_simulations=n_simulations,
+ experiment_length=experiment_length,
+ ):
+ for power in powers:
+ mde = self._normal_mde_calculation(
+ alpha=alpha, std_error=std_error_mean, power=power
+ )
+ results.append(
+ {"power": power, "mde": mde, "experiment_length": n_days}
+ )
+ return results
+
+ def power_line(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effects: Iterable[float] = (),
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ ) -> Dict[float, float]:
+ """
+ Run power analysis by simulation, using standard errors from the analysis.
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effects: Average effects to test.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ alpha = self.alpha if alpha is None else alpha
+
+ std_error_mean = self._get_average_standard_error(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ n_simulations=n_simulations,
+ )
+
+ return {
+ effect: self._normal_power_calculation(
+ alpha=alpha, std_error=std_error_mean, average_effect=effect
+ )
+ for effect in average_effects
+ }
+
+ def power_analysis(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: float = 0.0,
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ ) -> float:
+ """
+ Run power analysis by simulation, using standard errors from the analysis.
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ return self.power_line(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effects=[average_effect],
+ n_simulations=n_simulations,
+ alpha=alpha,
+ )[average_effect]
+
+ def log_nulls(self, df: pd.DataFrame) -> None:
+ """Warns about dropping nulls in treatment column"""
+ n_nulls = len(df.query(f"{self.treatment_col}.isnull()", engine="python"))
+ if n_nulls > 0:
+ logging.warning(
+ f"There are {n_nulls} null values in treatment, dropping them"
+ )
+
+ @classmethod
+ def from_dict(cls, config_dict: dict) -> "NormalPowerAnalysis":
+ """Constructs PowerAnalysis from dictionary"""
+ config = PowerConfig(**config_dict)
+ return cls.from_config(config)
+
+ @classmethod
+ def from_config(cls, config: PowerConfig) -> "NormalPowerAnalysis":
+ """Constructs PowerAnalysis from PowerConfig"""
+ splitter_cls = _get_mapping_key(splitter_mapping, config.splitter)
+ analysis_cls = _get_mapping_key(analysis_mapping, config.analysis)
+ cupac_cls = _get_mapping_key(cupac_model_mapping, config.cupac_model)
+ return cls(
+ splitter=splitter_cls.from_config(config),
+ analysis=analysis_cls.from_config(config),
+ cupac_model=cupac_cls.from_config(config),
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ control=config.control,
+ n_simulations=config.n_simulations,
+ alpha=config.alpha,
+ features_cupac_model=config.features_cupac_model,
+ seed=config.seed,
+ hypothesis=config.hypothesis,
+ time_col=config.time_col,
+ )
+
+ def check_treatment_col(self):
+ """Checks consistency of treatment column"""
+ assert (
+ self.analysis.treatment_col == self.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in PowerAnalysis ({self.treatment_col})"
+
+ assert (
+ self.analysis.treatment_col == self.splitter.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in splitter ({self.splitter.treatment_col})"
+
+ def check_target_col(self):
+ assert (
+ self.analysis.target_col == self.target_col
+ ), f"target_col in analysis ({self.analysis.target_col}) must be the same as target_col in PowerAnalysis ({self.target_col})"
+
+ def check_treatment(self):
+ assert (
+ self.analysis.treatment == self.treatment
+ ), f"treatment in analysis ({self.analysis.treatment}) must be the same as treatment in PowerAnalysis ({self.treatment})"
+
+ assert (
+ self.analysis.treatment in self.splitter.treatments
+ ), f"treatment in analysis ({self.analysis.treatment}) must be in treatments in splitter ({self.splitter.treatments})"
+
+ assert (
+ self.control in self.splitter.treatments
+ ), f"control in power analysis ({self.control}) must be in treatments in splitter ({self.splitter.treatments})"
+
+ def check_covariates(self):
+ if hasattr(self.analysis, "covariates"):
+ cupac_in_covariates = (
+ self.cupac_handler.cupac_outcome_name in self.analysis.covariates
+ )
+
+ assert cupac_in_covariates or not self.cupac_handler.is_cupac, (
+ f"covariates in analysis must contain {self.cupac_handler.cupac_outcome_name} if cupac_model is not None. "
+ f"If you want to use cupac_model, you must add the cupac outcome to the covariates of the analysis "
+ f"You may want to do covariates=['{self.cupac_handler.cupac_outcome_name}'] in your analysis method or your config"
+ )
+
+ if hasattr(self.splitter, "cluster_cols"):
+ if set(self.analysis.covariates).intersection(
+ set(self.splitter.cluster_cols)
+ ):
+ logging.warning(
+ f"covariates in analysis ({self.analysis.covariates}) are also cluster_cols in splitter ({self.splitter.cluster_cols}). "
+ f"Be specially careful when using switchback splitters, since the time splitter column is being overriden"
+ )
+
+ def check_clusters(self):
+ has_analysis_clusters = hasattr(self.analysis, "cluster_cols")
+ has_splitter_clusters = hasattr(self.splitter, "cluster_cols")
+ not_cluster_cols_cond = not has_analysis_clusters or not has_splitter_clusters
+ assert (
+ not_cluster_cols_cond
+ or self.analysis.cluster_cols == self.splitter.cluster_cols
+ ), f"cluster_cols in analysis ({self.analysis.cluster_cols}) must be the same as cluster_cols in splitter ({self.splitter.cluster_cols})"
+
+ assert (
+ has_splitter_clusters
+ or not has_analysis_clusters
+ or not self.analysis.cluster_cols
+ or isinstance(self.splitter, RepeatedSampler)
+ ), "analysis has cluster_cols but splitter does not."
+
+ assert (
+ has_analysis_clusters
+ or not has_splitter_clusters
+ or not self.splitter.cluster_cols
+ ), "splitter has cluster_cols but analysis does not."
+
+ has_time_col = hasattr(self.splitter, "time_col")
+ assert not (
+ has_time_col
+ and has_splitter_clusters
+ and self.splitter.time_col not in self.splitter.cluster_cols
+ ), "in switchback splitters, time_col must be in cluster_cols"
+
+ def check_inputs(self):
+ self.check_covariates()
+ self.check_treatment_col()
+ self.check_target_col()
+ self.check_treatment()
+ self.check_clusters()
+check_treatment_col(self)
+
+
+¶Checks consistency of treatment column
+ +cluster_experiments/power_analysis.pydef check_treatment_col(self):
+ """Checks consistency of treatment column"""
+ assert (
+ self.analysis.treatment_col == self.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in PowerAnalysis ({self.treatment_col})"
+
+ assert (
+ self.analysis.treatment_col == self.splitter.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in splitter ({self.splitter.treatment_col})"
+from_config(config)
+
+
+ classmethod
+
+
+¶Constructs PowerAnalysis from PowerConfig
+ +cluster_experiments/power_analysis.py@classmethod
+def from_config(cls, config: PowerConfig) -> "NormalPowerAnalysis":
+ """Constructs PowerAnalysis from PowerConfig"""
+ splitter_cls = _get_mapping_key(splitter_mapping, config.splitter)
+ analysis_cls = _get_mapping_key(analysis_mapping, config.analysis)
+ cupac_cls = _get_mapping_key(cupac_model_mapping, config.cupac_model)
+ return cls(
+ splitter=splitter_cls.from_config(config),
+ analysis=analysis_cls.from_config(config),
+ cupac_model=cupac_cls.from_config(config),
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ control=config.control,
+ n_simulations=config.n_simulations,
+ alpha=config.alpha,
+ features_cupac_model=config.features_cupac_model,
+ seed=config.seed,
+ hypothesis=config.hypothesis,
+ time_col=config.time_col,
+ )
+from_dict(config_dict)
+
+
+ classmethod
+
+
+¶Constructs PowerAnalysis from dictionary
+ +cluster_experiments/power_analysis.py@classmethod
+def from_dict(cls, config_dict: dict) -> "NormalPowerAnalysis":
+ """Constructs PowerAnalysis from dictionary"""
+ config = PowerConfig(**config_dict)
+ return cls.from_config(config)
+log_nulls(self, df)
+
+
+¶Warns about dropping nulls in treatment column
+ +cluster_experiments/power_analysis.pydef log_nulls(self, df: pd.DataFrame) -> None:
+ """Warns about dropping nulls in treatment column"""
+ n_nulls = len(df.query(f"{self.treatment_col}.isnull()", engine="python"))
+ if n_nulls > 0:
+ logging.warning(
+ f"There are {n_nulls} null values in treatment, dropping them"
+ )
+mde(self, df, pre_experiment_df=None, verbose=False, power=0.8, n_simulations=None, alpha=None)
+
+
+¶Returns the minimum detectable effect of the analysis.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
power |
+ float |
+ Power of the analysis. |
+ 0.8 |
+
n_simulations |
+ Optional[int] |
+ Number of simulations to run. |
+ None |
+
alpha |
+ Optional[float] |
+ Significance level. |
+ None |
+
cluster_experiments/power_analysis.pydef mde(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ power: float = 0.8,
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+) -> float:
+ """
+ Returns the minimum detectable effect of the analysis.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ power: Power of the analysis.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ return self.mde_power_line(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ powers=[power],
+ n_simulations=n_simulations,
+ alpha=alpha,
+ )[power]
+mde_power_line(self, df, pre_experiment_df=None, verbose=False, powers=(), n_simulations=None, alpha=None)
+
+
+¶Returns the minimum detectable effect of the analysis.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
power |
+ + | Power of the analysis. |
+ required | +
n_simulations |
+ Optional[int] |
+ Number of simulations to run. |
+ None |
+
alpha |
+ Optional[float] |
+ Significance level. |
+ None |
+
cluster_experiments/power_analysis.pydef mde_power_line(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ powers: Iterable[float] = (),
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+) -> Dict[float, float]:
+ """
+ Returns the minimum detectable effect of the analysis.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ power: Power of the analysis.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ alpha = self.alpha if alpha is None else alpha
+ std_error = self._get_average_standard_error(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ n_simulations=n_simulations,
+ )
+ return {
+ power: self._normal_mde_calculation(
+ alpha=alpha, std_error=std_error, power=power
+ )
+ for power in powers
+ }
+power_analysis(self, df, pre_experiment_df=None, verbose=False, average_effect=0.0, n_simulations=None, alpha=None)
+
+
+¶Run power analysis by simulation, using standard errors from the analysis.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
average_effect |
+ float |
+ Average effect of treatment. |
+ 0.0 |
+
n_simulations |
+ Optional[int] |
+ Number of simulations to run. |
+ None |
+
alpha |
+ Optional[float] |
+ Significance level. |
+ None |
+
cluster_experiments/power_analysis.pydef power_analysis(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: float = 0.0,
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+) -> float:
+ """
+ Run power analysis by simulation, using standard errors from the analysis.
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ return self.power_line(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effects=[average_effect],
+ n_simulations=n_simulations,
+ alpha=alpha,
+ )[average_effect]
+power_line(self, df, pre_experiment_df=None, verbose=False, average_effects=(), n_simulations=None, alpha=None)
+
+
+¶Run power analysis by simulation, using standard errors from the analysis.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
average_effects |
+ Iterable[float] |
+ Average effects to test. |
+ () |
+
n_simulations |
+ Optional[int] |
+ Number of simulations to run. |
+ None |
+
alpha |
+ Optional[float] |
+ Significance level. |
+ None |
+
cluster_experiments/power_analysis.pydef power_line(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effects: Iterable[float] = (),
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+) -> Dict[float, float]:
+ """
+ Run power analysis by simulation, using standard errors from the analysis.
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effects: Average effects to test.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ alpha = self.alpha if alpha is None else alpha
+
+ std_error_mean = self._get_average_standard_error(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ n_simulations=n_simulations,
+ )
+
+ return {
+ effect: self._normal_power_calculation(
+ alpha=alpha, std_error=std_error_mean, average_effect=effect
+ )
+ for effect in average_effects
+ }
+power_time_line(self, df, pre_experiment_df=None, verbose=False, average_effects=(), experiment_length=(), n_simulations=None, alpha=None)
+
+
+¶Run power analysis by simulation, using standard errors from the analysis.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
average_effects |
+ Iterable[float] |
+ Average effects to test. |
+ () |
+
experiment_length |
+ Iterable[int] |
+ Length of the experiment in days. |
+ () |
+
n_simulations |
+ Optional[int] |
+ Number of simulations to run. |
+ None |
+
alpha |
+ Optional[float] |
+ Significance level. |
+ None |
+
cluster_experiments/power_analysis.pydef power_time_line(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effects: Iterable[float] = (),
+ experiment_length: Iterable[int] = (),
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+) -> List[Dict]:
+ """
+ Run power analysis by simulation, using standard errors from the analysis.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effects: Average effects to test.
+ experiment_length: Length of the experiment in days.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ alpha = self.alpha if alpha is None else alpha
+
+ results = []
+ for std_error_mean, n_days in self.run_average_standard_error(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ n_simulations=n_simulations,
+ experiment_length=experiment_length,
+ ):
+ for effect in average_effects:
+ power = self._normal_power_calculation(
+ alpha=alpha, std_error=std_error_mean, average_effect=effect
+ )
+ results.append(
+ {"effect": effect, "power": power, "experiment_length": n_days}
+ )
+
+ return results
+run_average_standard_error(self, df, pre_experiment_df=None, verbose=False, n_simulations=None, experiment_length=())
+
+
+¶Run power analysis by simulation, using standard errors from the analysis.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
n_simulations |
+ Optional[int] |
+ Number of simulations to run. |
+ None |
+
experiment_length |
+ Iterable[int] |
+ Length of the experiment in days. |
+ () |
+
cluster_experiments/power_analysis.pydef run_average_standard_error(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ n_simulations: Optional[int] = None,
+ experiment_length: Iterable[int] = (),
+) -> Generator[Tuple[float, int], None, None]:
+ """
+ Run power analysis by simulation, using standard errors from the analysis.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ n_simulations: Number of simulations to run.
+ experiment_length: Length of the experiment in days.
+ """
+ n_simulations = self.n_simulations if n_simulations is None else n_simulations
+
+ for n_days in experiment_length:
+ df_time = df.copy()
+ experiment_start = df_time[self.time_col].min()
+ df_time = df_time.loc[
+ df_time[self.time_col] < experiment_start + pd.Timedelta(days=n_days)
+ ]
+ std_error_mean = self._get_average_standard_error(
+ df=df_time,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ n_simulations=n_simulations,
+ )
+ yield std_error_mean, n_days
+
+PowerAnalysis
+
+
+
+¶Class used to run Power analysis. It does so by running simulations. In each simulation: +1. Assign treatment to dataframe randomly +2. Perturbate dataframe +3. Add pre-experiment data if needed +4. Run analysis
+Finally it returns the power of the analysis by counting how many times the effect was detected.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
perturbator |
+ Perturbator |
+ Perturbator class to perturbate dataframe with treatment assigned. |
+ required | +
splitter |
+ RandomSplitter |
+ RandomSplitter class to randomly assign treatment to dataframe. |
+ required | +
analysis |
+ ExperimentAnalysis |
+ ExperimentAnalysis class to use for analysis. |
+ required | +
cupac_model |
+ Optional[sklearn.base.BaseEstimator] |
+ Sklearn estimator class to add pre-experiment data to dataframe. If None, no pre-experiment data will be added. |
+ None |
+
target_col |
+ str |
+ Name of the column with the outcome variable. |
+ 'target' |
+
treatment_col |
+ str |
+ Name of the column with the treatment variable. |
+ 'treatment' |
+
treatment |
+ str |
+ value of treatment_col considered to be treatment (not control) |
+ 'B' |
+
control |
+ str |
+ value of treatment_col considered to be control (not treatment) |
+ 'A' |
+
n_simulations |
+ int |
+ Number of simulations to run. |
+ 100 |
+
alpha |
+ float |
+ Significance level. |
+ 0.05 |
+
features_cupac_model |
+ Optional[List[str]] |
+ Covariates to be used in cupac model |
+ None |
+
seed |
+ Optional[int] |
+ Optional. Seed to use for the splitter. |
+ None |
+
Usage: +
from datetime import date
+
+import numpy as np
+import pandas as pd
+from cluster_experiments.experiment_analysis import GeeExperimentAnalysis
+from cluster_experiments.perturbator import ConstantPerturbator
+from cluster_experiments.power_analysis import PowerAnalysis
+from cluster_experiments.random_splitter import ClusteredSplitter
+
+N = 1_000
+users = [f"User {i}" for i in range(1000)]
+clusters = [f"Cluster {i}" for i in range(100)]
+dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(1, 32)]
+df = pd.DataFrame(
+ {
+ "cluster": np.random.choice(clusters, size=N),
+ "target": np.random.normal(0, 1, size=N),
+ "user": np.random.choice(users, size=N),
+ "date": np.random.choice(dates, size=N),
+ }
+)
+
+experiment_dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(15, 32)]
+sw = ClusteredSplitter(
+ cluster_cols=["cluster", "date"],
+)
+
+perturbator = ConstantPerturbator()
+
+analysis = GeeExperimentAnalysis(
+ cluster_cols=["cluster", "date"],
+)
+
+pw = PowerAnalysis(
+ perturbator=perturbator, splitter=sw, analysis=analysis, n_simulations=50
+)
+
+power = pw.power_analysis(df, average_effect=0.1)
+print(f"{power = }")
+cluster_experiments/power_analysis.pyclass PowerAnalysis:
+ """
+ Class used to run Power analysis. It does so by running simulations. In each simulation:
+ 1. Assign treatment to dataframe randomly
+ 2. Perturbate dataframe
+ 3. Add pre-experiment data if needed
+ 4. Run analysis
+
+ Finally it returns the power of the analysis by counting how many times the effect was detected.
+
+ Args:
+ perturbator: Perturbator class to perturbate dataframe with treatment assigned.
+ splitter: RandomSplitter class to randomly assign treatment to dataframe.
+ analysis: ExperimentAnalysis class to use for analysis.
+ cupac_model: Sklearn estimator class to add pre-experiment data to dataframe. If None, no pre-experiment data will be added.
+ target_col: Name of the column with the outcome variable.
+ treatment_col: Name of the column with the treatment variable.
+ treatment: value of treatment_col considered to be treatment (not control)
+ control: value of treatment_col considered to be control (not treatment)
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ features_cupac_model: Covariates to be used in cupac model
+ seed: Optional. Seed to use for the splitter.
+
+ Usage:
+ ```python
+ from datetime import date
+
+ import numpy as np
+ import pandas as pd
+ from cluster_experiments.experiment_analysis import GeeExperimentAnalysis
+ from cluster_experiments.perturbator import ConstantPerturbator
+ from cluster_experiments.power_analysis import PowerAnalysis
+ from cluster_experiments.random_splitter import ClusteredSplitter
+
+ N = 1_000
+ users = [f"User {i}" for i in range(1000)]
+ clusters = [f"Cluster {i}" for i in range(100)]
+ dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(1, 32)]
+ df = pd.DataFrame(
+ {
+ "cluster": np.random.choice(clusters, size=N),
+ "target": np.random.normal(0, 1, size=N),
+ "user": np.random.choice(users, size=N),
+ "date": np.random.choice(dates, size=N),
+ }
+ )
+
+ experiment_dates = [f"{date(2022, 1, i):%Y-%m-%d}" for i in range(15, 32)]
+ sw = ClusteredSplitter(
+ cluster_cols=["cluster", "date"],
+ )
+
+ perturbator = ConstantPerturbator()
+
+ analysis = GeeExperimentAnalysis(
+ cluster_cols=["cluster", "date"],
+ )
+
+ pw = PowerAnalysis(
+ perturbator=perturbator, splitter=sw, analysis=analysis, n_simulations=50
+ )
+
+ power = pw.power_analysis(df, average_effect=0.1)
+ print(f"{power = }")
+ ```
+ """
+
+ def __init__(
+ self,
+ perturbator: Perturbator,
+ splitter: RandomSplitter,
+ analysis: ExperimentAnalysis,
+ cupac_model: Optional[BaseEstimator] = None,
+ target_col: str = "target",
+ treatment_col: str = "treatment",
+ treatment: str = "B",
+ control: str = "A",
+ n_simulations: int = 100,
+ alpha: float = 0.05,
+ features_cupac_model: Optional[List[str]] = None,
+ seed: Optional[int] = None,
+ hypothesis: str = "two-sided",
+ ):
+ self.perturbator = perturbator
+ self.splitter = splitter
+ self.analysis = analysis
+ self.n_simulations = n_simulations
+ self.target_col = target_col
+ self.treatment = treatment
+ self.control = control
+ self.treatment_col = treatment_col
+ self.alpha = alpha
+ self.hypothesis = hypothesis
+
+ self.cupac_handler = CupacHandler(
+ cupac_model=cupac_model,
+ target_col=target_col,
+ features_cupac_model=features_cupac_model,
+ )
+ if seed is not None:
+ random.seed(seed) # seed for splitter
+ np.random.seed(seed) # seed for the binary perturbator
+ # may need to seed other stochasticity sources if added
+
+ self.check_inputs()
+
+ def _simulate_perturbed_df(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: Optional[float] = None,
+ n_simulations: int = 100,
+ ) -> Generator[pd.DataFrame, None, None]:
+ """Yields splitted + perturbated dataframe for each iteration of the simulation."""
+ df = df.copy()
+ df = self.cupac_handler.add_covariates(df, pre_experiment_df)
+
+ for _ in tqdm(range(n_simulations), disable=not verbose):
+ yield self._split_and_perturbate(df, average_effect)
+
+ def simulate_pvalue(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: Optional[float] = None,
+ n_simulations: int = 100,
+ ) -> Generator[float, None, None]:
+ """
+ Yields p-values for each iteration of the simulation.
+ In general, this is to be used in power_analysis method. However,
+ if you're interested in the distribution of p-values, you can use this method to generate them.
+ Args:
+ df: Dataframe with outcome variable.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ """
+ for perturbed_df in self._simulate_perturbed_df(
+ df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effect=average_effect,
+ n_simulations=n_simulations,
+ ):
+ yield self.analysis.get_pvalue(perturbed_df)
+
+ def running_power_analysis(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: Optional[float] = None,
+ n_simulations: int = 100,
+ ) -> Generator[float, None, None]:
+ """
+ Yields running power for each iteration of the simulation.
+ if you're interested in getting the power at each iteration, you can use this method to generate them.
+ Args:
+ df: Dataframe with outcome variable.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ """
+ n_rejected = 0
+ for i, perturbed_df in enumerate(
+ self._simulate_perturbed_df(
+ df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effect=average_effect,
+ n_simulations=n_simulations,
+ )
+ ):
+ p_value = self.analysis.get_pvalue(perturbed_df)
+ n_rejected += int(p_value < self.alpha)
+ yield n_rejected / (i + 1)
+
+ def simulate_point_estimate(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: Optional[float] = None,
+ n_simulations: int = 100,
+ ) -> Generator[float, None, None]:
+ """
+ Yields point estimates for each iteration of the simulation.
+ In general, this is to be used in power_analysis method. However,
+ if you're interested in the distribution of point estimates, you can use this method to generate them.
+
+ This is an experimental feature and it might change in the future.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ """
+ for perturbed_df in self._simulate_perturbed_df(
+ df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effect=average_effect,
+ n_simulations=n_simulations,
+ ):
+ yield self.analysis.get_point_estimate(perturbed_df)
+
+ def power_analysis(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: Optional[float] = None,
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ n_jobs: int = 1,
+ ) -> float:
+ """
+ Run power analysis by simulation
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ n_jobs: Number of jobs to run in parallel. If 1, it will run in serial.
+ """
+ n_simulations = self.n_simulations if n_simulations is None else n_simulations
+ alpha = self.alpha if alpha is None else alpha
+
+ df = df.copy()
+ df = self.cupac_handler.add_covariates(df, pre_experiment_df)
+
+ if n_jobs == 1:
+ return self._non_parallel_loop(
+ df, average_effect, n_simulations, alpha, verbose
+ )
+ elif n_jobs > 1 or n_jobs == -1:
+ return self._parallel_loop(
+ df, average_effect, n_simulations, alpha, verbose, n_jobs
+ )
+ else:
+ raise ValueError("n_jobs must be greater than 0, or -1.")
+
+ def _split(self, df: pd.DataFrame) -> pd.DataFrame:
+ """
+ Split dataframe.
+ Args:
+ df: Dataframe with outcome variable
+ """
+ treatment_df = self.splitter.assign_treatment_df(df)
+ self.log_nulls(treatment_df)
+ treatment_df = treatment_df.query(
+ f"{self.treatment_col}.notnull()", engine="python"
+ ).query(
+ f"{self.treatment_col}.isin(['{self.treatment}', '{self.control}'])",
+ engine="python",
+ )
+
+ return treatment_df
+
+ def _perturbate(
+ self, treatment_df: pd.DataFrame, average_effect: Optional[float]
+ ) -> pd.DataFrame:
+ """
+ Perturbate dataframe using perturbator.
+ Args:
+ df: Dataframe with outcome variable
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ """
+
+ perturbed_df = self.perturbator.perturbate(
+ treatment_df, average_effect=average_effect
+ )
+ return perturbed_df
+
+ def _split_and_perturbate(
+ self, df: pd.DataFrame, average_effect: Optional[float]
+ ) -> pd.DataFrame:
+ treatment_df = self._split(df)
+ perturbed_df = self._perturbate(
+ treatment_df=treatment_df, average_effect=average_effect
+ )
+ return perturbed_df
+
+ def _run_simulation(self, args: Tuple[pd.DataFrame, Optional[float]]) -> float:
+ df, average_effect = args
+ perturbed_df = self._split_and_perturbate(df, average_effect)
+ return self.analysis.get_pvalue(perturbed_df)
+
+ def _non_parallel_loop(
+ self,
+ df: pd.DataFrame,
+ average_effect: Optional[float],
+ n_simulations: int,
+ alpha: float,
+ verbose: bool,
+ ) -> float:
+ """
+ Run power analysis by simulation in serial
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ """
+ n_detected_mde = 0
+ for _ in tqdm(range(n_simulations), disable=not verbose):
+ p_value = self._run_simulation((df, average_effect))
+ if verbose:
+ print(f"p_value of simulation run: {p_value:.3f}")
+ n_detected_mde += p_value < alpha
+
+ return n_detected_mde / n_simulations
+
+ def _parallel_loop(
+ self,
+ df: pd.DataFrame,
+ average_effect: Optional[float],
+ n_simulations: int,
+ alpha: float,
+ verbose: bool,
+ n_jobs: int,
+ ) -> float:
+ """
+ Run power analysis by simulation in parallel
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ n_jobs: Number of jobs to run in parallel.
+ """
+ from multiprocessing import Pool, cpu_count
+
+ n_jobs = n_jobs if n_jobs != -1 else cpu_count()
+
+ n_detected_mde = 0
+ with Pool(processes=n_jobs) as pool:
+ args = [(df, average_effect) for _ in range(n_simulations)]
+ results = pool.imap_unordered(self._run_simulation, args)
+ for p_value in tqdm(results, total=n_simulations, disable=not verbose):
+ n_detected_mde += p_value < alpha
+
+ return n_detected_mde / n_simulations
+
+ def power_line(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effects: Iterable[float] = (),
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ n_jobs: int = 1,
+ ) -> Dict[float, float]:
+ """Runs power analysis with multiple average effects
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effects: Average effects to test.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ n_jobs: Number of jobs to run in parallel.
+
+ Returns:
+ Dictionary with average effects as keys and power as values.
+ """
+ return {
+ effect: self.power_analysis(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effect=effect,
+ n_simulations=n_simulations,
+ alpha=alpha,
+ n_jobs=n_jobs,
+ )
+ for effect in tqdm(
+ list(average_effects), disable=not verbose, desc="Effects loop"
+ )
+ }
+
+ def log_nulls(self, df: pd.DataFrame) -> None:
+ """Warns about dropping nulls in treatment column"""
+ n_nulls = len(df.query(f"{self.treatment_col}.isnull()", engine="python"))
+ if n_nulls > 0:
+ logging.warning(
+ f"There are {n_nulls} null values in treatment, dropping them"
+ )
+
+ @classmethod
+ def from_dict(cls, config_dict: dict) -> "PowerAnalysis":
+ """Constructs PowerAnalysis from dictionary"""
+ config = PowerConfig(**config_dict)
+ return cls.from_config(config)
+
+ @classmethod
+ def from_config(cls, config: PowerConfig) -> "PowerAnalysis":
+ """Constructs PowerAnalysis from PowerConfig"""
+ perturbator_cls = _get_mapping_key(perturbator_mapping, config.perturbator)
+ splitter_cls = _get_mapping_key(splitter_mapping, config.splitter)
+ analysis_cls = _get_mapping_key(analysis_mapping, config.analysis)
+ cupac_cls = _get_mapping_key(cupac_model_mapping, config.cupac_model)
+ return cls(
+ perturbator=perturbator_cls.from_config(config),
+ splitter=splitter_cls.from_config(config),
+ analysis=analysis_cls.from_config(config),
+ cupac_model=cupac_cls.from_config(config),
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ control=config.control,
+ n_simulations=config.n_simulations,
+ alpha=config.alpha,
+ features_cupac_model=config.features_cupac_model,
+ seed=config.seed,
+ hypothesis=config.hypothesis,
+ )
+
+ def check_treatment_col(self):
+ """Checks consistency of treatment column"""
+ assert (
+ self.analysis.treatment_col == self.perturbator.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in perturbator ({self.perturbator.treatment_col})"
+
+ assert (
+ self.analysis.treatment_col == self.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in PowerAnalysis ({self.treatment_col})"
+
+ assert (
+ self.analysis.treatment_col == self.splitter.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in splitter ({self.splitter.treatment_col})"
+
+ def check_target_col(self):
+ assert (
+ self.analysis.target_col == self.perturbator.target_col
+ ), f"target_col in analysis ({self.analysis.target_col}) must be the same as target_col in perturbator ({self.perturbator.target_col})"
+
+ assert (
+ self.analysis.target_col == self.target_col
+ ), f"target_col in analysis ({self.analysis.target_col}) must be the same as target_col in PowerAnalysis ({self.target_col})"
+
+ def check_treatment(self):
+ assert (
+ self.analysis.treatment == self.perturbator.treatment
+ ), f"treatment in analysis ({self.analysis.treatment}) must be the same as treatment in perturbator ({self.perturbator.treatment})"
+
+ assert (
+ self.analysis.treatment == self.treatment
+ ), f"treatment in analysis ({self.analysis.treatment}) must be the same as treatment in PowerAnalysis ({self.treatment})"
+
+ assert (
+ self.analysis.treatment in self.splitter.treatments
+ ), f"treatment in analysis ({self.analysis.treatment}) must be in treatments in splitter ({self.splitter.treatments})"
+
+ assert (
+ self.control in self.splitter.treatments
+ ), f"control in power analysis ({self.control}) must be in treatments in splitter ({self.splitter.treatments})"
+
+ def check_covariates(self):
+ if hasattr(self.analysis, "covariates"):
+ cupac_in_covariates = (
+ self.cupac_handler.cupac_outcome_name in self.analysis.covariates
+ )
+
+ assert cupac_in_covariates or not self.cupac_handler.is_cupac, (
+ f"covariates in analysis must contain {self.cupac_handler.cupac_outcome_name} if cupac_model is not None. "
+ f"If you want to use cupac_model, you must add the cupac outcome to the covariates of the analysis "
+ f"You may want to do covariates=['{self.cupac_handler.cupac_outcome_name}'] in your analysis method or your config"
+ )
+
+ if hasattr(self.splitter, "cluster_cols"):
+ if set(self.analysis.covariates).intersection(
+ set(self.splitter.cluster_cols)
+ ):
+ logging.warning(
+ f"covariates in analysis ({self.analysis.covariates}) are also cluster_cols in splitter ({self.splitter.cluster_cols}). "
+ f"Be specially careful when using switchback splitters, since the time splitter column is being overriden"
+ )
+
+ def check_clusters(self):
+ has_analysis_clusters = hasattr(self.analysis, "cluster_cols")
+ has_splitter_clusters = hasattr(self.splitter, "cluster_cols")
+ not_cluster_cols_cond = not has_analysis_clusters or not has_splitter_clusters
+ assert (
+ not_cluster_cols_cond
+ or self.analysis.cluster_cols == self.splitter.cluster_cols
+ ), f"cluster_cols in analysis ({self.analysis.cluster_cols}) must be the same as cluster_cols in splitter ({self.splitter.cluster_cols})"
+
+ assert (
+ has_splitter_clusters
+ or not has_analysis_clusters
+ or not self.analysis.cluster_cols
+ or isinstance(self.splitter, RepeatedSampler)
+ ), "analysis has cluster_cols but splitter does not."
+
+ assert (
+ has_analysis_clusters
+ or not has_splitter_clusters
+ or not self.splitter.cluster_cols
+ ), "splitter has cluster_cols but analysis does not."
+
+ has_time_col = hasattr(self.splitter, "time_col")
+ assert not (
+ has_time_col
+ and has_splitter_clusters
+ and self.splitter.time_col not in self.splitter.cluster_cols
+ ), "in switchback splitters, time_col must be in cluster_cols"
+
+ def check_inputs(self):
+ self.check_covariates()
+ self.check_treatment_col()
+ self.check_target_col()
+ self.check_treatment()
+ self.check_clusters()
+check_treatment_col(self)
+
+
+¶Checks consistency of treatment column
+ +cluster_experiments/power_analysis.pydef check_treatment_col(self):
+ """Checks consistency of treatment column"""
+ assert (
+ self.analysis.treatment_col == self.perturbator.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in perturbator ({self.perturbator.treatment_col})"
+
+ assert (
+ self.analysis.treatment_col == self.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in PowerAnalysis ({self.treatment_col})"
+
+ assert (
+ self.analysis.treatment_col == self.splitter.treatment_col
+ ), f"treatment_col in analysis ({self.analysis.treatment_col}) must be the same as treatment_col in splitter ({self.splitter.treatment_col})"
+from_config(config)
+
+
+ classmethod
+
+
+¶Constructs PowerAnalysis from PowerConfig
+ +cluster_experiments/power_analysis.py@classmethod
+def from_config(cls, config: PowerConfig) -> "PowerAnalysis":
+ """Constructs PowerAnalysis from PowerConfig"""
+ perturbator_cls = _get_mapping_key(perturbator_mapping, config.perturbator)
+ splitter_cls = _get_mapping_key(splitter_mapping, config.splitter)
+ analysis_cls = _get_mapping_key(analysis_mapping, config.analysis)
+ cupac_cls = _get_mapping_key(cupac_model_mapping, config.cupac_model)
+ return cls(
+ perturbator=perturbator_cls.from_config(config),
+ splitter=splitter_cls.from_config(config),
+ analysis=analysis_cls.from_config(config),
+ cupac_model=cupac_cls.from_config(config),
+ target_col=config.target_col,
+ treatment_col=config.treatment_col,
+ treatment=config.treatment,
+ control=config.control,
+ n_simulations=config.n_simulations,
+ alpha=config.alpha,
+ features_cupac_model=config.features_cupac_model,
+ seed=config.seed,
+ hypothesis=config.hypothesis,
+ )
+from_dict(config_dict)
+
+
+ classmethod
+
+
+¶Constructs PowerAnalysis from dictionary
+ +cluster_experiments/power_analysis.py@classmethod
+def from_dict(cls, config_dict: dict) -> "PowerAnalysis":
+ """Constructs PowerAnalysis from dictionary"""
+ config = PowerConfig(**config_dict)
+ return cls.from_config(config)
+log_nulls(self, df)
+
+
+¶Warns about dropping nulls in treatment column
+ +cluster_experiments/power_analysis.pydef log_nulls(self, df: pd.DataFrame) -> None:
+ """Warns about dropping nulls in treatment column"""
+ n_nulls = len(df.query(f"{self.treatment_col}.isnull()", engine="python"))
+ if n_nulls > 0:
+ logging.warning(
+ f"There are {n_nulls} null values in treatment, dropping them"
+ )
+power_analysis(self, df, pre_experiment_df=None, verbose=False, average_effect=None, n_simulations=None, alpha=None, n_jobs=1)
+
+
+¶Run power analysis by simulation
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
average_effect |
+ Optional[float] |
+ Average effect of treatment. If None, it will use the perturbator average effect. |
+ None |
+
n_simulations |
+ Optional[int] |
+ Number of simulations to run. |
+ None |
+
alpha |
+ Optional[float] |
+ Significance level. |
+ None |
+
n_jobs |
+ int |
+ Number of jobs to run in parallel. If 1, it will run in serial. |
+ 1 |
+
cluster_experiments/power_analysis.pydef power_analysis(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: Optional[float] = None,
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ n_jobs: int = 1,
+) -> float:
+ """
+ Run power analysis by simulation
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ n_jobs: Number of jobs to run in parallel. If 1, it will run in serial.
+ """
+ n_simulations = self.n_simulations if n_simulations is None else n_simulations
+ alpha = self.alpha if alpha is None else alpha
+
+ df = df.copy()
+ df = self.cupac_handler.add_covariates(df, pre_experiment_df)
+
+ if n_jobs == 1:
+ return self._non_parallel_loop(
+ df, average_effect, n_simulations, alpha, verbose
+ )
+ elif n_jobs > 1 or n_jobs == -1:
+ return self._parallel_loop(
+ df, average_effect, n_simulations, alpha, verbose, n_jobs
+ )
+ else:
+ raise ValueError("n_jobs must be greater than 0, or -1.")
+power_line(self, df, pre_experiment_df=None, verbose=False, average_effects=(), n_simulations=None, alpha=None, n_jobs=1)
+
+
+¶Runs power analysis with multiple average effects
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
average_effects |
+ Iterable[float] |
+ Average effects to test. |
+ () |
+
n_simulations |
+ Optional[int] |
+ Number of simulations to run. |
+ None |
+
alpha |
+ Optional[float] |
+ Significance level. |
+ None |
+
n_jobs |
+ int |
+ Number of jobs to run in parallel. |
+ 1 |
+
Returns:
+| Type | +Description | +
|---|---|
Dict[float, float] |
+ Dictionary with average effects as keys and power as values. |
+
cluster_experiments/power_analysis.pydef power_line(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effects: Iterable[float] = (),
+ n_simulations: Optional[int] = None,
+ alpha: Optional[float] = None,
+ n_jobs: int = 1,
+) -> Dict[float, float]:
+ """Runs power analysis with multiple average effects
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effects: Average effects to test.
+ n_simulations: Number of simulations to run.
+ alpha: Significance level.
+ n_jobs: Number of jobs to run in parallel.
+
+ Returns:
+ Dictionary with average effects as keys and power as values.
+ """
+ return {
+ effect: self.power_analysis(
+ df=df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effect=effect,
+ n_simulations=n_simulations,
+ alpha=alpha,
+ n_jobs=n_jobs,
+ )
+ for effect in tqdm(
+ list(average_effects), disable=not verbose, desc="Effects loop"
+ )
+ }
+running_power_analysis(self, df, pre_experiment_df=None, verbose=False, average_effect=None, n_simulations=100)
+
+
+¶Yields running power for each iteration of the simulation. +if you're interested in getting the power at each iteration, you can use this method to generate them.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome variable. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
average_effect |
+ Optional[float] |
+ Average effect of treatment. If None, it will use the perturbator average effect. |
+ None |
+
n_simulations |
+ int |
+ Number of simulations to run. |
+ 100 |
+
cluster_experiments/power_analysis.pydef running_power_analysis(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: Optional[float] = None,
+ n_simulations: int = 100,
+) -> Generator[float, None, None]:
+ """
+ Yields running power for each iteration of the simulation.
+ if you're interested in getting the power at each iteration, you can use this method to generate them.
+ Args:
+ df: Dataframe with outcome variable.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ """
+ n_rejected = 0
+ for i, perturbed_df in enumerate(
+ self._simulate_perturbed_df(
+ df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effect=average_effect,
+ n_simulations=n_simulations,
+ )
+ ):
+ p_value = self.analysis.get_pvalue(perturbed_df)
+ n_rejected += int(p_value < self.alpha)
+ yield n_rejected / (i + 1)
+simulate_point_estimate(self, df, pre_experiment_df=None, verbose=False, average_effect=None, n_simulations=100)
+
+
+¶Yields point estimates for each iteration of the simulation. +In general, this is to be used in power_analysis method. However, +if you're interested in the distribution of point estimates, you can use this method to generate them.
+This is an experimental feature and it might change in the future.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome and treatment variables. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
average_effect |
+ Optional[float] |
+ Average effect of treatment. If None, it will use the perturbator average effect. |
+ None |
+
n_simulations |
+ int |
+ Number of simulations to run. |
+ 100 |
+
cluster_experiments/power_analysis.pydef simulate_point_estimate(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: Optional[float] = None,
+ n_simulations: int = 100,
+) -> Generator[float, None, None]:
+ """
+ Yields point estimates for each iteration of the simulation.
+ In general, this is to be used in power_analysis method. However,
+ if you're interested in the distribution of point estimates, you can use this method to generate them.
+
+ This is an experimental feature and it might change in the future.
+
+ Args:
+ df: Dataframe with outcome and treatment variables.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ """
+ for perturbed_df in self._simulate_perturbed_df(
+ df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effect=average_effect,
+ n_simulations=n_simulations,
+ ):
+ yield self.analysis.get_point_estimate(perturbed_df)
+simulate_pvalue(self, df, pre_experiment_df=None, verbose=False, average_effect=None, n_simulations=100)
+
+
+¶Yields p-values for each iteration of the simulation. +In general, this is to be used in power_analysis method. However, +if you're interested in the distribution of p-values, you can use this method to generate them.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ Dataframe with outcome variable. |
+ required | +
pre_experiment_df |
+ Optional[pandas.core.frame.DataFrame] |
+ Dataframe with pre-experiment data. |
+ None |
+
verbose |
+ bool |
+ Whether to show progress bar. |
+ False |
+
average_effect |
+ Optional[float] |
+ Average effect of treatment. If None, it will use the perturbator average effect. |
+ None |
+
n_simulations |
+ int |
+ Number of simulations to run. |
+ 100 |
+
cluster_experiments/power_analysis.pydef simulate_pvalue(
+ self,
+ df: pd.DataFrame,
+ pre_experiment_df: Optional[pd.DataFrame] = None,
+ verbose: bool = False,
+ average_effect: Optional[float] = None,
+ n_simulations: int = 100,
+) -> Generator[float, None, None]:
+ """
+ Yields p-values for each iteration of the simulation.
+ In general, this is to be used in power_analysis method. However,
+ if you're interested in the distribution of p-values, you can use this method to generate them.
+ Args:
+ df: Dataframe with outcome variable.
+ pre_experiment_df: Dataframe with pre-experiment data.
+ verbose: Whether to show progress bar.
+ average_effect: Average effect of treatment. If None, it will use the perturbator average effect.
+ n_simulations: Number of simulations to run.
+ """
+ for perturbed_df in self._simulate_perturbed_df(
+ df,
+ pre_experiment_df=pre_experiment_df,
+ verbose=verbose,
+ average_effect=average_effect,
+ n_simulations=n_simulations,
+ ):
+ yield self.analysis.get_pvalue(perturbed_df)
+
+PowerAnalysisWithPreExperimentData (PowerAnalysis)
+
+
+
+
+¶This is intended to work mainly for diff-in-diff or synthetic control-like estimators, and NOT for cases of CUPED/CUPAC. +Same as PowerAnalysis, but allowing a perturbation only at experiment period and keeping pre-experiment df intact. +Using this class, the pre experiment df is also available when the class is instantiated.
+ +cluster_experiments/power_analysis.pyclass PowerAnalysisWithPreExperimentData(PowerAnalysis):
+ """
+ This is intended to work mainly for diff-in-diff or synthetic control-like estimators, and NOT for cases of CUPED/CUPAC.
+ Same as PowerAnalysis, but allowing a perturbation only at experiment period and keeping pre-experiment df intact.
+ Using this class, the pre experiment df is also available when the class is instantiated.
+ """
+
+ def _perturbate(
+ self, treatment_df: pd.DataFrame, average_effect: Optional[float]
+ ) -> pd.DataFrame:
+ if not hasattr(self.analysis, "_split_pre_experiment_df"):
+ raise AttributeError(
+ "The PowerAnalysisWithPreExperimentData is intended to work mainly for diff-in-diff or synthetic control-like estimators."
+ "For other cases use the PowerAnalysis"
+ )
+
+ df, pre_experiment_df = self.analysis._split_pre_experiment_df(treatment_df)
+
+ perturbed_df = self.perturbator.perturbate(df, average_effect=average_effect)
+
+ return pd.concat([perturbed_df, pre_experiment_df])
+from cluster_experiments.power_config import *¶
+PowerConfig
+
+
+
+ dataclass
+
+
+¶Dataclass to create a power analysis from.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
splitter |
+ str |
+ Splitter object to use |
+ required | +
perturbator |
+ str |
+ Perturbator object to use, defaults to "" for normal power analysis |
+ '' |
+
analysis |
+ str |
+ ExperimentAnalysis object to use |
+ required | +
washover |
+ str |
+ Washover object to use, defaults to "" |
+ '' |
+
cupac_model |
+ str |
+ CUPAC model to use |
+ '' |
+
n_simulations |
+ int |
+ number of simulations to run |
+ 100 |
+
cluster_cols |
+ Optional[List[str]] |
+ list of columns to use as clusters |
+ None |
+
target_col |
+ str |
+ column to use as target |
+ 'target' |
+
treatment_col |
+ str |
+ column to use as treatment |
+ 'treatment' |
+
treatment |
+ str |
+ what value of treatment_col should be considered as treatment |
+ 'B' |
+
control |
+ str |
+ what value of treatment_col should be considered as control |
+ 'A' |
+
strata_cols |
+ Optional[List[str]] |
+ columns to stratify with |
+ None |
+
splitter_weights |
+ Optional[List[float]] |
+ weights to use for the splitter, should have the same length as treatments, each weight should correspond to an element in treatments |
+ None |
+
switch_frequency |
+ Optional[str] |
+ how often to switch treatments |
+ None |
+
time_col |
+ Optional[str] |
+ column to use as time in switchback splitter |
+ None |
+
washover_time_delta |
+ Union[datetime.timedelta, int] |
+ optional, int indicating the washover time in minutes or datetime.timedelta object |
+ None |
+
covariates |
+ Optional[List[str]] |
+ list of columns to use as covariates |
+ None |
+
average_effect |
+ Optional[float] |
+ average effect to use in the perturbator |
+ None |
+
scale |
+ Optional[float] |
+ scale to use in stochastic perturbators |
+ None |
+
range_min |
+ Optional[float] |
+ minimum value of the target range for relative beta perturbator, must be >-1 |
+ None |
+
range_max |
+ Optional[float] |
+ maximum value of the target range for relative beta perturbator |
+ None |
+
reduce_variance |
+ Optional[bool] |
+ whether to reduce variance in the BetaRelative perturbator |
+ None |
+
segment_cols |
+ Optional[List[str]] |
+ list of segmentation columns for segmented perturbator |
+ None |
+
treatments |
+ Optional[List[str]] |
+ list of treatments to use |
+ None |
+
alpha |
+ float |
+ alpha value to use in the power analysis |
+ 0.05 |
+
agg_col |
+ str |
+ column to use for aggregation in the CUPAC model |
+ '' |
+
smoothing_factor |
+ float |
+ smoothing value to use in the CUPAC model |
+ 20 |
+
features_cupac_model |
+ Optional[List[str]] |
+ list of features to use in the CUPAC model |
+ None |
+
seed |
+ Optional[int] |
+ seed to make the power analysis reproducible |
+ None |
+
Usage:
+from cluster_experiments.power_config import PowerConfig
+from cluster_experiments.power_analysis import PowerAnalysis, NormalPowerAnalysis
+
+p = PowerConfig(
+ analysis="gee",
+ splitter="clustered_balance",
+ perturbator="constant",
+ cluster_cols=["city"],
+ n_simulations=100,
+ alpha=0.05,
+)
+power_analysis = PowerAnalysis.from_config(p)
+
+normal_power_analysis = NormalPowerAnalysis.from_config(p)
+cluster_experiments/power_config.pyclass PowerConfig:
+ """
+ Dataclass to create a power analysis from.
+
+ Arguments:
+ splitter: Splitter object to use
+ perturbator: Perturbator object to use, defaults to "" for normal power analysis
+ analysis: ExperimentAnalysis object to use
+ washover: Washover object to use, defaults to ""
+ cupac_model: CUPAC model to use
+ n_simulations: number of simulations to run
+ cluster_cols: list of columns to use as clusters
+ target_col: column to use as target
+ treatment_col: column to use as treatment
+ treatment: what value of treatment_col should be considered as treatment
+ control: what value of treatment_col should be considered as control
+ strata_cols: columns to stratify with
+ splitter_weights: weights to use for the splitter, should have the same length as treatments, each weight should correspond to an element in treatments
+ switch_frequency: how often to switch treatments
+ time_col: column to use as time in switchback splitter
+ washover_time_delta: optional, int indicating the washover time in minutes or datetime.timedelta object
+ covariates: list of columns to use as covariates
+ average_effect: average effect to use in the perturbator
+ scale: scale to use in stochastic perturbators
+ range_min: minimum value of the target range for relative beta perturbator, must be >-1
+ range_max: maximum value of the target range for relative beta perturbator
+ reduce_variance: whether to reduce variance in the BetaRelative perturbator
+ segment_cols: list of segmentation columns for segmented perturbator
+ treatments: list of treatments to use
+ alpha: alpha value to use in the power analysis
+ agg_col: column to use for aggregation in the CUPAC model
+ smoothing_factor: smoothing value to use in the CUPAC model
+ features_cupac_model: list of features to use in the CUPAC model
+ seed: seed to make the power analysis reproducible
+
+ Usage:
+
+ ```python
+ from cluster_experiments.power_config import PowerConfig
+ from cluster_experiments.power_analysis import PowerAnalysis, NormalPowerAnalysis
+
+ p = PowerConfig(
+ analysis="gee",
+ splitter="clustered_balance",
+ perturbator="constant",
+ cluster_cols=["city"],
+ n_simulations=100,
+ alpha=0.05,
+ )
+ power_analysis = PowerAnalysis.from_config(p)
+
+ normal_power_analysis = NormalPowerAnalysis.from_config(p)
+ ```
+ """
+
+ # mappings
+ splitter: str
+ analysis: str
+ perturbator: str = ""
+ washover: str = ""
+
+ # Needed
+ cluster_cols: Optional[List[str]] = None
+
+ # optional mappings
+ cupac_model: str = ""
+
+ # Shared
+ target_col: str = "target"
+ treatment_col: str = "treatment"
+ treatment: str = "B"
+
+ # Perturbator
+ average_effect: Optional[float] = None
+ scale: Optional[float] = None
+ range_min: Optional[float] = None
+ range_max: Optional[float] = None
+ reduce_variance: Optional[bool] = None
+ segment_cols: Optional[List[str]] = None
+
+ # Splitter
+ treatments: Optional[List[str]] = None
+ strata_cols: Optional[List[str]] = None
+ splitter_weights: Optional[List[float]] = None
+ switch_frequency: Optional[str] = None
+ # Switchback
+ time_col: Optional[str] = None
+ washover_time_delta: Optional[Union[datetime.timedelta, int]] = None
+
+ # Analysis
+ covariates: Optional[List[str]] = None
+ hypothesis: str = "two-sided"
+
+ # Power analysis
+ n_simulations: int = 100
+ alpha: float = 0.05
+ control: str = "A"
+
+ # Cupac
+ agg_col: str = ""
+ smoothing_factor: float = 20
+ features_cupac_model: Optional[List[str]] = None
+
+ seed: Optional[int] = None
+
+ def __post_init__(self):
+ if "switchback" not in self.splitter:
+ if self._are_different(self.switch_frequency, None):
+ self._set_and_log("switch_frequency", None, "splitter")
+ if self._are_different(self.washover_time_delta, None):
+ self._set_and_log("washover_time_delta", None, "splitter")
+ if self._are_different(self.washover, ""):
+ self._set_and_log("washover", "", "splitter")
+ # an exception is made when we have no perturbator (normal power analysis)
+ if self._are_different(self.time_col, None) and self.perturbator != "":
+ self._set_and_log("time_col", None, "splitter")
+
+ if self.perturbator not in {"normal", "beta_relative_positive"}:
+ if self._are_different(self.scale, None):
+ self._set_and_log("scale", None, "perturbator")
+
+ if self.perturbator not in {"beta_relative", "segmented_beta_relative"}:
+ if self._are_different(self.range_min, None):
+ self._set_and_log("range_min", None, "perturbator")
+ if self._are_different(self.range_max, None):
+ self._set_and_log("range_max", None, "perturbator")
+ if self._are_different(self.reduce_variance, None):
+ self._set_and_log("reduce_variance", None, "perturbator")
+
+ if self.perturbator not in {"segmented_beta_relative"}:
+ if self._are_different(self.segment_cols, None):
+ self._set_and_log("segment_cols", None, "perturbator")
+
+ if "stratified" not in self.splitter and "paired_ttest" not in self.analysis:
+ if self._are_different(self.strata_cols, None):
+ self._set_and_log("strata_cols", None, "splitter")
+
+ if "stratified" in self.splitter or "balanced" in self.splitter:
+ if self._are_different(self.splitter_weights, None):
+ self._set_and_log("splitter_weights", None, "splitter")
+
+ if self.cupac_model != "mean_cupac_model":
+ if self._are_different(self.agg_col, ""):
+ self._set_and_log("agg_col", "", "cupac_model")
+ if self._are_different(self.smoothing_factor, 20):
+ self._set_and_log("smoothing_factor", 20, "cupac_model")
+ # for now, features_cupac_model are not used
+ if self._are_different(self.features_cupac_model, None):
+ self._set_and_log("features_cupac_model", None, "cupac_model")
+
+ if "ttest" in self.analysis:
+ if self._are_different(self.covariates, None):
+ self._set_and_log("covariates", None, "analysis")
+
+ if "segmented" in self.perturbator:
+ self._raise_error_if_missing("segment_cols", "perturbator")
+
+ def _are_different(self, arg1, arg2) -> bool:
+ return arg1 != arg2
+
+ def _set_and_log(self, attr, value, other_attr):
+ logging.warning(
+ f"{attr} = {getattr(self, attr)} has no effect with "
+ f"{other_attr} = {getattr(self, other_attr)}. "
+ f"Overriding {attr} to {value}."
+ )
+ setattr(self, attr, value)
+
+ def _raise_error_if_missing(self, attr, other_attr):
+ if getattr(self, attr) is None:
+ raise MissingArgumentError(
+ f"{attr} is required when using "
+ f"{other_attr} = {getattr(self, other_attr)}."
+ )
+from cluster_experiments.random_splitter import *¶
+BalancedClusteredSplitter (ClusteredSplitter)
+
+
+
+
+¶Like ClusteredSplitter, but ensures that treatments are balanced among clusters. That is, if we have +25 clusters and 2 treatments, 13 clusters should have treatment A and 12 clusters should have treatment B.
+ +cluster_experiments/random_splitter.pyclass BalancedClusteredSplitter(ClusteredSplitter):
+ """Like ClusteredSplitter, but ensures that treatments are balanced among clusters. That is, if we have
+ 25 clusters and 2 treatments, 13 clusters should have treatment A and 12 clusters should have treatment B."""
+
+ def sample_treatment(
+ self,
+ cluster_df: pd.DataFrame,
+ ) -> List[str]:
+ """
+ Samples treatments for each cluster
+
+ Arguments:
+ cluster_df: dataframe to assign treatments to
+ """
+ n_clusters = len(cluster_df)
+ n_treatments = len(self.treatments)
+ n_per_treatment = n_clusters // n_treatments
+ n_extra = n_clusters % n_treatments
+ treatments = []
+ for i in range(n_treatments):
+ treatments += [self.treatments[i]] * (n_per_treatment + (i < n_extra))
+ random.shuffle(treatments)
+ return treatments
+sample_treatment(self, cluster_df)
+
+
+¶Samples treatments for each cluster
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_df |
+ DataFrame |
+ dataframe to assign treatments to |
+ required | +
cluster_experiments/random_splitter.pydef sample_treatment(
+ self,
+ cluster_df: pd.DataFrame,
+) -> List[str]:
+ """
+ Samples treatments for each cluster
+
+ Arguments:
+ cluster_df: dataframe to assign treatments to
+ """
+ n_clusters = len(cluster_df)
+ n_treatments = len(self.treatments)
+ n_per_treatment = n_clusters // n_treatments
+ n_extra = n_clusters % n_treatments
+ treatments = []
+ for i in range(n_treatments):
+ treatments += [self.treatments[i]] * (n_per_treatment + (i < n_extra))
+ random.shuffle(treatments)
+ return treatments
+
+BalancedSwitchbackSplitter (BalancedClusteredSplitter, SwitchbackSplitter)
+
+
+
+
+¶Like SwitchbackSplitter, but ensures that treatments are balanced among clusters. That is, if we have +25 clusters and 2 treatments, 13 clusters should have treatment A and 12 clusters should have treatment B.
+ +cluster_experiments/random_splitter.pyclass BalancedSwitchbackSplitter(BalancedClusteredSplitter, SwitchbackSplitter):
+ """
+ Like SwitchbackSplitter, but ensures that treatments are balanced among clusters. That is, if we have
+ 25 clusters and 2 treatments, 13 clusters should have treatment A and 12 clusters should have treatment B.
+ """
+
+ pass
+
+ClusteredSplitter (RandomSplitter)
+
+
+
+
+¶Splits randomly using clusters
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_cols |
+ List[str] |
+ List of columns to use as clusters |
+ required | +
treatments |
+ Optional[List[str]] |
+ list of treatments |
+ None |
+
treatment_col |
+ str |
+ Name of the column with the treatment variable. |
+ 'treatment' |
+
splitter_weights |
+ Optional[List[float]] |
+ weights to use for the splitter, should have the same length as treatments, each weight should correspond to an element in treatments |
+ None |
+
Usage: +
import pandas as pd
+from cluster_experiments.random_splitter import ClusteredSplitter
+splitter = ClusteredSplitter(cluster_cols=["city"])
+df = pd.DataFrame({"city": ["A", "B", "C"]})
+df = splitter.assign_treatment_df(df)
+print(df)
+cluster_experiments/random_splitter.pyclass ClusteredSplitter(RandomSplitter):
+ """
+ Splits randomly using clusters
+
+ Arguments:
+ cluster_cols: List of columns to use as clusters
+ treatments: list of treatments
+ treatment_col: Name of the column with the treatment variable.
+ splitter_weights: weights to use for the splitter, should have the same length as treatments, each weight should correspond to an element in treatments
+
+ Usage:
+ ```python
+ import pandas as pd
+ from cluster_experiments.random_splitter import ClusteredSplitter
+ splitter = ClusteredSplitter(cluster_cols=["city"])
+ df = pd.DataFrame({"city": ["A", "B", "C"]})
+ df = splitter.assign_treatment_df(df)
+ print(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ cluster_cols: List[str],
+ treatments: Optional[List[str]] = None,
+ treatment_col: str = "treatment",
+ splitter_weights: Optional[List[float]] = None,
+ ) -> None:
+ self.treatments = treatments or ["A", "B"]
+ self.cluster_cols = cluster_cols
+ self.treatment_col = treatment_col
+ self.splitter_weights = splitter_weights
+
+ def assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+ ) -> pd.DataFrame:
+ """
+ Takes a df, randomizes treatments and adds the treatment column to the dataframe
+
+ Arguments:
+ df: dataframe to assign treatments to
+ """
+ df = df.copy()
+
+ # raise error if any nulls in cluster_cols
+ if df[self.cluster_cols].isnull().values.any():
+ raise ValueError(
+ f"Null values found in cluster_cols: {self.cluster_cols}. "
+ "Please remove nulls before running the splitter."
+ )
+
+ clusters_df = df.loc[:, self.cluster_cols].drop_duplicates()
+ clusters_df[self.treatment_col] = self.sample_treatment(clusters_df)
+ df = df.merge(clusters_df, on=self.cluster_cols, how="left")
+ return df
+
+ def sample_treatment(
+ self,
+ cluster_df: pd.DataFrame,
+ ) -> List[str]:
+ """
+ Samples treatments for each cluster
+
+ Arguments:
+ cluster_df: dataframe to assign treatments to
+ """
+ return random.choices(
+ self.treatments, k=len(cluster_df), weights=self.splitter_weights
+ )
+assign_treatment_df(self, df)
+
+
+¶Takes a df, randomizes treatments and adds the treatment column to the dataframe
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe to assign treatments to |
+ required | +
cluster_experiments/random_splitter.pydef assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+) -> pd.DataFrame:
+ """
+ Takes a df, randomizes treatments and adds the treatment column to the dataframe
+
+ Arguments:
+ df: dataframe to assign treatments to
+ """
+ df = df.copy()
+
+ # raise error if any nulls in cluster_cols
+ if df[self.cluster_cols].isnull().values.any():
+ raise ValueError(
+ f"Null values found in cluster_cols: {self.cluster_cols}. "
+ "Please remove nulls before running the splitter."
+ )
+
+ clusters_df = df.loc[:, self.cluster_cols].drop_duplicates()
+ clusters_df[self.treatment_col] = self.sample_treatment(clusters_df)
+ df = df.merge(clusters_df, on=self.cluster_cols, how="left")
+ return df
+sample_treatment(self, cluster_df)
+
+
+¶Samples treatments for each cluster
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_df |
+ DataFrame |
+ dataframe to assign treatments to |
+ required | +
cluster_experiments/random_splitter.pydef sample_treatment(
+ self,
+ cluster_df: pd.DataFrame,
+) -> List[str]:
+ """
+ Samples treatments for each cluster
+
+ Arguments:
+ cluster_df: dataframe to assign treatments to
+ """
+ return random.choices(
+ self.treatments, k=len(cluster_df), weights=self.splitter_weights
+ )
+
+FixedSizeClusteredSplitter (ClusteredSplitter)
+
+
+
+
+¶This class represents a splitter that splits clusters into treatment groups with a predefined number of +treatment clusters. This is particularly useful for synthetic control analysis, where we only want 1 cluster ( +unit) to be in treatment group and the rest in control The cluster that receives treatment remains random.
+ +Attributes:
+| Name | +Type | +Description | +
|---|---|---|
cluster_cols |
+ List[str] |
+ List of columns to use as clusters. |
+
n_treatment_clusters |
+ int |
+ The predefined number of treatment clusters. |
+
cluster_experiments/random_splitter.pyclass FixedSizeClusteredSplitter(ClusteredSplitter):
+ """
+ This class represents a splitter that splits clusters into treatment groups with a predefined number of
+ treatment clusters. This is particularly useful for synthetic control analysis, where we only want 1 cluster (
+ unit) to be in treatment group and the rest in control The cluster that receives treatment remains random.
+
+ Attributes:
+ cluster_cols (List[str]): List of columns to use as clusters.
+ n_treatment_clusters (int): The predefined number of treatment clusters.
+
+ """
+
+ def __init__(self, cluster_cols: List[str], n_treatment_clusters: int):
+ super().__init__(cluster_cols=cluster_cols)
+ self.n_treatment_clusters = n_treatment_clusters
+
+ def sample_treatment(
+ self,
+ cluster_df: pd.DataFrame,
+ ) -> List[str]:
+ """
+ Samples treatments for each cluster.
+
+ Args:
+ cluster_df (pd.DataFrame): Dataframe to assign treatments to.
+
+ Returns:
+ List[str]: A list of treatments for each cluster.
+ """
+ n_control_treatment = [
+ len(cluster_df) - self.n_treatment_clusters,
+ self.n_treatment_clusters,
+ ]
+
+ sample_treatment = [
+ treatment
+ for treatment, count in zip(self.treatments, n_control_treatment)
+ for _ in range(count)
+ ]
+ random.shuffle(sample_treatment)
+ return sample_treatment
+sample_treatment(self, cluster_df)
+
+
+¶Samples treatments for each cluster.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_df |
+ pd.DataFrame |
+ Dataframe to assign treatments to. |
+ required | +
Returns:
+| Type | +Description | +
|---|---|
List[str] |
+ A list of treatments for each cluster. |
+
cluster_experiments/random_splitter.pydef sample_treatment(
+ self,
+ cluster_df: pd.DataFrame,
+) -> List[str]:
+ """
+ Samples treatments for each cluster.
+
+ Args:
+ cluster_df (pd.DataFrame): Dataframe to assign treatments to.
+
+ Returns:
+ List[str]: A list of treatments for each cluster.
+ """
+ n_control_treatment = [
+ len(cluster_df) - self.n_treatment_clusters,
+ self.n_treatment_clusters,
+ ]
+
+ sample_treatment = [
+ treatment
+ for treatment, count in zip(self.treatments, n_control_treatment)
+ for _ in range(count)
+ ]
+ random.shuffle(sample_treatment)
+ return sample_treatment
+
+NonClusteredSplitter (RandomSplitter)
+
+
+
+
+¶Splits randomly without clusters
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
treatments |
+ Optional[List[str]] |
+ list of treatments |
+ None |
+
treatment_col |
+ str |
+ Name of the column with the treatment variable. |
+ 'treatment' |
+
Usage: +
import pandas as pd
+from cluster_experiments.random_splitter import NonClusteredSplitter
+splitter = NonClusteredSplitter(
+ treatments=["A", "B"],
+)
+df = pd.DataFrame({"city": ["A", "B", "C"]})
+df = splitter.assign_treatment_df(df)
+print(df)
+cluster_experiments/random_splitter.pyclass NonClusteredSplitter(RandomSplitter):
+ """
+ Splits randomly without clusters
+
+ Arguments:
+ treatments: list of treatments
+ treatment_col: Name of the column with the treatment variable.
+
+ Usage:
+ ```python
+ import pandas as pd
+ from cluster_experiments.random_splitter import NonClusteredSplitter
+ splitter = NonClusteredSplitter(
+ treatments=["A", "B"],
+ )
+ df = pd.DataFrame({"city": ["A", "B", "C"]})
+ df = splitter.assign_treatment_df(df)
+ print(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ treatments: Optional[List[str]] = None,
+ treatment_col: str = "treatment",
+ splitter_weights: Optional[List[float]] = None,
+ ) -> None:
+ self.treatments = treatments or ["A", "B"]
+ self.treatment_col = treatment_col
+ self.splitter_weights = splitter_weights
+
+ def assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+ ) -> pd.DataFrame:
+ """
+ Takes a df, randomizes treatments and adds the treatment column to the dataframe
+
+ Arguments:
+ df: dataframe to assign treatments to
+ """
+ df = df.copy()
+ df[self.treatment_col] = random.choices(
+ self.treatments, k=len(df), weights=self.splitter_weights
+ )
+ return df
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a NonClusteredSplitter from a PowerConfig"""
+ return cls(
+ treatments=config.treatments,
+ treatment_col=config.treatment_col,
+ splitter_weights=config.splitter_weights,
+ )
+assign_treatment_df(self, df)
+
+
+¶Takes a df, randomizes treatments and adds the treatment column to the dataframe
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe to assign treatments to |
+ required | +
cluster_experiments/random_splitter.pydef assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+) -> pd.DataFrame:
+ """
+ Takes a df, randomizes treatments and adds the treatment column to the dataframe
+
+ Arguments:
+ df: dataframe to assign treatments to
+ """
+ df = df.copy()
+ df[self.treatment_col] = random.choices(
+ self.treatments, k=len(df), weights=self.splitter_weights
+ )
+ return df
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a NonClusteredSplitter from a PowerConfig
+ +cluster_experiments/random_splitter.py@classmethod
+def from_config(cls, config):
+ """Creates a NonClusteredSplitter from a PowerConfig"""
+ return cls(
+ treatments=config.treatments,
+ treatment_col=config.treatment_col,
+ splitter_weights=config.splitter_weights,
+ )
+
+RandomSplitter (ABC)
+
+
+
+
+¶Abstract class to split instances in a switchback or clustered way. It can be used to create a calendar/split of clusters +or to run a power analysis.
+In order to create your own RandomSplitter, you should write your own assign_treatment_df method, that takes a dataframe as an input and returns the same dataframe with the treatment_col column.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_cols |
+ Optional[List[str]] |
+ List of columns to use as clusters |
+ None |
+
treatments |
+ Optional[List[str]] |
+ list of treatments |
+ None |
+
treatment_col |
+ str |
+ Name of the column with the treatment variable. |
+ 'treatment' |
+
splitter_weights |
+ Optional[List[float]] |
+ weights to use for the splitter, should have the same length as treatments, each weight should correspond to an element in treatments |
+ None |
+
cluster_experiments/random_splitter.pyclass RandomSplitter(ABC):
+ """
+ Abstract class to split instances in a switchback or clustered way. It can be used to create a calendar/split of clusters
+ or to run a power analysis.
+
+ In order to create your own RandomSplitter, you should write your own assign_treatment_df method, that takes a dataframe as an input and returns the same dataframe with the treatment_col column.
+
+ Arguments:
+ cluster_cols: List of columns to use as clusters
+ treatments: list of treatments
+ treatment_col: Name of the column with the treatment variable.
+ splitter_weights: weights to use for the splitter, should have the same length as treatments, each weight should correspond to an element in treatments
+
+ """
+
+ def __init__(
+ self,
+ cluster_cols: Optional[List[str]] = None,
+ treatments: Optional[List[str]] = None,
+ treatment_col: str = "treatment",
+ splitter_weights: Optional[List[float]] = None,
+ ) -> None:
+ self.treatments = treatments or ["A", "B"]
+ self.cluster_cols = cluster_cols or []
+ self.treatment_col = treatment_col
+ self.splitter_weights = splitter_weights
+
+ @abstractmethod
+ def assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+ ) -> pd.DataFrame:
+ """
+ Takes a df, randomizes treatments and adds the treatment column to the dataframe
+
+ Arguments:
+ df: dataframe to assign treatments to
+ """
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a RandomSplitter from a PowerConfig"""
+ return cls(
+ treatments=config.treatments,
+ cluster_cols=config.cluster_cols,
+ treatment_col=config.treatment_col,
+ splitter_weights=config.splitter_weights,
+ )
+assign_treatment_df(self, df)
+
+
+¶Takes a df, randomizes treatments and adds the treatment column to the dataframe
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe to assign treatments to |
+ required | +
cluster_experiments/random_splitter.py@abstractmethod
+def assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+) -> pd.DataFrame:
+ """
+ Takes a df, randomizes treatments and adds the treatment column to the dataframe
+
+ Arguments:
+ df: dataframe to assign treatments to
+ """
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a RandomSplitter from a PowerConfig
+ +cluster_experiments/random_splitter.py@classmethod
+def from_config(cls, config):
+ """Creates a RandomSplitter from a PowerConfig"""
+ return cls(
+ treatments=config.treatments,
+ cluster_cols=config.cluster_cols,
+ treatment_col=config.treatment_col,
+ splitter_weights=config.splitter_weights,
+ )
+
+RepeatedSampler (RandomSplitter)
+
+
+
+
+¶Doesn't actually split the data, but repeatedly samples (i.e. duplicates) all rows for all treatments. +This is useful for backtesting, where we assume to have access to all counterfactuals.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
treatments |
+ Optional[List[str]] |
+ list of treatments |
+ None |
+
treatment_col |
+ str |
+ Name of the column with the treatment variable. |
+ 'treatment' |
+
Usage: +
import pandas as pd
+from cluster_experiments.random_splitter import RepeatedSampler
+splitter = RepeatedSampler(
+ treatments=["A", "B"],
+)
+df = pd.DataFrame({"city": ["A", "B", "C"]})
+df = splitter.assign_treatment_df(df)
+print(df)
+cluster_experiments/random_splitter.pyclass RepeatedSampler(RandomSplitter):
+ """
+ Doesn't actually split the data, but repeatedly samples (i.e. duplicates) all rows for all treatments.
+ This is useful for backtesting, where we assume to have access to all counterfactuals.
+
+ Arguments:
+ treatments: list of treatments
+ treatment_col: Name of the column with the treatment variable.
+
+ Usage:
+ ```python
+ import pandas as pd
+ from cluster_experiments.random_splitter import RepeatedSampler
+ splitter = RepeatedSampler(
+ treatments=["A", "B"],
+ )
+ df = pd.DataFrame({"city": ["A", "B", "C"]})
+ df = splitter.assign_treatment_df(df)
+ print(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ treatments: Optional[List[str]] = None,
+ treatment_col: str = "treatment",
+ ) -> None:
+ self.treatments = treatments or ["A", "B"]
+ self.treatment_col = treatment_col
+
+ def assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+ ) -> pd.DataFrame:
+ df = df.copy()
+
+ dfs = []
+ for treatment in self.treatments:
+ df_treat = df.copy().assign(**{self.treatment_col: treatment})
+ dfs.append(df_treat)
+
+ return pd.concat(dfs).reset_index(drop=True)
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a RepeatedSampler from a PowerConfig"""
+ return cls(
+ treatments=config.treatments,
+ treatment_col=config.treatment_col,
+ )
+assign_treatment_df(self, df)
+
+
+¶Takes a df, randomizes treatments and adds the treatment column to the dataframe
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe to assign treatments to |
+ required | +
cluster_experiments/random_splitter.pydef assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+) -> pd.DataFrame:
+ df = df.copy()
+
+ dfs = []
+ for treatment in self.treatments:
+ df_treat = df.copy().assign(**{self.treatment_col: treatment})
+ dfs.append(df_treat)
+
+ return pd.concat(dfs).reset_index(drop=True)
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a RepeatedSampler from a PowerConfig
+ +cluster_experiments/random_splitter.py@classmethod
+def from_config(cls, config):
+ """Creates a RepeatedSampler from a PowerConfig"""
+ return cls(
+ treatments=config.treatments,
+ treatment_col=config.treatment_col,
+ )
+
+StratifiedClusteredSplitter (RandomSplitter)
+
+
+
+
+¶Splits randomly with clusters, ensuring a balanced allocation of treatment groups across clusters and strata. +To be used, for example, when having days as clusters and days of the week as stratus. This splitter will make sure +that we won't have all Sundays in treatment and no Sundays in control.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
cluster_cols |
+ Optional[List[str]] |
+ List of columns to use as clusters |
+ None |
+
treatments |
+ Optional[List[str]] |
+ list of treatments |
+ None |
+
treatment_col |
+ str |
+ Name of the column with the treatment variable. |
+ 'treatment' |
+
strata_cols |
+ Optional[List[str]] |
+ List of columns to use as strata |
+ None |
+
Usage: +
import pandas as pd
+from cluster_experiments.random_splitter import StratifiedClusteredSplitter
+splitter = StratifiedClusteredSplitter(cluster_cols=["city"],strata_cols=["country"])
+df = pd.DataFrame({"city": ["A", "B", "C","D"], "country":["C1","C2","C2","C1"]})
+df = splitter.assign_treatment_df(df)
+print(df)
+cluster_experiments/random_splitter.pyclass StratifiedClusteredSplitter(RandomSplitter):
+ """
+ Splits randomly with clusters, ensuring a balanced allocation of treatment groups across clusters and strata.
+ To be used, for example, when having days as clusters and days of the week as stratus. This splitter will make sure
+ that we won't have all Sundays in treatment and no Sundays in control.
+
+ Arguments:
+ cluster_cols: List of columns to use as clusters
+ treatments: list of treatments
+ treatment_col: Name of the column with the treatment variable.
+ strata_cols: List of columns to use as strata
+
+ Usage:
+ ```python
+ import pandas as pd
+ from cluster_experiments.random_splitter import StratifiedClusteredSplitter
+ splitter = StratifiedClusteredSplitter(cluster_cols=["city"],strata_cols=["country"])
+ df = pd.DataFrame({"city": ["A", "B", "C","D"], "country":["C1","C2","C2","C1"]})
+ df = splitter.assign_treatment_df(df)
+ print(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ cluster_cols: Optional[List[str]] = None,
+ treatments: Optional[List[str]] = None,
+ treatment_col: str = "treatment",
+ strata_cols: Optional[List[str]] = None,
+ ) -> None:
+ super().__init__(
+ cluster_cols=cluster_cols,
+ treatments=treatments,
+ treatment_col=treatment_col,
+ )
+ if not strata_cols or strata_cols == [""]:
+ raise ValueError(
+ f"Splitter {self.__class__.__name__} requires strata_cols,"
+ f" got {strata_cols = }"
+ )
+ self.strata_cols = strata_cols
+
+ def assign_treatment_df(self, df: pd.DataFrame) -> pd.DataFrame:
+ df = df.copy()
+ df_unique_shuffled = (
+ df.loc[:, list(set(self.cluster_cols + self.strata_cols))]
+ .drop_duplicates()
+ .sample(frac=1)
+ .reset_index(drop=True)
+ )
+
+ # check that, for a given cluster, there is only 1 strata
+ for strata_col in self.strata_cols:
+ if (
+ df_unique_shuffled.groupby(self.cluster_cols)[strata_col]
+ .nunique()
+ .max()
+ > 1
+ ):
+ raise ValueError(
+ f"There are multiple values in {strata_col} for the same cluster item \n"
+ "You cannot stratify on this column",
+ )
+
+ # random shuffling
+ random_sorted_treatments = list(np.random.permutation(self.treatments))
+
+ df_unique_shuffled[self.treatment_col] = (
+ df_unique_shuffled.groupby(self.strata_cols, as_index=False)
+ .cumcount()
+ .mod(len(random_sorted_treatments))
+ .map(dict(enumerate(random_sorted_treatments)))
+ )
+
+ df = df.merge(
+ df_unique_shuffled, on=self.cluster_cols + self.strata_cols, how="left"
+ )
+
+ return df
+
+ @classmethod
+ def from_config(cls, config):
+ """Creates a StratifiedClusteredSplitter from a PowerConfig"""
+ return cls(
+ treatments=config.treatments,
+ cluster_cols=config.cluster_cols,
+ strata_cols=config.strata_cols,
+ treatment_col=config.treatment_col,
+ )
+assign_treatment_df(self, df)
+
+
+¶Takes a df, randomizes treatments and adds the treatment column to the dataframe
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe to assign treatments to |
+ required | +
cluster_experiments/random_splitter.pydef assign_treatment_df(self, df: pd.DataFrame) -> pd.DataFrame:
+ df = df.copy()
+ df_unique_shuffled = (
+ df.loc[:, list(set(self.cluster_cols + self.strata_cols))]
+ .drop_duplicates()
+ .sample(frac=1)
+ .reset_index(drop=True)
+ )
+
+ # check that, for a given cluster, there is only 1 strata
+ for strata_col in self.strata_cols:
+ if (
+ df_unique_shuffled.groupby(self.cluster_cols)[strata_col]
+ .nunique()
+ .max()
+ > 1
+ ):
+ raise ValueError(
+ f"There are multiple values in {strata_col} for the same cluster item \n"
+ "You cannot stratify on this column",
+ )
+
+ # random shuffling
+ random_sorted_treatments = list(np.random.permutation(self.treatments))
+
+ df_unique_shuffled[self.treatment_col] = (
+ df_unique_shuffled.groupby(self.strata_cols, as_index=False)
+ .cumcount()
+ .mod(len(random_sorted_treatments))
+ .map(dict(enumerate(random_sorted_treatments)))
+ )
+
+ df = df.merge(
+ df_unique_shuffled, on=self.cluster_cols + self.strata_cols, how="left"
+ )
+
+ return df
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a StratifiedClusteredSplitter from a PowerConfig
+ +cluster_experiments/random_splitter.py@classmethod
+def from_config(cls, config):
+ """Creates a StratifiedClusteredSplitter from a PowerConfig"""
+ return cls(
+ treatments=config.treatments,
+ cluster_cols=config.cluster_cols,
+ strata_cols=config.strata_cols,
+ treatment_col=config.treatment_col,
+ )
+
+StratifiedSwitchbackSplitter (StratifiedClusteredSplitter, SwitchbackSplitter)
+
+
+
+
+¶Splits randomly with clusters, ensuring a balanced allocation of treatment groups across clusters and strata. +To be used, for example, when having days as clusters and days of the week as stratus. This splitter will make sure +that we won't have all Sundays in treatment and no Sundays in control.
+It can be created using the time_col and switch_frequency arguments, just like the SwitchbackSplitter.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
time_col |
+ str |
+ Name of the column with the time variable. |
+ 'date' |
+
switch_frequency |
+ str |
+ Frequency of the switchback. Must be a string (e.g. "1D") |
+ '1D' |
+
cluster_cols |
+ Optional[List[str]] |
+ List of columns to use as clusters |
+ None |
+
treatments |
+ Optional[List[str]] |
+ list of treatments |
+ None |
+
treatment_col |
+ str |
+ Name of the column with the treatment variable. |
+ 'treatment' |
+
splitter_weights |
+ Optional[List[float]] |
+ List of weights for the treatments. If None, all treatments will have the same weight. |
+ None |
+
strata_cols |
+ Optional[List[str]] |
+ List of columns to use as strata |
+ None |
+
Usage: +
import pandas as pd
+from cluster_experiments.random_splitter import StratifiedSwitchbackSplitter
+splitter = StratifiedSwitchbackSplitter(time_col="date",switch_frequency="1D",strata_cols=["country"], cluster_cols=["country", "date"])
+df = pd.DataFrame({"date": ["2020-01-01", "2020-01-02", "2020-01-03","2020-01-04"], "country":["C1","C2","C2","C1"]})
+df = splitter.assign_treatment_df(df)
+print(df)
+cluster_experiments/random_splitter.pyclass StratifiedSwitchbackSplitter(StratifiedClusteredSplitter, SwitchbackSplitter):
+ """
+ Splits randomly with clusters, ensuring a balanced allocation of treatment groups across clusters and strata.
+ To be used, for example, when having days as clusters and days of the week as stratus. This splitter will make sure
+ that we won't have all Sundays in treatment and no Sundays in control.
+
+ It can be created using the time_col and switch_frequency arguments, just like the SwitchbackSplitter.
+
+ Arguments:
+ time_col: Name of the column with the time variable.
+ switch_frequency: Frequency of the switchback. Must be a string (e.g. "1D")
+ cluster_cols: List of columns to use as clusters
+ treatments: list of treatments
+ treatment_col: Name of the column with the treatment variable.
+ splitter_weights: List of weights for the treatments. If None, all treatments will have the same weight.
+ strata_cols: List of columns to use as strata
+
+ Usage:
+ ```python
+ import pandas as pd
+ from cluster_experiments.random_splitter import StratifiedSwitchbackSplitter
+ splitter = StratifiedSwitchbackSplitter(time_col="date",switch_frequency="1D",strata_cols=["country"], cluster_cols=["country", "date"])
+ df = pd.DataFrame({"date": ["2020-01-01", "2020-01-02", "2020-01-03","2020-01-04"], "country":["C1","C2","C2","C1"]})
+ df = splitter.assign_treatment_df(df)
+ print(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ time_col: str = "date",
+ switch_frequency: str = "1D",
+ cluster_cols: Optional[List[str]] = None,
+ treatments: Optional[List[str]] = None,
+ treatment_col: str = "treatment",
+ splitter_weights: Optional[List[float]] = None,
+ washover: Optional[Washover] = None,
+ strata_cols: Optional[List[str]] = None,
+ ) -> None:
+ # Inherit init from SwitchbackSplitter
+ SwitchbackSplitter.__init__(
+ self,
+ time_col=time_col,
+ switch_frequency=switch_frequency,
+ cluster_cols=cluster_cols,
+ treatments=treatments,
+ treatment_col=treatment_col,
+ splitter_weights=splitter_weights,
+ washover=washover,
+ )
+ self.strata_cols = strata_cols or ["strata"]
+
+ def assign_treatment_df(self, df: pd.DataFrame) -> pd.DataFrame:
+ df = df.copy()
+ df = self._prepare_switchback_df(df)
+ df = StratifiedClusteredSplitter.assign_treatment_df(self, df)
+ return self.washover.washover(
+ df=df,
+ treatment_col=self.treatment_col,
+ truncated_time_col=self.time_col,
+ cluster_cols=self.cluster_cols,
+ )
+
+ @classmethod
+ def from_config(cls, config) -> "StratifiedSwitchbackSplitter":
+ """Creates a StratifiedSwitchbackSplitter from a PowerConfig"""
+ washover_cls = _get_mapping_key(washover_mapping, config.washover)
+ return cls(
+ treatments=config.treatments,
+ cluster_cols=config.cluster_cols,
+ strata_cols=config.strata_cols,
+ treatment_col=config.treatment_col,
+ time_col=config.time_col,
+ switch_frequency=config.switch_frequency,
+ splitter_weights=config.splitter_weights,
+ washover=washover_cls.from_config(config),
+ )
+assign_treatment_df(self, df)
+
+
+¶Creates the switchback column, adds it to cluster_cols and then calls ClusteredSplitter assign_treatment_df
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe to assign treatments to |
+ required | +
cluster_experiments/random_splitter.pydef assign_treatment_df(self, df: pd.DataFrame) -> pd.DataFrame:
+ df = df.copy()
+ df = self._prepare_switchback_df(df)
+ df = StratifiedClusteredSplitter.assign_treatment_df(self, df)
+ return self.washover.washover(
+ df=df,
+ treatment_col=self.treatment_col,
+ truncated_time_col=self.time_col,
+ cluster_cols=self.cluster_cols,
+ )
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a StratifiedSwitchbackSplitter from a PowerConfig
+ +cluster_experiments/random_splitter.py@classmethod
+def from_config(cls, config) -> "StratifiedSwitchbackSplitter":
+ """Creates a StratifiedSwitchbackSplitter from a PowerConfig"""
+ washover_cls = _get_mapping_key(washover_mapping, config.washover)
+ return cls(
+ treatments=config.treatments,
+ cluster_cols=config.cluster_cols,
+ strata_cols=config.strata_cols,
+ treatment_col=config.treatment_col,
+ time_col=config.time_col,
+ switch_frequency=config.switch_frequency,
+ splitter_weights=config.splitter_weights,
+ washover=washover_cls.from_config(config),
+ )
+
+SwitchbackSplitter (ClusteredSplitter)
+
+
+
+
+¶Splits randomly using clusters and time column
+It is a clustered splitter but one of the cluster columns is obtained by truncating the time column to the switch frequency.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
time_col |
+ Optional[str] |
+ Name of the column with the time variable. |
+ None |
+
switch_frequency |
+ Optional[str] |
+ Frequency to switch treatments. Uses pandas frequency aliases (https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases) |
+ None |
+
cluster_cols |
+ Optional[List[str]] |
+ List of columns to use as clusters |
+ None |
+
treatments |
+ Optional[List[str]] |
+ list of treatments |
+ None |
+
treatment_col |
+ str |
+ Name of the column with the treatment variable. |
+ 'treatment' |
+
splitter_weights |
+ Optional[List[float]] |
+ weights to use for the splitter, should have the same length as treatments, each weight should correspond to an element in treatments |
+ None |
+
Usage: +
import pandas as pd
+from cluster_experiments.random_splitter import SwitchbackSplitter
+splitter = SwitchbackSplitter(time_col="date", switch_frequency="1D", cluster_cols=["date"])
+df = pd.DataFrame({"date": pd.date_range("2020-01-01", "2020-01-03")})
+df = splitter.assign_treatment_df(df)
+print(df)
+cluster_experiments/random_splitter.pyclass SwitchbackSplitter(ClusteredSplitter):
+ """
+ Splits randomly using clusters and time column
+
+ It is a clustered splitter but one of the cluster columns is obtained by truncating the time column to the switch frequency.
+
+ Arguments:
+ time_col: Name of the column with the time variable.
+ switch_frequency: Frequency to switch treatments. Uses pandas frequency aliases (https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases)
+ cluster_cols: List of columns to use as clusters
+ treatments: list of treatments
+ treatment_col: Name of the column with the treatment variable.
+ splitter_weights: weights to use for the splitter, should have the same length as treatments, each weight should correspond to an element in treatments
+
+ Usage:
+ ```python
+ import pandas as pd
+ from cluster_experiments.random_splitter import SwitchbackSplitter
+ splitter = SwitchbackSplitter(time_col="date", switch_frequency="1D", cluster_cols=["date"])
+ df = pd.DataFrame({"date": pd.date_range("2020-01-01", "2020-01-03")})
+ df = splitter.assign_treatment_df(df)
+ print(df)
+ ```
+ """
+
+ def __init__(
+ self,
+ time_col: Optional[str] = None,
+ switch_frequency: Optional[str] = None,
+ cluster_cols: Optional[List[str]] = None,
+ treatments: Optional[List[str]] = None,
+ treatment_col: str = "treatment",
+ splitter_weights: Optional[List[float]] = None,
+ washover: Optional[Washover] = None,
+ ) -> None:
+ self.time_col = time_col or "date"
+ self.switch_frequency = switch_frequency or "1D"
+ self.cluster_cols = cluster_cols or []
+ self.treatments = treatments or ["A", "B"]
+ self.treatment_col = treatment_col
+ self.splitter_weights = splitter_weights
+ self.washover = washover or EmptyWashover()
+ self._check_clusters()
+
+ def _check_clusters(self):
+ """Check if time_col is in cluster_cols"""
+ assert (
+ self.time_col in self.cluster_cols
+ ), "in switchback splitters, time_col must be in cluster_cols"
+
+ def _get_time_col_cluster(self, df: pd.DataFrame) -> pd.Series:
+ df = df.copy()
+ df[self.time_col] = pd.to_datetime(df[self.time_col])
+ # Given the switch frequency, truncate the time column to the switch frequency
+ # Using pandas frequency aliases: https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases
+ if "W" in self.switch_frequency or "M" in self.switch_frequency:
+ return df[self.time_col].dt.to_period(self.switch_frequency).dt.start_time
+ return df[self.time_col].dt.floor(self.switch_frequency)
+
+ def _prepare_switchback_df(self, df: pd.DataFrame) -> pd.DataFrame:
+ df = df.copy()
+ # Build time_col switchback column
+ # Overwriting column, this is the worst! If we use the column as a covariate, we're screwed. Needs improvement
+ df[_original_time_column(self.time_col)] = df[self.time_col]
+ df[self.time_col] = self._get_time_col_cluster(df)
+ return df
+
+ def assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+ ) -> pd.DataFrame:
+ """
+ Creates the switchback column, adds it to cluster_cols and then calls ClusteredSplitter assign_treatment_df
+
+ Arguments:
+ df: dataframe to assign treatments to
+ """
+ df = df.copy()
+ df = self._prepare_switchback_df(df)
+ df = super().assign_treatment_df(df)
+ df = self.washover.washover(
+ df,
+ truncated_time_col=self.time_col,
+ treatment_col=self.treatment_col,
+ cluster_cols=self.cluster_cols,
+ )
+ return df
+
+ @classmethod
+ def from_config(cls, config) -> "SwitchbackSplitter":
+ washover_cls = _get_mapping_key(washover_mapping, config.washover)
+ return cls(
+ time_col=config.time_col,
+ switch_frequency=config.switch_frequency,
+ cluster_cols=config.cluster_cols,
+ treatments=config.treatments,
+ treatment_col=config.treatment_col,
+ splitter_weights=config.splitter_weights,
+ washover=washover_cls.from_config(config),
+ )
+assign_treatment_df(self, df)
+
+
+¶Creates the switchback column, adds it to cluster_cols and then calls ClusteredSplitter assign_treatment_df
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ DataFrame |
+ dataframe to assign treatments to |
+ required | +
cluster_experiments/random_splitter.pydef assign_treatment_df(
+ self,
+ df: pd.DataFrame,
+) -> pd.DataFrame:
+ """
+ Creates the switchback column, adds it to cluster_cols and then calls ClusteredSplitter assign_treatment_df
+
+ Arguments:
+ df: dataframe to assign treatments to
+ """
+ df = df.copy()
+ df = self._prepare_switchback_df(df)
+ df = super().assign_treatment_df(df)
+ df = self.washover.washover(
+ df,
+ truncated_time_col=self.time_col,
+ treatment_col=self.treatment_col,
+ cluster_cols=self.cluster_cols,
+ )
+ return df
+from_config(config)
+
+
+ classmethod
+
+
+¶Creates a RandomSplitter from a PowerConfig
+ +cluster_experiments/random_splitter.py@classmethod
+def from_config(cls, config) -> "SwitchbackSplitter":
+ washover_cls = _get_mapping_key(washover_mapping, config.washover)
+ return cls(
+ time_col=config.time_col,
+ switch_frequency=config.switch_frequency,
+ cluster_cols=config.cluster_cols,
+ treatments=config.treatments,
+ treatment_col=config.treatment_col,
+ splitter_weights=config.splitter_weights,
+ washover=washover_cls.from_config(config),
+ )
+from cluster_experiments.inference.variant import *¶
+Variant
+
+
+
+ dataclass
+
+
+¶A class used to represent a Variant with a name and a control flag.
+name : str + The name of the variant +is_control : bool + A boolean indicating if the variant is a control variant
+ +cluster_experiments/inference/variant.pyclass Variant:
+ """
+ A class used to represent a Variant with a name and a control flag.
+
+ Attributes
+ ----------
+ name : str
+ The name of the variant
+ is_control : bool
+ A boolean indicating if the variant is a control variant
+ """
+
+ name: str
+ is_control: bool
+
+ def __post_init__(self):
+ """
+ Validates the inputs after initialization.
+ """
+ self._validate_inputs()
+
+ def _validate_inputs(self):
+ """
+ Validates the inputs for the Variant class.
+
+ Raises
+ ------
+ TypeError
+ If the name is not a string or if is_control is not a boolean.
+ """
+ if not isinstance(self.name, str):
+ raise TypeError("Variant name must be a string")
+ if not isinstance(self.is_control, bool):
+ raise TypeError("Variant is_control must be a boolean")
+__post_init__(self)
+
+
+ special
+
+
+¶Validates the inputs after initialization.
+ +cluster_experiments/inference/variant.pydef __post_init__(self):
+ """
+ Validates the inputs after initialization.
+ """
+ self._validate_inputs()
+from cluster_experiments.washover import *¶
+ConstantWashover (Washover)
+
+
+
+
+¶Constant washover - we drop all rows in the washover period when +there is a switch where the treatment is different.
+ +cluster_experiments/washover.pyclass ConstantWashover(Washover):
+ """Constant washover - we drop all rows in the washover period when
+ there is a switch where the treatment is different."""
+
+ def __init__(self, washover_time_delta: datetime.timedelta):
+ self.washover_time_delta = washover_time_delta
+
+ def washover(
+ self,
+ df: pd.DataFrame,
+ truncated_time_col: str,
+ treatment_col: str,
+ cluster_cols: List[str],
+ original_time_col: Optional[str] = None,
+ ) -> pd.DataFrame:
+ """Constant washover - we drop all rows in the washover period when
+ there is a switch where the treatment is different.
+
+ Args:
+ df (pd.DataFrame): Input dataframe.
+ truncated_time_col (str): Name of the truncated time column.
+ treatment_col (str): Name of the treatment column.
+ cluster_cols (List[str]): List of clusters of experiment.
+ original_time_col (Optional[str], optional): Name of the original time column.
+
+ Returns:
+ pd.DataFrame: Same dataframe as input without the rows in the washover period.
+
+ Usage:
+ ```python
+ import numpy as np
+ import pandas as pd
+ from datetime import datetime, timedelta
+
+ from cluster_experiments import ConstantWashover
+
+ np.random.seed(42)
+
+ num_rows = 10
+
+ def random_timestamp(start_time, end_time):
+ time_delta = end_time - start_time
+ random_seconds = np.random.randint(0, time_delta.total_seconds())
+ return start_time + timedelta(seconds=random_seconds)
+
+ def generate_data(start_time, end_time, treatment):
+ data = {
+ 'order_id': np.random.randint(10**9, 10**10, size=num_rows),
+ 'city_code': 'VAL',
+ 'activation_time_local': [random_timestamp(start_time, end_time) for _ in range(num_rows)],
+ 'bin_start_time_local': start_time,
+ 'treatment': treatment
+ }
+ return pd.DataFrame(data)
+
+ start_times = [datetime(2024, 1, 22, 9, 0), datetime(2024, 1, 22, 11, 0),
+ datetime(2024, 1, 22, 13, 0), datetime(2024, 1, 22, 15, 0)]
+
+ treatments = ['control', 'variation', 'variation', 'control']
+
+ dataframes = [generate_data(start, start + timedelta(hours=2), treatment) for start, treatment in zip(start_times, treatments)]
+
+ df = pd.concat(dataframes).sort_values(by='activation_time_local').reset_index(drop=True)
+
+ ## Define washover with 30 min duration
+ washover = ConstantWashover(washover_time_delta=timedelta(minutes=30))
+
+ ## Apply washover to the dataframe, the orders with activation time within the first 30 minutes after every change in the treatment column, clustering by city and 2h time bin, will be dropped
+ df_analysis_washover = washover.washover(
+ df=df,
+ truncated_time_col='bin_start_time_local',
+ treatment_col='treatment',
+ cluster_cols=['city_code','bin_start_time_local'],
+ original_time_col='activation_time_local',
+ )
+ ```
+ """
+ # Set original time column
+ original_time_col = (
+ original_time_col
+ if original_time_col
+ else _original_time_column(truncated_time_col)
+ )
+
+ # Validate columns
+ self._validate_columns(df, truncated_time_col, cluster_cols, original_time_col)
+
+ # Cluster columns that do not involve time
+ non_time_cols = list(set(cluster_cols) - set([truncated_time_col]))
+ # For each cluster, we need to check if treatment has changed wrt last time
+ df_agg = df.sort_values([original_time_col]).copy()
+ df_agg = df_agg.drop_duplicates(subset=cluster_cols + [treatment_col])
+
+ if non_time_cols:
+ df_agg["__changed"] = (
+ df_agg.groupby(non_time_cols)[treatment_col].shift(1)
+ != df_agg[treatment_col]
+ )
+ else:
+ df_agg["__changed"] = (
+ df_agg[treatment_col].shift(1) != df_agg[treatment_col]
+ )
+ df_agg = df_agg.loc[:, cluster_cols + ["__changed"]]
+ return (
+ df.merge(df_agg, on=cluster_cols, how="inner")
+ .assign(
+ __time_since_switch=lambda x: x[original_time_col].astype(
+ "datetime64[ns]"
+ )
+ - x[truncated_time_col].astype("datetime64[ns]"),
+ __after_washover=lambda x: x["__time_since_switch"]
+ > self.washover_time_delta,
+ )
+ # add not changed in query
+ .query("__after_washover or not __changed")
+ .drop(columns=["__time_since_switch", "__after_washover", "__changed"])
+ )
+
+ @classmethod
+ def from_config(cls, config) -> "Washover":
+ if not config.washover_time_delta:
+ raise ValueError(
+ f"Washover time delta must be specified for ConstantWashover, while it is {config.washover_time_delta = }"
+ )
+
+ washover_time_delta = config.washover_time_delta
+ if isinstance(washover_time_delta, int):
+ washover_time_delta = datetime.timedelta(minutes=config.washover_time_delta)
+ return cls(washover_time_delta=washover_time_delta)
+washover(self, df, truncated_time_col, treatment_col, cluster_cols, original_time_col=None)
+
+
+¶Constant washover - we drop all rows in the washover period when +there is a switch where the treatment is different.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ pd.DataFrame |
+ Input dataframe. |
+ required | +
truncated_time_col |
+ str |
+ Name of the truncated time column. |
+ required | +
treatment_col |
+ str |
+ Name of the treatment column. |
+ required | +
cluster_cols |
+ List[str] |
+ List of clusters of experiment. |
+ required | +
original_time_col |
+ Optional[str] |
+ Name of the original time column. |
+ None |
+
Returns:
+| Type | +Description | +
|---|---|
pd.DataFrame |
+ Same dataframe as input without the rows in the washover period. |
+
Usage: +
import numpy as np
+import pandas as pd
+from datetime import datetime, timedelta
+
+from cluster_experiments import ConstantWashover
+
+np.random.seed(42)
+
+num_rows = 10
+
+def random_timestamp(start_time, end_time):
+ time_delta = end_time - start_time
+ random_seconds = np.random.randint(0, time_delta.total_seconds())
+ return start_time + timedelta(seconds=random_seconds)
+
+def generate_data(start_time, end_time, treatment):
+ data = {
+ 'order_id': np.random.randint(10**9, 10**10, size=num_rows),
+ 'city_code': 'VAL',
+ 'activation_time_local': [random_timestamp(start_time, end_time) for _ in range(num_rows)],
+ 'bin_start_time_local': start_time,
+ 'treatment': treatment
+ }
+ return pd.DataFrame(data)
+
+start_times = [datetime(2024, 1, 22, 9, 0), datetime(2024, 1, 22, 11, 0),
+ datetime(2024, 1, 22, 13, 0), datetime(2024, 1, 22, 15, 0)]
+
+treatments = ['control', 'variation', 'variation', 'control']
+
+dataframes = [generate_data(start, start + timedelta(hours=2), treatment) for start, treatment in zip(start_times, treatments)]
+
+df = pd.concat(dataframes).sort_values(by='activation_time_local').reset_index(drop=True)
+
+## Define washover with 30 min duration
+washover = ConstantWashover(washover_time_delta=timedelta(minutes=30))
+
+## Apply washover to the dataframe, the orders with activation time within the first 30 minutes after every change in the treatment column, clustering by city and 2h time bin, will be dropped
+df_analysis_washover = washover.washover(
+ df=df,
+ truncated_time_col='bin_start_time_local',
+ treatment_col='treatment',
+ cluster_cols=['city_code','bin_start_time_local'],
+ original_time_col='activation_time_local',
+)
+cluster_experiments/washover.pydef washover(
+ self,
+ df: pd.DataFrame,
+ truncated_time_col: str,
+ treatment_col: str,
+ cluster_cols: List[str],
+ original_time_col: Optional[str] = None,
+) -> pd.DataFrame:
+ """Constant washover - we drop all rows in the washover period when
+ there is a switch where the treatment is different.
+
+ Args:
+ df (pd.DataFrame): Input dataframe.
+ truncated_time_col (str): Name of the truncated time column.
+ treatment_col (str): Name of the treatment column.
+ cluster_cols (List[str]): List of clusters of experiment.
+ original_time_col (Optional[str], optional): Name of the original time column.
+
+ Returns:
+ pd.DataFrame: Same dataframe as input without the rows in the washover period.
+
+ Usage:
+ ```python
+ import numpy as np
+ import pandas as pd
+ from datetime import datetime, timedelta
+
+ from cluster_experiments import ConstantWashover
+
+ np.random.seed(42)
+
+ num_rows = 10
+
+ def random_timestamp(start_time, end_time):
+ time_delta = end_time - start_time
+ random_seconds = np.random.randint(0, time_delta.total_seconds())
+ return start_time + timedelta(seconds=random_seconds)
+
+ def generate_data(start_time, end_time, treatment):
+ data = {
+ 'order_id': np.random.randint(10**9, 10**10, size=num_rows),
+ 'city_code': 'VAL',
+ 'activation_time_local': [random_timestamp(start_time, end_time) for _ in range(num_rows)],
+ 'bin_start_time_local': start_time,
+ 'treatment': treatment
+ }
+ return pd.DataFrame(data)
+
+ start_times = [datetime(2024, 1, 22, 9, 0), datetime(2024, 1, 22, 11, 0),
+ datetime(2024, 1, 22, 13, 0), datetime(2024, 1, 22, 15, 0)]
+
+ treatments = ['control', 'variation', 'variation', 'control']
+
+ dataframes = [generate_data(start, start + timedelta(hours=2), treatment) for start, treatment in zip(start_times, treatments)]
+
+ df = pd.concat(dataframes).sort_values(by='activation_time_local').reset_index(drop=True)
+
+ ## Define washover with 30 min duration
+ washover = ConstantWashover(washover_time_delta=timedelta(minutes=30))
+
+ ## Apply washover to the dataframe, the orders with activation time within the first 30 minutes after every change in the treatment column, clustering by city and 2h time bin, will be dropped
+ df_analysis_washover = washover.washover(
+ df=df,
+ truncated_time_col='bin_start_time_local',
+ treatment_col='treatment',
+ cluster_cols=['city_code','bin_start_time_local'],
+ original_time_col='activation_time_local',
+ )
+ ```
+ """
+ # Set original time column
+ original_time_col = (
+ original_time_col
+ if original_time_col
+ else _original_time_column(truncated_time_col)
+ )
+
+ # Validate columns
+ self._validate_columns(df, truncated_time_col, cluster_cols, original_time_col)
+
+ # Cluster columns that do not involve time
+ non_time_cols = list(set(cluster_cols) - set([truncated_time_col]))
+ # For each cluster, we need to check if treatment has changed wrt last time
+ df_agg = df.sort_values([original_time_col]).copy()
+ df_agg = df_agg.drop_duplicates(subset=cluster_cols + [treatment_col])
+
+ if non_time_cols:
+ df_agg["__changed"] = (
+ df_agg.groupby(non_time_cols)[treatment_col].shift(1)
+ != df_agg[treatment_col]
+ )
+ else:
+ df_agg["__changed"] = (
+ df_agg[treatment_col].shift(1) != df_agg[treatment_col]
+ )
+ df_agg = df_agg.loc[:, cluster_cols + ["__changed"]]
+ return (
+ df.merge(df_agg, on=cluster_cols, how="inner")
+ .assign(
+ __time_since_switch=lambda x: x[original_time_col].astype(
+ "datetime64[ns]"
+ )
+ - x[truncated_time_col].astype("datetime64[ns]"),
+ __after_washover=lambda x: x["__time_since_switch"]
+ > self.washover_time_delta,
+ )
+ # add not changed in query
+ .query("__after_washover or not __changed")
+ .drop(columns=["__time_since_switch", "__after_washover", "__changed"])
+ )
+
+EmptyWashover (Washover)
+
+
+
+
+¶No washover - assumes no spill-over effects from one treatment to another.
+ +cluster_experiments/washover.pyclass EmptyWashover(Washover):
+ """No washover - assumes no spill-over effects from one treatment to another."""
+
+ def washover(
+ self,
+ df: pd.DataFrame,
+ truncated_time_col: str,
+ treatment_col: str,
+ cluster_cols: List[str],
+ original_time_col: Optional[str] = None,
+ ) -> pd.DataFrame:
+ """No washover - returns the same dataframe as input.
+
+ Args:
+ df (pd.DataFrame): Input dataframe.
+ truncated_time_col (str): Name of the truncated time column.
+ treatment_col (str): Name of the treatment column.
+ cluster_cols (List[str]): List of clusters of experiment.
+ original_time_col (Optional[str], optional): Name of the original time column.
+
+ Returns:
+ pd.DataFrame: Same dataframe as input.
+
+ Usage:
+ ```python
+ from cluster_experiments import SwitchbackSplitter
+ from cluster_experiments import EmptyWashover
+
+ washover = EmptyWashover()
+
+ n = 10
+ df = pd.DataFrame(
+ {
+ # Random time each minute in 2022-01-01, length 10
+ "time": pd.date_range("2022-01-01", "2022-01-02", freq="1min")[
+ np.random.randint(24 * 60, size=n)
+ ],
+ "city": random.choices(["TGN", "NYC", "LON", "REU"], k=n),
+ }
+ )
+
+
+ splitter = SwitchbackSplitter(
+ washover=washover,
+ time_col="time",
+ cluster_cols=["city", "time"],
+ treatment_col="treatment",
+ switch_frequency="30T",
+ )
+
+ out_df = splitter.assign_treatment_df(df=washover_split_df)
+ ```
+ """
+ return df
+washover(self, df, truncated_time_col, treatment_col, cluster_cols, original_time_col=None)
+
+
+¶No washover - returns the same dataframe as input.
+ +Parameters:
+| Name | +Type | +Description | +Default | +
|---|---|---|---|
df |
+ pd.DataFrame |
+ Input dataframe. |
+ required | +
truncated_time_col |
+ str |
+ Name of the truncated time column. |
+ required | +
treatment_col |
+ str |
+ Name of the treatment column. |
+ required | +
cluster_cols |
+ List[str] |
+ List of clusters of experiment. |
+ required | +
original_time_col |
+ Optional[str] |
+ Name of the original time column. |
+ None |
+
Returns:
+| Type | +Description | +
|---|---|
pd.DataFrame |
+ Same dataframe as input. |
+
Usage: +
from cluster_experiments import SwitchbackSplitter
+from cluster_experiments import EmptyWashover
+
+washover = EmptyWashover()
+
+n = 10
+df = pd.DataFrame(
+ {
+ # Random time each minute in 2022-01-01, length 10
+ "time": pd.date_range("2022-01-01", "2022-01-02", freq="1min")[
+ np.random.randint(24 * 60, size=n)
+ ],
+ "city": random.choices(["TGN", "NYC", "LON", "REU"], k=n),
+ }
+)
+
+
+splitter = SwitchbackSplitter(
+ washover=washover,
+ time_col="time",
+ cluster_cols=["city", "time"],
+ treatment_col="treatment",
+ switch_frequency="30T",
+)
+
+out_df = splitter.assign_treatment_df(df=washover_split_df)
+cluster_experiments/washover.pydef washover(
+ self,
+ df: pd.DataFrame,
+ truncated_time_col: str,
+ treatment_col: str,
+ cluster_cols: List[str],
+ original_time_col: Optional[str] = None,
+) -> pd.DataFrame:
+ """No washover - returns the same dataframe as input.
+
+ Args:
+ df (pd.DataFrame): Input dataframe.
+ truncated_time_col (str): Name of the truncated time column.
+ treatment_col (str): Name of the treatment column.
+ cluster_cols (List[str]): List of clusters of experiment.
+ original_time_col (Optional[str], optional): Name of the original time column.
+
+ Returns:
+ pd.DataFrame: Same dataframe as input.
+
+ Usage:
+ ```python
+ from cluster_experiments import SwitchbackSplitter
+ from cluster_experiments import EmptyWashover
+
+ washover = EmptyWashover()
+
+ n = 10
+ df = pd.DataFrame(
+ {
+ # Random time each minute in 2022-01-01, length 10
+ "time": pd.date_range("2022-01-01", "2022-01-02", freq="1min")[
+ np.random.randint(24 * 60, size=n)
+ ],
+ "city": random.choices(["TGN", "NYC", "LON", "REU"], k=n),
+ }
+ )
+
+
+ splitter = SwitchbackSplitter(
+ washover=washover,
+ time_col="time",
+ cluster_cols=["city", "time"],
+ treatment_col="treatment",
+ switch_frequency="30T",
+ )
+
+ out_df = splitter.assign_treatment_df(df=washover_split_df)
+ ```
+ """
+ return df
+
+Washover (ABC)
+
+
+
+
+¶Abstract class to model washovers in the switchback splitter.
+ +cluster_experiments/washover.pyclass Washover(ABC):
+ """Abstract class to model washovers in the switchback splitter."""
+
+ def _validate_columns(
+ self,
+ df: pd.DataFrame,
+ truncated_time_col: str,
+ cluster_cols: List[str],
+ original_time_col: str,
+ ):
+ """Validate that all the columns required for the washover are present in the dataframe.
+
+ Args:
+ df (pd.DataFrame): Input dataframe.
+ truncated_time_col (str): Name of the truncated time column.
+ cluster_cols (List[str]): List of clusters of experiment.
+ original_time_col (str): Name of the original time column.
+
+ Returns:
+ None: This method does not return any data; it only performs validation.
+
+ """
+ if original_time_col not in df.columns:
+ raise ValueError(
+ f"{original_time_col = } is not in the dataframe columns and/or not specified as an input."
+ )
+ if truncated_time_col not in cluster_cols:
+ raise ValueError(f"{truncated_time_col = } is not in the cluster columns.")
+ for col in cluster_cols:
+ if col not in df.columns:
+ raise ValueError(f"{col = } cluster is not in the dataframe columns.")
+
+ @abstractmethod
+ def washover(
+ self,
+ df: pd.DataFrame,
+ truncated_time_col: str,
+ treatment_col: str,
+ cluster_cols: List[str],
+ original_time_col: Optional[str] = None,
+ ) -> pd.DataFrame:
+ """Abstract method to add washvover to the dataframe."""
+
+ @classmethod
+ def from_config(cls, config) -> "Washover":
+ return cls()
+washover(self, df, truncated_time_col, treatment_col, cluster_cols, original_time_col=None)
+
+
+¶Abstract method to add washvover to the dataframe.
+ +cluster_experiments/washover.py@abstractmethod
+def washover(
+ self,
+ df: pd.DataFrame,
+ truncated_time_col: str,
+ treatment_col: str,
+ cluster_cols: List[str],
+ original_time_col: Optional[str] = None,
+) -> pd.DataFrame:
+ """Abstract method to add washvover to the dataframe."""
+\n {translation(\"search.result.term.missing\")}: {...missing}\n
\n }\n \n