diff --git a/azure-pipelines-steps.yml b/azure-pipelines-steps.yml index 448484735..24e3d9807 100644 --- a/azure-pipelines-steps.yml +++ b/azure-pipelines-steps.yml @@ -24,7 +24,7 @@ steps: condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) # Install the package -- script: 'python -m pip install --upgrade pip && pip install --upgrade setuptools && pip install ${{ parameters.package }}' +- script: 'python -m pip install --upgrade pip && pip install --upgrade setuptools wheel && pip install ${{ parameters.package }}' displayName: 'Install dependencies' - ${{ parameters.body }} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 17c056584..e3fb7791e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -103,40 +103,40 @@ jobs: testRunTitle: 'Notebooks' condition: succeededOrFailed() -- job: 'AutoML' - dependsOn: 'EvalChanges' - condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True') - variables: - python.version: '3.6' - pool: - vmImage: 'ubuntu-16.04' - steps: - - template: azure-pipelines-steps.yml - parameters: - body: - - task: AzureCLI@2 - displayName: 'AutoML tests' - inputs: - azureSubscription: 'automl' - scriptLocation: 'inlineScript' - scriptType: 'pscore' - powerShellIgnoreLASTEXITCODE: '' # string for now due to https://github.com/microsoft/azure-pipelines-tasks/issues/12266 - inlineScript: | - $env:SUBSCRIPTION_ID = az account show --query id -o tsv - python setup.py pytest - env: - WORKSPACE_NAME: 'testWorkspace' - RESOURCE_GROUP: 'testingAutoMLEconML' - PYTEST_ADDOPTS: '-m "automl" -n 0' - COVERAGE_PROCESS_START: 'setup.cfg' - - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/test-results.xml' - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'AutoML' - condition: succeededOrFailed() - package: '.[automl]' +# - job: 'AutoML' +# dependsOn: 'EvalChanges' +# condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True') +# variables: +# python.version: '3.6' +# pool: +# vmImage: 'ubuntu-16.04' +# steps: +# - template: azure-pipelines-steps.yml +# parameters: +# body: +# - task: AzureCLI@2 +# displayName: 'AutoML tests' +# inputs: +# azureSubscription: 'automl' +# scriptLocation: 'inlineScript' +# scriptType: 'pscore' +# powerShellIgnoreLASTEXITCODE: '' # string for now due to https://github.com/microsoft/azure-pipelines-tasks/issues/12266 +# inlineScript: | +# $env:SUBSCRIPTION_ID = az account show --query id -o tsv +# python setup.py pytest +# env: +# WORKSPACE_NAME: 'testWorkspace' +# RESOURCE_GROUP: 'testingAutoMLEconML' +# PYTEST_ADDOPTS: '-m "automl" -n 0' +# COVERAGE_PROCESS_START: 'setup.cfg' + +# - task: PublishTestResults@2 +# displayName: 'Publish Test Results **/test-results.xml' +# inputs: +# testResultsFiles: '**/test-results.xml' +# testRunTitle: 'AutoML' +# condition: succeededOrFailed() +# package: '.[automl]' - job: 'Linting' dependsOn: 'EvalChanges' @@ -185,6 +185,15 @@ jobs: Windows, Python 3.7: imageName: 'vs2017-win2016' python.version: '3.7' + Linux, Python 3.8: + imageName: 'ubuntu-16.04' + python.version: '3.8' + macOS, Python 3.8: + imageName: 'macOS-10.15' + python.version: '3.8' + Windows, Python 3.8: + imageName: 'vs2017-win2016' + python.version: '3.8' pool: vmImage: $(imageName) diff --git a/doc/spec/estimation/dml.rst b/doc/spec/estimation/dml.rst index ac24e6743..f67e98005 100644 --- a/doc/spec/estimation/dml.rst +++ b/doc/spec/estimation/dml.rst @@ -696,9 +696,7 @@ the case where this matrix has low rank: all the products can be embedded in som space and the cross-price elasticities is a linear function of these low dimensional embeddings. This corresponds to well-studied latent factor models in pricing. Our framework can easily handle this by using a nuclear norm regularized multi-task regression in the final stage. For instance the -lightning package implements such a class: - -.. testcode:: +lightning package implements such a class:: from econml.dml import DMLCateEstimator from sklearn.preprocessing import PolynomialFeatures @@ -714,8 +712,3 @@ lightning package implements such a class: te_pred = est.const_marginal_effect(np.median(X, axis=0, keepdims=True)) print(te_pred) print(np.linalg.svd(te_pred[0])) - -.. testoutput:: - :hide: - - ... \ No newline at end of file diff --git a/doc/spec/estimation/forest.rst b/doc/spec/estimation/forest.rst index 5ded4181e..4177395f9 100644 --- a/doc/spec/estimation/forest.rst +++ b/doc/spec/estimation/forest.rst @@ -376,7 +376,7 @@ Similarly, we can call :class:`.DiscreteTreatmentOrthoForest`: >>> est.fit(Y, T, W, W) >>> print(est.effect(W[:2])) - [1.01... 1.25...] + [0.99... 1.35...] Let's now look at a more involved example with a high-dimensional set of confounders :math:`W` and with more realistic noisy data. In this case we can just use the default parameters diff --git a/econml/_ortho_learner.py b/econml/_ortho_learner.py index bb922e676..074c8d132 100644 --- a/econml/_ortho_learner.py +++ b/econml/_ortho_learner.py @@ -335,8 +335,7 @@ def score(self, Y, T, W=None, nuisances=None): >>> est.score(y, X[:, 0], W=X[:, 1:]) 0.00727995... >>> est.model_final.model - LinearRegression(copy_X=True, fit_intercept=False, n_jobs=None, - normalize=False) + LinearRegression(fit_intercept=False) >>> est.model_final.model.coef_ array([1.023649...]) @@ -388,15 +387,15 @@ def score(self, Y, T, W=None, nuisances=None): est.fit(y, T, W=W) >>> est.score_ - 0.00316040... + 0.00673015... >>> est.const_marginal_effect() - array([[1.001231...]]) + array([[1.008401...]]) >>> est.effect() - array([1.001231...]) + array([1.008401...]) >>> est.score(y, T, W=W) - 0.00256958... + 0.00310431... >>> est.model_final.model.coef_[0] - 1.00123158... + 1.00840170... Attributes ---------- diff --git a/econml/_rlearner.py b/econml/_rlearner.py index 074703335..9d6f62f10 100644 --- a/econml/_rlearner.py +++ b/econml/_rlearner.py @@ -156,22 +156,15 @@ def predict(self, X): >>> est.score(y, X[:, 0], X=np.ones((X.shape[0], 1)), W=X[:, 1:]) 9.73638006...e-05 >>> est.model_final.model - LinearRegression(copy_X=True, fit_intercept=False, n_jobs=None, - normalize=False) + LinearRegression(fit_intercept=False) >>> est.model_final.model.coef_ array([0.999631...]) >>> est.score_ 9.82623204...e-05 >>> [mdl._model for mdl in est.models_y] - [LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, - normalize=False), - LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, - normalize=False)] + [LinearRegression(), LinearRegression()] >>> [mdl._model for mdl in est.models_t] - [LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, - normalize=False), - LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, - normalize=False)] + [LinearRegression(), LinearRegression()] Attributes ---------- diff --git a/econml/drlearner.py b/econml/drlearner.py index bd4d014a5..1223ead3b 100644 --- a/econml/drlearner.py +++ b/econml/drlearner.py @@ -168,30 +168,30 @@ class takes as input the parameter ``model_regressor``, which is an arbitrary sc est.fit(y, T, X=X, W=None) >>> est.const_marginal_effect(X[:2]) - array([[0.527611..., 1.043938...], - [0.345923..., 0.422289...]]) + array([[0.511640..., 1.144004...], + [0.378140..., 0.613143...]]) >>> est.effect(X[:2], T0=0, T1=1) - array([0.527611..., 0.345923...]) + array([0.511640..., 0.378140...]) >>> est.score_ - 6.48100436... + 5.11238581... >>> est.score(y, T, X=X) - 4.58598642... + 5.78673506... >>> est.model_cate(T=1).coef_ - array([0.413288..., 0.02370... , 0.021575...]) + array([0.434910..., 0.010226..., 0.047913...]) >>> est.model_cate(T=2).coef_ - array([ 0.920586..., 0.0963652..., -0.060305...]) + array([ 0.863723..., 0.086946..., -0.022288...]) >>> est.cate_feature_names() >>> [mdl.coef_ for mdl in est.models_regression] - [array([ 1.435973...e+00, 3.342106...e-04, -7.102984...e-03, 6.707922...e-01, - 1.984256...e+00]), array([ 1.494633...e+00, -2.463273...e-03, 2.009746...e-03, 6.828204...e-01, - 2.034977...e+00])] + [array([ 1.472104...e+00, 1.984419...e-03, -1.103451...e-02, 6.984376...e-01, + 2.049695...e+00]), array([ 1.455654..., -0.002110..., 0.005488..., 0.677090..., 1.998648...])] >>> [mdl.coef_ for mdl in est.models_propensity] - [array([[-1.005830..., 0.087684..., 0.110012... ], - [ 0.087689..., 0.034947..., -0.088753...], - [ 0.918140..., -0.122632..., -0.021259...]]), array([[-0.742430..., 0.067423..., -0.080428...], - [ 0.046120..., -0.030004..., -0.076622...], - [ 0.696310..., -0.037418..., 0.157051...]])] + [array([[-0.747137..., 0.153419..., -0.018412...], + [ 0.083807..., -0.110360..., -0.076003...], + [ 0.663330..., -0.043058... , 0.094416...]]), + array([[-1.048348...e+00, 2.248997...e-04, 3.228087...e-02], + [ 1.911900...e-02, 1.241337...e-01, -8.196211...e-02], + [ 1.029229...e+00, -1.243586...e-01, 4.968123...e-02]])] Beyond default models: @@ -215,19 +215,19 @@ class takes as input the parameter ``model_regressor``, which is an arbitrary sc est.fit(y, T, X=X, W=None) >>> est.score_ - 1.9... + 1.7... >>> est.const_marginal_effect(X[:3]) - array([[0.66..., 1.16...], - [0.56..., 0.86...], - [0.34..., 0.20...]]) + array([[0.68..., 1.10...], + [0.56..., 0.79...], + [0.34..., 0.10...]]) >>> est.model_cate(T=2).coef_ - array([ 0.71..., -0. , -0. ]) + array([0.74..., 0. , 0. ]) >>> est.model_cate(T=2).intercept_ 1.9... >>> est.model_cate(T=1).coef_ - array([0.23..., 0. , 0. ]) + array([0.24..., 0.00..., 0. ]) >>> est.model_cate(T=1).intercept_ - 0.92... + 0.94... Attributes ---------- @@ -605,18 +605,17 @@ class LinearDRLearner(StatsModelsCateEstimatorDiscreteMixin, DRLearner): est.fit(y, T, X=X, W=None, inference='statsmodels') >>> est.effect(X[:3]) - array([ 0.454507..., 0.324469..., -0.070401...]) + array([ 0.409743..., 0.312604..., -0.127394...]) >>> est.effect_interval(X[:3]) - (array([ 0.186553..., -0.117521..., -0.589221...]), - array([0.722462..., 0.766459..., 0.448419...])) + (array([ 0.120682..., -0.102543..., -0.663246...]), array([0.698803..., 0.727753..., 0.408458...])) >>> est.coef_(T=1) - array([0.409764... , 0.019722..., 0.053648...]) + array([ 0.450779..., -0.003214... , 0.063884... ]) >>> est.coef__interval(T=1) - (array([ 0.188595..., -0.168478..., -0.139291...]), array([0.630934..., 0.207922..., 0.246588...])) + (array([ 0.202646..., -0.207195..., -0.104558...]), array([0.698911..., 0.200767..., 0.232326...])) >>> est.intercept_(T=1) - 0.86450983... + 0.88425066... >>> est.intercept__interval(T=1) - (0.67765526..., 1.05136440...) + (0.68655813..., 1.08194320...) Attributes ---------- @@ -801,19 +800,17 @@ class SparseLinearDRLearner(DebiasedLassoCateEstimatorDiscreteMixin, DRLearner): est.fit(y, T, X=X, W=None, inference='debiasedlasso') >>> est.effect(X[:3]) - array([ 0.461389..., 0.319324..., -0.074323...]) + array([ 0.418400..., 0.306400..., -0.130733...]) >>> est.effect_interval(X[:3]) - (array([ 0.119569..., -0.165439..., -0.649570...]), - array([0.803210..., 0.804087..., 0.500923...])) + (array([ 0.056783..., -0.206438..., -0.739296...]), array([0.780017..., 0.819239..., 0.477828...])) >>> est.coef_(T=1) - array([0.409848..., 0.026783..., 0.053017...]) + array([0.449779..., 0.004807..., 0.061954...]) >>> est.coef__interval(T=1) - (array([ 0.213627..., -0.158139..., -0.137547...]), - array([0.606069..., 0.211706..., 0.243582...])) + (array([ 0.242194... , -0.190825..., -0.139646...]), array([0.657365..., 0.200440..., 0.263556...])) >>> est.intercept_(T=1) - 0.86461883... + 0.88436847... >>> est.intercept__interval(T=1) - (0.67790198..., 1.05133569...) + (0.68683788..., 1.08189907...) Attributes ---------- diff --git a/econml/sklearn_extensions/ensemble.py b/econml/sklearn_extensions/ensemble.py index 365fcb864..f9c17c505 100644 --- a/econml/sklearn_extensions/ensemble.py +++ b/econml/sklearn_extensions/ensemble.py @@ -312,12 +312,7 @@ class SubsampledHonestForest(ForestRegressor, RegressorMixin): n_estimators=1000) >>> regr.fit(X_train, y_train) - SubsampledHonestForest(criterion='mse', honest=True, max_depth=None, - max_features='auto', max_leaf_nodes=None, - min_impurity_decrease=0.0, min_samples_leaf=1, - min_samples_split=2, min_weight_fraction_leaf=0.0, - n_estimators=1000, n_jobs=None, random_state=0, - subsample_fr='auto', verbose=0, warm_start=False) + SubsampledHonestForest(n_estimators=1000, random_state=0) >>> regr.feature_importances_ array([0.40..., 0.35..., 0.11..., 0.11...]) >>> regr.predict(np.ones((1, 4))) diff --git a/econml/tests/test_dml.py b/econml/tests/test_dml.py index 5fcadbe68..b7fa1f668 100644 --- a/econml/tests/test_dml.py +++ b/econml/tests/test_dml.py @@ -34,18 +34,22 @@ class TestDML(unittest.TestCase): def test_cate_api(self): """Test that we correctly implement the CATE API.""" - n = 20 + n_c = 20 # number of rows for continuous models + n_d = 30 # number of rows for discrete models - def make_random(is_discrete, d): + def make_random(n, is_discrete, d): if d is None: return None sz = (n, d) if d >= 0 else (n,) if is_discrete: while True: arr = np.random.choice(['a', 'b', 'c'], size=sz) - # ensure that we've got at least two of every element + # ensure that we've got at least 6 of every element + # 2 outer splits, 3 inner splits when model_t is 'auto' and treatment is discrete + # NOTE: this number may need to change if the default number of folds in + # WeightedStratifiedKFold changes _, counts = np.unique(arr, return_counts=True) - if len(counts) == 3 and counts.min() > 1: + if len(counts) == 3 and counts.min() > 5: return arr else: return np.random.normal(size=sz) @@ -55,7 +59,8 @@ def make_random(is_discrete, d): for d_y in [3, 1, -1]: for d_x in [2, None]: for d_w in [2, None]: - W, X, Y, T = [make_random(is_discrete, d) + n = n_d if is_discrete else n_c + W, X, Y, T = [make_random(n, is_discrete, d) for is_discrete, d in [(False, d_w), (False, d_x), (False, d_y), @@ -699,7 +704,7 @@ def test_can_custom_splitter(self): def test_can_use_featurizer(self): "Test that we can use a featurizer, and that fit is only called during training" dml = LinearDMLCateEstimator(LinearRegression(), LinearRegression(), - fit_cate_intercept=False, featurizer=OneHotEncoder(n_values='auto', sparse=False)) + fit_cate_intercept=False, featurizer=OneHotEncoder(sparse=False)) T = np.tile([1, 2, 3], 6) Y = np.array([1, 2, 3, 1, 2, 3]) diff --git a/econml/tests/test_drlearner.py b/econml/tests/test_drlearner.py index 89f6f2517..587e844f8 100644 --- a/econml/tests/test_drlearner.py +++ b/econml/tests/test_drlearner.py @@ -678,7 +678,7 @@ def test_sparse(self): n_x = 50 n_nonzero = 1 n_w = 5 - n = 1000 + n = 2000 # Treatment effect coef a = np.zeros(n_x) nonzero_idx = np.random.choice(n_x, size=n_nonzero, replace=False) @@ -713,7 +713,7 @@ def test_sparse(self): y_lower, y_upper = sparse_dml.effect_interval(x_test, T0=0, T1=1) in_CI = ((y_lower < true_eff) & (true_eff < y_upper)) # Check that a majority of true effects lie in the 5-95% CI - self.assertTrue(in_CI.mean() > 0.8) + self.assertGreater(in_CI.mean(), 0.8) def _test_te(self, learner_instance, tol, te_type="const"): if te_type not in ["const", "heterogeneous"]: diff --git a/econml/tests/test_orf.py b/econml/tests/test_orf.py index 114062a2b..ada59691c 100644 --- a/econml/tests/test_orf.py +++ b/econml/tests/test_orf.py @@ -184,6 +184,20 @@ def test_effect_shape(self): def test_nuisance_model_has_weights(self): """Test whether the correct exception is being raised if model_final doesn't have weights.""" + + # Create a wrapper around Lasso that doesn't support weights + # since Lasso does natively support them starting in sklearn 0.23 + class NoWeightModel: + def __init__(self): + self.model = Lasso() + + def fit(self, X, y): + self.model.fit(X, y) + return self + + def predict(self, X): + return self.model.predict(X) + # Generate data with continuous treatments T = np.dot(TestOrthoForest.W[:, TestOrthoForest.support], TestOrthoForest.coefs_T) + \ TestOrthoForest.eta_sample(TestOrthoForest.n) @@ -192,14 +206,14 @@ def test_nuisance_model_has_weights(self): T * TE + TestOrthoForest.epsilon_sample(TestOrthoForest.n) # Instantiate model with most of the default parameters est = ContinuousTreatmentOrthoForest(n_jobs=4, n_trees=10, - model_T=Lasso(), - model_Y=Lasso()) + model_T=NoWeightModel(), + model_Y=NoWeightModel()) est.fit(Y=Y, T=T, X=TestOrthoForest.X, W=TestOrthoForest.W) weights_error_msg = ( "Estimators of type {} do not accept weights. " "Consider using the class WeightedModelWrapper from econml.utilities to build a weighted model." ) - self.assertRaisesRegexp(TypeError, weights_error_msg.format("Lasso"), + self.assertRaisesRegexp(TypeError, weights_error_msg.format("NoWeightModel"), est.effect, X=TestOrthoForest.X) def _test_te(self, learner_instance, expected_te, tol, treatment_type='continuous'): diff --git a/econml/utilities.py b/econml/utilities.py index c36a10df0..f583a0b21 100644 --- a/econml/utilities.py +++ b/econml/utilities.py @@ -708,8 +708,9 @@ def filter_inds(coords, data, n): for (c, d) in l: results[tuple(c[i] for i in coordMap)] += d - return sp.COO(np.array([k for k in results.keys()]).T, - np.array([v for v in results.values()]), + return sp.COO(np.array(list(results.keys())).T if results else + np.empty((len(outputs), 0)), + np.array(list(results.values())), [arrs[indMap[c][0][0]].shape[indMap[c][0][1]] for c in outputs]) diff --git a/setup.cfg b/setup.cfg index 689e6ce41..94d8292aa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ classifiers = Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: MIT License Operating System :: OS Independent @@ -34,15 +35,15 @@ classifiers = packages = find_namespace: setup_requires = pytest-runner - sphinx + sphinx < 3.2 sphinx_rtd_theme install_requires = numpy scipy != 1.4.0 - scikit-learn ~= 0.21.0 - keras - sparse != 0.10 - tensorflow == 1.* + scikit-learn > 0.21.0 + keras < 2.4 + sparse + tensorflow > 1.10, < 2.3 joblib >= 0.13.0 numba != 0.42.1 statsmodels >= 0.9 @@ -50,7 +51,7 @@ install_requires = matplotlib < 3.1; python_version <= '3.5' matplotlib; python_version > '3.5' llvmlite < 0.32; python_version <= '3.5' - pandas + pandas < 1.1 test_suite = econml.tests tests_require = pytest