-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: testing, comments and prepare for release (#13)
* feat: testing, comments and prepare for release * resolve comments
- Loading branch information
1 parent
db8c20b
commit 1f38faf
Showing
14 changed files
with
202 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
graft src/amplitude_experiment |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,20 +15,24 @@ Amplitude Python Server SDK for Experiment. | |
pip install amplitude-experiment | ||
``` | ||
|
||
## Quick Start | ||
## Remote Evaluation Quick Start | ||
```python | ||
from amplitude_experiment import Experiment, Config, Client, User | ||
from amplitude_experiment import Experiment, RemoteEvaluationConfig, RemoteEvaluationClient, User | ||
|
||
# (1) Get your deployment's API key | ||
apiKey = 'YOUR-API-KEY' | ||
|
||
# (2) Initialize the experiment client | ||
experiment = Experiment.initialize(api_key) | ||
# (2) Initialize the experiment remote evaluation | ||
experiment = Experiment.initialize_remote(api_key) | ||
|
||
# (3) Fetch variants for a user | ||
user = User(device_id="abcdefg", user_id="[email protected]", user_properties={ | ||
'premium': True | ||
}) | ||
user = User( | ||
device_id="abcdefg", | ||
user_id="[email protected]", | ||
user_properties={ | ||
'premium': True | ||
} | ||
) | ||
|
||
# (4) Lookup a flag's variant | ||
# | ||
|
@@ -52,8 +56,27 @@ def fetch_callback(user, variants): | |
else: | ||
# Flag is off | ||
|
||
``` | ||
|
||
## Local Evaluation Quick Start | ||
```python | ||
# (1) Initialize the local evaluation client with a server deployment key. | ||
experiment = Experiment.initialize_local(api_key) | ||
|
||
# (2) Start the local evaluation client. | ||
experiment.start() | ||
|
||
# (3) Evaluate a user. | ||
user = User( | ||
device_id="abcdefg", | ||
user_id="[email protected]", | ||
user_properties={ | ||
'premium': True | ||
} | ||
) | ||
variants = experiment.evaluate(user) | ||
``` | ||
|
||
## More Information | ||
Please visit our :100:[Developer Center](https://www.docs.developers.amplitude.com/experiment/sdks/python-sdk/) for more instructions on using our the SDK. | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
import random | ||
import time | ||
import unittest | ||
|
||
from src.amplitude_experiment import LocalEvaluationClient, User | ||
|
||
API_KEY = 'server-qz35UwzJ5akieoAdIgzM4m9MIiOLXLoz' | ||
|
||
|
||
def random_boolean(): | ||
return bool(random.getrandbits(1)) | ||
|
||
|
||
def measure(function, *args, **kwargs): | ||
start = time.time() | ||
function(*args, **kwargs) | ||
elapsed = (time.time() - start) * 1000 | ||
return elapsed | ||
|
||
|
||
def random_string(length): | ||
letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' | ||
return ''.join(random.choice(letters) for i in range(length)) | ||
|
||
|
||
def random_experiment_user(): | ||
n = 15 | ||
user = User(user_id=random_string(n)) | ||
if random_boolean(): | ||
user.device_id = random_string(n) | ||
if random_boolean(): | ||
user.platform = random_string(n) | ||
if random_boolean(): | ||
user.version = random_string(n) | ||
if random_boolean(): | ||
user.os = random_string(n) | ||
if random_boolean(): | ||
user.device_manufacturer = random_string(n) | ||
if random_boolean(): | ||
user.device_model = random_string(n) | ||
if random_boolean(): | ||
user.device_brand = random_string(n) | ||
if random_boolean(): | ||
user.user_properties = { | ||
'test': 'test' | ||
} | ||
return user | ||
|
||
|
||
def random_benchmark_flag(): | ||
n = random.randint(1, 4) | ||
return f"local-evaluation-benchmark-{n}" | ||
|
||
|
||
class BenchmarkTestCase(unittest.TestCase): | ||
_local_evaluation_client: LocalEvaluationClient = None | ||
|
||
@classmethod | ||
def setUpClass(cls) -> None: | ||
cls._local_evaluation_client = LocalEvaluationClient(API_KEY) | ||
cls._local_evaluation_client.start() | ||
|
||
@classmethod | ||
def tearDownClass(cls) -> None: | ||
cls._local_evaluation_client.stop() | ||
|
||
def test_evaluate_benchmark_1_flag_smaller_than_10_ms(self): | ||
user = random_experiment_user() | ||
flag = random_benchmark_flag() | ||
duration = measure(self._local_evaluation_client.evaluate, user, [flag]) | ||
self.assertTrue(duration < 10) | ||
|
||
def test_evaluate_benchmark_10_flag_smaller_than_10_ms(self): | ||
total = 0 | ||
for i in range(10): | ||
user = random_experiment_user() | ||
flag = random_benchmark_flag() | ||
duration = measure(self._local_evaluation_client.evaluate, user, [flag]) | ||
total += duration | ||
self.assertTrue(total < 10) | ||
|
||
def test_evaluate_benchmark_100_flag_smaller_than_100_ms(self): | ||
total = 0 | ||
for i in range(100): | ||
user = random_experiment_user() | ||
flag = random_benchmark_flag() | ||
duration = measure(self._local_evaluation_client.evaluate, user, [flag]) | ||
total += duration | ||
self.assertTrue(total < 100) | ||
|
||
def test_evaluate_benchmark_1000_flag_smaller_than_1000_ms(self): | ||
total = 0 | ||
for i in range(1000): | ||
user = random_experiment_user() | ||
flag = random_benchmark_flag() | ||
duration = measure(self._local_evaluation_client.evaluate, user, [flag]) | ||
total += duration | ||
self.assertTrue(total < 1000) | ||
|
||
|
||
if __name__ == '__main__': | ||
unittest.main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters