diff --git a/docs/Basic Usage.ipynb b/docs/Basic Usage.ipynb new file mode 100644 index 0000000..c371667 --- /dev/null +++ b/docs/Basic Usage.ipynb @@ -0,0 +1,291 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Recruitment Via Prolific\n", + "In this tutorial, you will learn how to set up a random dot motion (RDK) experiment, sample experimental conditions, collect data via the recruitment platform [Prolific](https://www.prolific.com/) and use linear regression to analyse the data. We will use the following components:\n", + "\n", + "- theorist: linear regression\n", + "- experiment-runner: serving the experiment via Firebase and recruiting participants via Prolific\n", + "- experimentalist: random sampling\n", + "\n", + "## Prerequisites\n", + "In this example, we assume that you have set up a RDK experiment on Firebase. For example, following the [cookiecutter tutorial](https://autoresearch.github.io/autora/user-cookiecutter/docs/)\n", + "Here, we focus on how to integrate recruitment via Prolific.\n", + "\n", + "\n", + "!!! Warning:\n", + "The `firebase-prolific-runner` will automatically set up a study on Prolific and recruit participants. It is highly recommended to test the experiment before recruiting participants, to have approval from an ethics committee, and to adhere to the ethics guidelines. For example include a consent page.\n", + "\n", + "\n", + "## Workflow\n" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "\n", + "\n", + "\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Variables And State\n", + "We set up an Autora experiment by defining variables and a state. Here, we will use the coherence of the random dot motion kinematogram as the dependent variable. This is the ratio of dots moving in a coherent direction as opposed to moving randomly." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "from autora.state import StandardState, on_state, Delta\n", + "from autora.variable import VariableCollection, Variable\n", + "\n", + "# *** Set up variables *** #\n", + "# independent variable is coherence in percent (0 - 100)\n", + "# dependent variable is accuracy (0 - 1)\n", + "variables = VariableCollection(\n", + " independent_variables=[Variable(name=\"coherence\", allowed_values=np.linspace(0, 1, 101))],\n", + " dependent_variables=[Variable(name=\"accuracy\", value_range=(0, 1))])\n", + "\n", + "# *** State *** #\n", + "# With the variables, we can set up a state. The state object represents the state of our\n", + "# closed loop experiment.\n", + "\n", + "state = StandardState(\n", + " variables=variables,\n", + ")" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "# Theorist\n", + "We use linear regression as a theorist and wrapt the regressor in on_state functionality to use it on the state" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "from sklearn.linear_model import LinearRegression\n", + "# ** Theorist ** #\n", + "theorist = LinearRegression()\n", + "\n", + "# To use the theorist on the state object, we wrap it with the on_state functionality and return a\n", + "# Delta object.\n", + "# Note: The if the input arguments of the theorist_on_state function are state-fields like\n", + "# experiment_data, variables, ... , then using this function on a state object will automatically\n", + "# use those state fields.\n", + "# The output of these functions is always a Delta object. The keyword argument in this case, tells\n", + "# the state object witch field to update.\n", + "\n", + "\n", + "@on_state()\n", + "def theorist_on_state(experiment_data, variables):\n", + " ivs = [iv.name for iv in variables.independent_variables]\n", + " dvs = [dv.name for dv in variables.dependent_variables]\n", + " x = experiment_data[ivs]\n", + " y = experiment_data[dvs]\n", + " return Delta(models=[theorist.fit(x, y)])" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Experimentalist\n", + "Here, we use a random pool and use the wrapper to create an on state function\n", + "Note: The argument num_samples is not a state field. Instead, we will pass it in when calling the function" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "from autora.experimentalist.random import pool\n", + "# ** Experimentalist ** #\n", + "@on_state()\n", + "def experimentalist_on_state(variables, num_samples):\n", + " return Delta(conditions=pool(variables, num_samples))" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Experiment Runner\n", + "Here, we will serve the experiment and store data via Firebase and recruit participants via Prolific. We assume that you already set up an experiment on Firebase. It is recommended to use the `autora-firebase-runner` and manually test the setup before using the `autora-firebase-prolific-runner`.\n" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# We will need json to parse the date from the runner\n", + "import json\n", + "from autora.experiment_runner.firebase_prolific import firebase_prolific_runner\n", + "\n", + "# We will run our experiment on firebase and need credentials. You will find them here:\n", + "# (https://console.firebase.google.com/)\n", + "# -> project -> project settings -> service accounts -> generate new private key\n", + "\n", + "firebase_credentials = {\n", + " \"type\": \"type\",\n", + " \"project_id\": \"project_id\",\n", + " \"private_key_id\": \"private_key_id\",\n", + " \"private_key\": \"private_key\",\n", + " \"client_email\": \"client_email\",\n", + " \"client_id\": \"client_id\",\n", + " \"auth_uri\": \"auth_uri\",\n", + " \"token_uri\": \"token_uri\",\n", + " \"auth_provider_x509_cert_url\": \"auth_provider_x509_cert_url\",\n", + " \"client_x509_cert_url\": \"client_x509_cert_url\"\n", + "}\n", + "\n", + "# Sleep time (seconds): The time between checks to the firebase database and updates of the prolific experiment\n", + "sleep_time = 30\n", + "\n", + "# Study name: This will be the name that will appear on prolific, participants that have participated in a study with the same name will be\n", + "# excluded automatically\n", + "study_name = 'my autora experiment'\n", + "\n", + "# Study description: This will appear as study description on prolific\n", + "study_description= 'RDK experiment'\n", + "\n", + "# Study Url: The url of your study (you can find this in the Firebase Console)\n", + "study_url = 'www.my-autora-experiment.com'\n", + "\n", + "# Study completion time (minutes): The estimated time a participant will take to finish your study. We use the compensation suggested by Prolific to calculate how much a participant will earn based on the completion time.\n", + "study_completion_time = 5\n", + "\n", + "# Prolific Token: You can generate a token on your Prolific account\n", + "prolific_token = 'my prolific token'\n", + "\n", + "# Completion code: The code a participant gets to prove they participated. If you are using the standard set up (with cookiecutter), please make sure this is the same code that you have providede in the .env file of the testing zone.\n", + "completion_code = 'my completion code'\n", + "\n", + "# Exclude Studies\n", + "\n", + "# simple experiment runner that runs the experiment on firebase\n", + "experiment_runner = firebase_prolific_runner(\n", + " firebase_credentials=firebase_credentials,\n", + " time_out=100,\n", + " sleep_time=5)\n", + "\n", + "experiment_runner = firebase_prolific_runner(\n", + " firebase_credentials=firebase_credentials,\n", + " sleep_time=sleep_time,\n", + " study_name=study_name,\n", + " study_description=study_description,\n", + " study_url=study_url,\n", + " study_completion_time=study_completion_time,\n", + " prolific_token=prolific_token,\n", + " completion_code=completion_code,\n", + " )\n", + "\n", + "\n", + "# We need to wrap the runner to use it on the state. Here, we send the raw conditions.\n", + "@on_state()\n", + "def runner_on_state(conditions):\n", + " data = experiment_runner(conditions)\n", + " # We parse the return value of the runner. The return value depends on the specific\n", + " # implementation of your online experiment (see testing_zone/src/design/main.js).\n", + " # In this example, the experiment runner returns a list of strings, that contain json formatted\n", + " # dictionaries.\n", + " # Example:\n", + " # data = ['{'coherence':.3, accuracy':.8}', ...]\n", + " result = []\n", + " for item in data:\n", + " result.append(json.loads(item))\n", + " return Delta(experiment_data=pd.DataFrame(result))" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Closed Loop\n", + "After setting up all components, we can use the runner just as other runners (see https://autoresearch.github.io/autora/)" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# Now, we can run our components\n", + "for _ in range(3):\n", + " state = experimentalist_on_state(state, num_samples=2) # Collect 2 conditions per iteration\n", + " state = runner_on_state(state) # This will collect data from two participants\n", + " state = theorist_on_state(state)" + ], + "metadata": { + "collapsed": false + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/src/autora/experiment_runner/firebase_prolific/__init__.py b/src/autora/experiment_runner/firebase_prolific/__init__.py index ca9b223..a5e7d62 100644 --- a/src/autora/experiment_runner/firebase_prolific/__init__.py +++ b/src/autora/experiment_runner/firebase_prolific/__init__.py @@ -14,7 +14,7 @@ get_submissions_incompleted, request_return_all, approve_all_no_code, - approve_all + approve_all, ) @@ -29,7 +29,9 @@ def _firebase_run(conditions, **kwargs): observations """ firebase_credentials = kwargs["firebase_credentials"] - time_out = kwargs["time_out"] + time_out = None + if "time_out" in kwargs: + time_out = kwargs["time_out"] sleep_time = kwargs["sleep_time"] # set up study on firebase @@ -61,14 +63,14 @@ def _firebase_prolific_run(conditions, **kwargs): exclude_studies = ["default"] else: exclude_studies = kwargs["exclude_studies"] - + if not 'approve_no_code' in kwargs: - print('Warning: Approving submissions with no code. Set approve_no_code to False if no code submissions should be requested to return') + print( + 'Warning: Approving submissions with no code. Set approve_no_code to False if no code submissions should be requested to return') approve_no_code = True else: approve_no_code = kwargs['approve_no_code'] - # set up study on firebase send_conditions("autora", conditions, kwargs["firebase_credentials"]) @@ -92,7 +94,7 @@ def _firebase_prolific_run(conditions, **kwargs): while True: # check firebase check_firebase = check_firebase_status( - "autora", kwargs["firebase_credentials"], time_out + "autora", kwargs["firebase_credentials"], None ) # check prolific if prolific_dict: @@ -104,12 +106,12 @@ def _firebase_prolific_run(conditions, **kwargs): incomplete_submissions = get_submissions_incompleted(study_id, kwargs["prolific_token"]) check_firebase = check_firebase_status( - "autora", kwargs["firebase_credentials"], time_out, incomplete_submissions + "autora", kwargs["firebase_credentials"], None, incomplete_submissions ) check_prolific = check_prolific_status(study_id, kwargs["prolific_token"]) if ( - check_prolific["number_of_submissions"] + check_prolific["number_of_submissions_finished"] >= check_prolific["total_available_places"] ): if check_firebase == "finished": @@ -117,14 +119,19 @@ def _firebase_prolific_run(conditions, **kwargs): observation_list = [observation[key] for key in sorted(observation.keys())] return observation_list else: - print("Warning: Number of collected participants was lower than submission number") + print( + "Warning: Number of collected participants was lower than submission number") observation = get_observations("autora", kwargs["firebase_credentials"]) observation_list = [observation[key] for key in sorted(observation.keys())] return observation_list # firebase places available if check_firebase == "finished": - pass - # return get_observations("autora", kwargs["firebase_credentials"]) + pause_study( + study_id=study_id, prolific_token=kwargs["prolific_token"] + ) + print('Warning: Firebase finished but prolific open') + return get_observations("autora", kwargs["firebase_credentials"]) + if check_firebase == "available": if check_prolific["status"] == "UNPUBLISHED": publish_study( @@ -160,10 +167,10 @@ def firebase_prolific_runner(**kwargs): the runner """ - def run(x): + def runner(x): return _firebase_prolific_run(x, **kwargs) - return run + return runner def firebase_runner(**kwargs):