sdk/python/examples/bayesianoptimization-katib-sdk.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from kubernetes.client import V1ObjectMeta\n",
    "import kubeflow.katib as kc\n",
    "from kubeflow.katib import constants\n",
    "from kubeflow.katib import utils\n",
    "from kubeflow.katib import V1alpha3AlgorithmSetting\n",
    "from kubeflow.katib import V1alpha3AlgorithmSetting\n",
    "from kubeflow.katib import V1alpha3AlgorithmSpec\n",
    "from kubeflow.katib import V1alpha3CollectorSpec\n",
    "from kubeflow.katib import V1alpha3EarlyStoppingSetting\n",
    "from kubeflow.katib import V1alpha3EarlyStoppingSpecs\n",
    "from kubeflow.katib import V1alpha3Experiment\n",
    "from kubeflow.katib import V1alpha3ExperimentCondition\n",
    "from kubeflow.katib import V1alpha3ExperimentList\n",
    "from kubeflow.katib import V1alpha3ExperimentSpec\n",
    "from kubeflow.katib import V1alpha3ExperimentStatus\n",
    "from kubeflow.katib import V1alpha3FeasibleSpace\n",
    "from kubeflow.katib import V1alpha3FileSystemPath\n",
    "from kubeflow.katib import V1alpha3FilterSpec\n",
    "from kubeflow.katib import V1alpha3GoTemplate\n",
    "from kubeflow.katib import V1alpha3GraphConfig\n",
    "from kubeflow.katib import V1alpha3Metric\n",
    "from kubeflow.katib import V1alpha3MetricsCollectorSpec\n",
    "from kubeflow.katib import V1alpha3NasConfig\n",
    "from kubeflow.katib import V1alpha3ObjectiveSpec\n",
    "from kubeflow.katib import V1alpha3Observation\n",
    "from kubeflow.katib import V1alpha3Operation\n",
    "from kubeflow.katib import V1alpha3OptimalTrial\n",
    "from kubeflow.katib import V1alpha3ParameterAssignment\n",
    "from kubeflow.katib import V1alpha3ParameterSpec\n",
    "from kubeflow.katib import V1alpha3SourceSpec\n",
    "from kubeflow.katib import V1alpha3Suggestion\n",
    "from kubeflow.katib import V1alpha3SuggestionCondition\n",
    "from kubeflow.katib import V1alpha3SuggestionList\n",
    "from kubeflow.katib import V1alpha3SuggestionSpec\n",
    "from kubeflow.katib import V1alpha3SuggestionStatus\n",
    "from kubeflow.katib import V1alpha3TemplateSpec\n",
    "from kubeflow.katib import V1alpha3Trial\n",
    "from kubeflow.katib import V1alpha3TrialAssignment\n",
    "from kubeflow.katib import V1alpha3TrialCondition\n",
    "from kubeflow.katib import V1alpha3TrialList\n",
    "from kubeflow.katib import V1alpha3TrialSpec\n",
    "from kubeflow.katib import V1alpha3TrialStatus\n",
    "from kubeflow.katib import V1alpha3TrialTemplate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "algorithmsettings = V1alpha3AlgorithmSetting(\n",
    "    name= \"random_state\",\n",
    "    value = \"10\"\n",
    "    )\n",
    "algorithm = V1alpha3AlgorithmSpec(\n",
    "    algorithm_name = \"bayesianoptimization\",\n",
    "    algorithm_settings = [algorithmsettings]\n",
    "  )\n",
    "\n",
    "# Objective\n",
    "objective = V1alpha3ObjectiveSpec(\n",
    "    goal = 0.9999,\n",
    "    objective_metric_name = \"Validation-accuracy\",\n",
    "    type = \"maximize\")\n",
    "\n",
    "# Parameters\n",
    "\n",
    "feasible_space_optimizer = V1alpha3FeasibleSpace(list = [\"sgd\", \"adam\", \"ftrl\"])\n",
    "feasible_space_lr = V1alpha3FeasibleSpace(min = \"0.01\", max = \"0.03\")\n",
    "feasible_space_layers = V1alpha3FeasibleSpace(min = \"2\", max = \"5\")\n",
    "\n",
    "parameters = [V1alpha3ParameterSpec(\n",
    "    feasible_space = feasible_space_optimizer, \n",
    "    name = \"--optimizer\",\n",
    "    parameter_type = \"categorical\"\n",
    "    ),\n",
    "    V1alpha3ParameterSpec(\n",
    "    feasible_space = feasible_space_lr, \n",
    "    name = \"--lr\",\n",
    "    parameter_type =\"double\"\n",
    "    ),\n",
    "    V1alpha3ParameterSpec(\n",
    "    feasible_space = feasible_space_layers, \n",
    "    name = \"--num-layers\",\n",
    "    parameter_type = \"int\"\n",
    "    )]\n",
    "\n",
    "# Trialtemplate\n",
    "go_template = V1alpha3GoTemplate(\n",
    "    raw_template =   \"apiVersion: \\\"batch/v1\\\"\\nkind: Job\\nmetadata:\\n  name: {{.Trial}}\\n  namespace: {{.NameSpace}}\\nspec:\\n  template:\\n    spec:\\n      containers:\\n      - name: {{.Trial}}\\n        image: docker.io/kubeflowkatib/mxnet-mnist\\n        command:\\n        - \\\"python3\\\"\\n        - \\\"/opt/mxnet-mnist/mnist.py\\\"\\n        - \\\"--batch-size=64\\\"\\n        {{- with .HyperParameters}}\\n        {{- range .}}\\n        - \\\"{{.Name}}={{.Value}}\\\"\\n        {{- end}}\\n        {{- end}}\\n      restartPolicy: Never\"\n",
    "    )\n",
    "\n",
    "\n",
    "trial_template= V1alpha3TrialTemplate(go_template=go_template)\n",
    "\n",
    "\n",
    "# Experiment\n",
    "experiment = V1alpha3Experiment(\n",
    "    api_version=\"kubeflow.org/v1alpha3\",\n",
    "    kind=\"Experiment\",\n",
    "    metadata=V1ObjectMeta(name=\"bayesianoptimization\",namespace=\"anonymous\"),\n",
    "\n",
    "    spec=V1alpha3ExperimentSpec(\n",
    "         algorithm = algorithm,\n",
    "         max_failed_trial_count=3,\n",
    "         max_trial_count=5,\n",
    "         objective = objective,\n",
    "         parallel_trial_count=5,\n",
    "         parameters = parameters ,\n",
    "         trial_template = trial_template\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "namespace = kc.utils.get_default_target_namespace()\n",
    "#experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "kclient = kc.KatibClient()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Katib Experiment link <a href=\"/_/katib/#/katib/hp_monitor/anonymous/bayesianoptimization\" target=\"_blank\">here</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'apiVersion': 'kubeflow.org/v1alpha3',\n",
       " 'kind': 'Experiment',\n",
       " 'metadata': {'creationTimestamp': '2020-03-04T10:46:49Z',\n",
       "  'generation': 1,\n",
       "  'name': 'bayesianoptimization',\n",
       "  'namespace': 'anonymous',\n",
       "  'resourceVersion': '7034916',\n",
       "  'selfLink': '/apis/kubeflow.org/v1alpha3/namespaces/anonymous/experiments/bayesianoptimization',\n",
       "  'uid': '7360157b-5e05-11ea-8d71-42010aa00012'},\n",
       " 'spec': {'algorithm': {'algorithmName': 'bayesianoptimization',\n",
       "   'algorithmSettings': [{'name': 'random_state', 'value': '10'}]},\n",
       "  'maxFailedTrialCount': 3,\n",
       "  'maxTrialCount': 5,\n",
       "  'metricsCollectorSpec': {'collector': {'kind': 'StdOut'}},\n",
       "  'objective': {'goal': 0.9999,\n",
       "   'objectiveMetricName': 'Validation-accuracy',\n",
       "   'type': 'maximize'},\n",
       "  'parallelTrialCount': 5,\n",
       "  'parameters': [{'feasibleSpace': {'list': ['sgd', 'adam', 'ftrl']},\n",
       "    'name': '--optimizer',\n",
       "    'parameterType': 'categorical'},\n",
       "   {'feasibleSpace': {'max': '0.03', 'min': '0.01'},\n",
       "    'name': '--lr',\n",
       "    'parameterType': 'double'},\n",
       "   {'feasibleSpace': {'max': '5', 'min': '2'},\n",
       "    'name': '--num-layers',\n",
       "    'parameterType': 'int'}],\n",
       "  'trialTemplate': {'goTemplate': {'rawTemplate': 'apiVersion: \"batch/v1\"\\nkind: Job\\nmetadata:\\n  name: {{.Trial}}\\n  namespace: {{.NameSpace}}\\nspec:\\n  template:\\n    spec:\\n      containers:\\n      - name: {{.Trial}}\\n        image: docker.io/kubeflowkatib/mxnet-mnist\\n        command:\\n        - \"python3\"\\n        - \"/opt/mxnet-mnist/mnist.py\"\\n        - \"--batch-size=64\"\\n        {{- with .HyperParameters}}\\n        {{- range .}}\\n        - \"{{.Name}}={{.Value}}\"\\n        {{- end}}\\n        {{- end}}\\n      restartPolicy: Never'}}}}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.create_experiment(experiment, namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Single Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'apiVersion': 'kubeflow.org/v1alpha3',\n",
       " 'kind': 'Experiment',\n",
       " 'metadata': {'creationTimestamp': '2020-03-04T10:46:49Z',\n",
       "  'finalizers': ['update-prometheus-metrics'],\n",
       "  'generation': 1,\n",
       "  'name': 'bayesianoptimization',\n",
       "  'namespace': 'anonymous',\n",
       "  'resourceVersion': '7034918',\n",
       "  'selfLink': '/apis/kubeflow.org/v1alpha3/namespaces/anonymous/experiments/bayesianoptimization',\n",
       "  'uid': '7360157b-5e05-11ea-8d71-42010aa00012'},\n",
       " 'spec': {'algorithm': {'algorithmName': 'bayesianoptimization',\n",
       "   'algorithmSettings': [{'name': 'random_state', 'value': '10'}]},\n",
       "  'maxFailedTrialCount': 3,\n",
       "  'maxTrialCount': 5,\n",
       "  'metricsCollectorSpec': {'collector': {'kind': 'StdOut'}},\n",
       "  'objective': {'goal': 0.9999,\n",
       "   'objectiveMetricName': 'Validation-accuracy',\n",
       "   'type': 'maximize'},\n",
       "  'parallelTrialCount': 5,\n",
       "  'parameters': [{'feasibleSpace': {'list': ['sgd', 'adam', 'ftrl']},\n",
       "    'name': '--optimizer',\n",
       "    'parameterType': 'categorical'},\n",
       "   {'feasibleSpace': {'max': '0.03', 'min': '0.01'},\n",
       "    'name': '--lr',\n",
       "    'parameterType': 'double'},\n",
       "   {'feasibleSpace': {'max': '5', 'min': '2'},\n",
       "    'name': '--num-layers',\n",
       "    'parameterType': 'int'}],\n",
       "  'trialTemplate': {'goTemplate': {'rawTemplate': 'apiVersion: \"batch/v1\"\\nkind: Job\\nmetadata:\\n  name: {{.Trial}}\\n  namespace: {{.NameSpace}}\\nspec:\\n  template:\\n    spec:\\n      containers:\\n      - name: {{.Trial}}\\n        image: docker.io/kubeflowkatib/mxnet-mnist\\n        command:\\n        - \"python3\"\\n        - \"/opt/mxnet-mnist/mnist.py\"\\n        - \"--batch-size=64\"\\n        {{- with .HyperParameters}}\\n        {{- range .}}\\n        - \"{{.Name}}={{.Value}}\"\\n        {{- end}}\\n        {{- end}}\\n      restartPolicy: Never'}}},\n",
       " 'status': {'completionTime': None,\n",
       "  'conditions': [{'lastTransitionTime': '2020-03-04T10:46:49Z',\n",
       "    'lastUpdateTime': '2020-03-04T10:46:49Z',\n",
       "    'message': 'Experiment is created',\n",
       "    'reason': 'ExperimentCreated',\n",
       "    'status': 'True',\n",
       "    'type': 'Created'}],\n",
       "  'currentOptimalTrial': {'bestTrialName': '',\n",
       "   'observation': {'metrics': None},\n",
       "   'parameterAssignments': None},\n",
       "  'startTime': '2020-03-04T10:46:49Z'}}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.get_experiment(name=\"bayesianoptimization\", namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get all Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'apiVersion': 'kubeflow.org/v1alpha3',\n",
       " 'items': [{'apiVersion': 'kubeflow.org/v1alpha3',\n",
       "   'kind': 'Experiment',\n",
       "   'metadata': {'creationTimestamp': '2020-03-04T10:46:49Z',\n",
       "    'finalizers': ['update-prometheus-metrics'],\n",
       "    'generation': 1,\n",
       "    'name': 'bayesianoptimization',\n",
       "    'namespace': 'anonymous',\n",
       "    'resourceVersion': '7034918',\n",
       "    'selfLink': '/apis/kubeflow.org/v1alpha3/namespaces/anonymous/experiments/bayesianoptimization',\n",
       "    'uid': '7360157b-5e05-11ea-8d71-42010aa00012'},\n",
       "   'spec': {'algorithm': {'algorithmName': 'bayesianoptimization',\n",
       "     'algorithmSettings': [{'name': 'random_state', 'value': '10'}]},\n",
       "    'maxFailedTrialCount': 3,\n",
       "    'maxTrialCount': 5,\n",
       "    'metricsCollectorSpec': {'collector': {'kind': 'StdOut'}},\n",
       "    'objective': {'goal': 0.9999,\n",
       "     'objectiveMetricName': 'Validation-accuracy',\n",
       "     'type': 'maximize'},\n",
       "    'parallelTrialCount': 5,\n",
       "    'parameters': [{'feasibleSpace': {'list': ['sgd', 'adam', 'ftrl']},\n",
       "      'name': '--optimizer',\n",
       "      'parameterType': 'categorical'},\n",
       "     {'feasibleSpace': {'max': '0.03', 'min': '0.01'},\n",
       "      'name': '--lr',\n",
       "      'parameterType': 'double'},\n",
       "     {'feasibleSpace': {'max': '5', 'min': '2'},\n",
       "      'name': '--num-layers',\n",
       "      'parameterType': 'int'}],\n",
       "    'trialTemplate': {'goTemplate': {'rawTemplate': 'apiVersion: \"batch/v1\"\\nkind: Job\\nmetadata:\\n  name: {{.Trial}}\\n  namespace: {{.NameSpace}}\\nspec:\\n  template:\\n    spec:\\n      containers:\\n      - name: {{.Trial}}\\n        image: docker.io/kubeflowkatib/mxnet-mnist\\n        command:\\n        - \"python3\"\\n        - \"/opt/mxnet-mnist/mnist.py\"\\n        - \"--batch-size=64\"\\n        {{- with .HyperParameters}}\\n        {{- range .}}\\n        - \"{{.Name}}={{.Value}}\"\\n        {{- end}}\\n        {{- end}}\\n      restartPolicy: Never'}}},\n",
       "   'status': {'completionTime': None,\n",
       "    'conditions': [{'lastTransitionTime': '2020-03-04T10:46:49Z',\n",
       "      'lastUpdateTime': '2020-03-04T10:46:49Z',\n",
       "      'message': 'Experiment is created',\n",
       "      'reason': 'ExperimentCreated',\n",
       "      'status': 'True',\n",
       "      'type': 'Created'}],\n",
       "    'currentOptimalTrial': {'bestTrialName': '',\n",
       "     'observation': {'metrics': None},\n",
       "     'parameterAssignments': None},\n",
       "    'startTime': '2020-03-04T10:46:49Z'}}],\n",
       " 'kind': 'ExperimentList',\n",
       " 'metadata': {'continue': '',\n",
       "  'resourceVersion': '7035035',\n",
       "  'selfLink': '/apis/kubeflow.org/v1alpha3/namespaces/anonymous/experiments'}}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.get_experiment()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get experiment status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Running'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.get_experiment_status(name=\"bayesianoptimization\", namespace=namespace)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Is experiment succeeded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.is_experiment_succeeded(name=\"bayesianoptimization\", namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## List Trials of an experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': 'bayesianoptimization-52tsscjq', 'status': 'Running'},\n",
       " {'name': 'bayesianoptimization-65fnsn5t', 'status': 'Running'},\n",
       " {'name': 'bayesianoptimization-d5blztvn', 'status': 'Running'},\n",
       " {'name': 'bayesianoptimization-gs7b6g25', 'status': 'Running'},\n",
       " {'name': 'bayesianoptimization-s8qwszvr', 'status': 'Running'}]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.list_trials(name=\"bayesianoptimization\", namespace=namespace)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## List all experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': 'bayesianoptimization', 'status': 'Running'}]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.list_experiments(namespace=namespace)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Optimal Hyperparameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'Succeeded',\n",
       " 'currentOptimalTrial': {'bestTrialName': 'bayesianoptimization-nkj9hd9z',\n",
       "  'observation': {'metrics': [{'name': 'Validation-accuracy',\n",
       "     'value': 0.980195}]},\n",
       "  'parameterAssignments': [{'name': '--optimizer', 'value': 'sgd'},\n",
       "   {'name': '--lr', 'value': '0.01721786775240965'},\n",
       "   {'name': '--num-layers', 'value': '3'}]}}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.get_optimal_hyperparmeters(name=\"bayesianoptimization\", namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delete experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    " kclient.delete_experiment(name=\"bayesianoptimization\", namespace=namespace)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}