Skip to content

Commit

Permalink
updated notebook
Browse files Browse the repository at this point in the history
  • Loading branch information
@SaschaHeyer
SaschaHeyer committed Jul 10, 2019
1 parent feda05d commit 87fade0
Showing 1 changed file with 61 additions and 31 deletions.
92 changes: 61 additions & 31 deletions notebooks/Pipeline.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@
},
{
"cell_type": "code",
"execution_count": 605,
"execution_count": 42,
"metadata": {},
"outputs": [],
"source": [
"EXPERIMENT_NAME = 'named-entity-recognition'"
"EXPERIMENT_NAME = 'named-entity-recognition'\n",
"BUCKET = \"your-bucket-name\""
]
},
{
Expand All @@ -25,7 +26,7 @@
},
{
"cell_type": "code",
"execution_count": 606,
"execution_count": 43,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -45,7 +46,7 @@
},
{
"cell_type": "code",
"execution_count": 607,
"execution_count": 44,
"metadata": {},
"outputs": [
{
Expand All @@ -59,7 +60,7 @@
"\n",
"Help on function train:\n",
"\n",
"train(input_x_uri:'GCSPath', input_y_uri:'GCSPath', input_job_dir_uri:'GCSPath', input_tags:'Integer', input_words:'Integer', output_model_uri_template:'GCSPath')\n",
"train(input_x_uri:'GCSPath', input_y_uri:'GCSPath', input_job_dir_uri:'GCSPath', input_tags:'Integer', input_words:'Integer', input_dropout, output_model_uri_template:'GCSPath')\n",
" Trains the NER Bi-LSTM.\n",
"\n",
"Help on function deploy:\n",
Expand All @@ -72,16 +73,16 @@
],
"source": [
"preprocess_operation = kfp.components.load_component_from_url(\n",
" 'https://storage.googleapis.com/spielwiese-sascha-heyer/components/preprocess/component.yaml')\n",
" 'https://storage.googleapis.com/{}/components/preprocess/component.yaml'.format(BUCKET))\n",
"help(preprocess_operation)\n",
"\n",
"train_operation = kfp.components.load_component_from_url(\n",
" 'https://storage.googleapis.com/spielwiese-sascha-heyer/components/train/component.yaml')\n",
" 'https://storage.googleapis.com/{}/components/train/component.yaml'.format(BUCKET))\n",
"help(train_operation)\n",
"\n",
"ai_platform_deploy_operation = comp.load_component_from_url(\n",
" \"https://storage.googleapis.com/spielwiese-sascha-heyer/components/deploy2/component.yaml\")\n",
"help(ai_platform_deploy_operation)\n"
" \"https://storage.googleapis.com/{}/components/deploy/component.yaml\".format(BUCKET))\n",
"help(ai_platform_deploy_operation)"
]
},
{
Expand All @@ -93,7 +94,7 @@
},
{
"cell_type": "code",
"execution_count": 608,
"execution_count": 45,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -104,20 +105,21 @@
"def pipeline():\n",
" \n",
" preprocess_task = preprocess_operation(\n",
" input_1_uri='gs://spielwiese-sascha-heyer/components/data/data.csv',\n",
" output_y_uri_template=\"gs://spielwiese-sascha-heyer/{{workflow.uid}}/preprocess/y/data\",\n",
" output_x_uri_template=\"gs://spielwiese-sascha-heyer/{{workflow.uid}}/preprocess/x/data\",\n",
" output_preprocessing_state_uri_template=\"gs://spielwiese-sascha-heyer/{{workflow.uid}}/model\"\n",
" input_1_uri='gs://{}/data/ner.csv'.format(BUCKET),\n",
" output_y_uri_template=\"gs://{}/{{workflow.uid}}/preprocess/y/data\".format(BUCKET),\n",
" output_x_uri_template=\"gs://{}/{{workflow.uid}}/preprocess/x/data\".format(BUCKET),\n",
" output_preprocessing_state_uri_template=\"gs://{}/{{workflow.uid}}/model\".format(BUCKET)\n",
" ).apply(kfp.gcp.use_gcp_secret('user-gcp-sa')) \n",
" \n",
" \n",
" train_task = train_operation(\n",
" input_x_uri=preprocess_task.outputs['output-x-uri'],\n",
" input_y_uri=preprocess_task.outputs['output-y-uri'],\n",
" input_job_dir_uri=\"gs://spielwiese-sascha-heyer/{{workflow.uid}}/job\",\n",
" input_job_dir_uri=\"gs://{}/{{workflow.uid}}/job\".format(BUCKET),\n",
" input_tags=preprocess_task.outputs['output-tags'],\n",
" input_words=preprocess_task.outputs['output-words'],\n",
" output_model_uri_template=\"gs://spielwiese-sascha-heyer/{{workflow.uid}}/model\"\n",
" input_dropout=0.1,\n",
" output_model_uri_template=\"gs://{}/{{workflow.uid}}/model\".format(BUCKET)\n",
" ).apply(kfp.gcp.use_gcp_secret('user-gcp-sa')) \n",
" \n",
" \n",
Expand All @@ -129,7 +131,7 @@
" model_runtime_version=\"1.13\",\n",
" model_prediction_class=\"model_prediction.CustomModelPrediction\",\n",
" model_python_version=\"3.5\",\n",
" model_package_uris=\"gs://spielwiese-sascha-heyer/components/data/custom_prediction_routine-0.2.tar.gz\"\n",
" model_package_uris=\"gs://{}/routine/custom_prediction_routine-0.2.tar.gz\".format(BUCKET)\n",
" ).apply(kfp.gcp.use_gcp_secret('user-gcp-sa'))"
]
},
Expand All @@ -142,7 +144,7 @@
},
{
"cell_type": "code",
"execution_count": 609,
"execution_count": 46,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -162,20 +164,18 @@
},
{
"cell_type": "code",
"execution_count": 610,
"execution_count": 47,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"Run link <a href=\"/pipeline/#/runs/details/717640c3-928d-11e9-ba22-42010a8001ad\" target=\"_blank\" >here</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
"name": "stdout",
"output_type": "stream",
"text": [
"{'created_at': datetime.datetime(2019, 7, 5, 10, 32, 13, tzinfo=tzlocal()),\n",
" 'description': None,\n",
" 'id': '84e88563-7774-4bae-aa33-4a67649c136a',\n",
" 'name': 'named-entity-recognition'}\n"
]
}
],
"source": [
Expand All @@ -198,9 +198,32 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 48,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/html": [
"Run link <a href=\"/pipeline/#/runs/details/705a2bc2-9f1c-11e9-9120-42010a800045\" target=\"_blank\" >here</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"84e88563-7774-4bae-aa33-4a67649c136a\n",
"pipeline run\n",
"pipeline.pipeline.zip\n",
"{}\n"
]
}
],
"source": [
"arguments = {}\n",
"\n",
Expand All @@ -215,6 +238,13 @@
"print(pipeline_filename)\n",
"print(arguments)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down

0 comments on commit 87fade0

Please sign in to comment.