Update component README generation for new pipeline interface (#684)

This PR updates the component generation to take into account the
component type and generate the appropriate usage example.

components/caption_images/README.md

@@ -28,19 +28,22 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-caption_images_op = ComponentOp.from_registry(
-    name="caption_images",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "caption_images",
     arguments={
         # Add arguments
         # "model_id": "Salesforce/blip-image-captioning-base",
         # "batch_size": 8,
         # "max_new_tokens": 50,
     }
 )
-pipeline.add_op(caption_images_op, dependencies=[...])  #Add previous component as dependency
 ```
 
 ### Testing

components/chunk_text/README.md

@@ -32,18 +32,21 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-chunk_text_op = ComponentOp.from_registry(
-    name="chunk_text",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "chunk_text",
     arguments={
         # Add arguments
         # "chunk_size": 0,
         # "chunk_overlap": 0,
     }
 )
-pipeline.add_op(chunk_text_op, dependencies=[...])  #Add previous component as dependency
 ```
 
 ### Testing

components/crop_images/README.md

@@ -44,17 +44,20 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-crop_images_op = ComponentOp.from_registry(
-    name="crop_images",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "crop_images",
     arguments={
         # Add arguments
         # "cropping_threshold": -30,
         # "padding": 10,
     }
 )
-pipeline.add_op(crop_images_op, dependencies=[...])  #Add previous component as dependency
 ```
 

components/download_images/README.md

@@ -42,11 +42,15 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-download_images_op = ComponentOp.from_registry(
-    name="download_images",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "download_images",
     arguments={
         # Add arguments
         # "timeout": 10,
@@ -59,7 +63,6 @@ download_images_op = ComponentOp.from_registry(
         # "max_aspect_ratio": 99.9,
     }
 )
-pipeline.add_op(download_images_op, dependencies=[...])  #Add previous component as dependency
 ```
 
 ### Testing

components/embed_images/README.md

@@ -27,17 +27,20 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-embed_images_op = ComponentOp.from_registry(
-    name="embed_images",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "embed_images",
     arguments={
         # Add arguments
         # "model_id": "openai/clip-vit-large-patch14",
         # "batch_size": 8,
     }
 )
-pipeline.add_op(embed_images_op, dependencies=[...])  #Add previous component as dependency
 ```
 

components/embed_text/README.md

@@ -29,11 +29,15 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-embed_text_op = ComponentOp.from_registry(
-    name="embed_text",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "embed_text",
     arguments={
         # Add arguments
         # "model_provider": "huggingface",
@@ -42,7 +46,6 @@ embed_text_op = ComponentOp.from_registry(
         # "auth_kwargs": {},
     }
 )
-pipeline.add_op(embed_text_op, dependencies=[...])  #Add previous component as dependency
 ```
 
 ### Testing

components/extract_image_resolution/README.md

@@ -24,15 +24,18 @@ This component takes no arguments.
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-extract_image_resolution_op = ComponentOp.from_registry(
-    name="extract_image_resolution",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "extract_image_resolution",
     arguments={
         # Add arguments
     }
 )
-pipeline.add_op(extract_image_resolution_op, dependencies=[...])  #Add previous component as dependency
 ```
 

components/filter_image_resolution/README.md

@@ -26,17 +26,20 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-filter_image_resolution_op = ComponentOp.from_registry(
-    name="filter_image_resolution",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "filter_image_resolution",
     arguments={
         # Add arguments
         # "min_image_dim": 0,
         # "max_aspect_ratio": 0.0,
     }
 )
-pipeline.add_op(filter_image_resolution_op, dependencies=[...])  #Add previous component as dependency
 ```
 

components/filter_language/README.md

@@ -24,17 +24,20 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-filter_language_op = ComponentOp.from_registry(
-    name="filter_language",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "filter_language",
     arguments={
         # Add arguments
         # "language": "en",
     }
 )
-pipeline.add_op(filter_language_op, dependencies=[...])  #Add previous component as dependency
 ```
 
 ### Testing

components/filter_text_length/README.md

@@ -25,18 +25,21 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-filter_text_length_op = ComponentOp.from_registry(
-    name="filter_text_length",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "filter_text_length",
     arguments={
         # Add arguments
         # "min_characters_length": 0,
         # "min_words_length": 0,
     }
 )
-pipeline.add_op(filter_text_length_op, dependencies=[...])  #Add previous component as dependency
 ```
 
 ### Testing

components/generate_minhash/README.md

@@ -26,17 +26,20 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-generate_minhash_op = ComponentOp.from_registry(
-    name="generate_minhash",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "generate_minhash",
     arguments={
         # Add arguments
         # "shingle_ngram_size": 3,
     }
 )
-pipeline.add_op(generate_minhash_op, dependencies=[...])  #Add previous component as dependency
 ```
 
 ### Testing

components/index_qdrant/README.md

@@ -39,11 +39,17 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-index_qdrant_op = ComponentOp.from_registry(
-    name="index_qdrant",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(...)
+
+dataset.write(
+    "index_qdrant",
     arguments={
         # Add arguments
         # "collection_name": ,
@@ -63,7 +69,6 @@ index_qdrant_op = ComponentOp.from_registry(
         # "force_disable_check_same_thread": False,
     }
 )
-pipeline.add_op(index_qdrant_op, dependencies=[...])  #Add previous component as dependency
 ```
 
 ### Testing

components/index_weaviate/README.md

@@ -31,11 +31,17 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-index_weaviate_op = ComponentOp.from_registry(
-    name="index_weaviate",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(...)
+
+dataset.write(
+    "index_weaviate",
     arguments={
         # Add arguments
         # "weaviate_url": "http://localhost:8080",
@@ -47,6 +53,5 @@ index_weaviate_op = ComponentOp.from_registry(
         # "vectorizer": ,
     }
 )
-pipeline.add_op(index_weaviate_op, dependencies=[...])  #Add previous component as dependency
 ```
 

components/load_from_files/README.md

@@ -27,17 +27,18 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-load_from_files_op = ComponentOp.from_registry(
-    name="load_from_files",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(
+    "load_from_files",
     arguments={
         # Add arguments
         # "directory_uri": ,
     }
 )
-pipeline.add_op(load_from_files_op, dependencies=[...])  #Add previous component as dependency
 ```
 
 ### Testing

components/load_from_hf_hub/README.md

@@ -28,11 +28,13 @@ The component takes the following arguments to alter its behavior:
 You can add this component to your pipeline using the following code:
 
 ```python
-from fondant.pipeline import ComponentOp
+from fondant.pipeline import Pipeline
 
 
-load_from_hf_hub_op = ComponentOp.from_registry(
-    name="load_from_hf_hub",
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(
+    "load_from_hf_hub",
     arguments={
         # Add arguments
         # "dataset_name": ,
@@ -42,6 +44,5 @@ load_from_hf_hub_op = ComponentOp.from_registry(
         # "index_column": ,
     }
 )
-pipeline.add_op(load_from_hf_hub_op, dependencies=[...])  #Add previous component as dependency
 ```