-
Notifications
You must be signed in to change notification settings - Fork 35
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
docs: add paddlepaddle instructions in "INTEGRATIONS"
- Loading branch information
1 parent
e41677a
commit f6b1a62
Showing
3 changed files
with
79 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
#!/usr/bin/env python3 | ||
# | ||
# Copyright 2021 Graviti. Licensed under MIT License. | ||
# | ||
|
||
# pylint: disable=pointless-string-statement | ||
# pylint: disable=wrong-import-position | ||
# pylint: disable=import-error | ||
# type: ignore | ||
|
||
"""This is the example code for using dataset in PaddlePaddle.""" | ||
|
||
"""Build a Segment class""" | ||
from paddle.io import DataLoader, Dataset | ||
from paddle.vision import transforms | ||
from PIL import Image | ||
|
||
from tensorbay import GAS | ||
from tensorbay.dataset import Dataset as TensorBayDataset | ||
|
||
|
||
class MNISTSegment(Dataset): | ||
"""class for wrapping a MNIST segment.""" | ||
|
||
def __init__(self, gas, segment_name, transform): | ||
super().__init__() | ||
self.dataset = TensorBayDataset("MNIST", gas) | ||
self.segment = self.dataset[segment_name] | ||
self.category_to_index = self.dataset.catalog.classification.get_category_to_index() | ||
self.transform = transform | ||
|
||
def __len__(self): | ||
return len(self.segment) | ||
|
||
def __getitem__(self, idx): | ||
data = self.segment[idx] | ||
with data.open() as fp: | ||
image_tensor = self.transform(Image.open(fp)) | ||
|
||
return image_tensor, self.category_to_index[data.label.classification.category] | ||
|
||
|
||
"""""" | ||
|
||
"""Build a dataloader and run it""" | ||
ACCESS_KEY = "Accesskey-*****" | ||
|
||
to_tensor = transforms.ToTensor() | ||
normalization = transforms.Normalize(mean=[0.485], std=[0.229]) | ||
my_transforms = transforms.Compose([to_tensor, normalization]) | ||
|
||
train_segment = MNISTSegment(GAS(ACCESS_KEY), segment_name="train", transform=my_transforms) | ||
train_dataloader = DataLoader(train_segment, batch_size=4, shuffle=True, num_workers=0) | ||
|
||
for index, (image, label) in enumerate(train_dataloader): | ||
print(f"{index}: {label}") | ||
"""""" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
############### | ||
PaddlePaddle | ||
############### | ||
|
||
This topic describes how to integrate TensorBay dataset with PaddlePaddle Pipeline | ||
using the `MNIST Dataset <https://gas.graviti.cn/dataset/data-decorators/MNIST>`_ as an example. | ||
|
||
The typical method to integrate TensorBay dataset with PaddlePaddle is to build a "Segment" class | ||
derived from ``paddle.io.Dataset``. | ||
|
||
.. literalinclude:: ../../../docs/code/use_dataset_in_paddlepaddle.py | ||
:language: python | ||
:start-after: """Build a Segment class""" | ||
:end-before: """""" | ||
|
||
Using the following code to create a PaddlePaddle dataloader and run it: | ||
|
||
.. literalinclude:: ../../../docs/code/use_dataset_in_paddlepaddle.py | ||
:language: python | ||
:start-after: """Build a dataloader and run it""" | ||
:end-before: """""" |