diff --git a/python/dllib/src/bigdl/dllib/feature/common.py b/python/dllib/src/bigdl/dllib/feature/common.py index 068f29542ee..e25fce0b018 100644 --- a/python/dllib/src/bigdl/dllib/feature/common.py +++ b/python/dllib/src/bigdl/dllib/feature/common.py @@ -246,10 +246,14 @@ def image_frame(cls, image_frame, memory_type="DRAM", bigdl_type="float"): def image_set(cls, imageset, memory_type="DRAM", bigdl_type="float"): """ Create FeatureSet from ImageFrame. - :param image_frame: ImageFrame + :param imageset: ImageSet :param memory_type: string, DRAM or PMEM If it's DRAM, will cache dataset into dynamic random-access memory If it's PMEM, will cache dataset into Intel Optane DC Persistent Memory + If it's a Int number n, will cache dataset into disk, and only hold 1/n + of the data into memory during the training. After going through the + 1/n, we will release the current cache, and load another 1/n into + memory. :param bigdl_type: numeric type :return: A feature set """ @@ -257,6 +261,24 @@ def image_set(cls, imageset, memory_type="DRAM", bigdl_type="float"): imageset.to_image_frame(), memory_type) return cls(jvalue=jvalue) + @classmethod + def sample_rdd(cls, rdd, memory_type="DRAM", bigdl_type="float"): + """ + Create FeatureSet from RDD[Sample]. + :param rdd: A RDD[Sample] + :param memory_type: string, DRAM or PMEM + If it's DRAM, will cache dataset into dynamic random-access memory + If it's PMEM, will cache dataset into Intel Optane DC Persistent Memory + If it's a Int number n, will cache dataset into disk, and only hold 1/n + of the data into memory during the training. After going through the + 1/n, we will release the current cache, and load another 1/n into + memory. + :param bigdl_type:numeric type + :return: A feature set + """ + jvalue = callBigDlFunc(bigdl_type, "createSampleFeatureSetFromRDD", rdd, memory_type) + return cls(jvalue=jvalue) + @classmethod def rdd(cls, rdd, memory_type="DRAM", bigdl_type="float"): """ diff --git a/python/dllib/src/bigdl/dllib/feature/image/imagePreprocessing.py b/python/dllib/src/bigdl/dllib/feature/image/imagePreprocessing.py index c0762546807..4dcf16c992c 100644 --- a/python/dllib/src/bigdl/dllib/feature/image/imagePreprocessing.py +++ b/python/dllib/src/bigdl/dllib/feature/image/imagePreprocessing.py @@ -41,6 +41,15 @@ def __init__(self, byte_key="bytes", image_codec=-1, bigdl_type="float"): super(ImageBytesToMat, self).__init__(bigdl_type, byte_key, image_codec) +class ImagePixelBytesToMat(ImagePreprocessing): + """ + Transform byte array(pixels in byte) to OpenCVMat + :param byte_key key that maps byte array + """ + def __init__(self, byte_key="bytes", bigdl_type="float"): + super(ImagePixelBytesToMat, self).__init__(bigdl_type, byte_key) + + class ImageResize(ImagePreprocessing): """ Resize image @@ -331,6 +340,14 @@ def __init__(self, bigdl_type="float"): super(ImageHFlip, self).__init__(bigdl_type) +class ImageMirror(ImagePreprocessing): + """ + Flip the image horizontally and vertically + """ + def __init__(self, bigdl_type="float"): + super(ImageMirror, self).__init__(bigdl_type) + + class ImageFeatureToTensor(Preprocessing): """ a Transformer that convert ImageFeature to a Tensor. @@ -339,6 +356,14 @@ def __init__(self, bigdl_type="float"): super(ImageFeatureToTensor, self).__init__(bigdl_type) +class ImageFeatureToSample(Preprocessing): + """ + A transformer that get Sample from ImageFeature. + """ + def __init__(self, bigdl_type="float"): + super(ImageFeatureToSample, self).__init__(bigdl_type) + + class RowToImageFeature(Preprocessing): """ a Transformer that converts a Spark Row to a BigDL ImageFeature.