Added new models to ModelHub: PoseNet and EfficientDet family

api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/loaders/TFModels.kt

@@ -205,7 +205,7 @@ public object TFModels {
     ): ImageRecognitionModel {
         modelHub as TFModelHub
         val model = modelHub.loadModel(modelType)
-
+        // TODO: this part is not needed for inference (if we could add manually Softmax at the end of the graph)
         model.compile(
             optimizer = Adam(),
             loss = Losses.MAE,
@@ -221,6 +221,7 @@ public object TFModels {
 }
 
 /** Basic interface for models loaded from S3. */
+// TODO: add information about T and U types
 public interface ModelType<T : InferenceModel, U : InferenceModel> {
     /** Relative path to model for local and S3 buckets storages. */
     public val modelRelativePath: String
@@ -256,4 +257,8 @@ public interface ModelType<T : InferenceModel, U : InferenceModel> {
     public fun model(modelHub: ModelHub): T {
         return modelHub.loadModel(this)
     }
+
+    public fun preInit(): InferenceModel {
+        TODO()
+    }
 }

api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/posedetection/DetectedPose.kt

@@ -0,0 +1,6 @@
+package org.jetbrains.kotlinx.dl.api.inference.posedetection
+
+public data class DetectedPose (
+    val poseLandmarks: List<PoseLandmark>,
+    val edges: List<PoseEdge>
+)

api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/posedetection/MultiPoseDetectionResult.kt

@@ -0,0 +1,7 @@
+package org.jetbrains.kotlinx.dl.api.inference.posedetection
+
+import org.jetbrains.kotlinx.dl.api.inference.objectdetection.DetectedObject
+
+public data class MultiPoseDetectionResult (
+    val multiplePoses: MutableList<Pair<DetectedObject, DetectedPose>>
+)

api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/posedetection/PoseEdge.kt

@@ -0,0 +1,12 @@
+package org.jetbrains.kotlinx.dl.api.inference.posedetection
+
+/**
+ * @property [poseEdgeLabel] The predicted pose edge label.
+ * @property [probability] The probability of the predicted class.
+ */
+public data class PoseEdge(
+    val poseEdgeLabel: String,
+    val probability: Float,
+    val start: PoseLandmark,
+    val end: PoseLandmark,
+)

api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/posedetection/PoseLandmark.kt

@@ -0,0 +1,12 @@
+package org.jetbrains.kotlinx.dl.api.inference.posedetection
+
+/**
+ * @property [poseLandmarkLabel] The predicted pose landmark label.
+ * @property [probability] The probability of the predicted class.
+ */
+public data class PoseLandmark (
+    val poseLandmarkLabel: String,
+    val probability: Float,
+    val x: Float,
+    val y: Float,
+)

api/src/main/kotlin/org/jetbrains/kotlinx/dl/dataset/handler/CocoUtils.kt

@@ -93,3 +93,93 @@ public val cocoCategories: Map<Int, String> = mapOf(
     79 to "hair drier",
     80 to "toothbrush"
 )
+
+
+/**
+ * 80 object categories in COCO dataset.
+ *
+ * @see <a href="https://cocodataset.org/#home">
+ *     COCO dataset</a>
+ */
+public val cocoCategoriesForEfficientDet: Map<Int, String> = mapOf(
+    1 to "person",
+    2 to "bicycle",
+    3 to "car",
+    4 to "motorcycle",
+    5 to "airplane",
+    6 to "bus",
+    7 to "train",
+    8 to "truck",
+    9 to "boat",
+    10 to "traffic light",
+    11 to "fire hydrant",
+    13 to "stop sign",
+    14 to "parking meter",
+    15 to "bench",
+    16 to "bird",
+    17 to "cat",
+    18 to "dog",
+    19 to "horse",
+    20 to "sheep",
+    21 to "cow",
+    22 to "elephant",
+    23 to "bear",
+    24 to "zebra",
+    25 to "giraffe",
+    27 to "backpack",
+    28 to "umbrella",
+    31 to "handbag",
+    32 to "tie",
+    33 to "suitcase",
+    34 to "frisbee",
+    35 to "skis",
+    36 to "snowboard",
+    37 to "sports ball",
+    38 to "kite",
+    39 to "baseball bat",
+    40 to "baseball glove",
+    41 to "skateboard",
+    42 to "surfboard",
+    43 to "tennis racket",
+    44 to "bottle",
+    46 to "wine glass",
+    47 to "cup",
+    48 to "fork",
+    49 to "knife",
+    50 to "spoon",
+    51 to "bowl",
+    52 to "banana",
+    53 to "apple",
+    54 to "sandwich",
+    55 to "orange",
+    56 to "broccoli",
+    57 to "carrot",
+    58 to "hot dog",
+    59 to "pizza",
+    60 to "donut",
+    61 to "cake",
+    62 to "chair",
+    63 to "couch",
+    64 to "potted plant",
+    65 to "bed",
+    67 to "dining table",
+    70 to "toilet",
+    72 to "tv",
+    73 to "laptop",
+    74 to "mouse",
+    75 to "remote",
+    76 to "keyboard",
+    77 to "cell phone",
+    78 to "microwave",
+    79 to "oven",
+    80 to "toaster",
+    81 to "sink",
+    82 to "refrigerator",
+    84 to "book",
+    85 to "clock",
+    86 to "vase",
+    87 to "scissors",
+    88 to "teddy bear",
+    89 to "hair drier",
+    90 to "toothbrush"
+)

examples/src/main/kotlin/examples/cnn/cifar10/VGG.kt

@@ -29,7 +29,7 @@ import org.jetbrains.kotlinx.dl.dataset.preprocessor.image.convert
 import java.io.File
 
 private const val PATH_TO_MODEL = "savedmodels/vgg11"
-private const val EPOCHS = 3
+private const val EPOCHS = 1 // 10, at least, is recommended
 private const val TRAINING_BATCH_SIZE = 128
 private const val TEST_BATCH_SIZE = 1000
 private const val NUM_LABELS = 10

examples/src/main/kotlin/examples/cnn/fashionmnist/VGG.kt

@@ -19,7 +19,7 @@ import org.jetbrains.kotlinx.dl.api.core.metric.Metrics
 import org.jetbrains.kotlinx.dl.api.core.optimizer.Adam
 import org.jetbrains.kotlinx.dl.dataset.fashionMnist
 
-private const val EPOCHS = 2
+private const val EPOCHS = 1
 private const val TRAINING_BATCH_SIZE = 100
 private const val TEST_BATCH_SIZE = 1000
 private const val NUM_LABELS = 10

examples/src/main/kotlin/examples/cnn/fsdd/SoundNet.kt

@@ -22,8 +22,8 @@ import org.jetbrains.kotlinx.dl.dataset.FSDD_SOUND_DATA_SIZE
 import org.jetbrains.kotlinx.dl.dataset.freeSpokenDigits
 import org.jetbrains.kotlinx.dl.dataset.handler.NUMBER_OF_CLASSES
 
-private const val EPOCHS = 10
-private const val TRAINING_BATCH_SIZE = 500
+private const val EPOCHS = 20 // 20, at least, is recommended
+private const val TRAINING_BATCH_SIZE = 64
 private const val TEST_BATCH_SIZE = 500
 private const val NUM_CHANNELS = 1L
 private const val SEED = 12L
@@ -122,6 +122,7 @@ private val soundNet = Sequential.of(
  */
 fun soundNet() {
     val (train, test) = freeSpokenDigits()
+    train.shuffle()
 
     soundNet.use {
         it.compile(

examples/src/main/kotlin/examples/cnn/mnist/VGG.kt

@@ -20,7 +20,7 @@ import org.jetbrains.kotlinx.dl.api.core.optimizer.Adam
 import org.jetbrains.kotlinx.dl.api.core.summary.logSummary
 import org.jetbrains.kotlinx.dl.dataset.mnist
 
-private const val EPOCHS = 5
+private const val EPOCHS = 1
 private const val TRAINING_BATCH_SIZE = 200
 private const val TEST_BATCH_SIZE = 1000
 private const val NUM_LABELS = 10

examples/src/main/kotlin/examples/inference/mnist/LeNetMnistExportImportWithJSON.kt

@@ -88,7 +88,7 @@ fun lenetOnMnistExportImportToJson() {
         it.fit(
             dataset = train,
             validationRate = 0.1,
-            epochs = 5,
+            epochs = 2,
             trainBatchSize = 1000,
             validationBatchSize = 100
         )

examples/src/main/kotlin/examples/onnx/cv/custom/efficientnet/EfficientNetB0.kt → examples/src/main/kotlin/examples/onnx/cv/efficicentnet/EfficientNetB0.kt

@@ -3,7 +3,7 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
  */
 
-package examples.onnx.cv.custom.efficientnet
+package examples.onnx.cv.efficicentnet
 
 import examples.transferlearning.getFileFromResource
 import org.jetbrains.kotlinx.dl.api.core.util.loadImageNetClassLabels

examples/src/main/kotlin/examples/onnx/cv/custom/efficientnet/easyEfficientNetB0.kt → examples/src/main/kotlin/examples/onnx/cv/efficicentnet/lightAPI/EfficientNetB0LightAPI.kt

@@ -2,7 +2,7 @@
  * Copyright 2020 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
  */
-package examples.onnx.cv.custom.efficientnet
+package examples.onnx.cv.efficicentnet.lightAPI
 
 import examples.transferlearning.getFileFromResource
 import org.jetbrains.kotlinx.dl.api.inference.imagerecognition.ImageRecognitionModel

examples/src/main/kotlin/examples/onnx/cv/custom/efficientnet/easyEfficientNetB7.kt → examples/src/main/kotlin/examples/onnx/cv/efficicentnet/lightAPI/EfficientNetB7LightAPI.kt

@@ -2,7 +2,7 @@
  * Copyright 2020 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
  */
-package examples.onnx.cv.custom.efficientnet
+package examples.onnx.cv.efficicentnet.lightAPI
 
 import examples.transferlearning.getFileFromResource
 import org.jetbrains.kotlinx.dl.api.inference.imagerecognition.ImageRecognitionModel
@@ -15,7 +15,7 @@ import java.io.File
  * - Model is obtained from [ONNXModelHub].
  * - Model predicts on a few images located in resources.
  */
-fun efficientNetB7EasyPrediction() {
+fun efficientNetB7LightAPIPrediction() {
     val modelHub =
         ONNXModelHub(cacheDirectory = File("cache/pretrainedModels"))
 
@@ -35,4 +35,4 @@ fun efficientNetB7EasyPrediction() {
 }
 
 /** */
-fun main(): Unit = efficientNetB7EasyPrediction()
+fun main(): Unit = efficientNetB7LightAPIPrediction()

examples/src/main/kotlin/examples/onnx/cv/custom/efficientnet/additionalTrainingWithTensorFlow_efficientNetB0.kt → examples/src/main/kotlin/examples/onnx/cv/efficicentnet/notop/EfficientNetB0additionalTrainingWithTensorFlow.kt

@@ -3,7 +3,7 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
  */
 
-package examples.onnx.cv.custom.efficientnet
+package examples.onnx.cv.efficicentnet.notop
 
 import org.jetbrains.kotlinx.dl.api.core.Sequential
 import org.jetbrains.kotlinx.dl.api.core.activation.Activations

examples/src/main/kotlin/examples/onnx/mnist/LenetOnMnist.kt → examples/src/main/kotlin/examples/onnx/cv/lenet/LenetOnMnist.kt

@@ -3,7 +3,7 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
  */
 
-package examples.onnx.mnist
+package examples.onnx.cv.lenet
 
 import org.jetbrains.kotlinx.dl.api.core.metric.Metrics
 import org.jetbrains.kotlinx.dl.api.inference.loaders.ONNXModelHub

examples/src/main/kotlin/examples/onnx/cv/custom/efficientnet/onnxAdditionalTraining.kt → examples/src/main/kotlin/examples/onnx/cv/onnxAdditionalTraining.kt

@@ -3,7 +3,7 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
  */
 
-package examples.onnx.cv.custom.efficientnet
+package examples.onnx.cv
 
 import org.jetbrains.kotlinx.dl.api.core.Sequential
 import org.jetbrains.kotlinx.dl.api.core.activation.Activations

examples/src/main/kotlin/examples/onnx/cv/custom/efficientnet/onnxPredictionRunner.kt → examples/src/main/kotlin/examples/onnx/cv/onnxPredictionRunner.kt

@@ -3,7 +3,7 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
  */
 
-package examples.onnx.cv.custom.efficientnet
+package examples.onnx.cv
 
 import examples.transferlearning.getFileFromResource
 import org.jetbrains.kotlinx.dl.api.core.util.loadImageNetClassLabels
@@ -16,8 +16,6 @@ import org.jetbrains.kotlinx.dl.dataset.preprocessor.*
 import org.jetbrains.kotlinx.dl.dataset.preprocessor.image.convert
 import org.jetbrains.kotlinx.dl.dataset.preprocessor.image.resize
 import java.io.File
-import java.net.URISyntaxException
-import java.net.URL
 
 fun runONNXImageRecognitionPrediction(
     modelType: ONNXModels.CV<out OnnxInferenceModel>,

examples/src/main/kotlin/examples/onnx/cv/resnet/easyResNet18.kt → examples/src/main/kotlin/examples/onnx/cv/resnet/ResNet18LightAPI.kt

@@ -6,7 +6,6 @@ package examples.onnx.cv.resnet
 
 import examples.transferlearning.getFileFromResource
 import org.jetbrains.kotlinx.dl.api.inference.imagerecognition.ImageRecognitionModel
-import org.jetbrains.kotlinx.dl.api.inference.keras.loaders.TFModelHub
 import org.jetbrains.kotlinx.dl.api.inference.loaders.ONNXModelHub
 import org.jetbrains.kotlinx.dl.api.inference.onnx.ONNXModels
 import java.io.File
@@ -16,7 +15,7 @@ import java.io.File
  * - Model is obtained from [ONNXModelHub].
  * - Model predicts on a few images located in resources.
  */
-fun resnet18easyPrediction() {
+fun resnet18LightAPIPrediction() {
     val modelHub =
         ONNXModelHub(cacheDirectory = File("cache/pretrainedModels"))
 
@@ -36,4 +35,4 @@ fun resnet18easyPrediction() {
 }
 
 /** */
-fun main(): Unit = resnet18easyPrediction()
+fun main(): Unit = resnet18LightAPIPrediction()

examples/src/main/kotlin/examples/onnx/cv/custom/ResNet50.kt → examples/src/main/kotlin/examples/onnx/cv/resnet/notop/ResNet50.kt

@@ -3,7 +3,7 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
  */
 
-package examples.onnx.cv.custom
+package examples.onnx.cv.resnet.notop
 
 import examples.transferlearning.getFileFromResource
 import org.jetbrains.kotlinx.dl.api.core.util.loadImageNetClassLabels

examples/src/main/kotlin/examples/onnx/cv/custom/additionalTrainingWithTensorFlow.kt → examples/src/main/kotlin/examples/onnx/cv/resnet/notop/ResNet50additionalTrainingWithTensorFlow.kt

@@ -3,7 +3,7 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
  */
 
-package examples.onnx.cv.custom
+package examples.onnx.cv.resnet.notop
 
 import org.jetbrains.kotlinx.dl.api.core.Sequential
 import org.jetbrains.kotlinx.dl.api.core.activation.Activations

examples/src/main/kotlin/examples/onnx/faces/Fan2D106.kt

@@ -49,7 +49,7 @@ fun main() {
             val inputData = modelType.preprocessInput(preprocessing)
 
             val yhat = it.predictRaw(inputData)
-            println(yhat.toTypedArray().contentDeepToString())
+            println(yhat.values.toTypedArray().contentDeepToString())
 
             visualiseLandMarks(imageFile, yhat)
         }
@@ -58,7 +58,7 @@ fun main() {
 
 fun visualiseLandMarks(
     imageFile: File,
-    landmarks: List<Array<*>>
+    landmarks: Map<String, Any>
 ) {
     val preprocessing: Preprocessing = preprocess {
         load {