Skip to content
This repository has been archived by the owner on Apr 4, 2023. It is now read-only.

Commit

Permalink
#699 Add ML Kit support (ML Kit demo++, features++, restored non-MLKi…
Browse files Browse the repository at this point in the history 
…t demo)
  • Loading branch information
@EddyVerbruggen
EddyVerbruggen committed May 15, 2018
1 parent dba179a commit 35c646d
Show file tree
Hide file tree
Showing 29 changed files with 518 additions and 365 deletions.
8 changes: 7 additions & 1 deletion demo-ng/app/App_Resources/Android/AndroidManifest.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,13 @@
android:label="@string/app_name"
android:theme="@style/AppTheme">

<meta-data android:name="com.facebook.sdk.ApplicationId" android:value="@string/facebook_app_id"/>
<meta-data
android:name="com.facebook.sdk.ApplicationId"
android:value="@string/facebook_app_id"/>

<meta-data
android:name="com.google.firebase.ml.vision.DEPENDENCIES"
android:value="text,barcode,face,label" />

<activity
android:name="com.tns.NativeScriptActivity"
Expand Down
5 changes: 5 additions & 0 deletions demo-ng/app/app.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,8 @@ button {
height: 280;
margin-top: 16;
}

Label.mlkit-result {
margin: 5;
text-align: left;
}
39 changes: 19 additions & 20 deletions demo-ng/app/tabs/mlkit/barcodescanning/barcodescanning.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,26 @@
<Label class="action-bar-title" text="Barcode scanning"></Label>
</ActionBar>

<ScrollView>
<StackLayout>
<GridLayout rows="auto, auto, *">

<Label text="The scanner has been configured to detect QR codes, EAN 8 and EAN 13. It processes every 10th frame. These settings can be tweaked in your usage of the plugin." textWrap="true"></Label>
<Label row="0" text="The scanner has been configured to detect QR codes, EAN 8 and EAN 13. It processes every 10th frame. These settings can be tweaked in your usage of the plugin." textWrap="true"></Label>

<MLKitBarcodeScanner
width="300"
height="340"
formats="QR_CODE, EAN_8, EAN_13"
processEveryNthFrame="10"
(scanResult)="onBarcodeScanResult($event)">
</MLKitBarcodeScanner>
<MLKitBarcodeScanner
row="1"
width="300"
height="340"
formats="QR_CODE, EAN_8, EAN_13"
processEveryNthFrame="10"
(scanResult)="onBarcodeScanResult($event)">
</MLKitBarcodeScanner>

<ListView [items]="barcodes" class="m-t-20">
<ng-template let-item="item">
<GridLayout columns="2*, 3*">
<Label col="0" class="m-5" [text]="item.format" horizontalAlignment="left"></Label>
<Label col="1" class="m-5" [text]="item.value" horizontalAlignment="left"></Label>
</GridLayout>
</ng-template>
</ListView>
<ListView row="2" [items]="barcodes" class="m-t-20">
<ng-template let-item="item">
<GridLayout columns="2*, 3*">
<Label col="0" class="mlkit-result" textWrap="true" [text]="item.format"></Label>
<Label col="1" class="mlkit-result" textWrap="true" [text]="item.value"></Label>
</GridLayout>
</ng-template>
</ListView>

</StackLayout>
</ScrollView>
</GridLayout>
1 change: 0 additions & 1 deletion demo-ng/app/tabs/mlkit/barcodescanning/barcodescanning.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ export class BarcodeScanningComponent {
}>;

onBarcodeScanResult(event): void {
console.log(">>> onBarcodeScanResult");
const result: MLKitScanBarcodesResult = event.value;
this.barcodes = result.barcodes;
console.log(">>> onBarcodeScanResult, this.barcodes: " + JSON.stringify(this.barcodes));
Expand Down
61 changes: 31 additions & 30 deletions demo-ng/app/tabs/mlkit/facedetection/facedetection.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,34 @@
<Label class="action-bar-title" text="Face detection"></Label>
</ActionBar>

<ScrollView>
<StackLayout>

<Label text="The scanner has been configured to detect faces every 5th frame. You can tweak this in your usage of the plugin." textWrap="true"></Label>

<MLKitFaceDetection
width="300"
height="340"
processEveryNthFrame="5"
(scanResult)="onFaceDetectionResult($event)">
</MLKitFaceDetection>

<Label [text]="mlKitAllOK" textWrap="true"></Label>

<Image [src]="mlKitLastMatchImg" *ngIf="mlKitLastMatchImg"></Image>

<!--<Image [src]="scannedImage" width="200" *ngIf="scannedImage"></Image>-->

<ListView [items]="faces" class="m-t-20">
<ng-template let-item="item">
<GridLayout columns="*, *, *">
<Label col="0" class="m-5" [text]="item.smilingProbability | number" horizontalAlignment="left"></Label>
<Label col="1" class="m-5" [text]="item.leftEyeOpenProbability | number" horizontalAlignment="left"></Label>
<Label col="2" class="m-5" [text]="item.rightEyeOpenProbability | number" horizontalAlignment="left"></Label>
</GridLayout>
</ng-template>
</ListView>

</StackLayout>
</ScrollView>
<GridLayout rows="auto, auto, auto, auto, *">

<Label row="0" text="The scanner has been configured to detect faces every 5th frame. You can tweak this in your usage of the plugin." textWrap="true"></Label>

<MLKitFaceDetection
row="1"
width="300"
height="340"
processEveryNthFrame="5"
(scanResult)="onFaceDetectionResult($event)">
</MLKitFaceDetection>

<Label row="2" [text]="mlKitAllOK" textWrap="true" class="m-t-10 c-purple"></Label>

<GridLayout row="3" columns="*, *, *" class="m-t-5">
<Label col="0" class="mlkit-result font-weight-bold" textWrap="true" text="Smiling"></Label>
<Label col="1" class="mlkit-result font-weight-bold" textWrap="true" text="Left 👁 open"></Label>
<Label col="2" class="mlkit-result font-weight-bold" textWrap="true" text="Right 👁 open"></Label>
</GridLayout>

<ListView row="4" [items]="faces">
<ng-template let-item="item">
<GridLayout columns="*, *, *">
<Label col="0" class="mlkit-result" textWrap="true" [text]="item.smilingProbability | number"></Label>
<Label col="1" class="mlkit-result" textWrap="true" [text]="item.leftEyeOpenProbability | number"></Label>
<Label col="2" class="mlkit-result" textWrap="true" [text]="item.rightEyeOpenProbability | number"></Label>
</GridLayout>
</ng-template>
</ListView>

</GridLayout>
31 changes: 7 additions & 24 deletions demo-ng/app/tabs/mlkit/facedetection/facedetection.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,43 +8,26 @@ import { ImageSource } from "tns-core-modules/image-source";
templateUrl: "./facedetection.component.html",
})
export class FaceDetectionComponent {
// TODO create a type (in the plugin) for these Arrays
faces: Array<{
smilingProbability: number;
leftEyeOpenProbability: number;
rightEyeOpenProbability: number;
smilingProbability?: number;
leftEyeOpenProbability?: number;
rightEyeOpenProbability?: number;
}>;

mlKitAllOK: string;
mlKitLastMatchImg: ImageSource;

onFaceDetectionResult(scanResult: any): any {
const value: MLKitDetectFacesResult = scanResult.value;
if (value.faces.length > 0) {
this.faces = value.faces;

let allSmilingAndEyesOpen = true;
value.faces.forEach(face => {
allSmilingAndEyesOpen = allSmilingAndEyesOpen && face.smilingProbability && face.leftEyeOpenProbability && face.rightEyeOpenProbability &&
face.smilingProbability > 0.7 && face.leftEyeOpenProbability > 0.7 && face.rightEyeOpenProbability > 0.7;
});
this.mlKitAllOK = `All smiling and eyes open? ${allSmilingAndEyesOpen ? 'Yes, screen grabbed:' : 'Nope. Sad.'}`;

// value.faces.map(face => {
// face.smilingProbability = FaceDetectionComponent.getPercentage(face.smilingProbability);
// face.leftEyeOpenProbability = FaceDetectionComponent.getPercentage(face.leftEyeOpenProbability);
// face.rightEyeOpenProbability = FaceDetectionComponent.getPercentage(face.rightEyeOpenProbability);
// });
this.faces = value.faces;

if (allSmilingAndEyesOpen && value.imageSource) {
this.mlKitLastMatchImg = value.imageSource;
}
this.mlKitAllOK = `All smiling and eyes open? ${allSmilingAndEyesOpen ? 'Yes!' : 'Nope'}`;
}
}

private static getPercentage(input: number): number {
if (isNaN(input)) {
return 0;
}
return Math.round(input * 100);
}

}
33 changes: 16 additions & 17 deletions demo-ng/app/tabs/mlkit/imagelabeling/imagelabeling.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,22 @@
<Label class="action-bar-title" text="Image labeling"></Label>
</ActionBar>

<ScrollView>
<StackLayout>
<GridLayout rows="auto, *">

<MLKitImageLabeling
width="300"
height="340"
(scanResult)="onImageLabeledResult($event)">
</MLKitImageLabeling>
<MLKitImageLabeling
row="0"
width="300"
height="340"
(scanResult)="onImageLabeledResult($event)">
</MLKitImageLabeling>

<ListView [items]="labels" class="m-t-20">
<ng-template let-item="item">
<GridLayout columns="2*, 3*">
<Label col="0" class="m-5" [text]="item.text" horizontalAlignment="left"></Label>
<Label col="1" class="m-5" [text]="item.confidence | number" horizontalAlignment="left"></Label>
</GridLayout>
</ng-template>
</ListView>
<ListView row="1" [items]="labels" class="m-t-20">
<ng-template let-item="item">
<GridLayout columns="3*, 2*">
<Label col="0" class="mlkit-result" textWrap="true" [text]="item.text"></Label>
<Label col="1" class="mlkit-result" textWrap="true" [text]="item.confidence | number"></Label>
</GridLayout>
</ng-template>
</ListView>

</StackLayout>
</ScrollView>
</GridLayout>
4 changes: 2 additions & 2 deletions demo-ng/app/tabs/mlkit/mlkit.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@

<StackLayout orientation="horizontal" horizontalAlignment="center">
<Button text="Cameraroll" (tap)="fromCameraroll()" class="button button-mlkit"></Button>
<Button text="Camera picture" (tap)="fromCameraPicture()" class="button button-mlkit"></Button>
<Button text="Camera feed" (tap)="fromCameraFeed()" class="button button-mlkit"></Button>
<Button text="Cam picture" (tap)="fromCameraPicture()" class="button button-mlkit"></Button>
<Button text="Cam feed" (tap)="fromCameraFeed()" class="button button-mlkit"></Button>
</StackLayout>

</StackLayout>
50 changes: 38 additions & 12 deletions demo-ng/app/tabs/mlkit/mlkit.component.ts
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
import { Component, NgZone } from "@angular/core";
import { Component } from "@angular/core";
import { ImageSource } from "tns-core-modules/image-source";

import { BarcodeFormat, MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
import { MLKitLandmarkRecognitionResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
import { MLKitDetectFacesResult } from "nativescript-plugin-firebase/mlkit/facedetection";
import { Image } from "tns-core-modules/ui/image";
import { action } from "tns-core-modules/ui/dialogs";
import { ImageAsset } from "tns-core-modules/image-asset";
import * as ImagePicker from "nativescript-imagepicker";
import * as Camera from "nativescript-camera";
import { RouterExtensions } from "nativescript-angular";
import { isIOS } from "tns-core-modules/platform";
import { MLKitImageLabelingResult } from "nativescript-plugin-firebase/mlkit/imagelabeling";

const firebase = require("nativescript-plugin-firebase");

Expand All @@ -19,18 +21,16 @@ const firebase = require("nativescript-plugin-firebase");
templateUrl: "./mlkit.component.html",
})
export class MLKitComponent {
public scannedImage: Image;

private mlkitFeatures: Array<string> = [
"Text recognition",
"Barcode scanning",
"Face detection",
"Image labeling",
"Landmark recognition"
"Landmark recognition (cloud)"
];

constructor(private zone: NgZone,
private routerExtensions: RouterExtensions) {
constructor(private routerExtensions: RouterExtensions) {
}

fromCameraFeed(): void {
Expand All @@ -48,6 +48,13 @@ export class MLKitComponent {
to = "/tabs/mlkit/facedetection";
} else if (pickedItem === "Image labeling") {
to = "/tabs/mlkit/imagelabeling";
} else if (pickedItem === "Landmark recognition (cloud)") {
alert({
title: `Not available`,
message: `Landmark recognition is currently cloud-only, so that would be a bit too taxing on your dataplan.`,
okButtonText: "Gotcha!"
});
return;
}
if (to !== undefined) {
this.routerExtensions.navigate([to],
Expand All @@ -64,9 +71,12 @@ export class MLKitComponent {
}

fromCameraPicture(): void {
if (!isIOS) {
Camera.requestPermissions();
}
Camera.takePicture({
width: 1000,
height: 1000,
width: 800,
height: 800,
keepAspectRatio: true,
saveToGallery: false,
cameraFacing: "rear"
Expand All @@ -91,8 +101,8 @@ export class MLKitComponent {

const selected = selection[0];
console.log(">>> selected: " + selected);
selected.options.height = 1000;
selected.options.width = 1000;
selected.options.height = 800;
selected.options.width = 800;
selected.options.keepAspectRatio = true;
selected.getImageAsync((image: any, error: any) => {
console.log(">>> error: " + error);
Expand Down Expand Up @@ -126,6 +136,8 @@ export class MLKitComponent {
this.detectFaces(imageSource);
} else if (pickedItem === "Image labeling") {
this.labelImage(imageSource);
} else if (pickedItem === "Landmark recognition (cloud)") {
this.recognizeLandmark(imageSource);
}
});
}
Expand All @@ -144,6 +156,20 @@ export class MLKitComponent {
.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
}

private recognizeLandmark(imageSource: ImageSource): void {
firebase.mlkit.landmarkrecognition.recognizeLandmark({
image: imageSource
}).then(
(result: MLKitLandmarkRecognitionResult) => {
alert({
title: `Result`,
message: JSON.stringify(result.landmarks),
okButtonText: "OK"
});
})
.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
}

private scanBarcode(imageSource: ImageSource): void {
firebase.mlkit.barcodescanning.scanBarcodes({
image: imageSource,
Expand Down Expand Up @@ -178,10 +204,10 @@ export class MLKitComponent {
image: imageSource,
confidenceThreshold: 0.3
}).then(
(result: MLKitDetectFacesResult) => {
(result: MLKitImageLabelingResult) => {
alert({
title: `Result`,
message: JSON.stringify(result.faces),
message: JSON.stringify(result.labels),
okButtonText: "OK"
});
})
Expand Down
Loading

0 comments on commit 35c646d

Please sign in to comment.