The Startup
Published in

The Startup

Object detection in Android using React Native & TensorFlow.js

$ git clone https://github.com/rickyhp/obj-detection.git
$ cd obj-detection
$ yarn install
import React, { Component } from 'react';
import { StyleSheet, Text, View, StatusBar, ActivityIndicator, TouchableOpacity, Image } from 'react-native'
import { ScrollView } from 'react-native-gesture-handler';

import Constants from 'expo-constants'
import * as Permissions from 'expo-permissions'
import * as ImagePicker from 'expo-image-picker'
import * as FileSystem from 'expo-file-system'

import * as jpeg from 'jpeg-js'

import * as tf from '@tensorflow/tfjs';
import '@tensorflow/tfjs-react-native';

// import * as mobilenet from '@tensorflow-models/mobilenet'
// You can try with other models, see https://github.com/tensorflow/tfjs-models
import * as cocossd from '@tensorflow-models/coco-ssd'
  1. Preparing TensorFlow engine
  2. Loading model
  3. User to choose image for model to predict, model does the prediction and result will be printed below the image
await tf.ready();
this.setState({ isTfReady: true,});
this.model = await cocossd.load();
this.setState({ isModelReady: true });
try {
let response = await ImagePicker.launchImageLibraryAsync({
mediaTypes: ImagePicker.MediaTypeOptions.All,
allowsEditing: true,
aspect: [4, 3]
})

if (!response.cancelled) {
const source = { uri: response.uri }
this.setState({ image: source })
this.detectObjects()
}
} catch (error) {
console.log(error)
}
detectObjects = async () => {
try {
const imageAssetPath = Image.resolveAssetSource(this.state.image)

console.log(imageAssetPath.uri);
const imgB64 = await FileSystem.readAsStringAsync(imageAssetPath.uri, {
encoding: FileSystem.EncodingType.Base64,
});
const imgBuffer = tf.util.encodeString(imgB64, 'base64').buffer;
const raw = new Uint8Array(imgBuffer)
const imageTensor = this.imageToTensor(raw);
console.log('imageTensor: ', imageTensor);
const predictions = await this.model.detect(imageTensor)

this.setState({ predictions: predictions })


console.log('----------- predictions: ', predictions);

} catch (error) {
console.log('Exception Error: ', error)
}
}
imageToTensor(rawImageData) {
const TO_UINT8ARRAY = true
const { width, height, data } = jpeg.decode(rawImageData, TO_UINT8ARRAY)

const buffer = new Uint8Array(width * height * 3)
let offset = 0 // offset into original data
for (let i = 0; i < buffer.length; i += 3) {
buffer[i] = data[offset]
buffer[i + 1] = data[offset + 1]
buffer[i + 2] = data[offset + 2]

offset += 4
}

return tf.tensor3d(buffer, [height, width, 3])
}
renderPrediction = (prediction, index) => {
const pclass = prediction.class;
const score = prediction.score;
const x = prediction.bbox[0];
const y = prediction.bbox[1];
const w = prediction.bbox[2];
const h = prediction.bbox[3];

return (
<Text key={index} style={styles.text}>
Prediction: {pclass} {', '} Probability: {score} {', '} Bbox: {x} {', '} {y} {', '} {w} {', '} {h}
</Text>
)
}

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Ricky H. Putra

Leading digitization initiatives in AwanTunai focusing on strengthening Indonesia MSME businesses with technology. Software Dev | Automation | Data Science | AI