Simple Rectangle Detection Using OpenCV on Android

Andi Ashari

7 min readApr 14, 2018

In this article, I will show you how to create a simple rectangle detector using OpenCV on native Android step by step.

Step 01: Initialize Android Application

Create new Android Project using Android Studio (File -> New -> New Project…)
Fill the form (Example):
Application name: OpenCVRectangleDetection
Company domain: code.aashari.id
Project location: /home/aashari/AndroidStudioProjects/OpenCVRectangleDetection
Click Next
Select the form minimum SDK, mine is API 22: Android 5.1 (Lollipop)
Click Next
Select Empty Activity
Click Next
Click Finish

Step 02: Add OpenCV Module to Android Project

Go to: https://opencv.org/releases.html
Download Android pack of newest version (I downloaded OpenCV 3.4.1)
Unzip downloaded file “opencv-3.4.1-android-sdk.zip”
Import OpenCV module to android project by clicking File -> New -> Import Module

Click the browse button (…) to browse OpenCV module, and browse extracted directory of OpenCV android sdk then locate sdk directory “opencv-3.4.1-android-sdk/OpenCV-android-sdk/sdk/java”

Then click Ok

Above field should be auto filled
Click Next
Check all the checkbox

Click Finish
Your Android Build Gradle should be error at this moment, it is because build.gradle of OpenCV conflicted with build.gradle of your application
Go to Project Mode

So you can see the project structure like this:

Open app/build.gradle copy below value to openCVLibrary341/build.gradle
compileSdkVersion
minSdkVersion
targetSdkVersion
So my application build.gradle and opencv build.gradle has the same value of compileSdkVersion, minSdkVersion, and targetSdkVersion:

Rebuild the project by clicking Try Again button above text editor.
There should be no error at this time
The next step is adding OpenCV dependencies to our android project by pressing F4 button or, Right Click on project root directory and click Open Module Setting

Select app module

Go to Dependencies tab

On the right bar click + button and select Module dependency
Then select :openCVLibrary341 and click Ok

Re-build your gradle and if there’s no error it means OpenCV module is successfully added to your android project
One last step is adding android opencv native library.
Go to extracted OpenCV directory “opencv-3.4.1-android-sdk/OpenCV-android-sdk/sdk/native”
Copy “libs” directory to your android project “OpenCVRectangleDetection/app/src/main”

Rename libs directory to jniLibs (Right Click on libs -> Refactor -> Rename) then enter “jniLibs”, make sure to un-check all the checkbox
Click Refactor

jniLibs contains all library your hardware device needs, if you already at this step then OpenCV library is ready to use.
If you build your android project to your device, it should be running smoothly without error.

Step 03: Displaying Camera

At this step we will use our camera device and display it in our application

Add using camera permission, open AndroidManifest.xml inside app/manifests directory, add below code inside manifest tag:

<uses-permission android:name="android.permission.CAMERA"></uses-permission>

Go to your main activity app/res/layout/activity_main.xml, remove hello world TextView, then add camera viewer:

<org.opencv.android.JavaCameraView
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:id="@+id/cameraViewer"/>

Go to MainActivity.java (app/java/id.aashari.code.opencvrectangledetection/MainActivity.java)
We should implement CameraBridgeViewBase.CvCameraViewListener2 to listen to our camera device (don’t forget to implement all methods)
Here is my MainActivity.java file to display camera on your application screen:

package id.aashari.code.opencvrectangledetection;

import android.support.v7.app.AppCompatActivity;
import android.os.Bundle;
import android.util.Log;
import android.view.SurfaceView;
import android.widget.Toast;

import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.JavaCameraView;
import org.opencv.android.LoaderCallbackInterface;
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Mat;

public class MainActivity extends AppCompatActivity implements CameraBridgeViewBase.CvCameraViewListener2 {

    //view holder
    CameraBridgeViewBase cameraBridgeViewBase;

    //camera listener callback
    BaseLoaderCallback baseLoaderCallback;

    //image holder
    Mat img;

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        cameraBridgeViewBase = (JavaCameraView) findViewById(R.id.cameraViewer);
        cameraBridgeViewBase.setVisibility(SurfaceView.VISIBLE);
        cameraBridgeViewBase.setCvCameraViewListener(this);

        //create camera listener callback
        baseLoaderCallback = new BaseLoaderCallback(this) {
            @Override
            public void onManagerConnected(int status) {
                switch (status) {
                    case LoaderCallbackInterface.SUCCESS:
                        Log.v("aashari-log", "Loader interface success");
                        cameraBridgeViewBase.enableView();
                        break;
                    default:
                        super.onManagerConnected(status);
                        break;
                }
            }
        };

    }

    @Override
    public void onCameraViewStarted(int width, int height) {

    }

    @Override
    public void onCameraViewStopped() {

    }

    @Override
    public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {

        //get rgba image
        img = inputFrame.rgba();

        //to get grayscale image using below line
        //img = inputFrame.gray();

        return img;

    }

    @Override
    protected void onPause() {
        super.onPause();
        if (cameraBridgeViewBase != null) {
            cameraBridgeViewBase.disableView();
        }
    }

    @Override
    protected void onResume() {
        super.onResume();
        if (!OpenCVLoader.initDebug()) {
            Toast.makeText(getApplicationContext(), "There is a problem", Toast.LENGTH_SHORT).show();
        } else {
            baseLoaderCallback.onManagerConnected(BaseLoaderCallback.SUCCESS);
        }
    }

    @Override
    protected void onDestroy() {
        super.onDestroy();
        if (cameraBridgeViewBase != null) {
            cameraBridgeViewBase.disableView();
        }
    }

}

If you get this message “It seems that you device does not support camera (or it is locked). Application will be closed”, go to Setting -> Apps -> OpenCVRectangleDetection -> Permissions, enable the Camera permission
Open your application, your application should be displaying your camera on your screen like this:

At this stage we should have understood how JavaCameraViewer works, every image on FPS will be stored on img variable inside onCameraFrame function

@Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {

    //get rgba image
    img = inputFrame.rgba();

    //to get grayscale image using below line
    //img = inputFrame.gray();

    return img;

}

Now we can process img variable as we want.

Step 04: Rectangle Detector

Now we modified the MainActivity.java file, here is my file:

package id.aashari.code.opencvrectangledetection;

import android.support.v7.app.AppCompatActivity;
import android.os.Bundle;
import android.util.Log;
import android.view.SurfaceView;
import android.widget.EditText;
import android.widget.Toast;

import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.JavaCameraView;
import org.opencv.android.LoaderCallbackInterface;
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfPoint;
import org.opencv.core.MatOfPoint2f;
import org.opencv.core.Point;
import org.opencv.core.Rect;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

public class MainActivity extends AppCompatActivity implements CameraBridgeViewBase.CvCameraViewListener2 {

    //view holder
    CameraBridgeViewBase cameraBridgeViewBase;

    //camera listener callback
    BaseLoaderCallback baseLoaderCallback;

    //image holder
    Mat bwIMG, hsvIMG, lrrIMG, urrIMG, dsIMG, usIMG, cIMG, hovIMG;
    MatOfPoint2f approxCurve;

    int threshold;

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        //initialize treshold
        threshold = 100;

        cameraBridgeViewBase = (JavaCameraView) findViewById(R.id.cameraViewer);
        cameraBridgeViewBase.setVisibility(SurfaceView.VISIBLE);
        cameraBridgeViewBase.setCvCameraViewListener(this);

        //create camera listener callback
        baseLoaderCallback = new BaseLoaderCallback(this) {
            @Override
            public void onManagerConnected(int status) {
                switch (status) {
                    case LoaderCallbackInterface.SUCCESS:
                        Log.v("aashari-log", "Loader interface success");
                        bwIMG = new Mat();
                        dsIMG = new Mat();
                        hsvIMG = new Mat();
                        lrrIMG = new Mat();
                        urrIMG = new Mat();
                        usIMG = new Mat();
                        cIMG = new Mat();
                        hovIMG = new Mat();
                        approxCurve = new MatOfPoint2f();
                        cameraBridgeViewBase.enableView();
                        break;
                    default:
                        super.onManagerConnected(status);
                        break;
                }
            }
        };

    }

    @Override
    public void onCameraViewStarted(int width, int height) {

    }

    @Override
    public void onCameraViewStopped() {

    }

    @Override
    public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {

        Mat gray = inputFrame.gray();
        Mat dst = inputFrame.rgba();

        Imgproc.pyrDown(gray, dsIMG, new Size(gray.cols() / 2, gray.rows() / 2));
        Imgproc.pyrUp(dsIMG, usIMG, gray.size());

        Imgproc.Canny(usIMG, bwIMG, 0, threshold);

        Imgproc.dilate(bwIMG, bwIMG, new Mat(), new Point(-1, 1), 1);

        List<MatOfPoint> contours = new ArrayList<MatOfPoint>();

        cIMG = bwIMG.clone();

        Imgproc.findContours(cIMG, contours, hovIMG, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);


        for (MatOfPoint cnt : contours) {

            MatOfPoint2f curve = new MatOfPoint2f(cnt.toArray());

            Imgproc.approxPolyDP(curve, approxCurve, 0.02 * Imgproc.arcLength(curve, true), true);

            int numberVertices = (int) approxCurve.total();

            double contourArea = Imgproc.contourArea(cnt);

            if (Math.abs(contourArea) < 100) {
                continue;
            }
        
            //Rectangle detected
            if (numberVertices >= 4 && numberVertices <= 6) {

                List<Double> cos = new ArrayList<>();

                for (int j = 2; j < numberVertices + 1; j++) {
                    cos.add(angle(approxCurve.toArray()[j % numberVertices], approxCurve.toArray()[j - 2], approxCurve.toArray()[j - 1]));
                }

                Collections.sort(cos);

                double mincos = cos.get(0);
                double maxcos = cos.get(cos.size() - 1);

                if (numberVertices == 4 && mincos >= -0.1 && maxcos <= 0.3) {
                    setLabel(dst, "X", cnt);
                }

            }


        }

        return dst;

    }

    @Override
    protected void onPause() {
        super.onPause();
        if (cameraBridgeViewBase != null) {
            cameraBridgeViewBase.disableView();
        }
    }

    @Override
    protected void onResume() {
        super.onResume();
        if (!OpenCVLoader.initDebug()) {
            Toast.makeText(getApplicationContext(), "There is a problem", Toast.LENGTH_SHORT).show();
        } else {
            baseLoaderCallback.onManagerConnected(BaseLoaderCallback.SUCCESS);
        }
    }

    @Override
    protected void onDestroy() {
        super.onDestroy();
        if (cameraBridgeViewBase != null) {
            cameraBridgeViewBase.disableView();
        }
    }

    private static double angle(Point pt1, Point pt2, Point pt0) {
        double dx1 = pt1.x - pt0.x;
        double dy1 = pt1.y - pt0.y;
        double dx2 = pt2.x - pt0.x;
        double dy2 = pt2.y - pt0.y;
        return (dx1 * dx2 + dy1 * dy2) / Math.sqrt((dx1 * dx1 + dy1 * dy1) * (dx2 * dx2 + dy2 * dy2) + 1e-10);
    }

    private void setLabel(Mat im, String label, MatOfPoint contour) {
        int fontface = Core.FONT_HERSHEY_SIMPLEX;
        double scale = 3;//0.4;
        int thickness = 3;//1;
        int[] baseline = new int[1];
        Size text = Imgproc.getTextSize(label, fontface, scale, thickness, baseline);
        Rect r = Imgproc.boundingRect(contour);
        Point pt = new Point(r.x + ((r.width - text.width) / 2),r.y + ((r.height + text.height) / 2));
        Imgproc.putText(im, label, pt, fontface, scale, new Scalar(255, 0, 0), thickness);
    }

}