Fixed errors in mobile application

Author: av-savchenko

mobile_app/app/build.gradle

@@ -2,13 +2,13 @@ apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
 
 android {
-    compileSdkVersion 31
-    buildToolsVersion "31.0.0"
+    compileSdkVersion 33
+    buildToolsVersion "33.0.0"
 
     defaultConfig {
         applicationId "com.asav.facialprocessing"
         minSdkVersion 24
-        targetSdkVersion 31
+        targetSdkVersion 33
         versionCode 1
         versionName "1.0"
     }
@@ -17,6 +17,9 @@ android {
         release {
             minifyEnabled false
             proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+            ndk {
+                abiFilters "arm64-v8a"
+            }
         }
     }
     aaptOptions {
@@ -37,10 +40,9 @@ dependencies {
     implementation fileTree(dir: "libs", include: ["*.jar"])
     implementation 'androidx.appcompat:appcompat:1.2.0'
     implementation 'androidx.constraintlayout:constraintlayout:2.0.2'
-    implementation 'org.pytorch:pytorch_android_lite:1.9.0'
-    implementation 'org.pytorch:pytorch_android_torchvision:1.9.0'
-    implementation 'org.tensorflow:tensorflow-android:+'
-    implementation 'org.tensorflow:tensorflow-lite-gpu:+'
+    implementation 'org.pytorch:pytorch_android_lite:+'
+    implementation 'org.pytorch:pytorch_android_torchvision_lite:+'
+    implementation 'org.tensorflow:tensorflow-lite:+'
     implementation 'org.tensorflow:tensorflow-lite-support:+'
     testImplementation 'junit:junit:4.12'
     androidTestImplementation 'androidx.test.ext:junit:1.1.2'

mobile_app/app/src/main/assets/mtcnn_model.pb

@@ -32,7 +32,7 @@ public class EmotionPyTorchClassifier{
     /** Tag for the {@link Log}. */
     private static final String TAG = "EmotionPyTorch";
 
-    private static final String MODEL_FILE = "enet_b0_8_best_vgaf.ptl";//"enet_b0_8_va_mtl.ptl";//"enet_b2_8.ptl";
+    private static final String MODEL_FILE = "enet_b0_8_va_mtl.ptl";//"enet_b0_8_best_vgaf.ptl";//"enet_b2_8.ptl";
     private List<String> labels;
     private Module module=null;
     //private int width=260, height=260;

mobile_app/app/src/main/assets/onet.tflite

@@ -22,7 +22,7 @@
 import android.widget.*;
 
 import com.asav.facialprocessing.mtcnn.Box;
-import com.asav.facialprocessing.mtcnn.MTCNNModel;
+import com.asav.facialprocessing.mtcnn.MTCNN;
 
 import java.io.*;
 import java.nio.ByteBuffer;
@@ -42,7 +42,7 @@ public class MainActivity extends AppCompatActivity {
     private Handler mBackgroundHandler=null;
 
     private static int minFaceSize=32;
-    private MTCNNModel mtcnnFaceDetector=null;
+    private MTCNN mtcnnFaceDetector=null;
     private AgeGenderEthnicityTfLiteClassifier facialAttributeClassifier=null;
     private EmotionTfLiteClassifier emotionClassifierTfLite =null;
     private EmotionPyTorchClassifier emotionClassifierPyTorch = null;
@@ -76,7 +76,7 @@ private void init(){
             Log.e(TAG, "Exception initializing EmotionPyTorchClassifier!", e);
         }
         try {
-            mtcnnFaceDetector =MTCNNModel.Companion.create(getAssets());
+            mtcnnFaceDetector =new MTCNN(getApplicationContext());
         } catch (final Exception e) {
             Log.e(TAG, "Exception initializing MTCNNModel!"+e);
         }

mobile_app/app/src/main/assets/pnet.tflite

@@ -9,17 +9,12 @@
 import android.util.Log;
 
 import org.tensorflow.lite.Interpreter;
-import org.tensorflow.lite.gpu.CompatibilityList;
 import org.tensorflow.lite.support.common.*;
-import org.tensorflow.lite.support.image.*;
-import org.tensorflow.lite.support.image.ops.*;
 
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -43,16 +38,7 @@ public abstract class TfLiteClassifier {
     Map<Integer, Object> outputMap = new HashMap<>();
 
     public TfLiteClassifier(final Context context, String model_path) throws IOException {
-        //GpuDelegate delegate = new GpuDelegate();
-        Interpreter.Options options = (new Interpreter.Options()).setNumThreads(4);//.addDelegate(delegate);
-        CompatibilityList compatList = new CompatibilityList();
-        boolean hasGPU=compatList.isDelegateSupportedOnThisDevice();
-        if (hasGPU) {
-            org.tensorflow.lite.gpu.GpuDelegate delegate = new org.tensorflow.lite.gpu.GpuDelegate();
-            options.addDelegate(delegate);
-        }
-
-
+        Interpreter.Options options = (new Interpreter.Options()).setNumThreads(4);
         MappedByteBuffer tfliteModel= FileUtil.loadMappedFile(context,model_path);
         tflite = new Interpreter(tfliteModel,options);
         tflite.allocateTensors();

mobile_app/app/src/main/assets/rnet.tflite

@@ -0,0 +1,32 @@
+package com.asav.facialprocessing.mtcnn;
+
+import android.graphics.Bitmap;
+import android.graphics.Matrix;
+import android.graphics.Point;
+
+/**
+ * 人脸对齐矫正
+ */
+public class Align {
+
+    /**
+     * 仿射变换
+     * @param bitmap 原图片
+     * @param landmarks landmarks
+     * @return 变换后的图片
+     */
+    public static Bitmap face_align(Bitmap bitmap, Point[] landmarks) {
+        float diffEyeX = landmarks[1].x - landmarks[0].x;
+        float diffEyeY = landmarks[1].y - landmarks[0].y;
+
+        float fAngle;
+        if (Math.abs(diffEyeY) < 1e-7) {
+            fAngle = 0.f;
+        } else {
+            fAngle = (float) (Math.atan(diffEyeY / diffEyeX) * 180.0f / Math.PI);
+        }
+        Matrix matrix = new Matrix();
+        matrix.setRotate(-fAngle);
+        return Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
+    }
+}

mobile_app/app/src/main/java/com/asav/facialprocessing/EmotionPyTorchClassifier.java

@@ -4,72 +4,108 @@
 import android.graphics.Rect;
 
 import static java.lang.Math.max;
-import static java.lang.Math.min;
 
+/**
+ * 人脸框
+ */
 public class Box {
-    public  int[] box;       //left:box[0],top:box[1],right:box[2],bottom:box[3]
-    public  float score;    //probability
-    public  float[] bbr;    //bounding box regression
-    public  boolean deleted;
-    public  Point[] landmark; //facial landmark.只有ONet输出Landmark
-    Box(){
-        box=new int[4];
-        bbr=new float[4];
-        deleted=false;
-        landmark=new Point[5];
+    public int[] box;           // left:box[0],top:box[1],right:box[2],bottom:box[3]
+    public float score;         // probability
+    public float[] bbr;         // bounding box regression
+    public boolean deleted;
+    public Point[] landmark;    // facial landmark.只有ONet输出Landmark
+
+    public Box() {
+        box = new int[4];
+        bbr = new float[4];
+        deleted = false;
+        landmark = new Point[5];
     }
-    public int left(){return box[0];}
-    public int right(){return box[2];}
-    public int top(){return box[1];}
-    public int bottom(){return box[3];}
-    public int width(){return box[2]-box[0]+1;}
-    public int height(){return box[3]-box[1]+1;}
-
-    public Rect transform2Rect(){
-        Rect rect=new Rect();
-        rect.left=Math.round(box[0]);
-        rect.top=Math.round(box[1]);
-        rect.right=Math.round(box[2]);
-        rect.bottom=Math.round(box[3]);
-        return  rect;
+
+    public int left() {
+        return box[0];
     }
 
-    public  int area(){
-        return width()*height();
+    public int right() {
+        return box[2];
     }
 
-    public void calibrate(){
-        int w=box[2]-box[0]+1;
-        int h=box[3]-box[1]+1;
-        box[0]=(int)(box[0]+w*bbr[0]);
-        box[1]=(int)(box[1]+h*bbr[1]);
-        box[2]=(int)(box[2]+w*bbr[2]);
-        box[3]=(int)(box[3]+h*bbr[3]);
-        for (int i=0;i<4;i++) bbr[i]=0.0f;
+    public int top() {
+        return box[1];
     }
 
-    public void toSquareShape(){
-        int w=width();
-        int h=height();
-        if (w>h){
-            box[1]-=(w-h)/2;
-            box[3]+=(w-h+1)/2;
-        }else{
-            box[0]-=(h-w)/2;
-            box[2]+=(h-w+1)/2;
+    public int bottom() {
+        return box[2];
+    }
+
+    public int width() {
+        return box[2] - box[0] + 1;
+    }
+
+    public int height() {
+        return box[3] - box[1] + 1;
+    }
+
+    // 转为rect
+    public Rect transform2Rect() {
+        Rect rect = new Rect();
+        rect.left = box[0];
+        rect.top = box[1];
+        rect.right = box[2];
+        rect.bottom = box[3];
+        return rect;
+    }
+
+    // 面积
+    public int area() {
+        return width() * height();
+    }
+
+    // Bounding Box Regression
+    public void calibrate() {
+        int w = box[2] - box[0] + 1;
+        int h = box[3] - box[1] + 1;
+        box[0] = (int) (box[0] + w * bbr[0]);
+        box[1] = (int) (box[1] + h * bbr[1]);
+        box[2] = (int) (box[2] + w * bbr[2]);
+        box[3] = (int) (box[3] + h * bbr[3]);
+        for (int i = 0; i < 4; i++) bbr[i] = 0.0f;
+    }
+
+    // 当前box转为正方形
+    public void toSquareShape() {
+        int w = width();
+        int h = height();
+        if (w > h) {
+            box[1] -= (w - h) / 2;
+            box[3] += (w - h + 1) / 2;
+        } else {
+            box[0] -= (h - w) / 2;
+            box[2] += (h - w + 1) / 2;
         }
     }
 
-    public void limit_square(int w,int h){
-        if (box[0]<0 || box[1]<0){
-            int len=max(-box[0],-box[1]);
-            box[0]+=len;
-            box[1]+=len;
+    // 防止边界溢出，并维持square大小
+    public void limitSquare(int w, int h) {
+        if (box[0] < 0 || box[1] < 0) {
+            int len = max(-box[0], -box[1]);
+            box[0] += len;
+            box[1] += len;
+        }
+        if (box[2] >= w || box[3] >= h) {
+            int len = max(box[2] - w + 1, box[3] - h + 1);
+            box[2] -= len;
+            box[3] -= len;
         }
-        if (box[2]>=w || box[3]>=h){
-            int len=max(box[2]-w+1,box[3]-h+1);
-            box[2]-=len;
-            box[3]-=len;
+    }
+
+    // 坐标是否越界
+    public boolean transbound(int w, int h) {
+        if (box[0] < 0 || box[1] < 0) {
+            return true;
+        } else if (box[2] >= w || box[3] >= h) {
+            return true;
         }
+        return false;
     }
 }