本文主要分为两部分，第一部分为使用moblienetv2 ssd lite训练一个识别扑克牌的模型，第2部分就是将训练好的模型部署到手机上运行，具体的运行效果如下所示：
pc端运行效果：
20220316
b站地址：视频链接
移动端：
扑克牌识别
app下载地址：下载地址
在这里分享一下数据集链接；
https://cloud.189.cn/t/eYRzu27ne6za (访问码:upa5)
该数据集制作不易，如有需要可前往下载。
1 网络训练
可以使用不同的网络，本文使用的是mobilenetv2-ssd-lite。具体可参考该github。
训练好模型以后将其转换成onnx格式。
onnx模型下载地址：
链接：https://pan.baidu.com/s/1kchAu_0B3KfJ3WkYY9IDXQ
提取码：g9fq
使用opencv加载onnx模型进行前向推理，代码如下所示：
import cv2
import numpy as np
label_dic=list({'card1':'0001','card2':'0002','card3':'0003','card4':'0004','card5':'0005',
'card6':'0006','card7':'0007','card8':'0008','card9':'0009','card10':'0010',
'card11':'0011','card12':'0012','card13':'0013',
'second1':'0014','second2':'0015','second3':'0016','second4':'0017','second5':'0018',
'second6':'0019','second7':'0020','second8':'0021','second9':'0022','second10':'0023',
'second11':'0024','second12':'0025','second13':'0026',
'three1':'0027','three2':'0028','three3':'0029','three4':'0030','three5':'0031','three6':'0032',
'three7':'0033','three8':'0034','three9':'0035','three10':'0036','three11':'0037','three12':'0038',
'three13':'0039',
'four1':'0040','four2':'0041','four3':'0042','four4':'0043','four5':'0044','four6':'0045','four7':'0046',
'four8':'0047','four9':'0048','four10':'0049','four11':'0050','four12':'0051','four13':'0052',
'xiao':'0053','da':'0054'
})
net=cv2.dnn.readNetFromONNX('./models/mb2-ssd-lite.onnx')
def dector(ori_image):
h, w = ori_image.shape[:2]
# ori_image=cv2.resize(ori_image,(300,300))
blob = cv2.dnn.blobFromImage(ori_image,
scalefactor=1.0 / 128,
size=(300, 300),
mean=[127, 127, 127],
swapRB=True,
crop=False)
# Run a model
net.setInput(blob)
# net.getUnconnectedOutLayersNames()
# boxes
aa = net.getUnconnectedOutLayersNames()
out = net.forward(['scores', 'boxes'])
#scores =>(3000,54) boxes=>(3000,4)
scores = out[0][0][...,1::]
boxes = out[1][0]
#3000
max_score=np.max(scores,axis=-1)
# mask=max_score>0.4
max_index=np.argmax(scores,axis=-1)
#
# pick_score=max_score[mask]
# pick_label=max_index[mask]
# pick_box=boxes[mask]
# out=detect(ori_image)
box_id = cv2.dnn.NMSBoxes(boxes.tolist(), max_score.tolist(), 0.4, 0.45)
if len(box_id)==0:
return ori_image
boxes = boxes[box_id]
confidences = max_score[box_id]
label=max_index[box_id]
for box,confidence,la in zip(boxes,confidences,label):
box=box*np.array([w,h,w,h])
box=box.astype(np.int32)
cv2.rectangle(ori_image,(box[0],box[1]),(box[2],box[3]),(255,255,0),2)
cv2.putText(ori_image,f'{label_dic[la]}:{round(confidence,3)}',
(box[0]+20, box[1]+40),
cv2.FONT_HERSHEY_SIMPLEX,
1, # font scale
(255, 0, 255),
2)
return ori_image
cap = cv2.VideoCapture(r'E:\card.mp4')
while True:
r,ori_image=cap.read()
# cv2.imshow('a231',ori_image)
# cv2.waitKey(0)
dector(ori_image)
cv2.imshow('a4232',ori_image)
cv2.waitKey(1)
**
2 移动端部署
使用opencv加载onnx模型，进行移动端部署。
**
mainactivaty.java代码
package com.myapp.puke;
import androidx.appcompat.app.AppCompatActivity;
import android.os.Bundle;
import android.Manifest;
import android.content.Context;
import android.content.res.AssetManager;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Paint;
import android.media.MediaPlayer;
import android.os.Build;
import android.os.Bundle;
import android.util.Log;
import android.view.View;
import android.view.Window;
import android.view.WindowManager;
import android.widget.Toast;
import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.CameraActivity;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.JavaCameraView;
import org.opencv.android.LoaderCallbackInterface;
import org.opencv.android.OpenCVLoader;
import org.opencv.android.Utils;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfFloat;
import org.opencv.core.MatOfInt;
import org.opencv.core.MatOfRect2d;
import org.opencv.core.Point;
import org.opencv.core.Rect2d;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgproc.Imgproc;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class MyCamera extends CameraActivity implements CameraBridgeViewBase.CvCameraViewListener2 {
static {
System.loadLibrary("native-lib");
}
private JavaCameraView mOpenCvCameraView;
private int M_REQUEST_CODE = 203;
private String[] permissions = {Manifest.permission.CAMERA};
// Initialize OpenCV manager.
private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {
@Override
public void onManagerConnected(int status) {
switch (status) {
case LoaderCallbackInterface.SUCCESS: {
mOpenCvCameraView.enableView();
break;
}
default:
break;
}
}
};
private MediaPlayer mediaPlayer;
private Mat src;
private Net net;
final double IN_SCALE_FACTOR = 0.0078125;
final double MEAN_VAL = 127.0;
final double THRESHOLD = 0.2;
final int IN_WIDTH = 300;
final int IN_HEIGHT = 300;
private Bitmap bp;
private long time1=0;
private long time2=0;
@Override
public void onResume() {
super.onResume();
if (!OpenCVLoader.initDebug()) {
} else {
mLoaderCallback.onManagerConnected(LoaderCallbackInterface.SUCCESS);
}
};
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_my_camera);
// 透明状态栏
if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.LOLLIPOP) {
Window window = getWindow();
window.clearFlags(WindowManager.LayoutParams.FLAG_TRANSLUCENT_STATUS
| WindowManager.LayoutParams.FLAG_TRANSLUCENT_NAVIGATION);
window.getDecorView().setSystemUiVisibility(View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN
| View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION
| View.SYSTEM_UI_FLAG_LAYOUT_STABLE);
window.addFlags(WindowManager.LayoutParams.FLAG_DRAWS_SYSTEM_BAR_BACKGROUNDS);
window.setStatusBarColor(Color.TRANSPARENT);
window.setNavigationBarColor(Color.TRANSPARENT);
}
// Set up camera listener.
mOpenCvCameraView = (JavaCameraView)findViewById(R.id.CameraView);
mOpenCvCameraView.setVisibility(CameraBridgeViewBase.VISIBLE);
mOpenCvCameraView.setCvCameraViewListener(this);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
requestPermissions(permissions, M_REQUEST_CODE);
}
// mediaPlayer= MediaPlayer.create(getApplicationContext(),R.raw.ye);
// mediaPlayer.start();
}
public void onCameraViewStarted(int width, int height) {
//加载模型
String proto=getPath("mb2-ssd-lite.onnx",this);
net= Dnn.readNetFromONNX(proto);
bp=Bitmap.createBitmap(width,height, Bitmap.Config.ARGB_8888);
}
private List<Integer> music_index=new ArrayList<>();
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
//局部放大
// mRgba=inputFrame.rgba();
// Size sizeRgba = mRgba.size();
// int rows = (int) sizeRgba.height;
// int cols = (int) sizeRgba.width;
//
// switch (statue){
// case 0:
// //Canny边缘检测
// mRgba = inputFrame.rgba();
// Imgproc.Canny(inputFrame.gray(), mTmp, 80, 100);
// Imgproc.cvtColor(mTmp, mRgba, Imgproc.COLOR_GRAY2RGBA, 4);
// break;
// case 1:
// //ZOOM放大镜
// Mat zoomCorner = mRgba.submat(0, rows / 2 - rows / 10, 0, cols / 2 - cols / 10);
// Mat mZoomWindow = mRgba.submat(rows / 2 - 9 * rows / 100, rows / 2 + 9 * rows / 100, cols / 2 - 9 * cols / 100, cols / 2 + 9 * cols / 100);
// Imgproc.resize(mZoomWindow, zoomCorner, zoomCorner.size());
// Size wsize = mZoomWindow.size();
// Imgproc.rectangle(mZoomWindow, new Point(1, 1), new Point(wsize.width - 2, wsize.height - 2), new Scalar(255, 0, 0, 255), 2);
// zoomCorner.release();
// mZoomWindow.release();
// break;
//
// }
//目标检测
time2=System.currentTimeMillis();
if (time2-time1>5000){
time1=time2;
music_index=new ArrayList<>();
}
src=inputFrame.rgba();
Imgproc.cvtColor(src,src,Imgproc.COLOR_RGBA2RGB);
Utils.matToBitmap(src,bp);
bp=detection(bp);
Utils.bitmapToMat(bp,src);
//开始播放
// mediaPlayer= MediaPlayer.create(getApplicationContext(),R.raw.m2);
// mediaPlayer.start();
List<Integer> index_class=new ArrayList<>();
//取出大于5个下标
for(int x=0;x<number_class.length;x++) {
if (number_class[x] >= 2) {
index_class.add(x);
}
}
if(index_class.size()>0 ){
for(int j:index_class){
if (music_index.contains(j)){
}else {
music_index.add(j);
mediaPlayer= MediaPlayer.create(getApplicationContext(),musices[j]);
mediaPlayer.start();
Log.i("aa",""+music_index);
}
// if (music_index!=j){
//
// }
}
number_class=new int[54];
}
return src;
}
public void onCameraViewStopped() {
// src.release();
}
@Override
public void onPause() {
super.onPause();
if (mOpenCvCameraView != null)
mOpenCvCameraView.disableView();
}
public void onDestroy() {
super.onDestroy();
mOpenCvCameraView.disableView();
}
@Override
protected List<? extends CameraBridgeViewBase> getCameraViewList() {
return Collections.singletonList(mOpenCvCameraView);
}
private static String getPath(String file, Context context) {
AssetManager assetManager = context.getAssets();
BufferedInputStream inputStream = null;
try {
// Read data from assets.
inputStream = new BufferedInputStream(assetManager.open(file));
byte[] data = new byte[inputStream.available()];
inputStream.read(data);
inputStream.close();
// Create copy file in storage.
File outFile = new File(context.getFilesDir(), file);
FileOutputStream os = new FileOutputStream(outFile);
os.write(data);
os.close();
// Return a path to file which may be read in common way.
return outFile.getAbsolutePath();
} catch (IOException ex) {
}
return "";
}
// private static final String TAG = "OpenCV/Sample/MobileNet";
private static final String[] classNames = {"黑桃A",
"黑桃2", "黑桃3", "黑桃4", "黑桃5","黑桃6", "黑桃7", "黑桃8", "黑桃9","黑桃10", "黑桃J", "黑桃Q", "黑桃K",
"红桃A", "红桃2","红桃3","红桃4","红桃5","红桃6","红桃7","红桃8","红桃9","红桃10","红桃J","红桃Q","红桃K",
"方块A","方块2","方块3","方块4","方块5","方块6","方块7","方块8","方块9","方块10","方块J","方块Q","方块K",
"梅花A","梅花2","梅花3","梅花4","梅花5","梅花6","梅花7","梅花8","梅花9","梅花10","梅花J","梅花Q","梅花K",
"小王","大王"};
private static final int[] musices = {R.raw.m1,
R.raw.m2, R.raw.m3, R.raw.m4, R.raw.m5,R.raw.m6, R.raw.m7, R.raw.m8, R.raw.m9,R.raw.m10, R.raw.m11, R.raw.m12,R.raw.m13,
R.raw.m14, R.raw.m15, R.raw.m16, R.raw.m17,R.raw.m18, R.raw.m19, R.raw.m20, R.raw.m21,R.raw.m22, R.raw.m23, R.raw.m24,R.raw.m25,R.raw.m26,
R.raw.m27, R.raw.m28, R.raw.m29, R.raw.m30,R.raw.m31, R.raw.m32, R.raw.m33, R.raw.m34,R.raw.m35, R.raw.m36, R.raw.m37,R.raw.m38,R.raw.m39,
R.raw.m40, R.raw.m41, R.raw.m42, R.raw.m43,R.raw.m44, R.raw.m45, R.raw.m46, R.raw.m47,R.raw.m48, R.raw.m49, R.raw.m50,R.raw.m51,R.raw.m52,
R.raw.m53,R.raw.m54};
public native String stringFromJNI();
//统计预测的结果
private int[] number_class=new int[54];
public Bitmap detection(Bitmap bp){
Canvas can=new Canvas();
Paint p=new Paint();
android.graphics.Bitmap.Config bitmapConfig = bp.getConfig();
bp = bp.copy(bitmapConfig, true);
can=new Canvas(bp);
p.setAntiAlias(true);
//不填充，默认填充
p.setStyle(Paint.Style.STROKE);
//设置线条宽度
p.setStrokeWidth(5);
//设置颜色
p.setColor(0xFF33FFFF);
p.setTextAlign(Paint.Align.LEFT);
p.setTextSize(50);
Mat blob = Dnn.blobFromImage(src, IN_SCALE_FACTOR,
new Size(IN_WIDTH, IN_HEIGHT),
new Scalar(MEAN_VAL, MEAN_VAL, MEAN_VAL), false);
net.setInput(blob);
blob.release();
//获取输出层的名字
List<String> outnames=net.getUnconnectedOutLayersNames();
// Log.i("aa",String.valueOf(outnames));
//创建输出矩阵集合
List<Mat> detections = new ArrayList<Mat>();
net.forward(detections,outnames);
//获取输出的盒子和置信度
Mat scores=detections.get(0);
Mat boxes= detections.get(1);
scores= scores.reshape(1,3000).colRange(1,55);
boxes= boxes.reshape(1,3000);
Size ss=scores.size();
// Log.i("aa",String.valueOf(scores));
// Log.i("aa",String.valueOf(boxes));
List<Rect2d> rect2dList=new ArrayList<>();//box信息
List<Float> confList=new ArrayList<>();//置信度
List<Integer> objIndexList=new ArrayList<>();//对象类别索引
for(int i=0; i<scores.rows();i++){
Mat one_row=scores.rowRange(i,i+1);
Core.MinMaxLocResult max_index=Core.minMaxLoc(one_row);
double max_value=max_index.maxVal;
Point location=max_index.maxLoc;
if(max_value>0.4){
confList.add((float) max_value);
objIndexList.add((int)location.x);
Mat box_one=boxes.rowRange(i,i+1);
float[] aa=new float[4];
box_one.get(0,0,aa);
double x1=aa[0];
double y1=aa[1];
double x2=aa[2];
double y2=aa[3];
rect2dList.add(new Rect2d(x1,y1,x2,y2));
}
}
//去重
//去重后的索引值
MatOfInt index=new MatOfInt();
//转换box的结果集
MatOfRect2d boxe=new MatOfRect2d(rect2dList.toArray(new Rect2d[0]));
//转换置信度结果集
float[] confArr=new float[confList.size()];
for(int j=0;j<confList.size();j++){
confArr[j]=confList.get(j);
}
MatOfFloat con=new MatOfFloat(confArr);
//使用nms去重
Dnn.NMSBoxes(boxe,con,0.4f,0.45f,index);
if (index.empty()){
return bp;
}
//画框
int[] ints=index.toArray();
for(int x:ints){
// Log.i("aa",String.valueOf(x));
double[] aa=new double[4];
boxe.get(x,0,aa);
//Log.i("aa",String.valueOf(aa[0]));
//Imgproc.rectangle(src,new Point(aa[0]*src.width(),aa[1]*src.height()-70),new Point(aa[2]*src.width()+200,aa[1]*src.height()),new Scalar(255,255,255),-1);
//Imgproc.rectangle(src,new Point(aa[0]*src.width(),aa[1]*src.height()),new Point(aa[2]*src.width(),aa[3]*src.height()),new Scalar(255,255,0),10);
//Imgproc.putText(src,""+classNames[objIndexList.get(x)]+":"+confList.get(x),new Point(aa[0]*src.width(),aa[1]*src.height()),Imgproc.FONT_HERSHEY_SIMPLEX, 3, new Scalar(0, 0, 0));
//画框
can.drawRect((float) aa[0]*src.width(),(float) aa[1]*src.height(),(float)aa[2]*src.width(),(float)aa[3]*src.height(),p);
//绘制填充框
p.setStyle(Paint.Style.FILL);
p.setColor(0xFFFFCC00);
can.drawRect((float) aa[0]*src.width(),(float) aa[1]*src.height()-60,(float)aa[2]*src.width()+150,(float)aa[1]*src.height(),p);
//写字
p.setColor(0xFFFF0000);
can.drawText(classNames[objIndexList.get(x)]+": "+String.format("%.3f", confList.get(x)),(float)aa[0]*src.width(),(float) aa[1]*src.height()-10,p);
p.setStyle(Paint.Style.STROKE);
p.setColor(0xFF33FFFF);
//统计类别信息
number_class[objIndexList.get(x)]+=1;
}
scores.release();
boxe.release();
index.release();
boxes.release();
con.release();
rect2dList.clear();
confList.clear();
objIndexList.clear();
return bp;
}
}
布局代码：
<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:opencv="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:visibility="visible">
<org.opencv.android.JavaCameraView
android:id="@+id/CameraView"
android:layout_width="match_parent"
android:layout_height="match_parent"
opencv:camera_id="any"
opencv:show_fps="true" />
</RelativeLayout>