diff --git a/app.js b/app.js
index 8958abf..38725ae 100644
--- a/app.js
+++ b/app.js
@@ -1,16 +1,31 @@
import updateManager from './common/updateManager';
-import * as paddlejs from '@paddlejs/paddlejs-core';
-import '@paddlejs/paddlejs-backend-webgl';
+var fetchWechat = require('fetch-wechat');
+var tf = require('@tensorflow/tfjs-core');
+var webgl = require('@tensorflow/tfjs-backend-webgl');
+var plugin = requirePlugin('tfjsPlugin');
-// const paddlejs = require('paddlejs');
-const plugin = requirePlugin("paddlejs-plugin");
-
-let pdjs;
App({
+ globalData: {
+ localStorageIO: plugin.localStorageIO,
+ systemInfo: {}
+ },
onLaunch: function () {
- plugin.register(paddlejs, wx);
+ plugin.configPlugin({
+ // polyfill fetch function
+ fetchFunc: fetchWechat.fetchFunc(),
+ // inject tfjs runtime
+ tf,
+ // inject webgl backend
+ webgl,
+ // provide webgl canvas
+ canvas: wx.createOffscreenCanvas()
+ },
+ true);
+
+ const systemInfo = wx.getSystemInfoSync();
+ this.globalData.systemInfo = wx.getSystemInfoSync();
},
onShow: function () {
updateManager();
diff --git a/app.json b/app.json
index ca88026..da29495 100644
--- a/app.json
+++ b/app.json
@@ -52,9 +52,9 @@
}
},
"plugins": {
- "paddlejs-plugin": {
- "version": "2.0.1",
- "provider": "wx7138a7bb793608c3"
+ "tfjsPlugin": {
+ "version": "0.2.0",
+ "provider": "wx6afed118d9e81df9"
}
}
}
\ No newline at end of file
diff --git a/package.json b/package.json
index 863026c..f601b27 100644
--- a/package.json
+++ b/package.json
@@ -23,8 +23,10 @@
"dayjs": "^1.9.3",
"tdesign-miniprogram": "1.0.1",
"tslib": "^1.11.1",
- "@paddlejs/paddlejs-backend-webgl": "^1.0.7",
- "@paddlejs/paddlejs-core": "^2.0.7"
+ "@tensorflow/tfjs-core": "3.5.0",
+ "@tensorflow/tfjs-converter": "3.5.0",
+ "@tensorflow/tfjs-backend-webgl": "3.5.0",
+ "fetch-wechat": "0.0.3"
},
"devDependencies": {
"eslint": "^6.8.0",
diff --git a/pages/home/camera/index.js b/pages/home/camera/index.js
index a1fb089..aa840af 100644
--- a/pages/home/camera/index.js
+++ b/pages/home/camera/index.js
@@ -1,10 +1,6 @@
// pages/home/camera/index.js
-import {
- paddlejs
-} from '@paddlejs/paddlejs-core';
-import {
- Paddlejs
-} from '../../../services/_utils/ocr';
+
+import * as model from '../../../services/tf/model'
Page({
@@ -13,44 +9,79 @@ Page({
*/
data: {
isAuth: false,
- src: ''
+ src: '',
+ fps: 0,
+ predicting: false,
+ cameraContext: null,
+ cameraListener: null,
+ lastTime: 0
},
/**
* 生命周期函数--监听页面加载
*/
onLoad(options) {
- const _this = this
- wx.getSetting({
- success: res => {
- if (res.authSetting['scope.camera']) {
- // 用户已经授权
- _this.setData({
- isAuth: true
- })
- } else {
- // 用户还没有授权，向用户发起授权请求
- wx.authorize({
- scope: 'scope.camera',
- success() { // 用户同意授权
- _this.setData({
- isAuth: true
- })
- },
- fail() { // 用户不同意授权
- _this.openSetting().then(res => {
- _this.setData({
- isAuth: true
- })
- })
- }
- })
- }
- },
- fail: res => {
- console.log('获取用户授权信息失败')
- }
+ this.setData({
+ lastTime: Date.now()
})
+ this.initModel();
+ },
+
+ initModel: async function () {
+ this.showLoadingToast();
+
+ await model.load();
+
+ this.hideLoadingToast();
+
+ if (!model.isReady()) {
+ wx.showToast({
+ title: '网络连接异常',
+ icon: 'none'
+ });
+ }
+ },
+
+ showLoadingToast: function () {
+ wx.showLoading({
+ title: '拼命加载模型',
+ });
+ },
+
+ hideLoadingToast: function () {
+ wx.hideLoading()
+ },
+
+ openCamera() {
+ this.setData({
+ cameraContext: wx.createCameraContext()
+ })
+
+ this.setData({
+ cameraListener: this.data.cameraContext.onCameraFrame(frame => {
+ this.executePredict(frame)
+ })
+ })
+ this.data.cameraListener.start();
+ },
+
+ executePredict(frame) {
+ if (!this.data.predicting && model.isReady()) {
+ this.setData({
+ predicting: true
+ }, async () => {
+ const now = Date.now()
+ // model.predict(frame)
+ // const predictionResults = await model.classify(frame)
+ // console.log(now - this.data.lastTime);
+ model.predict(frame)
+ this.setData({
+ predicting: false,
+ fps: (1000 / (now - this.data.lastTime)).toFixed(2),
+ lastTime: now
+ })
+ })
+ }
},
openSetting() {
@@ -86,81 +117,43 @@ Page({
return promise;
},
- takePhoto() {
- console.log("开始识别图片");
- const ctx = wx.createCameraContext()
- ctx.takePhoto({
- quality: 'high',
- success: (res) => {
- console.log(res);
- this.setData({
- src: res.tempImagePath
- })
- // Paddlejs.predict(res.tempImagePath).then(res => {
- // console.log(res);
- // })
- var pic0 = new Image();
- pic0.src = res.tempImagePath;
- const this_ = this
- pic0.onload = () => {
- var img0 = tf.browser.fromPixels(pic0);
- this_.predict(img0);
- }
-
- // 获取到图片的像素信息
- // wx.getImageInfo({
- // src: res.tempImagePath,
- // success: (imgInfo) => {
- // const {
- // width,
- // height,
- // path
- // } = imgInfo;
-
- // const canvasId = 'myCanvas';
- // const me = this;
- // // 获取页面中的canvas上下文，tips：canvas设置的宽高要大于选择的图片宽高，canvas位置可以绝对定位到视口不可以见
- // let ctx = canvas.getContext(canvasId);
- // ctx.drawImage(path, 0, 0, width, height);
- // ctx.draw(false, () => {
- // // API 1.9.0 获取图像数据
- // wx.canvasGetImageData({
- // canvasId: canvasId,
- // x: 0,
- // y: 0,
- // width: width,
- // height: height,
- // success(res) {
- // console.log(res);
- // me.predict({
- // data: res.data,
- // width: width,
- // height: height
- // });
- // }
- // });
- // });
- // }
- // });
- }
- })
- },
-
- predict(imgObj) {
- // 4. 在线预测计算
- const me = this;
- Paddlejs.predict(imgObj, function (data) {
- // 5. 对预测结果进行后处理
- const maxItem = pdjs.utils.getMaxItem(data);
- console.log(maxItem);
- });
- },
-
/**
* 生命周期函数--监听页面初次渲染完成
*/
onReady() {
-
+ const _this = this
+ wx.getSetting({
+ success: res => {
+ if (res.authSetting['scope.camera']) {
+ // 用户已经授权
+ _this.setData({
+ isAuth: true
+ })
+ _this.openCamera()
+ } else {
+ // 用户还没有授权，向用户发起授权请求
+ wx.authorize({
+ scope: 'scope.camera',
+ success() { // 用户同意授权
+ _this.setData({
+ isAuth: true
+ })
+ _this.openCamera()
+ },
+ fail() { // 用户不同意授权
+ _this.openSetting().then(res => {
+ _this.setData({
+ isAuth: false
+ })
+ })
+ }
+ })
+ }
+ },
+ fail: res => {
+ console.log('获取用户授权信息失败')
+ }
+ })
},
/**
@@ -181,7 +174,7 @@ Page({
* 生命周期函数--监听页面卸载
*/
onUnload() {
-
+ this.data.cameraListener.stop();
},
/**
diff --git a/pages/home/camera/index.wxml b/pages/home/camera/index.wxml
index 6db6e0f..fe3fa48 100644
--- a/pages/home/camera/index.wxml
+++ b/pages/home/camera/index.wxml
@@ -2,5 +2,5 @@
-
-
\ No newline at end of file
+ FPS: {{fps}}
+
\ No newline at end of file
diff --git a/project.config.json b/project.config.json
index c9c4f6d..503980b 100644
--- a/project.config.json
+++ b/project.config.json
@@ -43,7 +43,8 @@
"showES6CompileOption": false,
"useCompilerPlugins": false,
"ignoreUploadUnusedFiles": true,
- "useStaticServer": true
+ "useStaticServer": true,
+ "condition": false
},
"compileType": "miniprogram",
"libVersion": "2.23.1",
diff --git a/services/_utils/ocr.js b/services/_utils/ocr.js
deleted file mode 100644
index 6aa168f..0000000
--- a/services/_utils/ocr.js
+++ /dev/null
@@ -1,21 +0,0 @@
-import * as paddlejs from '@paddlejs/paddlejs-core';
-import '@paddlejs/paddlejs-backend-webgl';
-
-const plugin = requirePlugin("paddlejs-plugin");
-plugin.register(paddlejs, wx);
-
-export const Paddlejs = new paddlejs.Runner({
- modelPath: '/resources/paddle.json',
- feedShape: {
- fw: 224,
- fh: 224
- },
- fill: '#fff',
- targetSize: {
- height: 224,
- width: 224
- },
- mean: [0.485, 0.456, 0.406],
- std: [0.229, 0.224, 0.225],
- // needPreheat: true
-})
\ No newline at end of file
diff --git a/services/tf/model.js b/services/tf/model.js
new file mode 100644
index 0000000..b5f016c
--- /dev/null
+++ b/services/tf/model.js
@@ -0,0 +1,67 @@
+import * as tfc from '@tensorflow/tfjs-converter';
+import * as tf from '@tensorflow/tfjs-core';
+import '@tensorflow/tfjs-backend-webgl';
+
+const LOCAL_STORAGE_KEY = 'mobilenet_model';
+// const MODEL_URL = 'https://shao5.net/static/model/model.json';
+const MODEL_URL = 'https://webplus-cn-hangzhou-s-603871eef968dd14ced82ed5.oss-cn-hangzhou.aliyuncs.com/hextech/static/paddle/model.json';
+
+let model = tfc.GraphModel;
+const app = getApp();
+
+export async function load() {
+
+ // const localStorageHandler = getApp().globalData.localStorageIO(LOCAL_STORAGE_KEY);
+ // try {
+ // model = await tfc.loadGraphModel(localStorageHandler);
+ // } catch (e) {
+ // model =
+ // await tfc.loadGraphModel(MODEL_URL);
+ // model.save(localStorageHandler);
+ // }
+ model = await tfc.loadGraphModel(MODEL_URL);
+ console.log(model);
+}
+
+export const isReady = () => {
+ return !!model;
+};
+
+
+const getFrameSliceOptions = (frameWidth, frameHeight, displayWidth, displayHeight) => {
+ let result = {
+ start: [0, 0, 0],
+ size: [-1, -1, 3]
+ };
+
+ const ratio = displayHeight / displayWidth;
+
+ if (ratio > frameHeight / frameWidth) {
+ result.start = [0, Math.ceil((frameWidth - Math.ceil(frameHeight / ratio)) / 2), 0];
+ result.size = [-1, Math.ceil(frameHeight / ratio), 3];
+ } else {
+ result.start = [Math.ceil((frameHeight - Math.floor(ratio * frameWidth)) / 2), 0, 0];
+ result.size = [Math.ceil(ratio * frameWidth), -1, 3];
+ }
+
+ return result;
+}
+
+export const predict = async (frame) => {
+ const temp = tf.browser.fromPixels({
+ data: new Uint8Array(frame.data),
+ width: frame.width,
+ height: frame.height,
+ }, 4);
+ const sliceOptions = getFrameSliceOptions(frame.width, frame.height, app.globalData.systemInfo.windowWidth, app.globalData.systemInfo.windowWidth)
+
+ const pixels = await tf.tidy(() => {
+ return tf.image.resizeBilinear(tf.slice(temp, sliceOptions.start, sliceOptions.size), [224, 224]);
+ });
+ const tensor = tf.reshape(pixels, [-1, 224, 224, 3]);
+
+ const res = model.execute(tensor)
+ console.log(res);
+ temp.dispose();
+ pixels.dispose();
+}
\ No newline at end of file