提交代码

app.js

@@ -1,16 +1,31 @@
 import updateManager from './common/updateManager';
-import * as paddlejs from '@paddlejs/paddlejs-core';
-import '@paddlejs/paddlejs-backend-webgl';
 
+var fetchWechat = require('fetch-wechat');
+var tf = require('@tensorflow/tfjs-core');
+var webgl = require('@tensorflow/tfjs-backend-webgl');
+var plugin = requirePlugin('tfjsPlugin');
 
-// const paddlejs = require('paddlejs');
-const plugin = requirePlugin("paddlejs-plugin");
-
-let pdjs;
 
 App({
+  globalData: {
+    localStorageIO: plugin.localStorageIO,
+    systemInfo: {}
+  },
   onLaunch: function () {
-    plugin.register(paddlejs, wx);
+    plugin.configPlugin({
+        // polyfill fetch function
+        fetchFunc: fetchWechat.fetchFunc(),
+        // inject tfjs runtime
+        tf,
+        // inject webgl backend
+        webgl,
+        // provide webgl canvas
+        canvas: wx.createOffscreenCanvas()
+      },
+      true);
+
+    const systemInfo = wx.getSystemInfoSync();
+    this.globalData.systemInfo = wx.getSystemInfoSync();
   },
   onShow: function () {
     updateManager();

app.json

@@ -52,9 +52,9 @@
     }
   },
   "plugins": {
-    "paddlejs-plugin": {
-      "version": "2.0.1",
-      "provider": "wx7138a7bb793608c3"
+    "tfjsPlugin": {
+      "version": "0.2.0",
+      "provider": "wx6afed118d9e81df9"
     }
   }
 }

package.json

@@ -23,8 +23,10 @@
     "dayjs": "^1.9.3",
     "tdesign-miniprogram": "1.0.1",
     "tslib": "^1.11.1",
-    "@paddlejs/paddlejs-backend-webgl": "^1.0.7",
-    "@paddlejs/paddlejs-core": "^2.0.7"
+    "@tensorflow/tfjs-core": "3.5.0",
+    "@tensorflow/tfjs-converter": "3.5.0",
+    "@tensorflow/tfjs-backend-webgl": "3.5.0",
+    "fetch-wechat": "0.0.3"
   },
   "devDependencies": {
     "eslint": "^6.8.0",

pages/home/camera/index.js

@@ -1,10 +1,6 @@
 // pages/home/camera/index.js
-import {
-  paddlejs
-} from '@paddlejs/paddlejs-core';
-import {
-  Paddlejs
-} from '../../../services/_utils/ocr';
+
+import * as model from '../../../services/tf/model'
 
 Page({
 
@@ -13,44 +9,79 @@ Page({
    */
   data: {
     isAuth: false,
-    src: ''
+    src: '',
+    fps: 0,
+    predicting: false,
+    cameraContext: null,
+    cameraListener: null,
+    lastTime: 0
   },
 
   /**
    * 生命周期函数--监听页面加载
    */
   onLoad(options) {
-    const _this = this
-    wx.getSetting({
-      success: res => {
-        if (res.authSetting['scope.camera']) {
-          // 用户已经授权
-          _this.setData({
-            isAuth: true
-          })
-        } else {
-          // 用户还没有授权，向用户发起授权请求
-          wx.authorize({
-            scope: 'scope.camera',
-            success() { // 用户同意授权
-              _this.setData({
-                isAuth: true
-              })
-            },
-            fail() { // 用户不同意授权
-              _this.openSetting().then(res => {
-                _this.setData({
-                  isAuth: true
-                })
-              })
-            }
-          })
-        }
-      },
-      fail: res => {
-        console.log('获取用户授权信息失败')
-      }
+    this.setData({
+      lastTime: Date.now()
     })
+    this.initModel();
+  },
+
+  initModel: async function () {
+    this.showLoadingToast();
+
+    await model.load();
+
+    this.hideLoadingToast();
+
+    if (!model.isReady()) {
+      wx.showToast({
+        title: '网络连接异常',
+        icon: 'none'
+      });
+    }
+  },
+
+  showLoadingToast: function () {
+    wx.showLoading({
+      title: '拼命加载模型',
+    });
+  },
+
+  hideLoadingToast: function () {
+    wx.hideLoading()
+  },
+
+  openCamera() {
+    this.setData({
+      cameraContext: wx.createCameraContext()
+    })
+
+    this.setData({
+      cameraListener: this.data.cameraContext.onCameraFrame(frame => {
+        this.executePredict(frame)
+      })
+    })
+    this.data.cameraListener.start();
+  },
+
+  executePredict(frame) {
+    if (!this.data.predicting && model.isReady()) {
+      this.setData({
+        predicting: true
+      }, async () => {
+        const now = Date.now()
+        // model.predict(frame)
+        // const predictionResults = await model.classify(frame)
+        // console.log(now - this.data.lastTime);
+        model.predict(frame)
+        this.setData({
+          predicting: false,
+          fps: (1000 / (now - this.data.lastTime)).toFixed(2),
+          lastTime: now
+        })
+      })
+    }
   },
 
   openSetting() {
@@ -86,81 +117,43 @@ Page({
     return promise;
   },
 
-  takePhoto() {
-    console.log("开始识别图片");
-    const ctx = wx.createCameraContext()
-    ctx.takePhoto({
-      quality: 'high',
-      success: (res) => {
-        console.log(res);
-        this.setData({
-          src: res.tempImagePath
-        })
-        // Paddlejs.predict(res.tempImagePath).then(res => {
-        //   console.log(res);
-        // })
-        var pic0 = new Image();
-        pic0.src = res.tempImagePath;
-        const this_ = this
-        pic0.onload = () => {
-          var img0 = tf.browser.fromPixels(pic0);
-          this_.predict(img0);
-        }
-
-        // 获取到图片的像素信息
-        // wx.getImageInfo({
-        //   src: res.tempImagePath,
-        //   success: (imgInfo) => {
-        //     const {
-        //       width,
-        //       height,
-        //       path
-        //     } = imgInfo;
-
-        //     const canvasId = 'myCanvas';
-        //     const me = this;
-        //     // 获取页面中的canvas上下文，tips：canvas设置的宽高要大于选择的图片宽高，canvas位置可以绝对定位到视口不可以见
-        //     let ctx = canvas.getContext(canvasId);
-        //     ctx.drawImage(path, 0, 0, width, height);
-        //     ctx.draw(false, () => {
-        //       // API 1.9.0 获取图像数据
-        //       wx.canvasGetImageData({
-        //         canvasId: canvasId,
-        //         x: 0,
-        //         y: 0,
-        //         width: width,
-        //         height: height,
-        //         success(res) {
-        //           console.log(res);
-        //           me.predict({
-        //             data: res.data,
-        //             width: width,
-        //             height: height
-        //           });
-        //         }
-        //       });
-        //     });
-        //   }
-        // });
-      }
-    })
-  },
-
-  predict(imgObj) {
-    // 4. 在线预测计算
-    const me = this;
-    Paddlejs.predict(imgObj, function (data) {
-      // 5. 对预测结果进行后处理
-      const maxItem = pdjs.utils.getMaxItem(data);
-      console.log(maxItem);
-    });
-  },
-
   /**
    * 生命周期函数--监听页面初次渲染完成
    */
   onReady() {
-
+    const _this = this
+    wx.getSetting({
+      success: res => {
+        if (res.authSetting['scope.camera']) {
+          // 用户已经授权
+          _this.setData({
+            isAuth: true
+          })
+          _this.openCamera()
+        } else {
+          // 用户还没有授权，向用户发起授权请求
+          wx.authorize({
+            scope: 'scope.camera',
+            success() { // 用户同意授权
+              _this.setData({
+                isAuth: true
+              })
+              _this.openCamera()
+            },
+            fail() { // 用户不同意授权
+              _this.openSetting().then(res => {
+                _this.setData({
+                  isAuth: false
+                })
+              })
+            }
+          })
+        }
+      },
+      fail: res => {
+        console.log('获取用户授权信息失败')
+      }
+    })
   },
 
   /**
@@ -181,7 +174,7 @@ Page({
    * 生命周期函数--监听页面卸载
    */
   onUnload() {
-
+    this.data.cameraListener.stop();
   },
 
   /**

pages/home/camera/index.wxml

@@ -2,5 +2,5 @@
 <view class='camera'>
   <image src="/images/border.png" mode="widthFix"></image>
   <camera wx:if="{{isAuth}}" device-position="back" flash="off" binderror="error"></camera>
+  <text>FPS: {{fps}}</text>
 </view>
-<button class="takePhoto" type="primary" bindtap="takePhoto">拍照</button>

project.config.json

@@ -43,7 +43,8 @@
     "showES6CompileOption": false,
     "useCompilerPlugins": false,
     "ignoreUploadUnusedFiles": true,
-    "useStaticServer": true
+    "useStaticServer": true,
+    "condition": false
   },
   "compileType": "miniprogram",
   "libVersion": "2.23.1",

services/_utils/ocr.js

@@ -1,21 +0,0 @@
-import * as paddlejs from '@paddlejs/paddlejs-core';
-import '@paddlejs/paddlejs-backend-webgl';
-
-const plugin = requirePlugin("paddlejs-plugin");
-plugin.register(paddlejs, wx);
-
-export const Paddlejs = new paddlejs.Runner({
-  modelPath: '/resources/paddle.json',
-  feedShape: {
-    fw: 224,
-    fh: 224
-  },
-  fill: '#fff',
-  targetSize: {
-    height: 224,
-    width: 224
-  },
-  mean: [0.485, 0.456, 0.406],
-  std: [0.229, 0.224, 0.225],
-  // needPreheat: true
-})

services/tf/model.js

@@ -0,0 +1,67 @@
+import * as tfc from '@tensorflow/tfjs-converter';
+import * as tf from '@tensorflow/tfjs-core';
+import '@tensorflow/tfjs-backend-webgl';
+
+const LOCAL_STORAGE_KEY = 'mobilenet_model';
+// const MODEL_URL = 'https://shao5.net/static/model/model.json';
+const MODEL_URL = 'https://webplus-cn-hangzhou-s-603871eef968dd14ced82ed5.oss-cn-hangzhou.aliyuncs.com/hextech/static/paddle/model.json';
+
+let model = tfc.GraphModel;
+const app = getApp();
+
+export async function load() {
+
+  // const localStorageHandler = getApp().globalData.localStorageIO(LOCAL_STORAGE_KEY);
+  // try {
+  //   model = await tfc.loadGraphModel(localStorageHandler);
+  // } catch (e) {
+  //   model =
+  //     await tfc.loadGraphModel(MODEL_URL);
+  //   model.save(localStorageHandler);
+  // }
+  model = await tfc.loadGraphModel(MODEL_URL);
+  console.log(model);
+}
+
+export const isReady = () => {
+  return !!model;
+};
+
+
+const getFrameSliceOptions = (frameWidth, frameHeight, displayWidth, displayHeight) => {
+  let result = {
+    start: [0, 0, 0],
+    size: [-1, -1, 3]
+  };
+
+  const ratio = displayHeight / displayWidth;
+
+  if (ratio > frameHeight / frameWidth) {
+    result.start = [0, Math.ceil((frameWidth - Math.ceil(frameHeight / ratio)) / 2), 0];
+    result.size = [-1, Math.ceil(frameHeight / ratio), 3];
+  } else {
+    result.start = [Math.ceil((frameHeight - Math.floor(ratio * frameWidth)) / 2), 0, 0];
+    result.size = [Math.ceil(ratio * frameWidth), -1, 3];
+  }
+
+  return result;
+}
+
+export const predict = async (frame) => {
+  const temp = tf.browser.fromPixels({
+    data: new Uint8Array(frame.data),
+    width: frame.width,
+    height: frame.height,
+  }, 4);
+  const sliceOptions = getFrameSliceOptions(frame.width, frame.height, app.globalData.systemInfo.windowWidth, app.globalData.systemInfo.windowWidth)
+
+  const pixels = await tf.tidy(() => {
+    return tf.image.resizeBilinear(tf.slice(temp, sliceOptions.start, sliceOptions.size), [224, 224]);
+  });
+  const tensor = tf.reshape(pixels, [-1, 224, 224, 3]);
+
+  const res = model.execute(tensor)
+  console.log(res);
+  temp.dispose();
+  pixels.dispose();
+}