文字認識をして動かそう

手書き文字でロボットキットを操作します

このセクションでは、今までのブロックプログラムではなく、Html/JavaScript を使用します。

また、Cloud Vision という、Google が提供している画像認識サービスを利用します。

GoogleCloudApi のキーを取得する

このアプリには GoogleCloudApi の API キーが必要です.

  1. Google アカウントを取得します。

  2. Google Cloud Platform のダッシュボードに行きます
    API とサービスより,ライブラリを選択します

  3. API ライブラリの検索画面が出るので vision で検索します

  4. Cloud VisionAPI を選択します

  5. 有効にするボタンを押します

  6. 有効化がおわったら,API とサービスのダッシュボードに戻り,認証情報を選択します

  7. 認証情報を作成から,API キーを選択します

  8. 表示された API キーは後々使いますので、記録しておいてください

動かしてみる

プログラムはこちらになります。
obnizID を入力して、TestOpen を押してみましょう。

google api key を入力する欄がありますので、先程の API キーを入力し、Start ボタンを押すとカメラが起動します。
カメラにうまく映るように「すすめ」や「止まれ」などの文字を見せるとロボットキットもその動きをします。

<!-- HTML Example -->
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8" />
    <title>Video Capture Example</title>
    <meta name="viewport" content="width=device-width,initial-scale=1" />
    <script src="https://code.jquery.com/jquery-3.2.1.min.js"></script>
    <script src="https://unpkg.com/obniz@2.0.2/obniz.js"></script>
    <script src="https://unpkg.com/obniz-parts-kits@0.11.2/airobot/index.js"></script>
  </head>
  <body>
    <div id="obniz-debug"></div>

    <div>
      <div class="control">
        <input
          type="text"
          id="google_api_key"
          placeholder="google api key"
          value=""
        />
        <button id="startAndStop">Start</button>
      </div>
    </div>
    <p class="err" id="errorMessage"></p>
    <div style="display:flex;">
      <video
        id="videoInput"
        autoplay
        playsinline
        width="320"
        height="240"
        style="border:1px solid;width:320px;margin-right:10px"
      ></video>
      <canvas
        id="canvasOutput"
        width="320"
        height="240"
        style="-webkit-font-smoothing:none"
        hidden
        style="display:none"
      ></canvas>
      <div
        id="log"
        style="border:1px solid;height: 240px;overflow-y:scroll;width:320px"
      ></div>
    </div>

    <script type="text/javascript">
      let airobotkit;
      let power = 60;
      let inOperation = false;

      let mediaDevices =
        navigator.mediaDevices ||
        (navigator.mozGetUserMedia ||
        navigator.webkitGetUserMedia ||
        navigator.msGetUserMedia
          ? {
              getUserMedia(c) {
                return new Promise((y, n) => {
                  (
                    navigator.mozGetUserMedia || navigator.webkitGetUserMedia
                  ).call(navigator, c, y, n);
                });
              },
            }
          : null);

      obniz = new Obniz("OBNIZ_ID_HERE");
      obniz.onconnect = async () => {
        obniz.display.print("ready");

        airobotkit = obniz.wired("AIRobotKit", {});
      };

      function moveTo(lPower, rPower) {
        if (!airobotkit) {
          return;
        }

        airobotkit.move(lPower, rPower);
      }

      let streaming = false;
      let videoInput = document.getElementById("videoInput");
      let startAndStop = document.getElementById("startAndStop");
      let canvasOutput = document.getElementById("canvasOutput");
      let canvasContext = canvasOutput.getContext("2d");

      function successCallback(stream) {
        document.getElementById("videoInput").srcObject = stream;
        onVideoStarted();
      }

      function errorCallback(err) {
        console.error("mediaDevice.getUserMedia() error:", err);
      }

      startAndStop.addEventListener("click", () => {
        if (!streaming) {
          const medias = {
            audio: false,
            video: {
              facingMode: "user",
            },
          };

          mediaDevices
            .getUserMedia(medias)
            .then(successCallback)
            .catch(errorCallback);
        } else {
          onVideoStopped();
        }
      });

      function onVideoStarted() {
        streaming = true;
        startAndStop.innerText = "Stop";
        start();
      }

      function onVideoStopped() {
        streaming = false;
        canvasContext.clearRect(0, 0, canvasOutput.width, canvasOutput.height);
        startAndStop.innerText = "Start";
      }

      async function start() {
        const FPS = 100;

        function processVideo() {
          try {
            if (!streaming) {
              return;
            }

            let begin = Date.now();
            let base64String = capture();

            postToGoogle(base64String);

            setTimeout(processVideo, 0);
          } catch (err) {
            console.error(err);
          }
        }

        // schedule the first one.
        setTimeout(processVideo, 0);
      }

      function capture() {
        if (!streaming) {
          return null;
        }
        canvasContext.drawImage(
          videoInput,
          0,
          0,
          canvasOutput.width,
          canvasOutput.height
        );
        return canvasOutput.toDataURL("image/webp");
      }

      //引数はbase64形式の文字列
      function toBlob(base64) {
        var bin = atob(base64.replace(/^.*,/, ""));
        var buffer = new Uint8Array(bin.length);
        for (var i = 0; i < bin.length; i++) {
          buffer[i] = bin.charCodeAt(i);
        }
        // Blobを作成
        try {
          var blob = new Blob([buffer.buffer], {
            type: "image/png",
          });
        } catch (e) {
          return false;
        }
        return blob;
      }

      function postToGoogle(base64Img) {
        if (inOperation) {
          return;
        }
        inOperation = true;

        var base64Img = base64Img.split(",")[1];
        var apiKey = document.getElementById("google_api_key").value;
        var body = {
          requests: [
            {
              image: {
                content: base64Img,
              },
              features: [
                {
                  type: "LABEL_DETECTION",
                },
                {
                  type: "TEXT_DETECTION",
                  maxResults: 1,
                },
              ],
            },
          ],
        };

        //request to google api
        fetch(
          "https://vision.googleapis.com/v1/images:annotate?key=" + apiKey,
          {
            body: JSON.stringify(body),
            method: "POST",
            headers: {
              "Content-Type": "application/json",
            },
          }
        )
          .then(function(response) {
            return response.json();
          })
          .then(async function(json) {
            if (isKeywordsIncludes(json.responses, ["右", "みぎ", "right"])) {
              obniz.display.clear();
              obniz.display.print("みぎ!");
              _log("右むく");
              moveTo(power, -1 * power);
            } else if (
              isKeywordsIncludes(json.responses, ["left", "左", "ひだり"])
            ) {
              _log("左向く");
              obniz.display.clear();
              obniz.display.print("ひだり!");
              moveTo(-1 * power, power);
            } else if (
              isKeywordsIncludes(json.responses, ["stop", "止", "とまれ"])
            ) {
              obniz.display.clear();
              obniz.display.print("ぶれーき!");
              _log("止まる");
              moveTo(0, 0);
            } else if (
              isKeywordsIncludes(json.responses, [
                "すすめ",
                "進",
                "go",
                "前",
                "まえ",
                "forward",
              ])
            ) {
              _log("すすむ");
              obniz.display.clear();
              obniz.display.print("まっすぐ!");
              moveTo(power, power);
            } else if (
              isKeywordsIncludes(json.responses, [
                "バック",
                "後",
                "back",
                "もどれ",
                "もどる",
              ])
            ) {
              _log("もどる");
              obniz.display.clear();
              obniz.display.print("ばっくします!");
              moveTo(-1 * power, -1 * power);
            } else if (
              isKeywordsIncludes(json.responses, ["こっち", "こちら", "here"])
            ) {
              let pos = isKeywordsIncludes(json.responses, [
                "こっち",
                "こちら",
                "here",
              ]);
              let sum = 0;
              for (let p of pos.vertices) {
                sum += p.x;
              }
              let avg = sum / 4;
              let rotate = (avg - canvasOutput.width / 2) / canvasOutput.width;
              moveTo(rotate * power, (1 - rotate) * power);
            } else {
              _log("keyward nothing");
              moveTo(0, 0);
            }
            inOperation = false;
          });
      }

      function isKeywordsIncludes(results, keywards) {
        if (!Array.isArray(keywards)) {
          keywards = [keywards];
        }
        for (let result of results) {
          if (result && result.labelAnnotations) {
            for (let anotation of result.labelAnnotations) {
              for (let keyward of keywards) {
                if (anotation.description.toLowerCase().indexOf(keyward) >= 0) {
                  return anotation.boundingPoly || true;
                }
              }
            }
          }
          if (result && result.textAnnotations) {
            for (let anotation of result.textAnnotations) {
              for (let keyward of keywards) {
                if (anotation.description.toLowerCase().indexOf(keyward) >= 0) {
                  return anotation.boundingPoly || true;
                }
              }
            }
          }
          if (result && result.fullTextAnnotation) {
            for (let keyward of keywards) {
              if (
                result.fullTextAnnotation.text.toLowerCase().indexOf(keyward) >=
                0
              ) {
                return result.fullTextAnnotation.boundingPoly || true;
              }
            }
          }
          return false;
        }
      }

      function _log(msg) {
        let log = document.getElementById("log");
        log.innerHTML += msg + "<br/>";
        log.scrollTop = log.scrollHeight;
      }
    </script>
  </body>
</html>




You will Get in Few Days

Circuit for Starter “obniz Board” is available on Amazon and other online stores.
You can get it at below

Our products and resellers

Forum

Visit our developer’s forum to discuss and discover technologies.

Forum

Contact

Feel free to contact out support and technical team.

Contact us