Operate the robot kit with handwritten characters
This section uses Html / JavaScript instead of the traditional block program.
We also use Cloud Vision, an image recognition service provided by Google.
Get the Google Cloud Api key
This app requires a Google Cloud Api API key.
2.Go to the Google Cloud Platform Dashboard.
Select a library from APIs and services.
3. The API library search screen will appear, so search by vision.
4. Cloud VisionAPIを選択します。
5. Press the button to enable.
6. Once activated, return to the API and Services dashboard and select your credentials.
7. From Create Credentials, select the API key.
8. Make a note of the displayed API key as we will use it later.
Try to move
The program is here.
Enter your obnizID and press TestOpen.
There is a field to enter the google api key, so enter the API key and press the Start button to start the camera.
If you show characters such as "Recommend" and "Stop" so that they can be seen well on the camera, the robot kit will also move.
<!-- HTML Example -->
<!DOCTYPE HTML>
<html>
<head>
<meta charset="utf-8">
<title>Video Capture Example</title>
<meta name="viewport" content="width=device-width,initial-scale=1">
<script src="https://obniz.com/js/jquery-3.2.1.min.js"></script>
<script src="https://unpkg.com/obniz@2.0.2/obniz.js"></script>
<script src="https://unpkg.com/obniz-parts-kits@0.15.2/airobot/index.js"></script>
</head>
<body>
<div id="obniz-debug"></div>
<div>
<div class="control">
<input type="text" id="google_api_key" placeholder="google api key" value=""/>
<button id="startAndStop">Start</button>
</div>
</div>
<p class="err" id="errorMessage"></p>
<div style="display:flex;">
<video id="videoInput" autoplay playsinline width=320 height=240 style="border:1px solid;width:320px;margin-right:10px"></video>
<canvas id="canvasOutput" width=320 height=240 style="-webkit-font-smoothing:none" hidden style="display:none"></canvas>
<div id="log" style="border:1px solid;height: 240px;overflow-y:scroll;width:320px"></div>
</div>
<script type="text/javascript">
let airobotkit;
let power = 60;
let inOperation = false;
let mediaDevices = navigator.mediaDevices || ((navigator.mozGetUserMedia || navigator.webkitGetUserMedia || navigator.msGetUserMedia) ? {
getUserMedia(c) {
return new Promise(((y, n) => {
(navigator.mozGetUserMedia || navigator.webkitGetUserMedia).call(navigator, c, y, n);
}));
}
} : null);
obniz = new Obniz("OBNIZ_ID_HERE");
obniz.onconnect = async () => {
obniz.display.print("ready");
airobotkit = obniz.wired("AIRobotKit",{});
};
function moveTo(lPower, rPower) {
if (!airobotkit) {
return
}
airobotkit.move(lPower,rPower);
}
let streaming = false;
let videoInput = document.getElementById('videoInput');
let startAndStop = document.getElementById('startAndStop');
let canvasOutput = document.getElementById('canvasOutput');
let canvasContext = canvasOutput.getContext('2d');
function successCallback(stream) {
document.getElementById("videoInput").srcObject = stream;
onVideoStarted();
}
function errorCallback(err) {
console.error('mediaDevice.getUserMedia() error:', err);
}
startAndStop.addEventListener('click', () => {
if (!streaming) {
const medias = {
audio: false,
video: {
facingMode: "user"
}
};
mediaDevices.getUserMedia(medias)
.then(successCallback).catch(errorCallback);
} else {
onVideoStopped();
}
});
function onVideoStarted() {
streaming = true;
startAndStop.innerText = 'Stop';
start();
}
function onVideoStopped() {
streaming = false;
canvasContext.clearRect(0, 0, canvasOutput.width, canvasOutput.height);
startAndStop.innerText = 'Start';
}
async function start() {
const FPS = 100;
function processVideo() {
try {
if (!streaming) {
return;
}
let begin = Date.now();
let base64String = capture();
postToGoogle(base64String);
setTimeout(processVideo, 0);
} catch (err) {
console.error(err);
}
}
// schedule the first one.
setTimeout(processVideo, 0);
}
function capture() {
if (!streaming) {
return null;
}
canvasContext.drawImage(videoInput, 0, 0, canvasOutput.width, canvasOutput.height);
return canvasOutput.toDataURL('image/webp');
}
//The argument is a base64 format string
function toBlob(base64) {
var bin = atob(base64.replace(/^.*,/, ''));
var buffer = new Uint8Array(bin.length);
for (var i = 0; i < bin.length; i++) {
buffer[i] = bin.charCodeAt(i);
}
// Create a blob
try {
var blob = new Blob([buffer.buffer], {
type: 'image/png'
});
} catch (e) {
return false;
}
return blob;
}
function postToGoogle(base64Img) {
if (inOperation) {
return;
}
inOperation = true;
var base64Img = base64Img.split(",")[1];
var apiKey = document.getElementById("google_api_key").value;
var body = {
"requests": [
{
"image": {
"content": base64Img
},
"features": [
{
"type": "LABEL_DETECTION"
},
{
"type": "TEXT_DETECTION",
"maxResults": 1
}
]
}
]
};
//request to google api
fetch("https://vision.googleapis.com/v1/images:annotate?key=" + apiKey, {
body: JSON.stringify(body),
method: 'POST',
headers: {
'Content-Type': 'application/json'
}
}).then(function (response) {
return response.json();
}).then(async function (json) {
if (isKeywordsIncludes(json.responses, ["右", "みぎ", "right"])) {
obniz.display.clear();
obniz.display.print("みぎ！");
_log("右むく");
moveTo(power, -1 * power);
}
else if (isKeywordsIncludes(json.responses, ["left", "左", "ひだり"])) {
_log("左向く");
obniz.display.clear();
obniz.display.print("ひだり！");
moveTo( -1 *power, power);
}
else if (isKeywordsIncludes(json.responses, ["stop", "止", "とまれ"])) {
obniz.display.clear();
obniz.display.print("ぶれーき！");
_log("止まる");
moveTo(0, 0);
}
else if (isKeywordsIncludes(json.responses, ["すすめ", "進", "go", "前", "まえ", "forward"])) {
_log("すすむ");
obniz.display.clear();
obniz.display.print("まっすぐ！");
moveTo(power, power);
}
else if (isKeywordsIncludes(json.responses, ["バック", "後", "back", "もどれ", "もどる"])) {
_log("もどる");
obniz.display.clear();
obniz.display.print("ばっくします！")
moveTo(-1 * power, -1 * power);
}
else if (isKeywordsIncludes(json.responses, ["こっち","こちら", "here"])) {
let pos = isKeywordsIncludes(json.responses, ["こっち", "こちら","here"]);
let sum = 0;
for (let p of pos.vertices) {
sum += p.x;
}
let avg = sum / 4;
let rotate = (avg - canvasOutput.width / 2) / canvasOutput.width;
moveTo(rotate * power, (1 - rotate) * power);
} else {
_log("keyward nothing");
moveTo(0, 0);
}
inOperation = false;
});
}
function isKeywordsIncludes(results, keywards) {
if (!Array.isArray(keywards)) {
keywards = [keywards];
}
for (let result of results) {
if (result && result.labelAnnotations) {
for (let anotation of result.labelAnnotations) {
for (let keyward of keywards) {
if (anotation.description.toLowerCase().indexOf(keyward) >= 0) {
return anotation.boundingPoly || true;
}
}
}
}
if (result && result.textAnnotations) {
for (let anotation of result.textAnnotations) {
for (let keyward of keywards) {
if (anotation.description.toLowerCase().indexOf(keyward) >= 0) {
return anotation.boundingPoly || true;
}
}
}
}
if (result && result.fullTextAnnotation) {
for (let keyward of keywards) {
if (result.fullTextAnnotation.text.toLowerCase().indexOf(keyward) >= 0) {
return result.fullTextAnnotation.boundingPoly || true;
}
}
}
return false;
}
}
function _log(msg) {
let log = document.getElementById("log");
log.innerHTML += msg + "<br/>";
log.scrollTop = log.scrollHeight;
}
</script>
</body>
</html>