Make Your Own Conversational AI/Social Robot with robokit — A Simple Approximation of Jibo

  • Hey, robo. What time is it? [ClockSkill.ts]
  • Hey robo. Tell me a joke. [JokeSkill.ts]
  • Hey robo. Who is your favorite robot? [FavoriteRobotSkill.ts]
robokit demo
  • Hotword detection using Snowboy
  • Cloud ASR using Microsoft’s Bing speech api
  • Cloud NLU using Microsoft LUIS (and google)
  • Cloud Text To Speech using Microsoft Bing TTS
  • Screen animation using Pixi.js — authored using Adobe Animate
  • A simple, extensible model for developing skills — with examples
  • Remote Operation Mode to enable Woz prototyping (Wizard of Oz)
  • All wrapped in a cross-platform Electron app
  • node v8.11 or better
  • yarn (latest)
  • sox (a linux audio library)
  • a Microsoft Azure Cognitive Services account (i.e. free trial)
yarn
yarn rebuild
yarn start
robokit eye idle
robokit architecture
src
├── main
│ └── main.ts
└── renderer
├── AsyncToken.ts
├── HotwordController.ts
├── NLUController.ts
├── ASRController.ts
├── TTSController.ts
├── log.ts
├── microsoft
│ ├── BingSpeechApiController.ts
│ ├── BingTTSController.ts
│ └── LUISController.ts
├── pixijs
│ └── PixijsManager.ts
├── renderer.ts
├── rom
│ ├── ClientCertificate.ts
│ ├── RomManager.ts
│ ├── SocketClient.ts
│ ├── SocketServer.ts
│ ├── commands
│ │ ├── BlinkCommandHandler.ts
│ │ ├── CommandHandler.ts
│ │ ├── IdentCommandHandler.ts
│ │ ├── LookAtCommandHandler.ts
│ │ └── TtsCommandHandler.ts
│ └── log.ts
├── skills
│ ├── ClockSkill.ts
│ ├── FavoriteRobotSkill.ts
│ ├── Hub.ts
│ ├── JokeSkill.ts
│ └── Skill.ts
├── snowboy
│ └── SnowboyController.ts
├── utils
│ └── Log.ts
└── ww
└── WwMusicController.ts
  • PixijsManager — screen animation rendering
  • Hub — central nervous system (i.e. skill life cycle management)
  • RomManager — remote operation via SocketServer
function startNLU(utterance: string) {
const nluController: NLUController = new LUISController();
let t: AsyncToken<NLUIntentAndEntities> =
nluController.getIntentAndEntities(utterance);
t.complete
.then((intentAndEntities: NLUIntentAndEntities) => {
console.log(`NLUIntentAndEntities: `,
intentAndEntities);
Hub.Instance().handleLaunchIntent(intentAndEntities,
utterance);
})
.catch((error: any) => {
console.log(error);
});
}
handleLaunchIntent(intentAndEntities: NLUIntentAndEntities, 
utterance: string): void {
let launchIntent = intentAndEntities.intent;
let skill: Skill | undefined =
this.launchIntentMap.get(launchIntent);
if (skill) {
skill.launch(intentAndEntities, utterance);
skill.running = true;
}
}
registerSkill(skill: Skill): void {
console.log(`HUB: registerSkill: `, skill);
this.skillMap.set(skill.id, skill);
this.launchIntentMap.set(skill.launchIntent, skill);
}
eyeInstance.gotoAndStop('idle');
eyeInstance.eye.eye_blue.visible = false;
export default abstract class Skill {    public id: string;
public launchIntent: string = '';
public running: boolean = false;
constructor(id: string, launchIntent: string) {
this.id = id;
this.launchIntent = launchIntent;
}
abstract launch(intentAndEntities: NLUIntentAndEntities,
utterance: string): void;
}
export default class ClockSkill extends Skill {    constructor() {
super('clockSkill', 'launchClock');
}
launch(intentAndEntities: NLUIntentAndEntities,
utterance: string) :void {
let time: Date = new Date();
let hours: number = time.getHours(); //'9';
if (hours > 12) {
hours -= 12;
}
let minutes: number = time.getMinutes(); //'35'
let minutesPrefix: string = (minutes < 10) ? 'oh' : '';
let timePrompt: string =
`The time is ${hours} ${minutesPrefix} ${minutes}`;
Hub.Instance().startTTS(timePrompt);
}
}
import { EventEmitter } from 'events';export default class AsyncToken<T> extends EventEmitter {    public complete: Promise<T>;    constructor() {
super();
}
}
export default abstract class ASRController {    abstract RecognizerStart(options?: any): AsyncToken<string>;}
export default abstract class TTSController {    abstract SynthesizerStart(text: string, options?: any): 
AsyncToken<string>;
}
export type NLUIntentAndEntities = {
intent: string;
entities: any;
}
export type NLURequestOptions = {
languageCode?: string;
contexts?: string[];
sessionId?: string;
}
export enum NLULanguageCode {
en_US = 'en-US'
}
export default abstract class NLUController { constructor() {
}
abstract set config(config: any); abstract call(query: string, languageCode: string,
context: string, sessionId?: string): Promise<any>;
abstract getEntitiesWithResponse(response: any): any | undefined; abstract getIntentAndEntities(utterance: string,
options?: NLURequestOptions): AsyncToken<NLUIntentAndEntities>;
}
getIntentAndEntities(utterance: string, 
options?: NLURequestOptions): AsyncToken<NLUIntentAndEntities> {
options = options || {};
let defaultOptions: NLURequestOptions = {
languageCode: NLULanguageCode.en_US,
contexts: undefined,
sessionId: undefined
}
options = Object.assign(defaultOptions, options);
let token = new AsyncToken<NLUIntentAndEntities>();
token.complete =
new Promise<NLUIntentAndEntities>((resolve, reject) => {
this.call(utterance)
.then((response: LUISResponse) => {
let intentAndEntities: NLUIntentAndEntities = {
intent: '',
entities: undefined
}
if (response && response.topScoringIntent) {
intentAndEntities = {
intent: response.topScoringIntent.intent,
entities:
this.getEntitiesWithResponse(response)
}
}
resolve(intentAndEntities);
})
.catch((err: any) => {
reject(err);
});
});
return token;
}
call(query: string): Promise<any> {
let endpoint = this.endpoint;
let luisAppId = this.luisAppId;
let queryParams = {
"subscription-key": this.subscriptionKey,
"timezoneOffset": "0",
"verbose": true,
"q": query
}
let luisRequest = endpoint + luisAppId + '?' +
querystring.stringify(queryParams);
return new Promise((resolve, reject) => {
request(luisRequest,
((error: string, response: any, body: any) => {
if (error) {
console.log(`error:`, response, error);
reject(error);
} else {
let body_obj: any = JSON.parse(body);
resolve(body_obj);
}
}));
});
}
getEntitiesWithResponse(response: LUISResponse): any {
let entitiesObject: any = {
user: 'Someone',
userOriginal: 'Someone',
thing: 'that',
thingOriginal: 'that'
};
response.entities.forEach((entity: LUISEntity) => {
entitiesObject[`${entity.type}Original`] = entity.entity;
if (entity.resolution && entity.resolution.values) {
entitiesObject[`${entity.type}`] =
entity.resolution.values[0];
}
});
return entitiesObject;
}
export type HotwordResult = {
hotword: string;
index?: number;
buffer?: any;
}
export default abstract class HotwordController { abstract RecognizerStart(options?: any):
AsyncToken<HotwordResult>;
}
const record = require('node-record-lpcm16');
import { Detector, Models } from 'snowboy';
...const modelPath: string =
path.resolve(root, 'resources/models/HeyRobo.pmdl');
...export default class SnowboyController extends HotwordController{ public models: Models
public detector: Detector;
public mic: any;
constructor() {
super();
this.models = new Models();
this.models.add({
file: modelPath,
sensitivity: '0.5',
hotwords : 'snowboy'
});
}
...}
const modelPath: string = 
path.resolve(root, 'resources/models/snowboy.pmdl');
RecognizerStart(options: any): AsyncToken<HotwordResult> {
let sampleRate = 16000;
if (options && options.sampleRate) {
sampleRate = options.sampleRate;
}
let token = new AsyncToken<HotwordResult>();
token.complete = new Promise<HotwordResult>((resolve: any,
reject: any) => {
process.nextTick(() => {token.emit('Listening');});
this.detector = new Detector({
resource: commonResPath,
models: this.models,
audioGain: 2.0,
applyFrontend: true
});
this.detector.on('silence', () => {
token.emit('silence');
});
this.detector.on('sound', (buffer) => {
token.emit('sound');
});
this.detector.on('error', (error: any) => {
console.log('error', error);
reject(error);
});
this.detector.on('hotword', function (index, hotword, buffer) {
record.stop();
token.emit('hotword');
resolve({hotword: hotword, index: index, buffer: buffer});
});
this.mic = record.start({
threshold: 0,
sampleRate: sampleRate,
verbose: true,
});
this.mic.pipe(this.detector as any);
});
return token;
}
cd [robokit]/tools
node test-snowboy.js
$ node test-snowboy.js 
/Users/.../github/wwlib/robokit/resources/models/HeyRobo.pmdl
/Users/.../github/wwlib/robokit/resources/common.res
Recording 1 channels with sample rate 16000...
Recording 8192 bytes
Recording 8192 bytes
Recording 8192 bytes
Recording 8192 bytes
Recording 8192 bytes
Recording 8192 bytes
Recording 8192 bytes
renderer: startHotword: on hotword:
HotWord: result: { hotword: 'snowboy',
index: 1,
buffer: <Buffer 9f fe b4 fe ca fe e5 fe 07 ff 39 ff 74 ff aa ff de ff 0b 00 46 00 6c 00 95 00 bf 00 dd 00 f4 00 fc 00 fe 00 eb 00 d1 00 c6 00 b9 00 9c 00 7e 00 64 00 ... > }
Recording 442 bytes
End Recording: 1940.446ms
{
"Microsoft": {
"BingSubscriptionKey": "<YOUR-BING-SUBSCRIPTION-KEY>",
"nluLUIS_endpoint": "<ENDPOINT-URL>",
"nluLUIS_appId": "<YOUR-LUIS-APP-ID>",
"nluLUIS_subscriptionKey": "<YOUR-LUIS-SUBSCRIPTION-KEY>"
}
}
test-bing-speech.js
test-bing-tts.js
test-luis-nlu.js
robokit Remote Operation Mode (i.e. via robocommander)

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store