UPDATE: Now available on GitHub: https://github.com/JanMattner/voice-control-openhab
I’d like to share my current state of my personal simple voice control.
My goals:
- Using voice instead of some UI (e.g. Main UI) seems more intuitive, easier and quicker to me
- I do not want any smart AI - just a way to trigger something via voice instead of a button (i.e. I know how my items are named, I do not need an AI to interpret natural language and extract semantics)
My setup:
- Raspberry Pi running OpenHabian
- Android App with the Speech-To-Text feature (by Google)
I have seen the Java Built-In interpreter and thought: well, that’s nice, but I’d like to have an easy way to add custom rules without building the Java code.
That’s why I just took those ideas and implemented it in JavaScript (the out of the box ES5).
So that’s the outcome:
Released under the terms of the MIT license:
MIT License
Copyright (c) 2021 Jan Mattner
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
Content of /automation/lib/javascript/personal/ruleBasedInterpreter.js:
// HOW TO load this file from a script/rule created in Main UI:
// var OPENHAB_CONF = Java.type('java.lang.System').getenv('OPENHAB_CONF');
// load(OPENHAB_CONF + '/automation/lib/javascript/personal/ruleBasedInterpreter.js');
'use strict';
(function(context) {
'use strict';
var logger = Java.type('org.slf4j.LoggerFactory').getLogger('org.openhab.core.automation.ruleBasedInterpreter');
// ***
// DATA
// ***
var expressionTypes = {
SEQUENCE: "sequence",
COMMAND: "command",
ALTERNATIVE: "alternative",
OPTIONAL: "optional",
ITEMLABEL: "itemlabel"
};
var rules = [];
// ***
// FUNCTIONS
// ***
function alt(params) {
return {
expressionType: expressionTypes.ALTERNATIVE,
value: params || []
}
}
function seq(params) {
return {
expressionType: expressionTypes.SEQUENCE,
value: params || []
}
}
function opt(expression) {
return {
expressionType: expressionTypes.OPTIONAL,
value: expression
}
}
function cmd(expression, command) {
return {
expressionType: expressionTypes.COMMAND,
value: expression,
command: command
}
}
function itemLabel() {
return {
expressionType: expressionTypes.ITEMLABEL
}
}
function addRule(expression, executeFunction) {
rules.push({
expression: expression,
executeFunction: executeFunction
});
}
function interpretUtterance(utterance) {
var normalizedUtterance = normalizeUtterance(utterance);
var tokens = tokenizeUtterance(normalizedUtterance);
logger.debug("input normalized utterance: " + normalizedUtterance);
logger.debug("input tokens: " + stringify(tokens));
for (var index = 0; index < rules.length; index++) {
logger.debug("check rule " + index);
var rule = rules[index];
logger.debug(stringify(rule));
var result = evaluateExpression(rule.expression, tokens.slice());
if (result.success) {
var executeFunction = result.executeFunction || rule.executeFunction;
if (!executeFunction) {
logger.debug("rule matched, but no function to execute found, continue");
continue;
}
executeFunction(result.executeParameter);
break;
}
}
}
function evaluateExpression(expression, tokens) {
if (tokens.length < 1) {
return createEvaluationResult(true, tokens, null);
}
if (typeof(expression) == "string") {
return evaluateStringExpression(expression, tokens);
}
switch (expression.expressionType) {
case expressionTypes.SEQUENCE:
return evaluateSequence(expression, tokens);
case expressionTypes.ALTERNATIVE:
return evaluateAlternative(expression, tokens);
case expressionTypes.OPTIONAL:
return evaluateOptional(expression, tokens);
case expressionTypes.COMMAND:
return evaluateCommand(expression, tokens);
case expressionTypes.ITEMLABEL:
return evaluateItemLabel(tokens);
default:
return createEvaluationResult(false, tokens, null, null);
}
}
/**
*
* @param {boolean} success - if evaluation was successful or not
* @param {string[]} remainingTokens
* @param {function} executeFunction - the function to execute in the end
* @param {object} executeParameter - the parameter inserted in the executeFunction. Should be a single object that can hold multiple parameters in its key/value pairs.
* @returns {object} - exactly the above parameters in an object
*/
function createEvaluationResult(success, remainingTokens, executeFunction, executeParameter) {
return {
success: success,
remainingTokens: remainingTokens,
executeFunction: executeFunction,
executeParameter: executeParameter
};
}
function evaluateStringExpression(expression, tokens) {
if (tokens.length < 1) {
return createEvaluationResult(false, tokens, null, null);
}
logger.debug("eval string: " + expression)
logger.debug("token: " + tokens[0]);
var hasMatch = tokens[0].match(expression) != null;
logger.debug("hasMatch: " + hasMatch)
return createEvaluationResult(hasMatch, tokens.slice(1), null, null);
}
function evaluateOptional(expression, tokens) {
logger.debug("eval opt: " + stringify(expression))
var result = evaluateExpression(expression.value, tokens.slice());
if (result.success) {
logger.debug("eval opt success")
// only return the reduced token array and other parameters if optional expression was successful.
return createEvaluationResult(true, result.remainingTokens, result.executeFunction, result.executeParameter);
}
logger.debug("eval opt fail")
// otherwise still return successful, but nothing from the optional expression result
return createEvaluationResult(true, tokens, null, null);
}
function evaluateCommand(expression, tokens) {
logger.debug("eval cmd: " + stringify(expression.value));
var result = evaluateExpression(expression.value, tokens);
logger.debug("eval cmd result: " + result.success)
if (!result.success) {
return createEvaluationResult(false, tokens, null, null);
}
var executeFunction = function(parameter) {
if (!parameter || typeof(parameter) != "object") {
logger.debug("Trying to send a command, but no proper object parameter found")
return;
}
var item = parameter.item;
if (!item) {
logger.debug("Trying to send a command, but no item parameter found")
return;
}
events.sendCommand(item, expression.command);
}
return createEvaluationResult(true, result.remainingTokens, executeFunction, result.executeParameter);
}
function evaluateItemLabel(tokens) {
logger.debug("eval item label with tokens: " + stringify(tokens))
if (tokens.length < 1) {
logger.debug("no tokens, eval item label fail")
return createEvaluationResult(false, tokens, null, null);
}
// get whole item registry; since that's only a Java list, convert it first to a JS array
// and by that way, normalize and tokenize the label for easier comparison
var allItems = Java.from(itemRegistry.getItems())
.map(function(i){
return {
item: i,
labelTokens: tokenizeUtterance(normalizeUtterance(i.getLabel()))
}
});
// we need a single exact match
// first try the regular labels
var checkLables = function(remainingItems) {
var tokenIndex = 0;
while (remainingItems.length > 1) {
if (tokens.length < tokenIndex + 1) {
// no tokens left, but still multiple possible items -> abort
return {remainingItems: remainingItems, tokenIndex: tokenIndex};
}
remainingItems = remainingItems.filter(function(entry) {
return (entry.labelTokens.length >= tokenIndex + 1) && entry.labelTokens[tokenIndex] == tokens[tokenIndex];
});
tokenIndex++;
}
return {remainingItems: remainingItems, tokenIndex: tokenIndex};
}
var matchResult = checkLables(allItems.slice());
logger.debug("eval item found matched labels: " + matchResult.remainingItems.length);
if (matchResult.remainingItems.length == 0) {
// either none or multiple matches found. Let's try the synonyms.
var checkSynonyms = function(allItems) {
var remainingItems = allItems.map(function(i){
return {
item: i.item,
synonymTokens: getSynonyms(i.item.getName()).map(function(s){ return tokenizeUtterance(normalizeUtterance(s));})
}
});
// remove items without synonyms
remainingItems = remainingItems.filter(function(i) {
return i.synonymTokens.length > 0;
});
var tokenIndex = 0;
while (remainingItems.length > 1) {
if (tokens.length < tokenIndex + 1) {
// no tokens left, but still multiple possible items -> abort
return {remainingItems: remainingItems, tokenIndex: tokenIndex};
}
// remove synonyms with fewer or non-matching tokens
remainingItems = remainingItems.map(function(i) {
i.synonymTokens = i.synonymTokens.filter(function(t) {
return (t.length >= tokenIndex + 1) && (t[tokenIndex] == tokens[tokenIndex]);
});
return i;
});
// remove items without synonyms
remainingItems = remainingItems.filter(function(i) {
return i.synonymTokens.length > 0;
});
tokenIndex++;
}
return {remainingItems: remainingItems, tokenIndex: tokenIndex};
}
matchResult = checkSynonyms(allItems.slice());
logger.debug("eval item found matched synonyms: " + matchResult.remainingItems.length);
}
if (matchResult.remainingItems.length == 1) {
logger.debug("eval item label success")
return createEvaluationResult(true, tokens.slice(matchResult.tokenIndex), null, {item: matchResult.remainingItems[0].item});
}
logger.debug("eval item label fail")
return createEvaluationResult(false, tokens, null, null);
}
/**
* Returns the metadata on the passed in item name with the given namespace.
* Credits to Rich Koshak.
* @param {string} itemName name of the item to search the metadata on
* @param {string} namespace namespace of the metadata to return
* @return {Metadata} the value and configuration or null if the metadata doesn't exist
*/
function getMetadata(itemName, namespace) {
var FrameworkUtil = Java.type("org.osgi.framework.FrameworkUtil");
var _bundle = FrameworkUtil.getBundle(scriptExtension.class);
var bundle_context = _bundle.getBundleContext()
var MetadataRegistry_Ref = bundle_context.getServiceReference("org.openhab.core.items.MetadataRegistry");
var MetadataRegistry = bundle_context.getService(MetadataRegistry_Ref);
var MetadataKey = Java.type("org.openhab.core.items.MetadataKey");
return MetadataRegistry.get(new MetadataKey(namespace, itemName));
}
function getSynonyms(itemName) {
var meta = getMetadata(itemName, "synonyms");
if (stringIsNullOrEmpty(meta)) {
return [];
}
return meta.value.split(",");
}
function evaluateSequence(expression, tokens) {
logger.debug("eval seq: " + stringify(expression));
var success = true;
var executeFunction = null;
var executeParameter = null;
var remainingTokens = tokens.slice();
for (var index = 0; index < expression.value.length; index++) {
var subexp = expression.value[index];
if (remainingTokens.length < 1) {
// no more tokens left, but another sub expression is required
// -> no match of full sequence possible, we can already abort at this point
var success = false;
break;
}
logger.debug("eval subexp " + index + "; subexp: " + stringify(subexp))
var result = evaluateExpression(subexp, remainingTokens);
if (!result.success) {
success = false;
break;
}
remainingTokens = result.remainingTokens;
executeFunction = result.executeFunction || executeFunction;
executeParameter = result.executeParameter || executeParameter;
}
logger.debug("eval seq: " + success)
return createEvaluationResult(success, remainingTokens, executeFunction, executeParameter);
}
function evaluateAlternative(expression, tokens) {
logger.debug("eval alt: " + stringify(expression));
logger.debug("for tokens: " + stringify(tokens));
if (tokens.length < 1) {
logger.debug("eval alt fail")
// no more tokens left, but at least one sub expression is required
// -> no match of any alternative possible, we can already abort at this point
return createEvaluationResult(false, tokens, null, null);
}
var success = false;
var executeFunction = null;
var remainingTokens = tokens;
var executeParameter = null;
for (var index = 0; index < expression.value.length; index++) {
var subexp = expression.value[index];
logger.debug("alt index: " + index + "; subexp: " + stringify(subexp));
var result = evaluateExpression(subexp, tokens.slice());
if (result.success) {
success = true;
remainingTokens = result.remainingTokens;
executeFunction = result.executeFunction || executeFunction;
executeParameter = result.executeParameter || executeParameter;
break;
}
}
logger.debug("eval alt: " + success)
return createEvaluationResult(success, remainingTokens, executeFunction, executeParameter);
}
function normalizeUtterance(utterance) {
return utterance.toLowerCase();
}
function tokenizeUtterance(utterance) {
return utterance.split(" ").filter(Boolean);
}
function stringify(obj) {
return JSON.stringify(obj, null, 2);
}
function stringIsNullOrEmpty(str) {
return str === undefined || str === null || str === "";
}
// ***
// EXPORTS
// ***
context.ruleBasedInterpreter = {
interpretUtterance: interpretUtterance,
alt: alt,
seq: seq,
opt: opt,
cmd: cmd,
itemLabel: itemLabel,
addRule: addRule
}
})(this);
Then I can load this and use the functions to define my rules in a separate file.
Content of /automation/lib/javascript/personal/voiceCommandRules.js:
(sorry, example only in German, but it is quite the same as in the Java interpreter linked above).
// HOW TO load this file from a script/rule created in Main UI:
// var OPENHAB_CONF = Java.type('java.lang.System').getenv('OPENHAB_CONF');
// load(OPENHAB_CONF + '/automation/lib/javascript/personal/voiceCommandRules.js');
var OPENHAB_CONF = Java.type('java.lang.System').getenv('OPENHAB_CONF');
load(OPENHAB_CONF + '/automation/lib/javascript/personal/ruleBasedInterpreter.js');
'use strict';
(function(context) {
'use strict';
var alt = ruleBasedInterpreter.alt;
var seq = ruleBasedInterpreter.seq;
var cmd = ruleBasedInterpreter.cmd;
var opt = ruleBasedInterpreter.opt;
var itemLabel = ruleBasedInterpreter.itemLabel;
var addRule = ruleBasedInterpreter.addRule;
var denDieDas = alt(["den", "die", "das"]);
var einAnAus = alt([cmd("ein", ON), cmd("an", ON), cmd("aus", OFF)]);
var schalte = alt(["schalte", "mache", "schalt", "mach"]);
var fahre = alt(["fahre", "fahr", "mache", "mach"]);
var hochRunter = alt([cmd("hoch", UP), cmd("runter", DOWN)]);
// ON OFF type
addRule(seq([schalte, opt(denDieDas), itemLabel(), einAnAus]));
// UP DOWN type
addRule(seq([fahre, opt(denDieDas), itemLabel(), hochRunter]));
})(this);
I then just set up a rule with “Update” trigger of the VoiceCommand variable and run the following script.
Content of a Main UI ES5 script:
var logger = Java.type('org.slf4j.LoggerFactory').getLogger('org.openhab.rule.' + ctx.ruleUID);
var OPENHAB_CONF = Java.type('java.lang.System').getenv('OPENHAB_CONF');
load(OPENHAB_CONF + '/automation/lib/javascript/personal/voiceCommandRules.js');
ruleBasedInterpreter.interpretUtterance(itemRegistry.getItem("VoiceCommand").getState().toString());
The code tries to get the item by its label (not the unique name, since spaces are not allowed there and thus tokenizing not possible) or by the user defined Synonyms (meta data). A single match is required, so the items should be labeled accordingly.
If a command and an item is found, it tries to send the defined command to the item. But user defined functions can be executed if a rule matches - and here it gets interesting. Any JavaScript function can be executed.
What do you think?
Useful or superfluous?
I’ve also seen the HABot, but also some posts referring to it as “old” - is HABot still a thing and actively used/developed and should this kind of voice control be done with HABot? (I have not set it up and no experience with it)
good idea, I start small, but I’ll come back to this with more sensors / data integration