Room Synesthesia
Published in
6 min readOct 29, 2019
Measuring the Great Indoors
Spending a large portion of our time indoors warrants a space that reads our habits and movements adjusting to optimise our interaction with the space on an emotional and synesthetic level. This first iteration of the ‘Room Synesthesia’ project uses a webcam, computer vision and Fiducial markers to detect three different scenarios and thus three different spatial use conditions. These three conditions are triggered using webhooks sent to IFTTT via a Processing script. Once the webhooks (HTTP requests) are sent, the Philips HUE bridge and bulbs are called to change to a certain colour tied to its spatial use condition.
The intended result is a seamless transition from condition to condition as a user moves from one action or state to another. Seen below is a short example of the system in action. Going forward, the script as well as the physical marker detection system will need to be developed to allow for a more responsive lighting system practically.
Future Developments
Moving forward, the project can begin to use the HUE bulbs more cohesively and can employ sounds and a projector for more compelling visualisations. The room can also take into account the time of day, motion, exterior lights, facial expressions etc. to offer more varied spatial use conditions and a more perceptive system. A potential avenue to explore is the use of audio/visual prompts to encourage, remind, soothe etc. The room can then transition between a space for super productivity and a space for meditation.
/*
TUIO 1.1 Demo for Processing
Copyright (c) 2005-2014 Martin Kaltenbrunner <martin@tuio.org>Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files
(the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of the Software,
and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*///Edited by Zeid Ghawi// import the TUIO library
import TUIO.*;
// declare a TuioProcessing client
TuioProcessing tuioClient;// these are some helper variables which are used
// to create scalable graphical feedback
float cursor_size = 15;
float object_size = 60;
float table_size = 760;
float scale_factor = 1;
PFont font;
float retrieveID;boolean verbose = false; // print console debug messages
boolean callback = true; // updates only after callbacksimport http.requests.*;
//update the below API key
String apiKey = "ot7veRhSfxbEm4sTREt8kVnxjt6kCGXjza0ZnvzuA19";
//update the event name to match the event name in Webhooks
String eventName1 = "zlights_to_red";
String eventName2 = "zlights_to_blue";
String eventName3 = "zlights_to_green";
String eventName4 = "z_test_event";void setup()
{
// GUI setup
noCursor();
size(displayWidth,displayHeight);
noStroke();
fill(0);
// periodic updates
if (!callback) {
frameRate(60);
loop();
} else noLoop(); // or callback updates
font = createFont("Arial", 18);
scale_factor = height/table_size;
// finally we create an instance of the TuioProcessing client
// since we add "this" class as an argument the TuioProcessing class expects
// an implementation of the TUIO callback methods in this class (see below)
tuioClient = new TuioProcessing(this);
}// within the draw method we retrieve an ArrayList of type <TuioObject>, <TuioCursor> or <TuioBlob>
// from the TuioProcessing client and then loops over all lists to draw the graphical feedback.
void draw()
{
background(255);
textFont(font,18*scale_factor);
float obj_size = object_size*scale_factor;
float cur_size = cursor_size*scale_factor;
ArrayList<TuioObject> tuioObjectList = tuioClient.getTuioObjectList();
for (int i=0;i<tuioObjectList.size();i++) {
TuioObject tobj = tuioObjectList.get(i);
retrieveID = tobj.getSymbolID();
if ((i == 1) && (i != 0)){
print ("Standard State - GREEN");
GetRequest get = new GetRequest("https://maker.ifttt.com/trigger/" + eventName3 + "/with/key/" + apiKey);
get.send();
background(0,255,127);
}
if (i == 0){
if (retrieveID == 0){
print ("Condition 1 - Working - RED");
GetRequest get = new GetRequest("https://maker.ifttt.com/trigger/" + eventName1 + "/with/key/" + apiKey);
get.send();
background(128,0,0);
} else if (retrieveID == 1) {
print ("Condition 2 - Playing - BLUE");
GetRequest get = new GetRequest("https://maker.ifttt.com/trigger/" + eventName2 + "/with/key/" + apiKey);
get.send();
background(65,105,225);
}
}
//https://maker.ifttt.com/use/ot7veRhSfxbEm4sTREt8kVnxjt6kCGXjza0ZnvzuA19
stroke(0);
fill(0,0,0);
pushMatrix();
translate(tobj.getScreenX(width),tobj.getScreenY(height));
rotate(tobj.getAngle());
rect(-obj_size/2,-obj_size/2,obj_size,obj_size);
popMatrix();
fill(255);
text(""+tobj.getSymbolID(), tobj.getScreenX(width), tobj.getScreenY(height));
}
ArrayList<TuioCursor> tuioCursorList = tuioClient.getTuioCursorList();
for (int i=0;i<tuioCursorList.size();i++) {
TuioCursor tcur = tuioCursorList.get(i);
ArrayList<TuioPoint> pointList = tcur.getPath();
if (pointList.size()>0) {
stroke(0,0,255);
TuioPoint start_point = pointList.get(0);
for (int j=0;j<pointList.size();j++) {
TuioPoint end_point = pointList.get(j);
line(start_point.getScreenX(width),start_point.getScreenY(height),end_point.getScreenX(width),end_point.getScreenY(height));
start_point = end_point;
}
stroke(192,192,192);
fill(192,192,192);
ellipse( tcur.getScreenX(width), tcur.getScreenY(height),cur_size,cur_size);
fill(0);
text(""+ tcur.getCursorID(), tcur.getScreenX(width)-5, tcur.getScreenY(height)+5);
}
}
ArrayList<TuioBlob> tuioBlobList = tuioClient.getTuioBlobList();
for (int i=0;i<tuioBlobList.size();i++) {
TuioBlob tblb = tuioBlobList.get(i);
stroke(0);
fill(0);
pushMatrix();
translate(tblb.getScreenX(width),tblb.getScreenY(height));
rotate(tblb.getAngle());
ellipse(-1*tblb.getScreenWidth(width)/2,-1*tblb.getScreenHeight(height)/2, tblb.getScreenWidth(width), tblb.getScreenWidth(width));
popMatrix();
fill(255);
text(""+tblb.getBlobID(), tblb.getScreenX(width), tblb.getScreenX(width));
}
}// --------------------------------------------------------------
// these callback methods are called whenever a TUIO event occurs
// there are three callbacks for add/set/del events for each object/cursor/blob type
// the final refresh callback marks the end of each TUIO frame// called when an object is added to the scene
void addTuioObject(TuioObject tobj) {
if (verbose) println("add obj "+tobj.getSymbolID()+" ("+tobj.getSessionID()+") "+tobj.getX()+" "+tobj.getY()+" "+tobj.getAngle());
}// called when an object is moved
void updateTuioObject (TuioObject tobj) {
if (verbose) println("set obj "+tobj.getSymbolID()+" ("+tobj.getSessionID()+") "+tobj.getX()+" "+tobj.getY()+" "+tobj.getAngle()
+" "+tobj.getMotionSpeed()+" "+tobj.getRotationSpeed()+" "+tobj.getMotionAccel()+" "+tobj.getRotationAccel());
}// called when an object is removed from the scene
void removeTuioObject(TuioObject tobj) {
if (verbose) println("del obj "+tobj.getSymbolID()+" ("+tobj.getSessionID()+")");
}// --------------------------------------------------------------
// called when a cursor is added to the scene
void addTuioCursor(TuioCursor tcur) {
if (verbose) println("add cur "+tcur.getCursorID()+" ("+tcur.getSessionID()+ ") " +tcur.getX()+" "+tcur.getY());
//redraw();
}// called when a cursor is moved
void updateTuioCursor (TuioCursor tcur) {
if (verbose) println("set cur "+tcur.getCursorID()+" ("+tcur.getSessionID()+ ") " +tcur.getX()+" "+tcur.getY()
+" "+tcur.getMotionSpeed()+" "+tcur.getMotionAccel());
//redraw();
}// called when a cursor is removed from the scene
void removeTuioCursor(TuioCursor tcur) {
if (verbose) println("del cur "+tcur.getCursorID()+" ("+tcur.getSessionID()+")");
//redraw()
}// --------------------------------------------------------------
// called when a blob is added to the scene
void addTuioBlob(TuioBlob tblb) {
if (verbose) println("add blb "+tblb.getBlobID()+" ("+tblb.getSessionID()+") "+tblb.getX()+" "+tblb.getY()+" "+tblb.getAngle()+" "+tblb.getWidth()+" "+tblb.getHeight()+" "+tblb.getArea());
//redraw();
}// called when a blob is moved
void updateTuioBlob (TuioBlob tblb) {
if (verbose) println("set blb "+tblb.getBlobID()+" ("+tblb.getSessionID()+") "+tblb.getX()+" "+tblb.getY()+" "+tblb.getAngle()+" "+tblb.getWidth()+" "+tblb.getHeight()+" "+tblb.getArea()
+" "+tblb.getMotionSpeed()+" "+tblb.getRotationSpeed()+" "+tblb.getMotionAccel()+" "+tblb.getRotationAccel());
//redraw()
}// called when a blob is removed from the scene
void removeTuioBlob(TuioBlob tblb) {
if (verbose) println("del blb "+tblb.getBlobID()+" ("+tblb.getSessionID()+")");
//redraw()
}// --------------------------------------------------------------
// called at the end of each TUIO frame
void refresh(TuioTime frameTime) {
if (verbose) println("frame #"+frameTime.getFrameID()+" ("+frameTime.getTotalMilliseconds()+")");
if (callback) redraw();
}
