Working with Xbox360 Kinect, Processing, and Arduino

November 9, 2013 Art

I spent the day tinkering with electronics for my Interactive Prototyping class.  We’ve got a project where we need to create an interactive installation, and our group wants to use the Kinect as the input device.  SO, I did a little experimenting to figure out what I need to do to make this work.  I started off in pure Arduino, just getting the sculpture to move, and making sure everything was connected correctly.

The next step was to figure out how to directly control the components using computer input in Processing.  I used the mouse position to change the parameters driving the sculpture.  To do this, you must first have the Arduino library for Processing.  Then you must run the Standard Firmata sketch by going to File>Examples>Firmata>StandardFirmata.  Then, you can run your Processing sketch, using the mouse as an input.

Finally, I used Processing and Simple Open NI to turn the motion data into usable input.  To set up the Kinect, you must first install the SDK and Developer Toolkit from Microsoft. Next, you must install the OpenNI SDK, and then Simple Open NI in the processing library to be able to read the kinect data.  Currently, there are new versions out of both Open NI and Simple Open NI that are significantly different than the previous versions.  Consequently, most of the sketches and tutorials I found on the web do not work.  After a considerable amount of time, I was finally able to get the “User 3D” example sketch running that is included with Simple Open NI.  From here, I figured out how the different points are being tracked.  I sloppily added a few functions into the code to make my sculpture work.

The code for these 3 iterations is posted below.  While the first 2 are original creations, the 3rd code is completely taken from the User 3D sketch at Simple Open NI.  Hope this helps!

// control speed of 3 servos with potentiometer.  the positions are constrained within a Hi and Lo limit
#include <Servo.h> 
// create servo object to control a servo a maximum of eight servo objects can be created 
Servo myservo[3];

int servoPins[3] = {9,10,11};
int ledPins[3] = {3,5,6};

int posHi = 105;
int posLo = 75;

int pos[3];
boolean dir[] = {true,true,false};

int ledBright[3];

int speedLo = 80;
int speedHi = 5;

int potPin = 0;

void setup() 
{ 
  Serial.begin(9600);
  for (int i=0;i<3;i++){
    myservo[i].attach(servoPins[i]);
  }

  pos[0] = posLo;
  pos[1] = (posHi+posLo)/2;
  pos[2] = posHi;
  servoWrite();

  ledBrightness();
  ledWrite();

} 

void loop() 
{ 
  int runSpeed = map(analogRead(potPin),0,1023,speedLo,speedHi);

  for (int i=0;i<3;i++){
    if ((pos[i] == posLo && dir[i] == false) || (pos[i] == posHi && dir[i] == true)){
      dir[i] = !dir[i]; 
    }
    if (dir[i] == true){
      pos[i]++;
    } 
    else {
      pos[i]--; 
    }
  }
  ledBrightness();
  ledWrite();

  servoWrite();
  delay(runSpeed);

  Serial.print(pos[0]);Serial.print(",");Serial.print(pos[1]);Serial.print(",");Serial.println(pos[2]);
}

void servoWrite(){
  for (int i=0;i<3;i++){
   myservo[i].write(pos[i]);
  } 
}

void ledBrightness(){
  for (int i=0;i<3;i++){
    if (pos[i] > (int((posLo+posHi)/2)-5) && pos[i] < (int((posLo+posHi)/2)+5)){
      ledBright[i] = 255;
    }
    else{
      ledBright[i] = 0;
    }
  }
}
void ledWrite(){
 for(int i=0;i<3;i++){
   analogWrite(ledPins[i],ledBright[i]);
 } 
}

Here is the Kinetic Sculpture 02 code in Processing:

import processing.serial.*;

import cc.arduino.*;

Arduino arduino;

void setup() {
  size(768, 200);

  println(Arduino.list());

  arduino = new Arduino(this, Arduino.list()[0], 57600);

  arduino.pinMode(9, Arduino.SERVO);
  arduino.pinMode(10, Arduino.SERVO);
  arduino.pinMode(11, Arduino.SERVO);

}

void draw() {
  background(constrain(mouseX / 3, 0, 255));

  if (mouseX <= 127){
    arduino.analogWrite(3, int(map(mouseX,0,127,0,255)));
    arduino.analogWrite(5, 0);
    arduino.analogWrite(6, 0);
  }
  else if (mouseX <= 255 && mouseX > 127){
    arduino.analogWrite(3, int(map(mouseX,128,255,255,0)));
    arduino.analogWrite(5, 0);
    arduino.analogWrite(6, 0);
  }
  else if (mouseX <= 383 && mouseX > 255){
    arduino.analogWrite(5, int(map(mouseX,256,383,0,255)));
    arduino.analogWrite(3, 0);
    arduino.analogWrite(6, 0);
  }
  else if (mouseX <= 511 && mouseX > 383){
    arduino.analogWrite(5, int(map(mouseX,384,511,255,0)));
    arduino.analogWrite(3, 0);
    arduino.analogWrite(6, 0);
  }
  else if (mouseX <= 639 && mouseX > 511){
    arduino.analogWrite(6, int(map(mouseX,512,639,0,255)));
    arduino.analogWrite(5, 0);
    arduino.analogWrite(3, 0);
  }
  else if (mouseX <= 767 && mouseX > 639){
    arduino.analogWrite(6, int(map(mouseX,640,767,255,0)));
    arduino.analogWrite(5, 0);
    arduino.analogWrite(3, 0);
  }

  arduino.servoWrite(9, int(map(mouseY,0,200,80,100)));
  arduino.servoWrite(10, int(map(mouseY,0,200,80,100)));
  arduino.servoWrite(11, int(map(mouseY,0,200,80,100)));

}

Here is the final code, again, most of which was ripped out of the User 3D example:

/* --------------------------------------------------------------------------
 * SimpleOpenNI User3d Test
 * --------------------------------------------------------------------------
 * Processing Wrapper for the OpenNI/Kinect 2 library
 * http://code.google.com/p/simple-openni
 * --------------------------------------------------------------------------
 * prog:  Max Rheiner / Interaction Design / Zhdk / http://iad.zhdk.ch/
 * date:  12/12/2012 (m/d/y)
 * ----------------------------------------------------------------------------
 */

import SimpleOpenNI.*;
import processing.serial.*;

import cc.arduino.*;

Arduino arduino;

SimpleOpenNI context;
float        zoomF =0.5f;
float        rotX = radians(180);  // by default rotate the hole scene 180deg around the x-axis, 
                                   // the data from openni comes upside down
float        rotY = radians(0);
boolean      autoCalib=true;

PVector      bodyCenter = new PVector();
PVector      bodyDir = new PVector();
PVector      com = new PVector();                                   
PVector      com2d = new PVector();                                   
color[]       userClr = new color[]{ color(255,0,0),
                                     color(0,255,0),
                                     color(0,0,255),
                                     color(255,255,0),
                                     color(255,0,255),
                                     color(0,255,255)
                                   };

void setup()
{
  size(1024,768,P3D);  // strange, get drawing error in the cameraFrustum if i use P3D, in opengl there is no problem

  // Prints out the available serial ports.
  println(Arduino.list());

  // Modify this line, by changing the "0" to the index of the serial
  // port corresponding to your Arduino board (as it appears in the list
  // printed by the line above).
  arduino = new Arduino(this, Arduino.list()[0], 57600);

  arduino.pinMode(9, Arduino.SERVO);
  //arduino.pinMode(10, Arduino.SERVO);
  arduino.pinMode(11, Arduino.SERVO);

  context = new SimpleOpenNI(this);
  if(context.isInit() == false)
  {
     println("Can't init SimpleOpenNI, maybe the camera is not connected!"); 
     exit();
     return;  
  }

  // disable mirror
  context.setMirror(false);

  // enable depthMap generation 
  context.enableDepth();

  // enable skeleton generation for all joints
  context.enableUser();

  stroke(255,255,255);
  smooth();  
  perspective(radians(45),
              float(width)/float(height),
              10,150000);
 }

void draw()
{
  // update the cam
  context.update();

  background(0,0,0);

  // set the scene pos
  translate(width/2, height/2, 0);
  rotateX(rotX);
  rotateY(rotY);
  scale(zoomF);

  int[]   depthMap = context.depthMap();
  int[]   userMap = context.userMap();
  int     steps   = 3;  // to speed up the drawing, draw every third point
  int     index;
  PVector realWorldPoint;

  translate(0,0,-1000);  // set the rotation center of the scene 1000 infront of the camera

  // draw the pointcloud
  beginShape(POINTS);
  for(int y=0;y < context.depthHeight();y+=steps)
  {
    for(int x=0;x < context.depthWidth();x+=steps)
    {
      index = x + y * context.depthWidth();
      if(depthMap[index] > 0)
      { 
        // draw the projected point
        realWorldPoint = context.depthMapRealWorld()[index];
        if(userMap[index] == 0)
          stroke(100); 
        else
          stroke(userClr[ (userMap[index] - 1) % userClr.length ]);        

        point(realWorldPoint.x,realWorldPoint.y,realWorldPoint.z);
      }
    } 
  } 
  endShape();

  // draw the skeleton if it's available
  int[] userList = context.getUsers();
  for(int i=0;i<userList.length;i++)
  {
    if(context.isTrackingSkeleton(userList[i]))
      drawSkeleton(userList[i]);

    // draw the center of mass
    if(context.getCoM(userList[i],com))
    {
      stroke(100,255,0);
      strokeWeight(1);
      beginShape(LINES);
        vertex(com.x - 15,com.y,com.z);
        vertex(com.x + 15,com.y,com.z);

        vertex(com.x,com.y - 15,com.z);
        vertex(com.x,com.y + 15,com.z);

        vertex(com.x,com.y,com.z - 15);
        vertex(com.x,com.y,com.z + 15);
      endShape();

      fill(0,255,100);
      text(Integer.toString(userList[i]),com.x,com.y,com.z);
    }

  }    

  // draw the kinect cam
  context.drawCamFrustum();
}

// draw the skeleton with the selected joints
void drawSkeleton(int userId)
{
  strokeWeight(3);

  // to get the 3d joint data
  drawLimb(userId, SimpleOpenNI.SKEL_HEAD, SimpleOpenNI.SKEL_NECK);

  drawLimb(userId, SimpleOpenNI.SKEL_NECK, SimpleOpenNI.SKEL_LEFT_SHOULDER);
  drawLimb(userId, SimpleOpenNI.SKEL_LEFT_SHOULDER, SimpleOpenNI.SKEL_LEFT_ELBOW);
  drawLimb(userId, SimpleOpenNI.SKEL_LEFT_ELBOW, SimpleOpenNI.SKEL_LEFT_HAND);

  drawLimb(userId, SimpleOpenNI.SKEL_NECK, SimpleOpenNI.SKEL_RIGHT_SHOULDER);
  drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_SHOULDER, SimpleOpenNI.SKEL_RIGHT_ELBOW);
  drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_ELBOW, SimpleOpenNI.SKEL_RIGHT_HAND);

  drawLimb(userId, SimpleOpenNI.SKEL_LEFT_SHOULDER, SimpleOpenNI.SKEL_TORSO);
  drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_SHOULDER, SimpleOpenNI.SKEL_TORSO);

  drawLimb(userId, SimpleOpenNI.SKEL_TORSO, SimpleOpenNI.SKEL_LEFT_HIP);
  drawLimb(userId, SimpleOpenNI.SKEL_LEFT_HIP, SimpleOpenNI.SKEL_LEFT_KNEE);
  drawLimb(userId, SimpleOpenNI.SKEL_LEFT_KNEE, SimpleOpenNI.SKEL_LEFT_FOOT);

  drawLimb(userId, SimpleOpenNI.SKEL_TORSO, SimpleOpenNI.SKEL_RIGHT_HIP);
  drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_HIP, SimpleOpenNI.SKEL_RIGHT_KNEE);
  drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_KNEE, SimpleOpenNI.SKEL_RIGHT_FOOT);  

  handCircle(userId,SimpleOpenNI.SKEL_RIGHT_HAND);
  handCircle(userId,SimpleOpenNI.SKEL_LEFT_HAND);

  drawbuttons(userId,SimpleOpenNI.SKEL_LEFT_HAND,SimpleOpenNI.SKEL_RIGHT_HAND,SimpleOpenNI.SKEL_HEAD); 
  // draw body direction
  getBodyDirection(userId,bodyCenter,bodyDir);

  bodyDir.mult(200);  // 200mm length
  bodyDir.add(bodyCenter);

  stroke(255,200,200);
  line(bodyCenter.x,bodyCenter.y,bodyCenter.z,
       bodyDir.x ,bodyDir.y,bodyDir.z);

  strokeWeight(1);

}
void drawbuttons(int userId,int jointType1,int jointType2,int jointType3){
  PVector jointPos1 = new PVector();
  PVector jointPos2 = new PVector();
  PVector jointPos3 = new PVector();
  float  confidence;
  confidence = context.getJointPositionSkeleton(userId,jointType1,jointPos1);
  confidence = context.getJointPositionSkeleton(userId,jointType2,jointPos2);
  confidence = context.getJointPositionSkeleton(userId,jointType3,jointPos3);

  if(dist(jointPos1.x,jointPos1.y,-600,500)<100){
    fill(255,255,0);
    arduino.analogWrite(6,255);
  }
  else{
    fill(0,255,255);
    arduino.analogWrite(6,0);
  }
  //fill(abs(constrain(jointPos1.x,0,255)), abs(constrain(jointPos1.y,0,255)), abs(constrain(jointPos1.z,0,255)));
  pushMatrix();
  translate(-100 ,100,0);
  ellipse(0,0,50,50);
  popMatrix();

  if(dist(jointPos2.x,jointPos2.y,600,500)<100){
    fill(255,255,0);
    arduino.analogWrite(3,255);
  }
  else{
    fill(0,255,255);
    arduino.analogWrite(3,0);
  }

  pushMatrix();
  translate(100 ,100,0);
  ellipse(0,0,50,50);
  popMatrix();
  print(jointPos1.z);print(",");print(jointPos2.z);print(",");println(jointPos3.z);

  if(dist(jointPos3.x,jointPos3.y,0,500)<100){
    arduino.analogWrite(5,255);
  }
  else{
    arduino.analogWrite(5,0);
  }

  arduino.servoWrite(11, int(map(abs(jointPos1.z),1000,2000,80,100)));
  //arduino.servoWrite(10, int(map(abs(jointPos3.z),1000,2000,80,100)));
  arduino.servoWrite(9, int(map(abs(jointPos2.z),1000,2000,80,100)));

}
void handCircle(int userId,int jointType1)
{
  PVector jointPos1 = new PVector();
  float  confidence;
  confidence = context.getJointPositionSkeleton(userId,jointType1,jointPos1);

  fill(200, 100, 0);
  pushMatrix();
  translate(jointPos1.x,jointPos1.y,jointPos1.z);
  ellipse(0,0,50,50);
  popMatrix();
}

void drawLimb(int userId,int jointType1,int jointType2)
{
  PVector jointPos1 = new PVector();
  PVector jointPos2 = new PVector();
  float  confidence;

  // draw the joint position
  confidence = context.getJointPositionSkeleton(userId,jointType1,jointPos1);
  confidence = context.getJointPositionSkeleton(userId,jointType2,jointPos2);

  stroke(255,0,0,confidence * 200 + 55);
  line(jointPos1.x,jointPos1.y,jointPos1.z,
       jointPos2.x,jointPos2.y,jointPos2.z);

  drawJointOrientation(userId,jointType1,jointPos1,50);
}

void drawJointOrientation(int userId,int jointType,PVector pos,float length)
{
  // draw the joint orientation  
  PMatrix3D  orientation = new PMatrix3D();
  float confidence = context.getJointOrientationSkeleton(userId,jointType,orientation);
  if(confidence < 0.001f) 
    // nothing to draw, orientation data is useless
    return;

  pushMatrix();
    translate(pos.x,pos.y,pos.z);

    // set the local coordsys
    applyMatrix(orientation);

    // coordsys lines are 100mm long
    // x - r
    stroke(255,0,0,confidence * 200 + 55);
    line(0,0,0,
         length,0,0);
    // y - g
    stroke(0,255,0,confidence * 200 + 55);
    line(0,0,0,
         0,length,0);
    // z - b    
    stroke(0,0,255,confidence * 200 + 55);
    line(0,0,0,
         0,0,length);
  popMatrix();
}

// -----------------------------------------------------------------
// SimpleOpenNI user events

void onNewUser(SimpleOpenNI curContext,int userId)
{
  println("onNewUser - userId: " + userId);
  println("\tstart tracking skeleton");

  context.startTrackingSkeleton(userId);
}

void onLostUser(SimpleOpenNI curContext,int userId)
{
  println("onLostUser - userId: " + userId);
}

void onVisibleUser(SimpleOpenNI curContext,int userId)
{
  //println("onVisibleUser - userId: " + userId);
}

// -----------------------------------------------------------------
// Keyboard events

void keyPressed()
{
  switch(key)
  {
  case ' ':
    context.setMirror(!context.mirror());
    break;
  }

  switch(keyCode)
  {
    case LEFT:
      rotY += 0.1f;
      break;
    case RIGHT:
      // zoom out
      rotY -= 0.1f;
      break;
    case UP:
      if(keyEvent.isShiftDown())
        zoomF += 0.01f;
      else
        rotX += 0.1f;
      break;
    case DOWN:
      if(keyEvent.isShiftDown())
      {
        zoomF -= 0.01f;
        if(zoomF < 0.01)
          zoomF = 0.01;
      }
      else
        rotX -= 0.1f;
      break;
  }
}

void getBodyDirection(int userId,PVector centerPoint,PVector dir)
{
  PVector jointL = new PVector();
  PVector jointH = new PVector();
  PVector jointR = new PVector();
  float  confidence;

  // draw the joint position
  confidence = context.getJointPositionSkeleton(userId,SimpleOpenNI.SKEL_LEFT_SHOULDER,jointL);
  confidence = context.getJointPositionSkeleton(userId,SimpleOpenNI.SKEL_HEAD,jointH);
  confidence = context.getJointPositionSkeleton(userId,SimpleOpenNI.SKEL_RIGHT_SHOULDER,jointR);

  // take the neck as the center point
  confidence = context.getJointPositionSkeleton(userId,SimpleOpenNI.SKEL_NECK,centerPoint);

  /*  // manually calc the centerPoint
  PVector shoulderDist = PVector.sub(jointL,jointR);
  centerPoint.set(PVector.mult(shoulderDist,.5));
  centerPoint.add(jointR);
  */

  PVector up = PVector.sub(jointH,centerPoint);
  PVector left = PVector.sub(jointR,centerPoint);

  dir.set(up.cross(left));
  dir.normalize();
}

Leave a Reply

Your email address will not be published. Required fields are marked *