The history, data, and code behind motion capture

We’ve all seen animated movie characters that appear life like in their movements and behavior. Often, their movements are modeled using Motion Capture, where the position of markers are capture with one or multiple cameras. For example, this can record the hands, arms, feet, legs, and head of a person. In this following video example, markers follow the movements of hands playing piano: In another example, one of the earliest recordings of applying motion can be seen driving the animated “Animac” character in 1962. They called the technology “Anthropometric programming” It took about a decade for researchers to start studying why these movements were so lifelike. In 1973, Swedish Psychologist Gunnar Johansson coined the term “Biological Motion” in his article “Visual perception of biological motion and a model for its analysis” published in the journal of Perception and Psychophysics. So how can programmers take advantage of motion capture data to simulate biological motion? One of the easiest ways is to store the locations of points in a list, and store a sequences of these lists (a “frame”) into some other data structure. In the following Processing program, a String data stores a series a frames separated by new line characters (“\n”), and each frame a series of points is separated by semicolons. Each point is a pair of points separated by a comma. Using the String.split function, this data can be split into arrays which are played back one at a time. Note that the Integer.parseInt function is also needed to convert substrings into numbers that can be used by the ellipse function. Each time a new frame is drawn, int frame increases by one, until it reaches the end of the array and loops back to zero (performed using the modulo of lines.length). In this way, the data behind motion capture is brought back to life. [raw] [/raw] The code for the above sketch is below, along with the String data that represents the movement:

int bars = 40;
void setup() {
  size(250,400);
}
void draw() {
  background(0);
  int frame = frameCount % lines.length;
  String line = lines[frame];
  String[] points = line.split(";");
  for(String point : points) {
    String[] coords = point.split(",");
    int col = Integer.parseInt(coords[0]); //use int(coords[0]) for old Processing.js versions
    int row = Integer.parseInt(coords[1]); //use int(coords[1]) for old Processing.js versions
    fill(255);
    noStroke();
    ellipse(row,col,5,5);
  }
  textAlign(LEFT,TOP);
  text("frame: "+frame,5,5);
  textAlign(LEFT,BOTTOM);
  text(line,5,height-65,width-10,50);
}

String data = "56,140;111,112;111,154;164,118;164,148;201,145;204,102;204,155;214,137;267,165;271,98;334,79;335,176\n"+
"56,140;111,112;112,154;164,118;164,148;201,145;203,156;204,102;214,137;267,166;271,99;331,75;333,179\n"+
"56,141;111,113;112,154;164,118;164,149;202,144;203,156;204,103;214,139;267,166;272,101;327,71;332,181\n"+
"56,141;111,113;112,154;164,118;164,149;202,144;203,156;204,103;214,139;267,166;272,101;327,71;332,181\n"+
"55,141;110,113;111,155;163,118;164,150;201,157;202,143;204,104;214,141;266,165;272,103;321,69;331,179\n"+
"55,141;110,113;111,155;163,118;164,150;201,157;202,143;204,104;214,141;266,165;272,103;321,69;331,179\n"+
"54,141;109,113;110,155;162,117;163,151;200,157;203,105;203,141;214,144;266,164;272,105;315,67;331,176\n"+
"53,141;107,113;110,156;161,116;163,153;199,157;202,106;204,138;213,148;265,162;272,108;310,67;332,172\n"+
"52,142;106,112;109,157;160,114;162,155;197,157;202,107;205,135;213,152;265,160;272,111;305,68;333,167\n"+
"51,142;105,112;108,157;159,113;162,157;196,157;201,107;206,131;212,156;265,157;271,114;301,70;334,161\n"+
"50,142;104,112;107,158;158,111;161,159;195,157;200,108;207,127;212,161;265,154;271,118;300,72;334,155\n"+
"50,142;104,111;107,159;158,109;161,161;195,157;200,109;208,122;212,166;265,151;271,122;301,76;336,149\n"+
"50,142;104,111;107,159;158,107;161,163;195,156;200,109;208,117;211,171;266,147;271,125;303,81;337,142\n"+
"50,142;104,110;107,160;158,105;161,165;195,155;200,110;209,112;211,176;266,143;270,129;308,86;337,135\n"+
"50,142;104,110;107,160;158,102;161,167;195,155;200,110;209,107;210,181;266,140;270,132;313,93;337,129\n"+
"51,142;105,109;108,160;159,100;162,169;196,154;201,111;209,186;210,102;267,136;320,101;337,122\n"+
"52,142;106,109;109,160;160,98;162,171;197,153;202,111;208,189;211,97;268,133;327,110;338,117\n"+
"53,141;107,108;110,161;161,96;163,172;199,152;202,111;206,193;211,93;268,129;269,141;333,120;338,112\n"+
"54,141;109,108;110,161;162,95;163,173;200,151;203,111;205,195;212,90;269,126;269,142;337,130;338,107\n"+
"55,141;110,107;111,161;163,93;164,174;201,150;204,111;204,197;213,87;269,144;270,124;338,104;339,141\n"+
"56,141;111,107;112,160;164,92;164,175;203,149;203,199;204,110;213,84;268,144;270,122;338,100;339,150\n"+
"56,140;111,106;112,160;164,91;164,175;201,199;203,148;204,110;213,83;268,144;271,120;336,96;337,158\n"+
"56,140;111,106;111,160;164,91;164,175;201,199;203,148;204,109;214,82;267,144;271,119;334,92;335,163\n"+
"56,140;111,160;112,106;164,91;164,174;201,199;203,109;204,147;214,82;267,143;271,119;331,88;333,167\n"+
"56,139;111,159;112,105;164,91;164,173;202,197;203,108;204,147;214,83;267,142;272,119;327,84;332,168\n"+
"55,139;110,159;111,105;163,172;164,91;201,107;202,195;204,146;214,85;266,140;272,120;321,81;331,166\n"+
"54,139;109,158;110,105;162,171;163,92;200,106;203,146;203,193;214,87;266,137;272,121;315,80;331,163\n"+
"53,138;107,158;110,105;161,169;163,93;199,105;202,146;204,190;213,90;265,135;272,123;310,79;332,159\n"+
"52,138;106,157;109,105;160,168;162,95;197,104;202,146;205,186;213,94;265,131;272,126;305,80;333,154\n"+
"51,138;105,156;108,106;159,166;162,96;196,103;201,146;206,182;212,98;265,128;271,128;301,82;334,149\n"+
"50,138;104,156;107,106;158,163;161,98;195,102;200,147;207,177;212,103;265,124;271,132;300,85;334,142\n"+
"50,138;104,155;107,106;158,161;161,100;195,102;200,147;208,172;212,108;265,121;271,135;301,89;336,136\n"+
"50,138;104,155;107,107;158,159;161,102;195,101;200,148;208,167;211,113;266,117;271,139;303,94;337,129\n"+
"50,138;104,154;107,107;158,157;161,105;195,100;200,148;209,162;211,118;266,113;270,142;308,99;337,122\n"+
"50,138;104,154;107,108;158,155;161,107;195,100;200,149;209,157;210,123;266,110;270,146;313,106;337,116\n"+
"51,138;105,154;108,108;159,153;162,109;196,100;201,150;209,127;210,152;267,107;270,150;320,114;337,110\n"+
"52,138;106,153;109,109;160,151;162,111;197,100;202,150;208,132;211,148;268,104;269,153;327,123;338,104\n"+
"53,138;107,153;110,110;161,150;163,113;199,100;202,151;206,135;211,145;268,102;269,156;333,133;338,99\n"+
"54,139;109,153;110,110;162,149;163,114;200,100;203,152;205,138;212,141;269,100;269,159;337,143;338,94\n"+
"55,139;110,153;111,111;163,148;164,116;201,100;204,141;204,153;213,139;269,161;270,99;338,91;339,153\n"+
"56,139;111,153;112,111;164,148;165,117;203,101;203,143;204,154;213,137;268,163;270,98;338,87;339,163";
String[] lines = data.split("\n");

Note that the original data for this video was parsed from a Video that was produced in 1971 by James Maas, then at Cornell University. This much more complicated Processing script uses a variant of flood-fill to identify and separate the various white points from each other.

import processing.video.*;
Movie myMovie;
int frames = 0;
void setup() {
  size(356, 384);
  background(0);
  myMovie = new Movie(this, "biologicalmotion.mp4");
  myMovie.play();
  
  frameRate(200);
}

void draw() {
  image(myMovie, 0, 0);
}

// Called every time a new frame is available to read
void movieEvent(Movie m) {
  m.read();
  print(++frames+":");
  loadPixels();
  boolean[][] visited = new boolean[height][width];
  boolean first = true;
  for(int row = 0; row < height; row++) {
    for(int col = 0; col < width; col++) { if(!visited[row][col] && brightness(pixels[row*width+col]) > 100) {
        if(!first) print(";");
        print(row+","+col);
        flood(visited,row,col);
        first = false;
      }
    }
  }
  println();
}

void flood(boolean[][] visited, int row, int col) {
  if(row < 0 || col < 0 || row >= height || col >= width) return;
  if(visited[row][col]) return;
  if(brightness(pixels[row*width+col]) < 100) return;
  int[] dr = {-1,1,0,0};
  int[] dc = {0,0,-1,1};
  visited[row][col] = true;
  for(int i = 0; i < 4; i++) flood(visited,row+dr[i],col+dc[i]);

}