Video cube in processing v4.4

Sorry for the lack of activity on this blog – I’ve been working on stuff for SSiC as well as some other heinous group project…

So the video cube has come along quite nicely but I have now run up against some more limitations. Basically the code below uses the processing.video library to read a quicktime .mov file into an array of PImages. Pixels are then picked out of the cube to create a view image (also a PImage object). This works fine for smaller movies, but larger ones run a bit jumpily on my aging computer (which is why the pic that follows is so tiny). The code processes input from 2 mice to proved an x-y center of affect, and a depth-radius (z-q) degree of affect. Mice are switchable by pressing the space key just in case you have a preferred mouse for x-y (or z-q) and it’s not the right device number for the ‘default’ way the code is set up. The proControll library is required to use this. See the processing website or the link in a previous post here.

It takes a while to load a movie into a cube, so I am trying to build a rudimentary system for saving the video cube array. Unfortunately PImage is not Serializable, so I either have to find my way around the processing method saveBytes() (or whatever it’s called…), or hack the PImage class, or make my own alternative SPImage class (for Serializable PImage). More on this in a sec.

I have also modified the video library so that it uses the class name ‘PVideo’ rather than just ‘Video’ as the library implements out-of-the-box. While I was doing that I added a couple of methods to retrieve the number of frames and an average frames-per-second value which seemed to be something that people on the processing discourse forum were after. It was a laborious process to figure out how to get xCode to make a working library, but now that I’ve figured it out I’ll have to post up the project file and build folder somewhere so others can use it. At the very least the library files are needed so the code below can be run. Get a zip from here. Unzip it and put the video folder inside the libraries folder in processing (but make sure you keep a copy of the original video folder somehwere 1st).

So given the headaches already encountered with modifying processing library code, I’m not too keen to hack the PImage class. I think my first attempt will be to re-purpose PImage as SPImage and make it an entirely separate package that accomplishes what PImage does (so use is mostly transferable for existing code), and probably add some sort of serialize and restore methods to the class. Anyway, here’s the code for version 4.4

//processing libraries (note that processing.video.* is customised)
import processing.video.*;
import procontroll.*;
//quicktime libraries
import quicktime.*;
import quicktime.std.*;
import quicktime.qd.*;
//java libraries (for the file I/O)
import java.io.*;
import java.util.*;
import java.io.ObjectOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;

//video variables and objects
PMovie m;
float pos = 0;
float step;
int frames;
float fps;
int movWidth;
int movHeight;
int currFrame;
PImage[] videoCube;
PImage view;
boolean cubeLoaded;
int left, right, top, btm;
double[][] distLookup;
String movieName;

//mouse variables and objects
ControllIO controll;
ControllSlider slider_m1X, slider_m1Y;
ControllSlider slider_m2X, slider_m2Y;
float m1X, m1Y;
float m2X, m2Y;
boolean miceSwapped;
float m2YscaleFactor;

void setup()
{
//movieName = “station”;//don’t include the .mov extension, it’s added if req’d
movieName = “titanfail”;
//movieName = “bmbwdscrn”;
//movieName = “spore04″;

//check if a cube is already available to load
boolean exists = (new File(sketchPath(“data/” + movieName + “.dat”))).exists();
if (exists)
{
//videoCube file exists so load it
cubeLoaded = true;
//loadVideoCube(movieName + “.ser”);
println(“cube file exists”);
} else {
//videoCube file does not exist, so load the video
println(“cube file does not exist”);
m = new PMovie(this, movieName + “.mov”);
frames = m.calcTotalFrames();
fps = m.getAverageFPS();
cubeLoaded = false;

try
{
movWidth = m.movie.getNaturalBoundsRect().getWidth();
movHeight = m.movie.getNaturalBoundsRect().getHeight();
}
catch (StdQTException e)
{
System.err.println(“Movie exception”);
e.printStackTrace();
}
}

step = 1.0 / fps;
currFrame = 0;

println(“frames:” + frames);
println(“fps: ” + fps);
println(“movWidth: ” + movWidth + ” movHeight: ” + movHeight);

size(movWidth, movHeight);
background(0);
stroke(255);
noFill();
frameRate(24);
noCursor();//works, but not always in this app.
colorMode(HSB);

//make an array to store the videocube in
videoCube = new PImage[frames];
//and make the display array
view = new PImage(movWidth, movHeight);

//make an array for distance lookup values
distLookup = new double[movWidth][movHeight];
makeLookupArray();

// The pause() command is problematic for frame advance,
// use this syntax instead.
m.speed(0);
m.loop();
m.jump(0);

//mouse/proControll stuff
controll = ControllIO.getInstance(this);
controll.printDevices();

ControllDevice device = controll.getDevice(1);//gets one mouse
slider_m1X = device.getSlider(0);
slider_m1Y = device.getSlider(1);
m1X = width/2;
m1Y = height/2;

device = controll.getDevice(2);//gets another mouse
slider_m2X = device.getSlider(0);
slider_m2Y = device.getSlider(1);
m2X = width/2;
m2Y = height/2;

miceSwapped = false;

//calculate the factor used to bring m2Y which is used for depth (z/t axis of video cube),
//into a range between 0 and the number of frames in movie (-1 because the frame count
//starts at 1, but all array and pixel counts start at 0)
m2YscaleFactor = (float)(frames – 1) / (float)height;
println(“m2YscaleFactor: ” + m2YscaleFactor);
}

void movieEvent(PMovie m)
{
m.read();
pos = (pos + step) % m.duration();
m.jump(pos);

if(currFrame = frames)
{
//movie read complete, so stop it
//m.noLoop();
m.stop();
cubeLoaded = true;
//now save the cube to file
saveVideoCube(movieName + “.dat”);//save fn not currently working
//m.dispose();
println(“videoCube.length: ” + videoCube.length);
}
}
}

void draw()
{
background(255);
mouseUpdate();

if(cubeLoaded)
{
//view = videoCube[0];
makeView();
rect(left, top, right – left, btm – top);
image(view, 0, 0);
}

//draw crosshairs for mouse 1 (for debugging purposes)
//line(m1X – 3.0, m1Y, m1X + 3.0, m1Y);
//line(m1X, m1Y – 3.0, m1X, m1Y + 3.0);

//draw tracking bars for mouse 2
line(m2X, height, m2X, height – 10);
line(0, m2Y, 10, m2Y);
}

void makeView()
{
//pick pixels from the videocube to display
//m2X is used for radius of affect (so a halved copy is used in this fn)
//m2Y is used for depth of affect
//m2Xd is a double cast from m2X which is a float, and the program
//has to do that because floats are req’d for the mouse stuff
//but a double is req’d for Math.sqrt() used here.
//m2Ynormalised brings the depth of affect to within the range of frames in
//the movie (the z or t dimension of the videocube)
//m2Xdnormalised is used to pick z pixels closer to the full depth of affect
//(as determined by m2Ynormalised) near the centre of the affect area,
//and towards 0 depth at the edge of the affect area
//so it produces a factor of 1 when distBetweenPixels == m2Xd,
//and produces a factor of 0 when distBetweenPixels == 0

int index;
double m2Ynormalised;
m2Ynormalised = ((double)m2Y * m2YscaleFactor);
float m2Xhalf = m2X * 0.5;
double m2Xd = (double)m2Xhalf;
double m2Xdnormalised = 1.0 / m2Xd;
double distBetweenPixels;

//get bounds of affected area
left = (int)(m1X – m2Xhalf);
right = (int)(m1X + m2Xhalf);
top = (int)(m1Y – m2Xhalf);
btm = (int)(m1Y + m2Xhalf);
//ensure bounds are within window display area
if(left width) right = width;
if(top height) btm = height;

//preload the view with the first frame
view.pixels = (int[])videoCube[0].pixels.clone();

//process the affect area
view.loadPixels();
//
for(int i = top; i = distBetweenPixels)
{
//pixel is inside circle so process it for depth from the centre
view.pixels[index] = videoCube[(int)((m2Xd - distBetweenPixels) * m2Xdnormalised * m2Ynormalised)].pixels[index];
//how this works: (m2Xd – distBetweenPixels) can range from 0 to m2Xd (note the if statement condition)
//m2Xdnormalised is a factor that will convert numbers ranging from 0 to m2Xd into a range between 0 and 1
//so this factor between 0 and 1 is then applied to m2Ynormalised to produce a z/t depth based on the distance
//from the centre
}
}
}
view.updatePixels();
}

void makeLookupArray()
{
int x, y;
double xd, yd;
for(x = 0; x width) m1X = width;
if(m1Y > height) m1Y = height;

m2X += slider_m2X.getValue();
m2Y += slider_m2Y.getValue();
if(m2X width) m2X = width;
if(m2Y > height) m2Y = height;
} else {
m2X += slider_m1X.getValue();
m2Y += slider_m1Y.getValue();
if(m2X width) m2X = width;
if(m2Y > height) m2Y = height;

m1X += slider_m2X.getValue();
m1Y += slider_m2Y.getValue();
if(m1X width) m1X = width;
if(m1Y > height) m1Y = height;
}
}

void keyPressed()
{
if(key == ‘ ‘)
{
//space bar toggles mouse assignments
miceSwapped = !miceSwapped;
}
}

void loadVideoCube(String filename)
{
//IMPORTANT! this fn needs to be written to set the following:
/*
cubeLoaded = true;
movWidth = ?;
movHeight = ?;
frames = ?;
fps = ?;

*/

FileInputStream fis = null;
ObjectInputStream in = null;
try
{
fis = new FileInputStream(filename);
in = new ObjectInputStream(fis);
//videoCube = (Array)in.readObject();
videoCube = (processing.core.PImage[])in.readObject();
in.close();
}
catch(IOException e)
{
e.printStackTrace();
}
catch(ClassNotFoundException e)
{
e.printStackTrace();
}
}

void saveVideoCube(String filename)
{
/* not currently working because PImage is not Serializable
FileOutputStream fos = null;
ObjectOutputStream out = null;
try
{
fos = new FileOutputStream(filename);
out = new ObjectOutputStream(fos);
out.writeObject(videoCube);
out.close();
}
catch(IOException e)
{
e.printStackTrace();
}
*/
}