jit.net.recv OpenCV.IplImage C#

Sep 17, 2013 at 12:15pm

jit.net.recv OpenCV.IplImage C#

I been trying to develop a way to send OpenCV.IplImage buffers from C# .Net app into Max via jit.net.recv

Mostly trying to port some version of

https://github.com/bakercp/ofxJitterNetworkSender

into C# using as reference some of the code from Mu (which interoperates with Unity3D)

http://disis.music.vt.edu/main/portfolio.php

I think i have a decent grasp of the structure and headers that i need to pass

http://cycling74.com/sdk/MaxSDK-6.1.3/html/chapter_jit_networking.html

and i managed to find some c# code to handle the whole little endian big endian thing.

After a few tries i finally managed to stream several frames (without any broken connections or data corrupt warnings) but all i seem to get is a whole lot of black regardless of what image i send. And i see no real way to debug it on Max side. Tried different image formats before sending and such but i’m limited to the OpenCV IplImage format.

I’m running out of ideas of what it could be. Does anyone have any thoughts on what could be the problem? Or any idea how to debug this?

Only smart idea i had was trying to sniff network packets and compare packets from jit.net.send with the external ones, but the only sniffers i found couldn’t handle 64bit windows aplications.

Some relevant code:
private int JIT_MATRIX_MAX_DIMCOUNT = 32;
private int JIT_MATRIX_MAX_PLANECOUNT = 32;
private int JIT_MATRIX_TYPE_CHAR = 0;
private int JIT_MATRIX_TYPE_LONG = 1;
private int JIT_MATRIX_TYPE_FLOAT32 = 2;
private int JIT_MATRIX_TYPE_FLOAT64 = 3;

private string JIT_MATRIX_PACKET_ID = "JMTX";
private string JIT_MATRIX_LATENCY_PACKET_ID = "JMLP";
private string JIT_MESSAGE_PACKET_ID = "JMMP";

double lastSent;

public enum Endianness
{
BigEndian,
LittleEndian
}

private static void MaybeAdjustEndianness(Type type, byte[] data, Endianness endianness, int startOffset = 0)
{
if ((BitConverter.IsLittleEndian) == (endianness == Endianness.LittleEndian))
{
// nothing to change => return
return;
}

foreach (var field in type.GetFields())
{
var fieldType = field.FieldType;
if (field.IsStatic)
// don't process static fields
continue;

if (fieldType == typeof(string))
// don't swap bytes for strings
continue;

var offset = Marshal.OffsetOf(type, field.Name).ToInt32();

// check for sub-fields to recurse if necessary
var subFields = fieldType.GetFields().Where(subField => subField.IsStatic == false).ToArray();

var effectiveOffset = startOffset + offset;

if (subFields.Length == 0)
{
Array.Reverse(data, effectiveOffset, Marshal.SizeOf(fieldType));
}
else
{
// recurse
MaybeAdjustEndianness(fieldType, data, endianness, effectiveOffset);
}
}
}

internal static T BytesToStruct<T>(byte[] rawData, Endianness endianness) where T : struct
{
T result = default(T);

MaybeAdjustEndianness(typeof(T), rawData, endianness);

GCHandle handle = GCHandle.Alloc(rawData, GCHandleType.Pinned);

try
{
IntPtr rawDataPtr = handle.AddrOfPinnedObject();
result = (T)Marshal.PtrToStructure(rawDataPtr, typeof(T));
}
finally
{
handle.Free();
}

return result;
}

internal static byte[] StructToBytes<T>(T data, Endianness endianness) where T : struct
{
byte[] rawData = new byte[Marshal.SizeOf(data)];
GCHandle handle = GCHandle.Alloc(rawData, GCHandleType.Pinned);
try
{
IntPtr rawDataPtr = handle.AddrOfPinnedObject();
Marshal.StructureToPtr(data, rawDataPtr, false);
}
finally
{
handle.Free();
}

MaybeAdjustEndianness(typeof(T), rawData, endianness);

return rawData;
}

public struct t_jit_net_packet_header {
public Int32 id;
public Int32 size; //size of packet to come
}
private t_jit_net_packet_header m_chunkHeader;
private t_jit_net_packet_header m_messageHeader;

public unsafe struct t_jit_net_packet_matrix {
public Int32 id;
public Int32 size;
public Int32 planecount;
public Int32 type; //0=char,1=long,2=float32,3=float64
public Int32 dimcount;
public fixed Int32 dim[32];
public fixed Int32 dimstride[32];
public Int32 datasize;
public double time; //64 bit double precision float
}
private t_jit_net_packet_matrix m_matrixHeader;

public struct t_jit_net_packet_latency {
public Int32 id;
public double client_time_original;
public double server_time_before_data;
public double server_time_after_data;
}
private t_jit_net_packet_latency m_latencyPacket;

private void sendFrame(IplImage input) {

// setup matrix

//byte[] temp = Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID);
//Array.Reverse(temp);
Int32 matrix_id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);

Int32 planecount = input.NumChannels;
Int32 typeSize = input.Depth;
Int32 type = JIT_MATRIX_TYPE_CHAR;
Int32 width = input.Width;
Int32 height = input.Height;

m_matrixHeader.id = matrix_id;
m_matrixHeader.size = 288;
m_matrixHeader.planecount = planecount;
m_matrixHeader.type = type;
m_matrixHeader.dimcount = 2;

unsafe
{
fixed (Int32* dim = m_matrixHeader.dim)
{
dim[0] = width;
dim[1] = height;

int i2 = 2;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dim[i2] = 0;
i2++;
}

fixed (Int32* dimstride = m_matrixHeader.dimstride)
{

dimstride[0] = typeSize * planecount;

dimstride[1] = (Int32)(dim[0] * dimstride[0]);
dimstride[2] = (Int32)(dim[1] * dimstride[1]);

i2 = 3;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dimstride[i2] = 0;
i2++;
}

m_matrixHeader.datasize = (Int32)( dimstride[1] * dim[1] );
}
}
}

// no clue why jit.net.recv expects 2 headers with same info, maybe for a possible multi-chunk stream?!
// according to other implementations it seems to ignore the size element of the chunkHeader
// and the id element of the matrixHeader

// send chunk header
m_chunkHeader.id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
m_chunkHeader.size = Marshal.SizeOf(m_matrixHeader);
var chunkHeader = StructToBytes(m_chunkHeader, Endianness.BigEndian);
if (myClient.Connected)
myStream.Write(chunkHeader, 0, chunkHeader.Length);

// send matrix header
var matrixHeader = StructToBytes(m_matrixHeader, Endianness.BigEndian);
if (myClient.Connected)
myStream.Write(matrixHeader, 0, matrixHeader.Length);

// send matrix array
int size = input.Height * input.Width * input.NumChannels;
byte[] managedArray = new byte[size];
Marshal.Copy(input.ImageData, managedArray, 0, size);
Array.Reverse(managedArray);
if (myClient.Connected)
myStream.Write(managedArray, 0, managedArray.Length);

}

#265557
Sep 20, 2013 at 6:24am

did a little progress, after a lot of wireshark sniffing, managed to get text messages transmitting from c# and receiving on jit.net.recv
just need to do the same sniffing trial and error to figure out what’s wrong with the larger matrix frames.
my sourcecode is fully available here:

https://bitbucket.org/artica/bonsai-with-jitnetsend/

#265793
Sep 23, 2013 at 9:52am

managed to make some progress, sent some test jit.net.send packets between 2 machines and captured packets for analysis with wireshark. Here is a run-down of my analysis:

https://bitbucket.org/artica/bonsai-with-jitnetsend/wiki/Example%204×4

I rewrote the code to match the actual packet example and managed to get 4×4 images sending, but the image glitches up on 8×8 and starts throwing errors in higher resolutions. I’m guessing it’s an IplImage format problem at this point. Here is the relevant code:

private string JIT_MATRIX_PACKET_ID = "JMTX";
public unsafe struct t_jit_net_packet_matrix {
public Int32 id;
public Int32 size;
public Int32 planecount;
public Int32 type; //0=char,1=long,2=float32,3=float64
public Int32 dimcount;
//public fixed Int32 dim[32];
//public fixed Int32 dimstride[32];
//public Int32 datasize;
//public double time; //64 bit double precision float
}
private t_jit_net_packet_matrix m_matrixHeader;

private void sendFrame(IplImage input) {

// setup matrix

Int32 matrix_id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);

Int32 planecount = input.NumChannels;
Int32 typeSize = input.Depth;
Int32 type = JIT_MATRIX_TYPE_CHAR;
Int32 width = 16;// input.Width;
Int32 height = 16;// input.Height;

m_matrixHeader.id = matrix_id;
m_matrixHeader.size = 288;
m_matrixHeader.planecount = planecount;
m_matrixHeader.type = type;
m_matrixHeader.dimcount = 2;

Int32[] dim = new Int32[32];
dim[0] = width;
dim[1] = height;
int i2 = 2;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dim[i2] = 1;
i2++;
}

Int32[] dimstride = new Int32[32];
dimstride[0] = planecount;
dimstride[1] = width * height;
i2 = 2;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dimstride[i2] = 0;
i2++;
}

Int32 datasize = planecount * width * height;

m_chunkHeader.id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
m_chunkHeader.size = sizeof(Int32) * (6 + 32 + 32) + sizeof(double); //should be 288 bytes
byte[] chunkHeader = StructToBytes(m_chunkHeader, Endianness.LittleEndian);
//Console.WriteLine(BitConverter.ToString(chunkHeader));

byte[] matrixHeader = StructToBytes(m_matrixHeader, Endianness.BigEndian);
//Console.WriteLine(BitConverter.ToString(matrixHeader));

byte[] dim_send = new byte[4 * 32];
byte[] dimstride_send = new byte[4 * 32];
for (int i = 0; i < 32; i++)
{
byte[] dimbytes = BitConverter.GetBytes(dim[i]);
Array.Reverse(dimbytes);
System.Buffer.BlockCopy(dimbytes, 0, dim_send, i * 4, dimbytes.Length);

byte[] dimstridebytes = BitConverter.GetBytes(dimstride[i]);
Array.Reverse(dimstridebytes);
System.Buffer.BlockCopy(dimstridebytes, 0, dimstride_send, i * 4, dimstridebytes.Length);
}
//Console.WriteLine(BitConverter.ToString(dim_send));
//Console.WriteLine(BitConverter.ToString(dimstride_send));

byte[] datasize_send = BitConverter.GetBytes(datasize);
Array.Reverse(datasize_send);
//Console.WriteLine(BitConverter.ToString(datasize_send));

double time = 0; //todo: should be elapsed time, not 0
byte[] time_send = BitConverter.GetBytes(time);
Array.Reverse(time_send);
//Console.WriteLine(BitConverter.ToString(time_send));

int size = width * height * 4 * 2;//input.Height * input.Width * input.NumChannels * input.Depth / 4;
byte[] managedArray = new byte[size];
Marshal.Copy(input.ImageData, managedArray, 0, size);
Array.Reverse(managedArray);

//Console.WriteLine(BitConverter.ToString(managedArray));

byte[] output = new byte[chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length + managedArray.Length];

// chunkheader
System.Buffer.BlockCopy(chunkHeader, 0, output, 0, chunkHeader.Length);
// matrixheader
System.Buffer.BlockCopy(matrixHeader, 0, output, chunkHeader.Length, matrixHeader.Length);
// dim
System.Buffer.BlockCopy(dim_send, 0, output, chunkHeader.Length + matrixHeader.Length, dim_send.Length);
// dimstride
System.Buffer.BlockCopy(dimstride_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length, dimstride_send.Length);
// datasize
System.Buffer.BlockCopy(datasize_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length, datasize_send.Length);
// time
System.Buffer.BlockCopy(time_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length, time_send.Length);
// matrix array
System.Buffer.BlockCopy(managedArray, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length, managedArray.Length);

Console.WriteLine(BitConverter.ToString(output));

if (myClient.Connected)
myStream.Write(output, 0, output.Length);

Console.WriteLine("looping");
//if (myClient.Connected)
// myStream.Write(managedArray, 0, managedArray.Length);

//Console.WriteLine(managedArray.ToString());

}

#266029
Oct 1, 2013 at 2:01pm

Figured it out on my own. Here is the working code in case anyone might care to take a look in the future:

private void sendFrame(IplImage input) {

try
{
// setup matrix

Int32 matrix_id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);

Int32 planecount = input.NumChannels;
Int32 typeSize = input.Depth;
Int32 type = JIT_MATRIX_TYPE_CHAR;
Int32 width = input.Width;
Int32 height = input.Height;

m_matrixHeader.id = matrix_id;
m_matrixHeader.size = 288;
m_matrixHeader.planecount = planecount;
m_matrixHeader.type = type;
m_matrixHeader.dimcount = 2;

Int32[] dim = new Int32[32];
dim[0] = width;
dim[1] = height;
int i2 = 2;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dim[i2] = 1;
i2++;
}

Int32[] dimstride = new Int32[32];
dimstride[0] = planecount;
dimstride[1] = width * planecount;
i2 = 2;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dimstride[i2] = 0;
i2++;
}

Int32 datasize = planecount * width * height;

m_chunkHeader.id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
m_chunkHeader.size = sizeof(Int32) * (6 + 32 + 32) + sizeof(double); //should be 288 bytes
byte[] chunkHeader = StructToBytes(m_chunkHeader, Endianness.LittleEndian);
//Console.WriteLine(BitConverter.ToString(chunkHeader));

byte[] matrixHeader = StructToBytes(m_matrixHeader, Endianness.BigEndian);
//Console.WriteLine(BitConverter.ToString(matrixHeader));

byte[] dim_send = new byte[4 * 32];
byte[] dimstride_send = new byte[4 * 32];
for (int i = 0; i < 32; i++)
{
byte[] dimbytes = BitConverter.GetBytes(dim[i]);
Array.Reverse(dimbytes);
System.Buffer.BlockCopy(dimbytes, 0, dim_send, i * 4, dimbytes.Length);

byte[] dimstridebytes = BitConverter.GetBytes(dimstride[i]);
Array.Reverse(dimstridebytes);
System.Buffer.BlockCopy(dimstridebytes, 0, dimstride_send, i * 4, dimstridebytes.Length);
}
//Console.WriteLine(BitConverter.ToString(dim_send));
//Console.WriteLine(BitConverter.ToString(dimstride_send));

byte[] datasize_send = BitConverter.GetBytes(datasize);
Array.Reverse(datasize_send);
//Console.WriteLine(BitConverter.ToString(datasize_send));

double time = 0; //todo: should be elapsed time, not 0
byte[] time_send = BitConverter.GetBytes(time);
Array.Reverse(time_send);
//Console.WriteLine(BitConverter.ToString(time_send));

int size = width * height * 4;//input.Height * input.Width * input.NumChannels * input.Depth / 4;
byte[] managedArray = new byte[size];
Marshal.Copy(input.ImageData, managedArray, 0, size);
Array.Reverse(managedArray);

byte[] output = new byte[chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length + managedArray.Length];

// chunkheader
System.Buffer.BlockCopy(chunkHeader, 0, output, 0, chunkHeader.Length);
// matrixheader
System.Buffer.BlockCopy(matrixHeader, 0, output, chunkHeader.Length, matrixHeader.Length);
// dim
System.Buffer.BlockCopy(dim_send, 0, output, chunkHeader.Length + matrixHeader.Length, dim_send.Length);
// dimstride
System.Buffer.BlockCopy(dimstride_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length, dimstride_send.Length);
// datasize
System.Buffer.BlockCopy(datasize_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length, datasize_send.Length);
// time
System.Buffer.BlockCopy(time_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length, time_send.Length);
// matrix array
System.Buffer.BlockCopy(managedArray, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length, managedArray.Length);

//Console.WriteLine(BitConverter.ToString(output));

if (myClient.Connected)
myStream.Write(output, 0, output.Length);

} catch ( Exception e )
{
Console.WriteLine("Exception: " + e.InnerException.Message);
}
}

#266766

You must be logged in to reply to this topic.