jit.net.recv OpenCV.IplImage C#


    Sep 17 2013 | 7:15 pm
    I been trying to develop a way to send OpenCV.IplImage buffers from C# .Net app into Max via jit.net.recv
    Mostly trying to port some version of
    https://github.com/bakercp/ofxJitterNetworkSender
    into C# using as reference some of the code from Mu (which interoperates with Unity3D)
    I think i have a decent grasp of the structure and headers that i need to pass
    http://cycling74.com/sdk/MaxSDK-6.1.3/html/chapter_jit_networking.html
    and i managed to find some c# code to handle the whole little endian big endian thing.
    After a few tries i finally managed to stream several frames (without any broken connections or data corrupt warnings) but all i seem to get is a whole lot of black regardless of what image i send. And i see no real way to debug it on Max side. Tried different image formats before sending and such but i'm limited to the OpenCV IplImage format.
    I'm running out of ideas of what it could be. Does anyone have any thoughts on what could be the problem? Or any idea how to debug this?
    Only smart idea i had was trying to sniff network packets and compare packets from jit.net.send with the external ones, but the only sniffers i found couldn't handle 64bit windows aplications.
    Some relevant code:
    private int JIT_MATRIX_MAX_DIMCOUNT = 32;
    private int JIT_MATRIX_MAX_PLANECOUNT = 32;
    private int JIT_MATRIX_TYPE_CHAR = 0;
    private int JIT_MATRIX_TYPE_LONG = 1;
    private int JIT_MATRIX_TYPE_FLOAT32 = 2;
    private int JIT_MATRIX_TYPE_FLOAT64 = 3;
    private string JIT_MATRIX_PACKET_ID = "JMTX";
    private string JIT_MATRIX_LATENCY_PACKET_ID = "JMLP";
    private string JIT_MESSAGE_PACKET_ID = "JMMP";
    double lastSent;
    public enum Endianness
    {
    BigEndian,
    LittleEndian
    }
    private static void MaybeAdjustEndianness(Type type, byte[] data, Endianness endianness, int startOffset = 0)
    {
    if ((BitConverter.IsLittleEndian) == (endianness == Endianness.LittleEndian))
    {
    // nothing to change => return
    return;
    }
    foreach (var field in type.GetFields())
    {
    var fieldType = field.FieldType;
    if (field.IsStatic)
    // don't process static fields
    continue;
    if (fieldType == typeof(string))
    // don't swap bytes for strings
    continue;
    var offset = Marshal.OffsetOf(type, field.Name).ToInt32();
    // check for sub-fields to recurse if necessary
    var subFields = fieldType.GetFields().Where(subField => subField.IsStatic == false).ToArray();
    var effectiveOffset = startOffset + offset;
    if (subFields.Length == 0)
    {
    Array.Reverse(data, effectiveOffset, Marshal.SizeOf(fieldType));
    }
    else
    {
    // recurse
    MaybeAdjustEndianness(fieldType, data, endianness, effectiveOffset);
    }
    }
    }
    internal static T BytesToStruct(byte[] rawData, Endianness endianness) where T : struct
    {
    T result = default(T);
    MaybeAdjustEndianness(typeof(T), rawData, endianness);
    GCHandle handle = GCHandle.Alloc(rawData, GCHandleType.Pinned);
    try
    {
    IntPtr rawDataPtr = handle.AddrOfPinnedObject();
    result = (T)Marshal.PtrToStructure(rawDataPtr, typeof(T));
    }
    finally
    {
    handle.Free();
    }
    return result;
    }
    internal static byte[] StructToBytes(T data, Endianness endianness) where T : struct
    {
    byte[] rawData = new byte[Marshal.SizeOf(data)];
    GCHandle handle = GCHandle.Alloc(rawData, GCHandleType.Pinned);
    try
    {
    IntPtr rawDataPtr = handle.AddrOfPinnedObject();
    Marshal.StructureToPtr(data, rawDataPtr, false);
    }
    finally
    {
    handle.Free();
    }
    MaybeAdjustEndianness(typeof(T), rawData, endianness);
    return rawData;
    }
    public struct t_jit_net_packet_header {
    public Int32 id;
    public Int32 size; //size of packet to come
    }
    private t_jit_net_packet_header m_chunkHeader;
    private t_jit_net_packet_header m_messageHeader;
    public unsafe struct t_jit_net_packet_matrix {
    public Int32 id;
    public Int32 size;
    public Int32 planecount;
    public Int32 type; //0=char,1=long,2=float32,3=float64
    public Int32 dimcount;
    public fixed Int32 dim[32];
    public fixed Int32 dimstride[32];
    public Int32 datasize;
    public double time; //64 bit double precision float
    }
    private t_jit_net_packet_matrix m_matrixHeader;
    public struct t_jit_net_packet_latency {
    public Int32 id;
    public double client_time_original;
    public double server_time_before_data;
    public double server_time_after_data;
    }
    private t_jit_net_packet_latency m_latencyPacket;
    private void sendFrame(IplImage input) {
    // setup matrix
    //byte[] temp = Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID);
    //Array.Reverse(temp);
    Int32 matrix_id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
    Int32 planecount = input.NumChannels;
    Int32 typeSize = input.Depth;
    Int32 type = JIT_MATRIX_TYPE_CHAR;
    Int32 width = input.Width;
    Int32 height = input.Height;
    m_matrixHeader.id = matrix_id;
    m_matrixHeader.size = 288;
    m_matrixHeader.planecount = planecount;
    m_matrixHeader.type = type;
    m_matrixHeader.dimcount = 2;
    unsafe
    {
    fixed (Int32* dim = m_matrixHeader.dim)
    {
    dim[0] = width;
    dim[1] = height;
    int i2 = 2;
    while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
    {
    dim[i2] = 0;
    i2++;
    }
    fixed (Int32* dimstride = m_matrixHeader.dimstride)
    {
    dimstride[0] = typeSize * planecount;
    dimstride[1] = (Int32)(dim[0] * dimstride[0]);
    dimstride[2] = (Int32)(dim[1] * dimstride[1]);
    i2 = 3;
    while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
    {
    dimstride[i2] = 0;
    i2++;
    }
    m_matrixHeader.datasize = (Int32)( dimstride[1] * dim[1] );
    }
    }
    }
    // no clue why jit.net.recv expects 2 headers with same info, maybe for a possible multi-chunk stream?!
    // according to other implementations it seems to ignore the size element of the chunkHeader
    // and the id element of the matrixHeader
    // send chunk header
    m_chunkHeader.id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
    m_chunkHeader.size = Marshal.SizeOf(m_matrixHeader);
    var chunkHeader = StructToBytes(m_chunkHeader, Endianness.BigEndian);
    if (myClient.Connected)
    myStream.Write(chunkHeader, 0, chunkHeader.Length);
    // send matrix header
    var matrixHeader = StructToBytes(m_matrixHeader, Endianness.BigEndian);
    if (myClient.Connected)
    myStream.Write(matrixHeader, 0, matrixHeader.Length);
    // send matrix array
    int size = input.Height * input.Width * input.NumChannels;
    byte[] managedArray = new byte[size];
    Marshal.Copy(input.ImageData, managedArray, 0, size);
    Array.Reverse(managedArray);
    if (myClient.Connected)
    myStream.Write(managedArray, 0, managedArray.Length);
    }

    • Sep 20 2013 | 1:24 pm
      did a little progress, after a lot of wireshark sniffing, managed to get text messages transmitting from c# and receiving on jit.net.recv
      just need to do the same sniffing trial and error to figure out what's wrong with the larger matrix frames.
      my sourcecode is fully available here:
    • Sep 23 2013 | 4:52 pm
      managed to make some progress, sent some test jit.net.send packets between 2 machines and captured packets for analysis with wireshark. Here is a run-down of my analysis:
      I rewrote the code to match the actual packet example and managed to get 4x4 images sending, but the image glitches up on 8x8 and starts throwing errors in higher resolutions. I'm guessing it's an IplImage format problem at this point. Here is the relevant code:
      private string JIT_MATRIX_PACKET_ID = "JMTX";
      public unsafe struct t_jit_net_packet_matrix {
      public Int32 id;
      public Int32 size;
      public Int32 planecount;
      public Int32 type; //0=char,1=long,2=float32,3=float64
      public Int32 dimcount;
      //public fixed Int32 dim[32];
      //public fixed Int32 dimstride[32];
      //public Int32 datasize;
      //public double time; //64 bit double precision float
      }
      private t_jit_net_packet_matrix m_matrixHeader;
      private void sendFrame(IplImage input) {
      // setup matrix
      Int32 matrix_id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
      Int32 planecount = input.NumChannels;
      Int32 typeSize = input.Depth;
      Int32 type = JIT_MATRIX_TYPE_CHAR;
      Int32 width = 16;// input.Width;
      Int32 height = 16;// input.Height;
      m_matrixHeader.id = matrix_id;
      m_matrixHeader.size = 288;
      m_matrixHeader.planecount = planecount;
      m_matrixHeader.type = type;
      m_matrixHeader.dimcount = 2;
      Int32[] dim = new Int32[32];
      dim[0] = width;
      dim[1] = height;
      int i2 = 2;
      while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
      {
      dim[i2] = 1;
      i2++;
      }
      Int32[] dimstride = new Int32[32];
      dimstride[0] = planecount;
      dimstride[1] = width * height;
      i2 = 2;
      while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
      {
      dimstride[i2] = 0;
      i2++;
      }
      Int32 datasize = planecount * width * height;
      m_chunkHeader.id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
      m_chunkHeader.size = sizeof(Int32) * (6 + 32 + 32) + sizeof(double); //should be 288 bytes
      byte[] chunkHeader = StructToBytes(m_chunkHeader, Endianness.LittleEndian);
      //Console.WriteLine(BitConverter.ToString(chunkHeader));
      byte[] matrixHeader = StructToBytes(m_matrixHeader, Endianness.BigEndian);
      //Console.WriteLine(BitConverter.ToString(matrixHeader));
      byte[] dim_send = new byte[4 * 32];
      byte[] dimstride_send = new byte[4 * 32];
      for (int i = 0; i < 32; i++)
      {
      byte[] dimbytes = BitConverter.GetBytes(dim[i]);
      Array.Reverse(dimbytes);
      System.Buffer.BlockCopy(dimbytes, 0, dim_send, i * 4, dimbytes.Length);
      byte[] dimstridebytes = BitConverter.GetBytes(dimstride[i]);
      Array.Reverse(dimstridebytes);
      System.Buffer.BlockCopy(dimstridebytes, 0, dimstride_send, i * 4, dimstridebytes.Length);
      }
      //Console.WriteLine(BitConverter.ToString(dim_send));
      //Console.WriteLine(BitConverter.ToString(dimstride_send));
      byte[] datasize_send = BitConverter.GetBytes(datasize);
      Array.Reverse(datasize_send);
      //Console.WriteLine(BitConverter.ToString(datasize_send));
      double time = 0; //todo: should be elapsed time, not 0
      byte[] time_send = BitConverter.GetBytes(time);
      Array.Reverse(time_send);
      //Console.WriteLine(BitConverter.ToString(time_send));
      int size = width * height * 4 * 2;//input.Height * input.Width * input.NumChannels * input.Depth / 4;
      byte[] managedArray = new byte[size];
      Marshal.Copy(input.ImageData, managedArray, 0, size);
      Array.Reverse(managedArray);
      //Console.WriteLine(BitConverter.ToString(managedArray));
      byte[] output = new byte[chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length + managedArray.Length];
      // chunkheader
      System.Buffer.BlockCopy(chunkHeader, 0, output, 0, chunkHeader.Length);
      // matrixheader
      System.Buffer.BlockCopy(matrixHeader, 0, output, chunkHeader.Length, matrixHeader.Length);
      // dim
      System.Buffer.BlockCopy(dim_send, 0, output, chunkHeader.Length + matrixHeader.Length, dim_send.Length);
      // dimstride
      System.Buffer.BlockCopy(dimstride_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length, dimstride_send.Length);
      // datasize
      System.Buffer.BlockCopy(datasize_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length, datasize_send.Length);
      // time
      System.Buffer.BlockCopy(time_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length, time_send.Length);
      // matrix array
      System.Buffer.BlockCopy(managedArray, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length, managedArray.Length);
      Console.WriteLine(BitConverter.ToString(output));
      if (myClient.Connected)
      myStream.Write(output, 0, output.Length);
      Console.WriteLine("looping");
      //if (myClient.Connected)
      // myStream.Write(managedArray, 0, managedArray.Length);
      //Console.WriteLine(managedArray.ToString());
      }
    • Oct 01 2013 | 9:01 pm
      Figured it out on my own. Here is the working code in case anyone might care to take a look in the future:
      private void sendFrame(IplImage input) {
      try
      {
      // setup matrix
      Int32 matrix_id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
      Int32 planecount = input.NumChannels;
      Int32 typeSize = input.Depth;
      Int32 type = JIT_MATRIX_TYPE_CHAR;
      Int32 width = input.Width;
      Int32 height = input.Height;
      m_matrixHeader.id = matrix_id;
      m_matrixHeader.size = 288;
      m_matrixHeader.planecount = planecount;
      m_matrixHeader.type = type;
      m_matrixHeader.dimcount = 2;
      Int32[] dim = new Int32[32];
      dim[0] = width;
      dim[1] = height;
      int i2 = 2;
      while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
      {
      dim[i2] = 1;
      i2++;
      }
      Int32[] dimstride = new Int32[32];
      dimstride[0] = planecount;
      dimstride[1] = width * planecount;
      i2 = 2;
      while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
      {
      dimstride[i2] = 0;
      i2++;
      }
      Int32 datasize = planecount * width * height;
      m_chunkHeader.id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
      m_chunkHeader.size = sizeof(Int32) * (6 + 32 + 32) + sizeof(double); //should be 288 bytes
      byte[] chunkHeader = StructToBytes(m_chunkHeader, Endianness.LittleEndian);
      //Console.WriteLine(BitConverter.ToString(chunkHeader));
      byte[] matrixHeader = StructToBytes(m_matrixHeader, Endianness.BigEndian);
      //Console.WriteLine(BitConverter.ToString(matrixHeader));
      byte[] dim_send = new byte[4 * 32];
      byte[] dimstride_send = new byte[4 * 32];
      for (int i = 0; i < 32; i++)
      {
      byte[] dimbytes = BitConverter.GetBytes(dim[i]);
      Array.Reverse(dimbytes);
      System.Buffer.BlockCopy(dimbytes, 0, dim_send, i * 4, dimbytes.Length);
      byte[] dimstridebytes = BitConverter.GetBytes(dimstride[i]);
      Array.Reverse(dimstridebytes);
      System.Buffer.BlockCopy(dimstridebytes, 0, dimstride_send, i * 4, dimstridebytes.Length);
      }
      //Console.WriteLine(BitConverter.ToString(dim_send));
      //Console.WriteLine(BitConverter.ToString(dimstride_send));
      byte[] datasize_send = BitConverter.GetBytes(datasize);
      Array.Reverse(datasize_send);
      //Console.WriteLine(BitConverter.ToString(datasize_send));
      double time = 0; //todo: should be elapsed time, not 0
      byte[] time_send = BitConverter.GetBytes(time);
      Array.Reverse(time_send);
      //Console.WriteLine(BitConverter.ToString(time_send));
      int size = width * height * 4;//input.Height * input.Width * input.NumChannels * input.Depth / 4;
      byte[] managedArray = new byte[size];
      Marshal.Copy(input.ImageData, managedArray, 0, size);
      Array.Reverse(managedArray);
      byte[] output = new byte[chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length + managedArray.Length];
      // chunkheader
      System.Buffer.BlockCopy(chunkHeader, 0, output, 0, chunkHeader.Length);
      // matrixheader
      System.Buffer.BlockCopy(matrixHeader, 0, output, chunkHeader.Length, matrixHeader.Length);
      // dim
      System.Buffer.BlockCopy(dim_send, 0, output, chunkHeader.Length + matrixHeader.Length, dim_send.Length);
      // dimstride
      System.Buffer.BlockCopy(dimstride_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length, dimstride_send.Length);
      // datasize
      System.Buffer.BlockCopy(datasize_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length, datasize_send.Length);
      // time
      System.Buffer.BlockCopy(time_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length, time_send.Length);
      // matrix array
      System.Buffer.BlockCopy(managedArray, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length, managedArray.Length);
      //Console.WriteLine(BitConverter.ToString(output));
      if (myClient.Connected)
      myStream.Write(output, 0, output.Length);
      } catch ( Exception e )
      {
      Console.WriteLine("Exception: " + e.InnerException.Message);
      }
      }
    • Jun 24 2014 | 4:07 am
    • Dec 29 2014 | 5:10 pm
      Your work is good. Hope there's a barcode function in C# .Net,
      because I need to scan barcode from C#.net.
      Looking forward your ultimate work.
      Best regards.