View Categories

SkeletonCSharp

Overview #

This is a C# implementation of a Real-time 3D Body Pose Estimation application using the LIPSBodyPose SDK(pro). It captures data from LIPSedge cameras, processes the skeleton tracking, and visualizes the results using the OpenCvSharp library.

Tutorial #

1. Imports the necessary libraries.

C#
using OpenCvSharp;
using System;
using System.Collections.Generic;

2. This code block defines a static List called connectionPairs for drawing a human skeleton.

C#
namespace Skeleton_csharp
{
class Program
{
    public static List < ( LIPS.Skeleton.PartEnum, LIPS.Skeleton.PartEnum ) > connectionPairs =
              new List < ( LIPS.Skeleton.PartEnum, LIPS.Skeleton.PartEnum ) >
    {
        // Face
        ( LIPS.Skeleton.PartEnum.R_EYE, LIPS.Skeleton.PartEnum.R_EAR ),
        ( LIPS.Skeleton.PartEnum.L_EYE, LIPS.Skeleton.PartEnum.L_EAR ),
        ( LIPS.Skeleton.PartEnum.R_EYE, LIPS.Skeleton.PartEnum.NOSE  ),
        ( LIPS.Skeleton.PartEnum.L_EYE, LIPS.Skeleton.PartEnum.NOSE  ),
        ( LIPS.Skeleton.PartEnum.R_EYE, LIPS.Skeleton.PartEnum.HEAD  ),
        ( LIPS.Skeleton.PartEnum.L_EYE, LIPS.Skeleton.PartEnum.HEAD  ),
        // Body
        ( LIPS.Skeleton.PartEnum.NOSE,        LIPS.Skeleton.PartEnum.NECK        ),
        ( LIPS.Skeleton.PartEnum.NECK,        LIPS.Skeleton.PartEnum.R_CLAVICLE  ),
        ( LIPS.Skeleton.PartEnum.R_CLAVICLE,  LIPS.Skeleton.PartEnum.R_SHOULDER  ),
        ( LIPS.Skeleton.PartEnum.NECK,        LIPS.Skeleton.PartEnum.L_CLAVICLE  ),
        ( LIPS.Skeleton.PartEnum.L_CLAVICLE,  LIPS.Skeleton.PartEnum.L_SHOULDER  ),
        ( LIPS.Skeleton.PartEnum.R_SHOULDER,  LIPS.Skeleton.PartEnum.R_HIP       ),
        ( LIPS.Skeleton.PartEnum.L_SHOULDER,  LIPS.Skeleton.PartEnum.L_HIP       ),
        ( LIPS.Skeleton.PartEnum.L_HIP,       LIPS.Skeleton.PartEnum.PELVIS      ),
        ( LIPS.Skeleton.PartEnum.PELVIS,      LIPS.Skeleton.PartEnum.R_HIP       ),
        ( LIPS.Skeleton.PartEnum.NECK,        LIPS.Skeleton.PartEnum.CHEST_SPINE ),
        ( LIPS.Skeleton.PartEnum.CHEST_SPINE, LIPS.Skeleton.PartEnum.NAVEL_SPINE ),
        ( LIPS.Skeleton.PartEnum.NAVEL_SPINE, LIPS.Skeleton.PartEnum.PELVIS      ),
        // Right arm
        ( LIPS.Skeleton.PartEnum.R_SHOULDER, LIPS.Skeleton.PartEnum.R_ELBOW     ),
        ( LIPS.Skeleton.PartEnum.R_ELBOW,    LIPS.Skeleton.PartEnum.R_WRIST     ),
        ( LIPS.Skeleton.PartEnum.R_WRIST,    LIPS.Skeleton.PartEnum.R_HAND      ),
        ( LIPS.Skeleton.PartEnum.R_WRIST,    LIPS.Skeleton.PartEnum.R_THUMB_TIP ),
        ( LIPS.Skeleton.PartEnum.R_HAND,     LIPS.Skeleton.PartEnum.R_HAND_TIP  ),
        // Left arm
        ( LIPS.Skeleton.PartEnum.L_SHOULDER, LIPS.Skeleton.PartEnum.L_ELBOW     ),
        ( LIPS.Skeleton.PartEnum.L_ELBOW,    LIPS.Skeleton.PartEnum.L_WRIST     ),
        ( LIPS.Skeleton.PartEnum.L_WRIST,    LIPS.Skeleton.PartEnum.L_HAND      ),
        ( LIPS.Skeleton.PartEnum.L_WRIST,    LIPS.Skeleton.PartEnum.L_THUMB_TIP ),
        ( LIPS.Skeleton.PartEnum.L_HAND,     LIPS.Skeleton.PartEnum.L_HAND_TIP  ),
        // Right leg
        ( LIPS.Skeleton.PartEnum.R_HIP,   LIPS.Skeleton.PartEnum.R_KNEE  ),
        ( LIPS.Skeleton.PartEnum.R_KNEE,  LIPS.Skeleton.PartEnum.R_ANKLE ),
        ( LIPS.Skeleton.PartEnum.R_ANKLE, LIPS.Skeleton.PartEnum.R_FOOT  ),
        // Left leg
        ( LIPS.Skeleton.PartEnum.L_HIP,   LIPS.Skeleton.PartEnum.L_KNEE  ),
        ( LIPS.Skeleton.PartEnum.L_KNEE,  LIPS.Skeleton.PartEnum.L_ANKLE ),
        ( LIPS.Skeleton.PartEnum.L_ANKLE, LIPS.Skeleton.PartEnum.L_FOOT  )
    };

3. This code defines the high-fidelity hand skeletal structure. While the previous “connectionPairs” handled the main body (shoulders, hips, legs), “fingerConnectionPairs” focuses exclusively on the intricate geometry of the human hand.

C#
    public static List < ( LIPS.Skeleton.PartEnum, LIPS.Skeleton.PartEnum ) > fingerConnectionPairs =
        new List < ( LIPS.Skeleton.PartEnum, LIPS.Skeleton.PartEnum ) >
    {
        // Right Thumb
        ( LIPS.Skeleton.PartEnum.R_WRIST,          LIPS.Skeleton.PartEnum.HAND_R_THUMB_CMC ),
        ( LIPS.Skeleton.PartEnum.HAND_R_THUMB_CMC, LIPS.Skeleton.PartEnum.HAND_R_THUMB_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_THUMB_MCP, LIPS.Skeleton.PartEnum.HAND_R_THUMB_IP  ),
        ( LIPS.Skeleton.PartEnum.HAND_R_THUMB_IP,  LIPS.Skeleton.PartEnum.HAND_R_THUMB_TIP ),
        // Right Index Finger
        ( LIPS.Skeleton.PartEnum.R_WRIST,          LIPS.Skeleton.PartEnum.HAND_R_INDEX_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_INDEX_MCP, LIPS.Skeleton.PartEnum.HAND_R_INDEX_PIP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_INDEX_PIP, LIPS.Skeleton.PartEnum.HAND_R_INDEX_DIP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_INDEX_DIP, LIPS.Skeleton.PartEnum.HAND_R_INDEX_TIP ),
        // Right Middle Finger
        ( LIPS.Skeleton.PartEnum.R_WRIST,           LIPS.Skeleton.PartEnum.HAND_R_MIDDLE_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_MIDDLE_MCP, LIPS.Skeleton.PartEnum.HAND_R_MIDDLE_PIP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_MIDDLE_PIP, LIPS.Skeleton.PartEnum.HAND_R_MIDDLE_DIP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_MIDDLE_DIP, LIPS.Skeleton.PartEnum.HAND_R_MIDDLE_TIP ),
        // Right Ring Finger
        ( LIPS.Skeleton.PartEnum.R_WRIST,         LIPS.Skeleton.PartEnum.HAND_R_RING_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_RING_MCP, LIPS.Skeleton.PartEnum.HAND_R_RING_PIP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_RING_PIP, LIPS.Skeleton.PartEnum.HAND_R_RING_DIP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_RING_DIP, LIPS.Skeleton.PartEnum.HAND_R_RING_TIP ),
        // Right Little Finger
        ( LIPS.Skeleton.PartEnum.R_WRIST,           LIPS.Skeleton.PartEnum.HAND_R_LITTLE_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_LITTLE_MCP, LIPS.Skeleton.PartEnum.HAND_R_LITTLE_PIP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_LITTLE_PIP, LIPS.Skeleton.PartEnum.HAND_R_LITTLE_DIP ),
        ( LIPS.Skeleton.PartEnum.HAND_R_LITTLE_DIP, LIPS.Skeleton.PartEnum.HAND_R_LITTLE_TIP ),
        // Left Thumb
        ( LIPS.Skeleton.PartEnum.L_WRIST,          LIPS.Skeleton.PartEnum.HAND_L_THUMB_CMC ),
        ( LIPS.Skeleton.PartEnum.HAND_L_THUMB_CMC, LIPS.Skeleton.PartEnum.HAND_L_THUMB_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_THUMB_MCP, LIPS.Skeleton.PartEnum.HAND_L_THUMB_IP  ),
        ( LIPS.Skeleton.PartEnum.HAND_L_THUMB_IP,  LIPS.Skeleton.PartEnum.HAND_L_THUMB_TIP ),
        // Left Index Finger
        ( LIPS.Skeleton.PartEnum.L_WRIST,          LIPS.Skeleton.PartEnum.HAND_L_INDEX_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_INDEX_MCP, LIPS.Skeleton.PartEnum.HAND_L_INDEX_PIP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_INDEX_PIP, LIPS.Skeleton.PartEnum.HAND_L_INDEX_DIP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_INDEX_DIP, LIPS.Skeleton.PartEnum.HAND_L_INDEX_TIP ),
        // Left Middle Finger
        ( LIPS.Skeleton.PartEnum.L_WRIST,           LIPS.Skeleton.PartEnum.HAND_L_MIDDLE_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_MIDDLE_MCP, LIPS.Skeleton.PartEnum.HAND_L_MIDDLE_PIP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_MIDDLE_PIP, LIPS.Skeleton.PartEnum.HAND_L_MIDDLE_DIP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_MIDDLE_DIP, LIPS.Skeleton.PartEnum.HAND_L_MIDDLE_TIP ),
        // Left Ring Finger
        ( LIPS.Skeleton.PartEnum.L_WRIST,         LIPS.Skeleton.PartEnum.HAND_L_RING_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_RING_MCP, LIPS.Skeleton.PartEnum.HAND_L_RING_PIP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_RING_PIP, LIPS.Skeleton.PartEnum.HAND_L_RING_DIP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_RING_DIP, LIPS.Skeleton.PartEnum.HAND_L_RING_TIP ),
        // Left Little Finger
        ( LIPS.Skeleton.PartEnum.L_WRIST,           LIPS.Skeleton.PartEnum.HAND_L_LITTLE_MCP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_LITTLE_MCP, LIPS.Skeleton.PartEnum.HAND_L_LITTLE_PIP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_LITTLE_PIP, LIPS.Skeleton.PartEnum.HAND_L_LITTLE_DIP ),
        ( LIPS.Skeleton.PartEnum.HAND_L_LITTLE_DIP, LIPS.Skeleton.PartEnum.HAND_L_LITTLE_TIP )
    };

4. This code block defines a List of Scalar objects, which serves as a color palette for the application.

C#
    public static List<Scalar> colorList = new List<Scalar>
    {
        new Scalar( 255, 100, 100 ),
        new Scalar( 255, 255, 100 ),
        new Scalar( 100, 255, 100 ),
        new Scalar( 100, 255, 255 ),
        new Scalar( 100, 100, 255 )
    };

5. The DrawSkeletons function is the “engine” that visualizes the skeletal data. It takes the abstract coordinates provided by the SDK and turns them into a visual representation on your BGR image:
(1)Start looping through all detected skeletons in the frame, calculates a colorIdx using the modulo operator against your colorList
(2)Use the two lists (connectionPairs and fingerConnectionPairs) you defined earlier. It iterates through these pairs to “draw lines” of body and finger.
(3)After the lines are drawn, the function draws the joints themselves

C#
    private static void DrawSkeletons( ref Mat colorMat, ref LIPS.Frame frame )
    {
        for( int skeletonIndex = 0; skeletonIndex < frame.Skeletons.Count; skeletonIndex++ )
        {
            int colorIdx = frame.Skeletons[skeletonIndex].Id % colorList.Count;
            Scalar lineColor = colorList[colorIdx];
            int lineWidth = 2;

            // draw line ( body )
            foreach( ( LIPS.Skeleton.PartEnum, LIPS.Skeleton.PartEnum ) connection in connectionPairs )
            {
                LIPS.Keypoint keypointA = frame.Skeletons[skeletonIndex][connection.Item1];
                LIPS.Keypoint keypointB = frame.Skeletons[skeletonIndex][connection.Item2];
                if( keypointA.IsValid && keypointB.IsValid )
                {
                    Point pointA = new Point( keypointA.Point2D.X, keypointA.Point2D.Y );
                    Point pointB = new Point( keypointB.Point2D.X, keypointB.Point2D.Y );
                    Cv2.Line( colorMat, pointA, pointB, lineColor, lineWidth );
                }
            }

            // draw line ( finger )
            foreach( ( LIPS.Skeleton.PartEnum, LIPS.Skeleton.PartEnum ) connection in fingerConnectionPairs )
            {
                LIPS.Keypoint keypointA = frame.Skeletons[skeletonIndex][connection.Item1];
                LIPS.Keypoint keypointB = frame.Skeletons[skeletonIndex][connection.Item2];
                if( keypointA.IsValid && keypointB.IsValid )
                {
                    Point pointA = new Point( keypointA.Point2D.X, keypointA.Point2D.Y );
                    Point pointB = new Point( keypointB.Point2D.X, keypointB.Point2D.Y );
                    Cv2.Line( colorMat, pointA, pointB, lineColor, lineWidth );
                }
            }

            // draw dot ( body & finger keypoints )
            Scalar circleColor = new Scalar( lineColor[0] + 100, lineColor[1] + 100, lineColor[2] + 100 );
            int circleRadius = 4;
            int circleLineWidth = -1;  // solid
            foreach( LIPS.Skeleton.PartEnum part in
                   ( LIPS.Skeleton.PartEnum[] )Enum.GetValues( typeof( LIPS.Skeleton.PartEnum ) ) )
            {
                if( part == LIPS.Skeleton.PartEnum.PART_TOTAL )
                    continue;

                if( frame.Skeletons[skeletonIndex][part].IsValid )
                {
                    Point point = new Point( frame.Skeletons[skeletonIndex][part].Point2D.X,
                                             frame.Skeletons[skeletonIndex][part].Point2D.Y );
                    Cv2.Circle( colorMat, point, circleRadius, circleColor, circleLineWidth );
                }
            }

        }
    }

6. This function, DrawNeckPoint, is a targeted visualization tool used to overlay precise 3D spatial data onto your 2D video feed. It bridges the gap between the raw 3D coordinates calculated by the LIPS SDK and the visual interface.

C#
    public static void DrawNeckPoint( ref Mat color, LIPS.Point3D neck3d, LIPS.Point2D neck2d )
    {
        string text = $"({Math.Round(neck3d.X,1)}, {Math.Round(neck3d.Y,1)}, {Math.Round(neck3d.Z, 1)})";
        Point cvNeck2d = new Point( neck2d.X, neck2d.Y );
        Cv2.PutText( color, text, cvNeck2d, HersheyFonts.HersheyComplex, 1, Scalar.Red );
    }

7. This function can draw the profile and show on the screen.

C#
    public static void DrawProfile( ref Mat color, float fps, float latency )
    {
        string text = $"FPS = {Math.Round(fps,1)}  Latency = {Math.Round(latency,2)}";
        Point textStart = new Point( 0, color.Height - 10 );
        Cv2.PutText( color, text, textStart, HersheyFonts.HersheyComplex, 1.2, Scalar.OrangeRed );
    }

8. The DrawMotionData function is a specialized diagnostic tool used to visualize the IMU (Inertial Measurement Unit) data from the camera. If your LIPSedge camera has an onboard accelerometer and gyroscope, this function allows you to see how the camera is moving or rotating in real-time.

C#
    public static void DrawMotionData( ref Mat color, float accel_x, float accel_y, float accel_z,
                                       float gyro_x, float gyro_y, float gyro_z )
    {
        string text = $"Accel = ({Math.Round(accel_x,2)}, {Math.Round(accel_y,2)}, {Math.Round(accel_z,2)})";
        Point textStart = new Point( 0, 25 );
        Cv2.PutText( color, text, textStart, HersheyFonts.HersheyComplex, 0.8, Scalar.OrangeRed );

        text = $"Gyro = ({Math.Round(gyro_x,2)}, {Math.Round(gyro_y,2)}, {Math.Round(gyro_z,2)})";
        textStart = new Point( 0, 50 );
        Cv2.PutText( color, text, textStart, HersheyFonts.HersheyComplex, 0.8, Scalar.OrangeRed );
    }

9. The ColorizeDepthMap function is your visualization translator. Raw depth data from a LIPSedge camera is typically stored as a 16-bit integer (representing distance in millimeters), which is not human-readable as an image. This function maps that data into a vibrant, color-coded visual format.

C#
    public static Mat ColorizeDepthMap( ref Mat depth, int displayMinZ = 300, int displayMaxZ = 3000 )
    {
        double normalizeAlpha = 255.0 / Convert.ToDouble( displayMaxZ - displayMinZ );
        double normalizeBeta = Convert.ToDouble( -255.0 * displayMinZ ) / Convert.ToDouble( displayMaxZ - displayMinZ );

        Mat output = new Mat();
        Mat depthNormal = new Mat();
        depth.ConvertTo( depthNormal, MatType.CV_8UC1, normalizeAlpha, normalizeBeta );
        Cv2.ApplyColorMap( depthNormal, output, ColormapTypes.Jet );

        return output;
    }

10.Create the Main Loop to :
(1)Starts the SDK and prints camera information.
(2)Enter a loop that grabs a new “frame” from the camera.
(3)Updates the display of depth image, rgb image, with the skeleton overlays and the profile information.
(4)Stop everything safely when you hit the ESC key.

C#
    static void Main( string[] args )
    {
        // initialize LIPSBodyPose & run bodypose with your config ( json format )
        LIPS.LIPSBodyPose bodyPose = new LIPS.LIPSBodyPose();
        LIPS.Frame frame;
        bodyPose.Run( @"{""runtime_dir"" : ""."", ""log_to"": ""CONSOLE"", ""profiling"": true}" );

        // get all config
        Console.WriteLine( bodyPose.DumpConfig() );

        // get datas every frame
        while( ( frame = bodyPose.ReadFrame() ) != null )
        {
            // images
            Mat colorMat = Mat.FromPixelData( new List<int> { frame.ColorImage_rgb.Height, frame.ColorImage_rgb.Width }, MatType.CV_8UC3, frame.ColorImage_rgb.Data );
            Cv2.CvtColor( colorMat, colorMat, ColorConversionCodes.RGB2BGR );
            Mat depthMat = Mat.FromPixelData( new List<int> { frame.DepthImage_mm.Height, frame.DepthImage_mm.Width }, MatType.CV_16UC1, frame.DepthImage_mm.Data );

            // skeleton 2D
            DrawSkeletons( ref colorMat, ref frame );

            // skeleton 2D & 3D ( 3D point unit : mm )
            for( int skeletonIndex = 0; skeletonIndex < frame.Skeletons.Count; skeletonIndex++ )
                DrawNeckPoint( ref colorMat,
                               frame.Skeletons[skeletonIndex][LIPS.Skeleton.PartEnum.NECK].Point3D,
                               frame.Skeletons[skeletonIndex][LIPS.Skeleton.PartEnum.NECK].Point2D );

            // profile
            DrawProfile( ref colorMat, frame.ProfileInfo.Fps, frame.ProfileInfo.LatencyMs );

            // camera motion data
            if( bodyPose.IsMotionDataAvailable() )
            {
                float accel_x, accel_y, accel_z, gyro_x, gyro_y, gyro_z;
                if( bodyPose.GetMotionData( out accel_x, out accel_y, out accel_z,
                                            out gyro_x, out gyro_y, out gyro_z ) )
                {
                    DrawMotionData( ref colorMat, accel_x, accel_y, accel_z, gyro_x, gyro_y, gyro_z );
                }
            }

            Cv2.ImShow( "Color", colorMat );
            Cv2.ImShow( "Depth", ColorizeDepthMap( ref depthMat ) );
            int inputKey = Cv2.WaitKey( 1 );

            // stop LIPSBodyPose
            if( inputKey == 27 )  // esc
            {
                Console.WriteLine( "Get exit signal." );
                bodyPose.Stop();
                break;
            }
        }
    }
}
}
Updated on 2026 年 6 月 22 日

What are your Feelings