ReFlex Logo
ReFlex (Title)
Documentation
Navigation

ReFlex Library: How add a new sensor

title image showing a pointcloud representation of two hands deforming the surface

Table of Contents

  1. Table of Contents
  2. General approach
  3. The IDepthCamera Interface
  4. Example Implementation: AzureKinectCamera
  5. Registering the Sensor with CameraManager

General approach

⬆ back to top

The IDepthCamera Interface

Method / Property Description
Id unique identifier, used to select the sensor. (string, readonly)
CameraType Enumeration of different Camera types, used to distinguish between physical camera and software-emulated
ModelDescription description of the sensor for server frontend (string, readonly)
State DepthCameraState for checking current state of the sensor (readonly, should be set by the sensor itself
StreamParameter description for currently active sensor mode (resolution, framerate, image format)
StateChanged event which should be triggered, when the camera state changes
FrameReady event which should be triggered, to broadcast an updated point cloud
DepthImageReady event which should be triggered, to broadcast the raw depth image as byte array
Initialize() Code for initial setup of the sensor (checking availability, etc.)
GetPossibleConfigurations() returns the list of available
EnableStream(StreamParameter parameter) returns list of available sensor modes
StartStream() starts tracking with the previously set configuration
StopStream() stops tracking
Dispose() cleanup code when the app is stopped (e.g. stop sensor and free resources)

⬆ back to top

Example Implementation: AzureKinectCamera

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

    [Plugin(PluginType = typeof(IDepthCamera))]
    public class AzureKinectCamera : IDepthCamera, IDisposable
    {
        // private fields
        
        private const int BytesPerChannel = sizeof(ushort);
        private const int NumChannels = 3;

        private readonly Device _device;

        private DepthCameraState _state;

        private bool _queryDepth;
        private Transformation _transform;

        private byte[] _transformedPixels;
        private Point3[] _convertedVertices;

        // Properties

        public string Id => _device?.SerialNum;

        public CameraType CameraType => CameraType.AzureKinect;

        public string ModelDescription => "Microsoft\u00A9 Azure Kinect";

        public DepthCameraState State
        {
            get => _state;
            private set
            {
                if (_state == value) return;

                _state = value;
                OnStateChanged(this, _state);
            }
        }
        public StreamParameter StreamParameter { get; private set; }

        // events

        public event EventHandler<DepthCameraState> StateChanged;

        public event EventHandler<ImageByteArray> DepthImageReady;

        public event EventHandler<DepthCameraFrame> FrameReady;

        // Constructor

        public AzureKinectCamera()
        {
            var numDevicesAvailable = Device.GetInstalledCount();
            _device = Device.Open();
            State = numDevicesAvailable > 0
                ? DepthCameraState.Connected
                : DepthCameraState.Disconnected;
        }

        // Methods

        public void Initialize()
        {
            var numDevicesAvailable = Device.GetInstalledCount();

            if (numDevicesAvailable <= 0) {
                State = DepthCameraState.Error;
            }
        }

        public void EnableStream(StreamParameter parameter)
        {
            StreamParameter = parameter;
        }

        public IList<StreamParameter> GetPossibleConfigurations()
        {
            return AzureKinectStreamParameterConverter.GetSupportedConfigurations();
        }

        public void StartStream()
        {
            var deviceConfiguration = new DeviceConfiguration
            {
                CameraFPS = AzureKinectStreamParameterConverter.GetFps(StreamParameter),
                ColorResolution = ColorResolution.Off,
                DepthMode = AzureKinectStreamParameterConverter.GetDepthMode(StreamParameter)
            };

            _device.StartCameras(deviceConfiguration);
            _transform = _device.GetCalibration().CreateTransformation();

            ArrayUtils.InitializeArray(out _transformedPixels, StreamParameter.Width * StreamParameter.Height * NumChannels * 2);
            ArrayUtils.InitializeArray(out _convertedVertices, StreamParameter.Width * StreamParameter.Height);

            State = DepthCameraState.Streaming;

            _queryDepth = true;

            Task.Run(QueryDepthStream);
        }

        public void StopStream()
        {
            _queryDepth = false;

            if (_device == null)
                return;

            _device.StopCameras();

            State = DepthCameraState.Connected;
        }
        
        public void Dispose()
        {
            _transform?.Dispose();
            _device?.Dispose();
        }

        // ... implementation details omitted 

    }

⬆ back to top

Registering the Sensor with CameraManager

To make camera available in the server, the module must be registered in the CameraManager of ReFlex.TrackingServer.

Example Code:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

    using NLog;
    using ReFlex.Core.Tracking.Interfaces;
    using ReFlex.Sensor.EmulatorModule;
    #if !NO_EXTERNAL_SENSORS
    using System.Runtime.InteropServices;
    using ReFlex.Sensor.AzureKinectModule;
    using ReFlex.Sensor.Kinect2Module;
    using ReFlex.Sensor.RealSenseD435Module;
    using ReFlex.Sensor.RealSenseL515Module;
    using ReFlex.Sensor.RealSenseR2Module;
    // place reference to new camera module here
    #endif

    namespace TrackingServer.Model
    {        
        public class CameraManager

        // ...

        public CameraManager()
        {

         _depthCameras = new List<IDepthCamera>();

            #if !NO_EXTERNAL_SENSORS

                try
                {
                    // register your new camera:
                    var myCustomCam = new CustomDepthSensor();
                    _depthCameras.Add(myCustomCam);
                    Logger.Info($"Successfully loaded {myCustomCam.ModelDescription} camera.");
                }
                catch (Exception exception)
                {
                    Logger.Error(exception);
                }

                // ...

            #endif
        }
    }

⬆ back to top