Custom Integration

Ultinous AI Suite is not only a standalone solution but can be used in an integrated system as well. All important information that belongs to an incident (eg. location, timestamp, detection coordinates, etc.) can be sent instantly to a third party software.

HTTP message

An external API endpoint can be set for each alarm. Whenever an alarm is triggered, a POST message will be sent to the API with metadata that belongs to the incident.

To enable snapshot sending, tick its checkbox under the respective Solution's HTTP webhook settings.

Message schema

The content type of messages is application/json. The content is described in proto in this document.

message Event{
  enum Type
  {
    UNKNOWN = 0;
    CROWD_DETECTION = 1;
    INTRUSION_DETECTION = 2;
    MULTI_OBJECT_DETECTION = 3;
    ZONE_CROSSING = 4;
    LOITERING_DETECTION = 5;
    STOPPED_DETECTION = 6;
    WRONG_DIRECTION_DETECTION = 7;
    PPE_DETECTION = 8;
  }

  string id = 1; ///< Global unique id of current event
  uint64 timestamp = 2; ///< Timestamp of the last video frame in window OR timestamp of heartbeat.
  string display_name = 3; ///< Human readable name of source alarm configuration
  Type type = 4; ///< Type of source alarm configuration
  string config_id = 5; ///< id of source alarm configuration
  bool heartbeat = 6; [deprecated=true];

  oneof value
  {
    CrowdDetectionDetails crowd_detection_details = 7; ///< Only for type=CROWD_DETECTION.
    IntrusionDetectionDetails intrusion_detection_details = 9; ///< Only for type=INTRUSION_DETECTION.
    IntrusionDetectionDetails multi_object_detection_details = 10; ///< Only for type=MULTI_OBJECT_DETECTION.
    IntrusionDetectionDetails zone_crossing_details = 11; ///< Only for type=ZONE_CROSSING.
    IntrusionDetectionDetails loitering_details = 12; ///< Only for type=LOITERING_DETECTION.
    IntrusionDetectionDetails stopped_details = 13; ///< Only for type=STOPPED_DETECTION.
    IntrusionDetectionDetails wrong_direction_details = 14; ///< Only for type=STOPPED_DETECTION.
    IntrusionDetectionDetails ppe_detection_details = 15; ///< Only for type=PPE_DETECTION.
  }

  uint64 schema_version = 8;

}

message CrowdDetectionDetails {
  int32 num_of_people = 1; ///< Number of detected people
  repeated CameraRecord sources = 2; ///< Detection details per camera
}

message IntrusionDetectionDetails {
  CameraRecord source = 1; ///< Detection detail from last frame
  bool end_of_event = 2; ///< If true, Intrusion is over
}

message CameraRecord {
  string camera_id = 1; ///< ID of current camera
  string camera_display_name = 2; ///< Display name of current camera
  string camera_technical_name = 3; ///< Technical name of current camera
  int32 width = 4; ///< Frame width of the video stream
  int32 height = 5; ///< Frame height of the video stream
  string source_topic = 6; ///< Source topic name of detection
  uint64 frame_timestamp = 7; ///< Timestamp of current video frame
  repeated Detection detections = 8; ///< Details of detections
  string snapshot = 9; ///< Jpeg snapshot Base64 encoded
}

message Detection {
  ObjectType type = 1; ///< Type of detected object.
  Rect bounding_box = 2; ///< Rectangular box containing the detection.
  float detection_confidence = 3; ///< Confidence of the detection. Range: [0..1]
}

enum ObjectType
{
  PERSON_HEAD = 0; ///< Head detection result type. Supported event type CROWD_DETECTION
  PERSON_FULL_BODY = 1; ///< Person detection result type. Supported event type INTRUSION_DETECTION
  CAR = 2; ///< Car detection result type. Supported event type INTRUSION_DETECTION
  BUS = 3; ///< Bus detection result type. Supported event type INTRUSION_DETECTION
  BOAT = 4; ///< Boat detection result type. Supported event type INTRUSION_DETECTION
  TRUCK = 5; ///< Truck detection result type. Supported event type INTRUSION_DETECTION
  MOTORCYCLE = 6; ///< Motorcycle detection result type. Supported event type INTRUSION_DETECTION
  BICYCLE = 7; ///< Bicycle detection result type. Supported event type INTRUSION_DETECTION
  TRAIN = 8; ///< Train detection result type. Supported event type INTRUSION_DETECTION
  AIRPLANE = 9; ///< Airplane detection result type. Supported event type INTRUSION_DETECTION
}

/**  Rectangle
unit: pixels
*/
message Rect {
  int32 x = 1; ///< Horizontal coordinate of the upper left corner.
  int32 y = 2; ///< Vertical coordinate of the upper left corner.
  uint32 width = 3; // Rectangle width in pixels.
  uint32 height = 4; // Rectangle height in pixels.
}

/** Two dimensional point
range: [(0,0)..(width,height)); (0,0) is upper left corner
unit: pixels
*/
message Point {
  int32 x = 1; ///< Horizontal coordinate
  int32 y = 2; ///< Vertical coordinate
}

Message examples

Event examples

Crowd

{
  "id":"1648549917244@gustave..unscheduled.alert89..Event.json",
  "timestamp":1648549917244,
  "display_name":"Crowd Example",
  "type":"CROWD_DETECTION",
  "config_id":"alert89",
  "heartbeat":false,
  "crowd_detection_details":{
    "num_of_people":1,
    "sources":[
      {
        "camera_id":"cam76",
        "camera_display_name":"Example Camera",
        "camera_technical_name":"Example-Camera-tech-name",
        "width":1920,
        "height":1080,
        "source_topic":"",
        "frame_timestamp":1648549917244,
        "detections":[
          {
            "type":"PERSON_FULL_BODY",
            "bounding_box":{
              "x":1440,
              "y":41,
              "width":129,
              "height":360
            },
            "detection_confidence":0.90866655
          }
        ],
        "snapshot":""
      }
    ]
  },
  "intrusion_detection_details":null,
  "multi_object_detection_details":null,
  "zone_crossing_details":null,
  "loitering_details":null,
  "stopped_details": null,
  "wrong_direction_details": null,
  "ppe_detection_details": null,,
  "schema_version":5
}

Intrusion

{
  "id":"1648549923245@gustave..unscheduled.alert85..Event.json",
  "timestamp":1648549923245,
  "display_name":"Intrusion example",
  "type":"INTRUSION_DETECTION",
  "config_id":"alert85",
  "heartbeat":false,
  "crowd_detection_details":null,
  "intrusion_detection_details":{
    "source":{
      "camera_id":"cam76",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width":1920,
      "height":1080,
      "source_topic":"",
      "frame_timestamp":1648549923245,
      "detections":[
        {
          "type":"PERSON_FULL_BODY",
          "bounding_box":{
            "x":398,
            "y":35,
            "width":101,
            "height":140
          },
          "detection_confidence":0.65235364
        }
      ],
      "snapshot":""
    },
    "end_of_event":false
  },
  "multi_object_detection_details":null,
  "zone_crossing_details":null,
  "loitering_details":null,
  "stopped_details": null,
  "wrong_direction_details": null,
  "ppe_detection_details": null,,
  "schema_version":5
}

Multi Object Detection

{
  "id":"1648549903243@gustave..unscheduled.alert87..Event.json",
  "timestamp":1648549903243,
  "display_name":"MOD example",
  "type":"MULTI_OBJECT_DETECTION",
  "config_id":"alert87",
  "heartbeat":false,
  "crowd_detection_details":null,
  "intrusion_detection_details":null,
  "multi_object_detection_details":{
    "source":{
      "camera_id":"cam76",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width":1920,
      "height":1080,
      "source_topic":"",
      "frame_timestamp":1648549903243,
      "detections":[
        {
          "type":"PERSON_FULL_BODY",
          "bounding_box":{
            "x":1283,
            "y":0,
            "width":62,
            "height":141
          },
          "detection_confidence":0.61611885
        }
      ],
      "snapshot":""
    },
    "end_of_event":false
  },
  "zone_crossing_details":null,
  "loitering_details":null,
  "stopped_details": null,
  "wrong_direction_details": null,
  "ppe_detection_details": null,,
  "schema_version":5
}

Zone Crossing

{
  "id":"1648549594902@crossingProc_alert91",
  "timestamp":1648549594902,
  "display_name":"Zone Crossing example",
  "type":"ZONE_CROSSING",
  "config_id":"alert91",
  "heartbeat":false,
  "crowd_detection_details":null,
  "intrusion_detection_details":null,
  "multi_object_detection_details":null,
  "zone_crossing_details":{
    "source":{
      "camera_id":"cam76",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width":1920,
      "height":1080,
      "source_topic":"",
      "frame_timestamp":1648549594902,
      "detections":[
        {
          "type":"PERSON_FULL_BODY",
          "bounding_box":{
            "x":1090,
            "y":0,
            "width":425,
            "height":0
          },
          "detection_confidence":0.8925944
        }
      ],
      "snapshot":""
    },
    "end_of_event":false
  },
  "loitering_details":null,
  "stopped_details": null,
  "wrong_direction_details": null,
  "ppe_detection_details": null,,
  "schema_version":5
}

Loitering

{
  "id":"1648549919844@loitering_alert93",
  "timestamp":"1648549919844",
  "display_name":"Loitering example",
  "type":"LOITERING_DETECTION",
  "config_id":"alert93",
  "heartbeat":false,
  "loitering_details":{
    "source":{
      "camera_id":"cam76",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width":1920,
      "height":1080,
      "source_topic":"",
      "frame_timestamp":"1648549919844",
      "detections":[
        {
          "type":"PERSON_FULL_BODY",
          "bounding_box":{
            "x":1041,
            "y":25,
            "width":528,
            "height":0
          },
          "detection_confidence":0.938471615
        }
      ],
      "snapshot":""
    },
    "end_of_event":false
  },
  "stopped_details":null,
  "wrong_direction_details":null,
  "ppe_detection_details":null,,
  "schema_version":5
}

Stopped

{
  "id": "1655736846227@Unnamed",
  "timestamp": 1655736846227,
  "display_name": "Unnamed",
  "type": "STOPPED_DETECTION",
  "config_id": "alert100",
  "heartbeat": false,
  "crowd_detection_details": null,
  "intrusion_detection_details": null,
  "multi_object_detection_details": null,
  "zone_crossing_details": null,
  "loitering_details": null,
  "stopped_details": {
    "source": {
      "camera_id": "cam83",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width": 1920,
      "height": 1080,
      "source_topic": "",
      "frame_timestamp": 1655736846227,
      "detections": [
        {
          "type": "CAR",
          "bounding_box": {
            "x": 1417,
            "y": 79,
            "width": 123,
            "height": 349
          },
          "detection_confidence": 0.90625
        }
      ],
      "snapshot": null
    },
    "end_of_event": false
  },
  "wrong_direction_details":null,
  "ppe_detection_details":null,
  "schema_version":5
}

Wrong Direction

{
  "id": "1655736622156@Unnamed",
  "timestamp": 1655736622156,
  "display_name": "Unnamed",
  "type": "WRONG_DIRECTION_DETECTION",
  "config_id": "alert98",
  "heartbeat": false,
  "crowd_detection_details": null,
  "intrusion_detection_details": null,
  "multi_object_detection_details": null,
  "zone_crossing_details": null,
  "loitering_details": null,
  "stopped_details": null,
  "wrong_direction_details": {
    "source": {
      "camera_id": "cam97",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width": 1920,
      "height": 1080,
      "source_topic": "",
      "frame_timestamp": 1655736622156,
      "detections": [
        {
          "type": "CAR",
          "bounding_box": {
            "x": 1094,
            "y": 119,
            "width": 90,
            "height": 95
          },
          "detection_confidence": 0.50878906
        }
      ],
      "snapshot": null
    },
    "end_of_event": false
  },
  "ppe_detection_details": null,
  "schema_version":5
}

PPE Detection

{
  "id" : "1655735004298@ppe",
  "timestamp" : 1655735004298,
  "display_name" : "ppe",
  "type" : "PPE_DETECTION",
  "config_id" : "alert5",
  "heartbeat" : false,
  "crowd_detection_details" : null,
  "intrusion_detection_details" : null,
  "multi_object_detection_details" : null,
  "zone_crossing_details" : null,
  "loitering_details" : null,
  "stopped_details" : null,
  "wrong_direction_details" : null,
  "ppe_detection_details" : {
    "source" : {
      "camera_id" : "cam1",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width" : 1920,
      "height" : 1080,
      "source_topic" : "",
      "frame_timestamp" : 1655735004298,
      "detections" : [ {
        "type" : "PERSON_FULL_BODY",
        "bounding_box" : {
          "x" : 1433,
          "y" : 95,
          "width" : 135,
          "height" : 365
        },
        "detection_confidence" : 0.9066977
      } ],
      "snapshot" : null
    },
    "end_of_event" : false
  },
  "schema_version":5
}

End of event

It is possible to include an End of Event timestamp, which triggers if 5 seconds have passed without incidents after the initial event that triggered the HTTP message. End of Event occurs once the incident is over.

NOTE 1: detections is an empty array, and end_of_event is true.

NOTE 2: the id of this event is unique and it does not refer to any former event. This means that all events with this display_name and type are to be considered as having ended.

End of Intrusion

{
  "id":"1648549933230@gustave..unscheduled.alert85..Event.json",
  "timestamp":1648549933230,
  "display_name":"Intrusion example",
  "type":"INTRUSION_DETECTION",
  "config_id":"alert85",
  "heartbeat":false,
  "crowd_detection_details":null,
  "intrusion_detection_details":{
    "source":{
      "camera_id":"cam76",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width":1920,
      "height":1080,
      "source_topic":"",
      "frame_timestamp":1648549933230,
      "detections":[],
      "snapshot":""
    },
    "end_of_event":true
  },
  "multi_object_detection_details":null,
  "zone_crossing_details":null,
  "loitering_details":null,
  "stopped_details": null,
  "wrong_direction_details": null,
  "ppe_detection_details": null,,
  "schema_version":5
}

End of Multi Object Detection

{
  "id":"1648549908244@gustave..unscheduled.alert87..Event.json",
  "timestamp":1648549908244,
  "display_name":"MOD example",
  "type":"MULTI_OBJECT_DETECTION",
  "config_id":"alert87",
  "heartbeat":false,
  "crowd_detection_details":null,
  "intrusion_detection_details":null,
  "multi_object_detection_details":{
    "source":{
      "camera_id":"cam76",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width":1920,
      "height":1080,
      "source_topic":"",
      "frame_timestamp":1648549908244,
      "detections":[],
      "snapshot":""
    },
    "end_of_event":true
  },
  "zone_crossing_details":null,
  "loitering_details":null,
  "stopped_details":null,
  "wrong_direction_details":null,
  "ppe_detection_details":null,,
  "schema_version":5
}

End of Zone Crossing

{
  "id":"1648549599103@crossingProc_alert91",
  "timestamp":1648549599103,
  "display_name":"Zone Crossing example",
  "type":"ZONE_CROSSING",
  "config_id":"alert91",
  "heartbeat":false,
  "crowd_detection_details":null,
  "intrusion_detection_details":null,
  "multi_object_detection_details":null,
  "zone_crossing_details":{
    "source":{
      "camera_id":"cam76",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width":1920,
      "height":1080,
      "source_topic":"",
      "frame_timestamp":1648549599103,
      "detections":[],
      "snapshot":""
    },
    "end_of_event":true
  },
  "loitering_details":null,
  "stopped_details": null,
  "wrong_direction_details": null,
  "ppe_detection_details": null,,
  "schema_version":5
}

End of Loitering

{
  "id":"1648549928925@loitering_alert93",
  "timestamp":"1648549928925",
  "display_name":"Loitering example",
  "type":"LOITERING_DETECTION",
  "config_id":"alert93",
  "heartbeat":false,
  "loitering_details":{
    "source":{
      "camera_id":"cam76",
      "camera_display_name":"Example Camera",
      "camera_technical_name":"Example-Camera-tech-name",
      "width":1920,
      "height":1080,
      "source_topic":"",
      "frame_timestamp":"1648549928925",
      "detections":[],
      "snapshot":""
    },
    "end_of_event":true
  },
  "stopped_details": null,
  "wrong_direction_details": null,
  "ppe_detection_details": null,,
  "schema_version":5
}

End of Stopped

{
    "id": "1655828448714@Unnamed",
    "timestamp": 1655828448714,
    "display_name": "Unnamed",
    "type": "STOPPED_DETECTION",
    "config_id": "alert135",
    "heartbeat": false,
    "crowd_detection_details": null,
    "intrusion_detection_details": null,
    "multi_object_detection_details": null,
    "zone_crossing_details": null,
    "loitering_details": null,
    "stopped_details": {
        "source": {
            "camera_id": "cam140",
            "camera_display_name":"Example Camera",
            "camera_technical_name":"Example-Camera-tech-name",
            "width": 1920,
            "height": 1080,
            "source_topic": "",
            "frame_timestamp": 1655828448714,
            "detections": [],
            "snapshot": null
        },
        "end_of_event": true
    },
    "wrong_direction_details": null,
    "ppe_detection_details": null,
    "schema_version": 5
}

End of Wrong Diection

{
    "id": "1655828578444@Unnamed",
    "timestamp": 1655828578444,
    "display_name": "Unnamed",
    "type": "WRONG_DIRECTION_DETECTION",
    "config_id": "alert142",
    "heartbeat": false,
    "crowd_detection_details": null,
    "intrusion_detection_details": null,
    "multi_object_detection_details": null,
    "zone_crossing_details": null,
    "loitering_details": null,
    "stopped_details": null,
    "wrong_direction_details": {
        "source": {
            "camera_id": "cam122",
            "camera_display_name":"Example Camera",
            "camera_technical_name":"Example-Camera-tech-name",
            "width": 1920,
            "height": 1080,
            "source_topic": "",
            "frame_timestamp": 1655828578444,
            "detections": [],
            "snapshot": null
        },
        "end_of_event": true
    },
    "ppe_detection_details": null,
    "schema_version": 5
}

Snapshot sending

If enabled, snapshots are sent to the following endpoint:

[PUT] YOUR_URL/snapshot/{event_is}/{c_t_n}

Its properties are as follows:

• Content-Type: multipart/form-data

• image media type: image/jpeg

Snapshot images, if enabled, are included in the schema in base64 encoded format, after the snapshot: part, in quotation marks (left empty in the examples).

Setup message sending in Ultinous AI Suite

1. Give unique technical names to cameras. The technical name will appear in the metadata and will aid the identification of the camera.
2. Fill in HTTP webhook settings for each alarm. Give the address, the security credentials in a custom message header (if any) and cert settings for the external API.