TTA
/
tta-ai-streamer
18
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'main'
${ noResults }
tta-ai-streamer/wayland_cam_app/wayland_cam_app copy 2.c

865 lines
28 KiB
Raw Permalink Normal View History Unescape

first commit
7 months ago
// cam_ws_app.c : Headless V4L2 → WebSocket 스트리머
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <signal.h>
#include <errno.h>
#include <pthread.h>
#include <linux/videodev2.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <turbojpeg.h>
#include <arm_neon.h>
#include <omp.h>
#include "uds_frame.h"
#include "uds_server.h"
#include "uds_det.h"
#include "ctrl_flags.h"
#include "v4l2_interface.h"
#include "nc_utils.h"
#include "nc_ts_fsync_flipflop_buffers.h"
#include "nc_app_config_parser.h"
#include "nc_cnn_aiware_runtime.h"
#include "nc_cnn_communicator.h"
#include "nc_cnn_worker_for_postprocess.h"
#include "nc_neon.h"
#ifdef AIWARE_DEVICE_SUPPORTED
#include "aiware/runtime/c/aiwaredevice.h"
#endif
#ifdef USE_8MP_VI
#include "nc_dsr_helper.h"
#include "nc_dsr_set.h"
#include "nc_dmabuf_ctrl_helper.h"
#endif
// -----------------------------
// 설정
// -----------------------------
#define VIS0_MAX_CH (0)
#define VIS1_MAX_CH (1)
#define VIDEO_MAX_CH (VIS0_MAX_CH + VIS1_MAX_CH)
#define VIDEO_BUFFER_NUM (3)
/* don't modify */
#define MAX_WIDTH_FOR_VDMA_CNN_DS (1920)
#define MAX_HEIGHT_FOR_VDMA_CNN_DS (1080)
#ifdef USE_8MP_VI
#define VIDEO_WIDTH MAX_WIDTH_FOR_VDMA_CNN_DS
#define VIDEO_HEIGHT MAX_HEIGHT_FOR_VDMA_CNN_DS
#else
#define VIDEO_WIDTH (1920)
#define VIDEO_HEIGHT (1080)
#endif
#define MQ_NAME_CNN_BUF "/cnn_data"
#define DEV_FILE_DSR "/dev/dsr"
#ifdef SHOW_FACE_DETECT
#define NETWORK_FILE_FACE_DET "/mnt/user_data/applications/misc/networks/cuuva/modified_yolov8n-face.aiwbin"
#endif
#ifdef SHOW_CUUVA_DETECT
#define NETWORK_FILE_CUUVA_DET "/mnt/user_data/applications/misc/networks/cuuva/yolov8s_coco_640x384_apache6sr250_aiw4939.aiwbin"
#endif
#ifdef SHOW_FIRE_DETECT
#define NETWORK_FILE_FIRE_DET "/mnt/user_data/applications/misc/networks/cuuva/modified_best_fire_detect.aiwbin"
#endif
#ifdef SHOW_LPR_DETECT
#define NETWORK_FILE_LPR_DET "/mnt/user_data/applications/misc/networks/cuuva/modified_best_lp_detect.aiwbin"
#endif
static volatile int g_running = 1;
st_nc_v4l2_config v4l2_config[VIDEO_MAX_CH];
st_npu_input_info npu_input_info;
#define FOLLOWUP_CLS_ID 0
#define NETWORK_OBJDET 0
#define NETWORK_ABNORM 1
#define NETWORK_FIRE 2
#define NETWORK_LPR 3
#define NETWORK_FACE 4
static uint32_t ch_seq[VIDEO_MAX_CH] = {0};
enum { SLOT_FREE=0, SLOT_WRITING=1, SLOT_READY=2 };
static inline void mem_fence(void) { __sync_synchronize(); }
int uds_server_start(const char* sock_path);
void uds_server_stop(const char* sock_path);
int uds_send_frame(int ch, const void* frame_ptr,
uint32_t w, uint32_t h, uint32_t stride,
uint32_t pixfmt, uint64_t ts_ns);
static inline uint64_t now_ns(void){
struct timespec ts; clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
return (uint64_t)ts.tv_sec*1000000000ull + ts.tv_nsec;
}
static inline uint64_t now_us(void){
struct timespec ts; clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
return (uint64_t)ts.tv_sec*1000000ull + ts.tv_nsec/1000ull;
}
static uint32_t fra_sec_cnt[VIDEO_MAX_CH] = {0};
static uint32_t fra_sec_cnt_total = 0;
static uint64_t fps_win_start_us = 0;
#ifdef USE_8MP_VI
size_t BUFF_SIZE;
int dsr_fd;
DSR_Data_t dsr_info;
dma_alloc_info dma_info;
st_nc_dsr_config dsr_config;
img_input_config dsr_input_config;
img_output_config dsr_output_config;
#endif
#ifdef USE_8MP_VI
static int dsr_init(void)
{
int input_fd = 0, output_fd = 0;
long page_size = sysconf(_SC_PAGESIZE);
dsr_input_config.format = IMG_FORMAT_RGB888;
dsr_input_config.width = MAX_WIDTH_FOR_VDMA_CNN_DS;
dsr_input_config.height = MAX_HEIGHT_FOR_VDMA_CNN_DS;
dsr_output_config.format = IMG_FORMAT_RGB888;
BUFF_SIZE = dsr_input_config.width * dsr_input_config.height * 3;
BUFF_SIZE = ((BUFF_SIZE + page_size - 1) / page_size) * page_size;
if(dsr_config_downscale(&dsr_config, 1, NPU_INPUT_WIDTH, NPU_INPUT_HEIGHT) < 0){
perror("Error: DSR config fail");
return -1;
}
if(open_device_and_dma_buffers(DEV_FILE_DSR, &dsr_fd, &input_fd, &output_fd, BUFF_SIZE) < 0){
perror("Error: DSR open fail");
return -1;
}
if(dsr_setup_buffer(&dsr_info, dsr_fd, &dma_info, input_fd, output_fd, BUFF_SIZE) < 0){
perror("Error: DSR setup fail");
return -1;
}
return 0;
}
static int dsr_deinit(void)
{
nc_dmabuf_ctrl_end_cpu_access(dma_info.dmabuf_fd_in);
nc_dmabuf_ctrl_end_cpu_access(dma_info.dmabuf_fd_out);
nc_dmabuf_ctrl_free_dma_fd(dma_info.dmabuf_fd_in);
nc_dmabuf_ctrl_free_dma_fd(dma_info.dmabuf_fd_out);
nc_dmabuf_ctrl_close();
for (int i = 0; i < BUFF_NUM; i++) {
if (dsr_info.dsr_in_buf[i] != MAP_FAILED) {
munmap(dsr_info.dsr_in_buf[i], BUFF_SIZE);
}
if (dsr_info.dsr_out_buf[i] != MAP_FAILED) {
munmap(dsr_info.dsr_out_buf[i], BUFF_SIZE);
}
}
dsr_device_deinit(&dsr_fd);
return 0;
}
#endif
// -----------------------------
// Ctrl+C 핸들러
// -----------------------------
static void on_sigint(int sig) {
(void)sig;
g_running = 0;
}
// -----------------------------
// 채널 설정
// -----------------------------
static void set_v4l2_config(void)
{
// VIS0 영역
for (int i = 0; i < VIS0_MAX_CH; i++) {
v4l2_config[i].video_buf.video_device_num = CNN_DEVICE_NUM(VISION0) + i;
v4l2_config[i].video_buf.video_fd = -1;
v4l2_config[i].dma_mode = INTERLEAVE;
v4l2_config[i].img_process = MODE_DS;
v4l2_config[i].pixformat = V4L2_PIX_FMT_RGB24;
v4l2_config[i].crop_x_start = 0;
v4l2_config[i].crop_y_start = 0;
v4l2_config[i].crop_width = 0;
v4l2_config[i].crop_height = 0;
v4l2_config[i].ds_width = VIDEO_WIDTH;
v4l2_config[i].ds_height = VIDEO_HEIGHT;
}
// VIS1 영역
for (int j = VIS0_MAX_CH; j < VIDEO_MAX_CH; j++) {
v4l2_config[j].video_buf.video_device_num = CNN_DEVICE_NUM(VISION1) + j;
v4l2_config[j].video_buf.video_fd = -1;
v4l2_config[j].dma_mode = INTERLEAVE;
v4l2_config[j].img_process = MODE_DS;
v4l2_config[j].pixformat = V4L2_PIX_FMT_RGB24;
v4l2_config[j].crop_x_start = 0;
v4l2_config[j].crop_y_start = 0;
v4l2_config[j].crop_width = 0;
v4l2_config[j].crop_height = 0;
v4l2_config[j].ds_width = VIDEO_WIDTH;
v4l2_config[j].ds_height = VIDEO_HEIGHT;
}
}
int send_cnn_buf (uint8_t *ptr_cnn_buf, uint64_t time_stamp_us, uint32_t cam_ch, E_NETWORK_UID net_id)
{
int ret = 0;
stCnnData *cnn_data;
struct mq_attr attr;
attr.mq_maxmsg = MAX_MQ_MSG_CNT;
attr.mq_msgsize = sizeof(stCnnData*);
int oflag = O_WRONLY | O_CREAT;
mqd_t mfd = mq_open(MQ_NAME_CNN_BUF, oflag, 0666, &attr);
if (mfd == -1) {
perror("mq open error");
return -1;
}
cnn_data = (stCnnData*)malloc(sizeof(stCnnData));
cnn_data->cam_ch = cam_ch;
cnn_data->ptr_cnn_buf = ptr_cnn_buf;
cnn_data->time_stamp_us = time_stamp_us;
cnn_data->net_id = net_id;
if ((ret = mq_send(mfd, (const char *)&cnn_data, attr.mq_msgsize, 1)) == -1) {
printf("errno of mq_send = %d\n", errno);
}
mq_close(mfd);
return ret;
}
int receive_cnn_buf (stCnnData **out_cnn_buf)
{
int ret = 0;
struct mq_attr attr;
attr.mq_maxmsg = MAX_MQ_MSG_CNT;
attr.mq_msgsize = sizeof(stCnnData *);
int oflag = O_RDONLY | O_CREAT;
mqd_t mfd = mq_open(MQ_NAME_CNN_BUF, oflag, 0666, &attr);
if (mfd == -1) {
perror("mq open error");
return -1;
}
if ((ret = (int32_t)mq_receive(mfd, (char*)out_cnn_buf, attr.mq_msgsize, NULL)) == -1) {
printf("errno of mq_receive = %d\n", errno);
}
mq_close(mfd);
return ret;
}
int npu_init(st_npu_input_info *npu_input_info)
{
/* Initialize CNN */
if (nc_aiw_init_cnn() < 0 ) {
fprintf(stderr, "nc_aiw_init_cnn() failure!!\n");
return -1;
}
#ifdef SHOW_FACE_DETECT
if (nc_aiw_add_network_to_builder(nc_localize_path((const char *)NETWORK_FILE_FACE_DET), NETWORK_FACE_DET, nc_postprocess_yolov8_inference_result) < 0) {
fprintf(stderr, "nc_aiw_add_network_to_builder() failure!!\n");
return -1;
}
#endif
#ifdef SHOW_CUUVA_DETECT
if (nc_aiw_add_network_to_builder(nc_localize_path((const char *)NETWORK_FILE_CUUVA_DET), NETWORK_CUUVA_DET, nc_postprocess_yolov8_inference_result) < 0) {
fprintf(stderr, "nc_aiw_add_network_to_builder() failure!!\n");
return -1;
}
#endif
#ifdef SHOW_FIRE_DETECT
if (nc_aiw_add_network_to_builder(nc_localize_path((const char *)NETWORK_FILE_FIRE_DET), NETWORK_FIRE_DET, nc_postprocess_yolov8_inference_result) < 0) {
fprintf(stderr, "nc_aiw_add_network_to_builder() failure!!\n");
return -1;
}
#endif
#ifdef SHOW_LPR_DETECT
if (nc_aiw_add_network_to_builder(nc_localize_path((const char *)NETWORK_FILE_LPR_DET), NETWORK_LPR_DET, nc_postprocess_yolov8_inference_result) < 0) {
fprintf(stderr, "nc_aiw_add_network_to_builder() failure!!\n");
return -1;
}
#endif
if(nc_aiw_finish_network_builder() < 0 ) {
fprintf(stderr, "nc_aiw_finish_network_builder() failure!!\n");
return -1;
}
// obtain the input resoltution for the CNN network
// get information of the input tensor
aiwTensorInfo in_tinfo;
#ifdef SHOW_FACE_DETECT
if(nc_get_cnn_network_input_resol(NETWORK_FACE_DET, &in_tinfo) < 0)
{
printf("failed to get the input resolution for the CNN network\n");
}
#endif
#ifdef SHOW_CUUVA_DETECT
if(nc_get_cnn_network_input_resol(NETWORK_CUUVA_DET, &in_tinfo) < 0)
{
printf("failed to get the input resolution for the CNN network\n");
}
#endif
#ifdef SHOW_FIRE_DETECT
if(nc_get_cnn_network_input_resol(NETWORK_FIRE_DET, &in_tinfo) < 0)
{
printf("failed to get the input resolution for the CNN network\n");
}
#endif
#ifdef SHOW_LPR_DETECT
if(nc_get_cnn_network_input_resol(NETWORK_LPR_DET, &in_tinfo) < 0)
{
printf("failed to get the input resolution for the CNN network\n");
}
#endif
npu_input_info->w = in_tinfo.dim.w;// network input width
npu_input_info->h = in_tinfo.dim.h;// network input height
npu_input_info->rgb_size = in_tinfo.dim.w * in_tinfo.dim.h * RGB_CNT;
printf("aiw finish\n");
return 0;
}
void *cnn_task(void *arg)
{
(void) arg;
printf("CNN TASK RUN!!\n");
while (g_running) {
stCnnData *cnn_data = NULL;
if(receive_cnn_buf(&cnn_data) != -1){
nc_aiw_run_cnn(cnn_data->ptr_cnn_buf, cnn_data->time_stamp_us, cnn_data->cam_ch, cnn_data->net_id);
if(cnn_data) {
free(cnn_data->ptr_cnn_buf);
free(cnn_data);
}
}
}
printf("EXIT CNN_TASK!!\n");
return NULL;
}
// 원본 버퍼 복제해서 MQ로 전송 (수신측 free)
static inline void dup_and_send(unsigned char* src, size_t sz,
uint64_t ts_us, uint32_t ch, E_NETWORK_UID net_uid){
unsigned char* p = (unsigned char*)malloc(sz);
if (!p) return;
memcpy(p, src, sz);
send_cnn_buf(p, ts_us, ch, net_uid);
}
static void dispatch_stage1_and_plan_followups(
uint32_t m,
uint32_t ch,
unsigned char* rgb_planar, size_t rgb_size,
uint64_t ts_us)
{
const int UID_DET = nc_get_cnn_networks_id_by_uid(NETWORK_CUUVA_DET);
const int UID_FIRE = nc_get_cnn_networks_id_by_uid(NETWORK_FIRE_DET);
if (CTRL_MODEL_ON(m, NETWORK_OBJDET) && UID_DET >= 0)
dup_and_send(rgb_planar, rgb_size, ts_us, ch, (E_NETWORK_UID)UID_DET);
if (CTRL_MODEL_ON(m, NETWORK_FIRE) && UID_FIRE >= 0)
dup_and_send(rgb_planar, rgb_size, ts_us, ch, (E_NETWORK_UID)UID_FIRE);
}
static void dispatch_stage2_followups(
uint32_t m, // ctrl_active_models_acquire() 결과(모델 비트)
uint32_t ch,
unsigned char* rgb_planar, size_t rgb_size,
uint64_t ts_us,
bool need_face,
bool need_lpr
){
const int UID_FACE = nc_get_cnn_networks_id_by_uid(NETWORK_FACE_DET);
const int UID_LPR = nc_get_cnn_networks_id_by_uid(NETWORK_LPR_DET);
// const int UID_FIRE = nc_get_cnn_networks_id_by_uid(NETWORK_FIRE_DET);
// const int UID_FIRE = nc_get_cnn_networks_id_by_uid(NETWORK_FIRE_DET);
if (need_face && CTRL_MODEL_ON(m, NETWORK_FACE) && UID_FACE >= 0){
dup_and_send(rgb_planar, rgb_size, ts_us, ch, (E_NETWORK_UID)UID_FACE);
}
if (need_lpr && CTRL_MODEL_ON(m, NETWORK_LPR) && UID_LPR >= 0){
dup_and_send(rgb_planar, rgb_size, ts_us, ch, (E_NETWORK_UID)UID_LPR);
}
// dup_and_send(rgb_planar, rgb_size, ts_us, ch, (E_NETWORK_UID)UID_FACE);
// if (need_abnorm && CTRL_MODEL_ON(m, NETWORK_ABNORM) && UID_ABNORM >= 0)
// dup_and_send(rgb_planar, rgb_size, ts_us, ch, (E_NETWORK_UID)UID_ABNORM);
}
// static inline void send_by_ctrl_mask(
// uint32_t m,
// uint32_t ch,
// unsigned char* rgb_planar, size_t rgb_size,
// uint64_t ts_us)
// {
// dispatch_stage1_and_plan_followups(m, ch, rgb_planar, rgb_size, ts_us);
// }
// -----------------------------
// V4L2 초기화
// -----------------------------
static int v4l2_initialize(void)
{
for (int i = 0; i < VIDEO_MAX_CH; i++) {
v4l2_config[i].video_buf.video_fd =
nc_v4l2_open(v4l2_config[i].video_buf.video_device_num, true);
if (v4l2_config[i].video_buf.video_fd == errno) {
fprintf(stderr, "[error] nc_v4l2_open() failure (ch=%d)\n", i);
return -1;
}
if (nc_v4l2_init_device_and_stream_on(&v4l2_config[i], VIDEO_BUFFER_NUM) < 0) {
fprintf(stderr, "[error] nc_v4l2_init_device_and_stream_on() failure (ch=%d)\n", i);
return -1;
}
}
nc_v4l2_show_user_config(&v4l2_config[0], VIDEO_MAX_CH);
return 0;
}
#ifndef ARRAY_LEN
#define ARRAY_LEN(a) ((uint32_t)(sizeof(a)/sizeof((a)[0])))
#endif
static void collect_and_append(
uint32_t ch,
const stCnnPostprocessingResults *R,
const char *tag,
bool *need_face_followup,
bool *need_lpr_followup,
uds_det_entry_t **total_items,
uint32_t *total_items_cnt
){
const uint32_t CLASS_CAP = ARRAY_LEN(R->class_objs);
uint32_t total = 0;
for (uint32_t ci = 0; ci < CLASS_CAP; ++ci) {
int cnt = R->class_objs[ci].obj_cnt;
if (cnt > 0) total += (uint32_t)cnt;
}
if (total == 0) return;
uds_det_entry_t *items = (uds_det_entry_t*)malloc((size_t)total * sizeof(uds_det_entry_t));
if (!items) return;
uint16_t tag_id = 0;
if (!strcmp(tag, "DET")) tag_id = UDS_TAG_DET;
else if (!strcmp(tag, "FIRE")) tag_id = UDS_TAG_FIRE;
else if (!strcmp(tag, "FACE")) tag_id = UDS_TAG_FACE;
else if (!strcmp(tag, "LPR")) tag_id = UDS_TAG_LPR;
uint32_t k = 0;
for (uint32_t ci = 0; ci < CLASS_CAP; ++ci) {
int n = R->class_objs[ci].obj_cnt;
if (n <= 0) continue;
const stObjInfo *objs = R->class_objs[ci].objs;
for (int j = 0; j < n; ++j) {
const stObjInfo *o = &objs[j];
uds_det_entry_t *dst = &items[k++];
dst->prob = (float)o->prob;
dst->x = (float)o->bbox.x;
dst->y = (float)o->bbox.y;
dst->w = (float)o->bbox.w;
dst->h = (float)o->bbox.h;
dst->cls = UDS_ENC_CLS(tag_id, (uint16_t)ci);
dst->reserved = UDS_RESERVED_PACK(ctrl_feat_mask_snapshot(), ctrl_active_models_acquire());
printf("[%s] ch=%u ci=%u j=%d k=%u prob=%.3f bbox=%.1f,%.1f,%.1f,%.1f\n",
tag, ch, ci, j, k, (float)o->prob,
(float)o->bbox.x, (float)o->bbox.y, (float)o->bbox.w, (float)o->bbox.h);
if ((uint32_t)ci == FOLLOWUP_CLS_ID && tag_id == UDS_TAG_DET) {
*need_face_followup = true;
}
if ((uint32_t)ci == FOLLOWUP_CLS_ID && tag_id == UDS_TAG_DET) {
*need_lpr_followup = true;
}
}
}
uds_det_entry_t *tmp = (uds_det_entry_t*)realloc(*total_items, (size_t)(*total_items_cnt + total) * sizeof(uds_det_entry_t));
if (tmp) {
*total_items = tmp;
memcpy(&(*total_items)[*total_items_cnt], items, (size_t)total * sizeof(uds_det_entry_t));
*total_items_cnt += total;
}
free(items);
}
static void read_and_collect_for(
uint32_t ch,
uint32_t network_offset,
const char *tag,
uint64_t *ts_ns_latest,
uds_det_entry_t **total_items,
uint32_t *total_items_cnt,
bool *need_face_followup,
bool *need_lpr_followup
){
uint64_t ts = 0;
pp_result_buf *det_buf = (pp_result_buf*)nc_tsfs_ff_get_readable_buffer_and_timestamp(ch + network_offset, &ts);
if (det_buf) {
stCnnPostprocessingResults *R = &det_buf->cnn_result;
collect_and_append(ch, R, tag, need_face_followup, need_lpr_followup, total_items, total_items_cnt);
uint64_t ts_ns = ts * 1000ULL;
if (ts_ns > *ts_ns_latest) *ts_ns_latest = ts_ns;
}
nc_tsfs_ff_finish_read_buf(ch + network_offset);
}
// static void send_face_followup_if_needed(
// bool need_face_followup,
// uint32_t ch,
// int *networkOrder,
// unsigned char *rgbdata_for_cnn,
// size_t rgb_size
// ){
// if (!need_face_followup) return;
// uint64_t ts_us = now_us();
// networkOrder[ch] = nc_get_cnn_networks_id_by_uid(NETWORK_YOLOV8_DET);
// unsigned char *rgbdata_for_face = (unsigned char*)malloc(rgb_size);
// if (!rgbdata_for_face) return;
// memcpy(rgbdata_for_face, rgbdata_for_cnn, rgb_size);
// send_cnn_buf(rgbdata_for_face, ts_us, (uint32_t)ch, (E_NETWORK_UID)networkOrder[ch]); // 수신측에서 free
// }
// -----------------------------
// 메인
// -----------------------------
int main(int argc, char** argv)
{
#ifdef USE_8MP_VI
puts("USE_8MP_VI defined (enabled)");
#else
puts("USE_8MP_VI not defined (disabled)");
#endif
(void)argc; (void)argv;
// SIGINT 핸들링
struct sigaction sa;
sa.sa_handler = on_sigint;
sigemptyset(&sa.sa_mask);
sa.sa_flags = SA_RESETHAND;
sigaction(SIGINT, &sa, NULL);
pthread_t p_thread[MAX_TASK_CNT];
int task_cnt = 0;
int thr_id;
int status;
// 채널/디바이스 설정
memset(v4l2_config, 0, sizeof(v4l2_config));
set_v4l2_config();
ctrl_uds_start();
ctrl_feature_on(FEAT_OBJDET);
// V4L2 시작
if (v4l2_initialize() < 0) {
fprintf(stderr, "v4l2_initialize failed\n");
return -1;
}
if (uds_server_start(UDS_SOCK_PATH) != 0) {
fprintf(stderr, "uds_server_start failed\n"); return -1;
}
#ifdef USE_8MP_VI
dsr_init();
#endif
// create thread-safe flip-flop buffer
for (int i = 0; i < VIDEO_MAX_CH; i++) {
if (v4l2_config[i].video_buf.video_fd == -1){
}
else{
#ifdef DETECT_NETWORK
int det_buf_size = sizeof(pp_result_buf);
if(nc_tsfs_ff_create_buffers(i+DETECT_NETWORK, det_buf_size) < 0) {
exit(1);
}
#endif
#ifdef FACE_NETWORK
int face_buf_size = sizeof(pp_result_buf);
if(nc_tsfs_ff_create_buffers(i+FACE_NETWORK, face_buf_size) < 0) {
exit(1);
}
#endif
#ifdef FIRE_NETWORK
int fire_buf_size = sizeof(pp_result_buf);
if(nc_tsfs_ff_create_buffers(i+FIRE_NETWORK, fire_buf_size) < 0) {
exit(1);
}
#endif
#ifdef LPR_NETWORK
int lpr_buf_size = sizeof(pp_result_buf);
if(nc_tsfs_ff_create_buffers(i+LPR_NETWORK, lpr_buf_size) < 0) {
exit(1);
}
#endif
}
}
mq_unlink(MQ_NAME_CNN_BUF);
if(npu_init(&npu_input_info) < 0) {
printf("failed to init NPU\n");
return -1;
}
printf("create tasks\n");
thr_id = pthread_create(&p_thread[task_cnt++], NULL, cnn_task, (void *)NULL);
if (thr_id < 0) {
perror("thread create error : cnn_task");
exit(1);
}
cnn_postprocess_arg cnn_post_param;
cnn_post_param.target_width = VIDEO_WIDTH;
cnn_post_param.target_height = VIDEO_HEIGHT;
thr_id = pthread_create(&p_thread[task_cnt++], NULL, nc_cnn_postprocess_task, (void *)&cnn_post_param);
if (thr_id < 0) {
perror("thread create error : nc_cnn_postprocess_task");
exit(1);
}
unsigned char* rgbdata_for_cnn = (unsigned char *)malloc(npu_input_info.rgb_size);
bool need_face_followup = false;
bool need_lpr_followup = false;
while (g_running) {
// int networkOrder[VIDEO_MAX_CH];
uint32_t m = ctrl_active_models_acquire();
need_face_followup = false;
need_lpr_followup = false;
uint64_t time_stamp_us = 0;
for (int ch = 0; ch < VIDEO_MAX_CH; ch++) {
if (v4l2_config[ch].video_buf.video_fd < 0) continue;
struct v4l2_buffer video_buf;
CLEAR(video_buf);
if (nc_v4l2_dequeue_buffer(v4l2_config[ch].video_buf.video_fd, &video_buf) == -1) {
// non-fatal
continue;
}
// size_t bytes = FRAME_BYTES(w,h);
void* frame_ptr = v4l2_config[ch].video_buf.buffers[video_buf.index].start;
time_stamp_us = now_us();
// writer_push(H, ch, frame_ptr, bytes, time_stamp_us);
uint32_t m = ctrl_active_models_acquire();
// networkOrder[ch] = nc_get_cnn_networks_id_by_uid(NETWORK_FIRE_DET);
// unsigned char* rgbdata_for_det = (unsigned char *)malloc(npu_input_info.rgb_size);
#ifdef USE_8MP_VI
memcpy(dsr_info.dsr_in_buf[0], (uint8_t *)v4l2_config[ch].video_buf.buffers[video_buf.index].start, MAX_WIDTH_FOR_VDMA_CNN_DS*MAX_HEIGHT_FOR_VDMA_CNN_DS*RGB_CNT);
dsr_downscale(dsr_fd, &dsr_info, dsr_input_config, dsr_output_config, dsr_config, 0);
nc_rgb_interleaved_to_planar_neon((unsigned char *)dsr_info.dsr_out_buf[0] \
,(unsigned char *)rgbdata_for_cnn \
,(unsigned char *)rgbdata_for_cnn + NPU_INPUT_WIDTH*NPU_INPUT_HEIGHT \
,(unsigned char *)rgbdata_for_cnn + NPU_INPUT_WIDTH*NPU_INPUT_HEIGHT*2 \
, NPU_INPUT_WIDTH, NPU_INPUT_HEIGHT);
#else
memcpy(rgbdata_for_cnn, (unsigned char *)v4l2_config[ch].video_buf.buffers[video_buf.index].start, npu_input_info.rgb_size); // 600us
#endif
// memcpy(rgbdata_for_cnn, rgbdata_for_det, npu_input_info.rgb_size);
// (내부에서 k_feat2model 사용)
// printf("[SEND] ch=%d mask=0x%08X (CUUVA=%d YOLOV8=%d FIRE=%d)\n",
// ch, m,
// nc_get_cnn_networks_id_by_uid(NETWORK_CUUVA_DET),
// nc_get_cnn_networks_id_by_uid(NETWORK_YOLOV8_DET),
// nc_get_cnn_networks_id_by_uid(NETWORK_FIRE_DET));
uint64_t t0 = now_us();
uds_send_frame(ch, frame_ptr, VIDEO_WIDTH, VIDEO_HEIGHT, VIDEO_WIDTH * 3, WS_PIXFMT_RGB24, time_stamp_us);
printf("[SEND][FRA] took %llu us\n", (unsigned long long)(now_us() - t0));
dispatch_stage1_and_plan_followups(m, ch, rgbdata_for_cnn, npu_input_info.rgb_size, time_stamp_us);
fra_sec_cnt[ch]++;
fra_sec_cnt_total++;
if (fps_win_start_us == 0) fps_win_start_us = time_stamp_us;
if (nc_v4l2_queue_buffer(v4l2_config[ch].video_buf.video_fd, video_buf.index) == -1) {
fprintf(stderr, "Error VIDIOC_QBUF buffer %d (ch=%d)\n", video_buf.index, ch);
}
}
for (uint32_t ch = 0; ch < VIDEO_MAX_CH; ch++) {
if (v4l2_config[ch].video_buf.video_fd == -1) continue;
uint64_t ts_ns_latest = 0;
uds_det_entry_t *total_items = NULL;
uint32_t total_items_cnt = 0;
if (CTRL_MODEL_ON(m, NETWORK_OBJDET)){
read_and_collect_for(
ch, DETECT_NETWORK, "DET",
&ts_ns_latest, &total_items, &total_items_cnt, &need_face_followup, &need_lpr_followup
);
}
if (CTRL_MODEL_ON(m, NETWORK_FIRE)){
read_and_collect_for(
ch, FIRE_NETWORK, "FIRE",
&ts_ns_latest, &total_items, &total_items_cnt, &need_face_followup, &need_lpr_followup
);
}
bool need_face = need_face_followup;
bool need_lpr = need_lpr_followup;
dispatch_stage2_followups(m, ch, rgbdata_for_cnn, npu_input_info.rgb_size, time_stamp_us, need_face, need_lpr);
if (CTRL_MODEL_ON(m, NETWORK_FACE) && need_face){
read_and_collect_for(
ch, FACE_NETWORK, "FACE",
&ts_ns_latest, &total_items, &total_items_cnt, &need_face_followup, &need_lpr_followup
);
}
if (CTRL_MODEL_ON(m, NETWORK_LPR) && need_lpr){
read_and_collect_for(
ch, LPR_NETWORK, "LPR",
&ts_ns_latest, &total_items, &total_items_cnt, &need_face_followup, &need_lpr_followup
);
}
// bool need_lpr = false;
// bool need_abnorm = need_face_followup;
// send_face_followup_if_needed(
// need_followup, ch, networkOrder, rgbdata_for_cnn, npu_input_info.rgb_size
// );
// if (need_face_followup) {
// read_and_collect_for(
// ch, FACE_NETWORK, "FACE",
// &ts_ns_latest, &total_items, &total_items_cnt, &need_followup
// );
// }
uds_det_entry_t dummy;
uint64_t det_ts_us = time_stamp_us;
if (ts_ns_latest > 0) {
det_ts_us = ts_ns_latest / 1000ULL;
}
uds_det_entry_t *send_items =
(total_items_cnt > 0) ? total_items : &dummy;
uint32_t send_cnt = total_items_cnt;
uint32_t seq = ch_seq[ch]++;
uint64_t t0 = now_us();
uds_send_dets_min(ch, seq, det_ts_us, send_items, send_cnt);
printf("[SEND][DET] took %llu us (cnt=%u)\n", (unsigned long long)(now_us() - t0), send_cnt);
if (total_items) free(total_items);
if (fps_win_start_us > 0 && (now_us() - fps_win_start_us) >= 1000000ULL) {
printf("[FPS] ");
for (uint32_t c = 0; c < VIDEO_MAX_CH; c++) {
if (v4l2_config[c].video_buf.video_fd >= 0) {
printf("ch%u=%u ", c, fra_sec_cnt[c]);
}
}
printf("| total=%u fps\n", fra_sec_cnt_total);
// reset window
memset(fra_sec_cnt, 0, sizeof(fra_sec_cnt));
fra_sec_cnt_total = 0;
fps_win_start_us = now_us();
}
}
}
for(int i =0; i< task_cnt; i++) {
pthread_join(p_thread[i], (void **)&status);
}
#ifdef USE_8MP_VI
dsr_deinit();
#endif
uds_server_stop(UDS_SOCK_PATH);
free(rgbdata_for_cnn);
for (int i = 0; i < VIDEO_MAX_CH; i++) {
if (v4l2_config[i].video_buf.video_fd >= 0) {
close(v4l2_config[i].video_buf.video_fd);
v4l2_config[i].video_buf.video_fd = -1;
}
}
return 0;
}