-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathinline_img2d.cpp
464 lines (413 loc) · 18 KB
/
inline_img2d.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
#include <iostream>
#include <thread>
#include <cmath>
#include <string>
#include <csignal>
#include <filesystem>
#include <chrono>
#define CL_HPP_TARGET_OPENCL_VERSION 210
#include <CL/opencl.hpp>
#include <gst/gst.h>
//#include <gst/gl/gl.h>
#include <opencv2/opencv.hpp>
#include <gst/app/gstappsink.h>
#include "stubserver.h"
#include <jsonrpccpp/server/connectors/httpserver.h>
#include "stabilization.h"
#define CROP_P 0.05
using namespace cv;
using namespace jsonrpc;
//GstElement *gltransformation0;
//GstElement *gltransformation1;
GstElement *appsrc;
GstElement *videocrop;
GstElement *x265enc0;
GstElement *x265enc1;
GstElement *udpsink0;
GstElement *udpsink1;
stabilizer st;
VideoCapture cap;
std::thread frames;
int width = 1920;
int height = 1080;
int width_t0 = 1280;
int height_t0 = 720;
int width_t1 = width_t0/2;
int height_t1 = height_t0/2;
int fps = 30;
int crop_x = width*CROP_P;
int crop_y = height*CROP_P;
int bitrate0 = 2000*1000;
int bitrate1 = bitrate0/3;
int stab = 3;
std::string clients0;
std::string clients1;
// Signal handler
void signalHandler(int signal){
}
class JSONServer : public stubserver {
public:
JSONServer(AbstractServerConnector &connector, serverVersion_t type);
virtual void setParam(const Json::Value &args);
virtual Json::Value getParam();
};
JSONServer::JSONServer(AbstractServerConnector &connector, serverVersion_t type) : stubserver(connector,type) {}
void JSONServer::setParam(const Json::Value &args){
//print parameters
auto itr = args.begin();
int i = 0;
for (itr = args.begin(); itr != args.end(); itr++){
auto name = args.getMemberNames()[i];
if (args[name].size() > 1) {
for (auto x:args[name]) {
std::cout << name << ": " << x << std::endl;
}
} else {
std::cout << name + " ";
std::cout << args[name] << std::endl;
}
i++;
}
if (args.isMember("bitrate0") && args["bitrate0"].isInt()){
bitrate0 = args["bitrate0"].asInt()*1000;
g_object_set(x265enc0, "bps", bitrate0, NULL);
}
if (args.isMember("bitrate1") && args["bitrate1"].isInt()){
bitrate1 = args["bitrate1"].asInt()*1000;
g_object_set(x265enc1, "bps", bitrate1, NULL);
}
if (args.isMember("stab") && args["stab"].isInt()) stab = args["stab"].asInt();
if (args.isMember("clients0") && args["clients0"].isString()){
clients0 = args["clients0"].asString();
g_object_set(G_OBJECT(udpsink0), "clients", clients0.c_str(), NULL);
}
if (args.isMember("clients1") && args["clients1"].isString()){
clients0 = args["clients1"].asString();
g_object_set(G_OBJECT(udpsink1), "clients", clients1.c_str(), NULL);
}
}
Json::Value JSONServer::getParam(){
Json::Value result;
result["bitrate0"] = bitrate0/1000;
result["bitrate1"] = bitrate1/1000;
result["stab"] = stab;
result["clients0"] = clients0;
result["clients1"] = clients1;
return result;
}
// OpenCL kernel for rotating an image
const char *rotateKernelSource = R"(
__kernel void rotate_image(
image2d_t srcImage, // Input image
image2d_t dstImage, // Output image
const float center_x, // center x
const float center_y, // center y
const float cosTheta, // Rotation angle in radians
const float sinTheta, // Rotation angle in radians
) {
// Get the global ID (x, y) for the current thread
const int x = get_global_id(0);
const int y = get_global_id(1);
// Compute the relative position of the pixel to the center
const float2 relPos = (float2)((float)x - center_x, (float)y - center_y);
// Apply the rotation to get the new position
const float2 newPos = (float2)(
cosTheta * relPos.x - sinTheta * relPos.y + center.x,
sinTheta * relPos.x + cosTheta * relPos.y + center.y
);
if (newPos.x >= 0 && newPos.x < (width*2) && newPos.y >= 0 && newPos.y < (height*2) {
// Read from the input image using bilinear interpolation
float4 pixel = read_imagef(srcImage, CLK_NORMALIZED_COORDS_FALSE | CLK_FILTER_LINEAR, newPos);
// Write the resulting pixel to the output image
write_imagef(dstImage, (int2)(x, y), pixel);
}
}
)";
// Callback to push data to appsrc
void process_frame() {
Mat frame;
Mat *image;
Mat prevFrame;
Mat output = Mat::zeros(width,height,CV_8UC4);
int frameSize = width*height*4;
std::cout << "Init OpenCL\n";
// Set up OpenCL environment
std::vector<cl::Platform> platforms;
cl::Platform::get(&platforms);
cl::Platform platform = platforms.front();
std::vector<cl::Device> devices;
platform.getDevices(CL_DEVICE_TYPE_GPU, &devices);
cl::Device device = devices.front();
cl::Context context(device);
cl::CommandQueue queue(context, device);
cl::Program program(context, rotateKernelSource, true);
cl::Kernel rotateKernel(program, "rotate_image");
cl::NDRange globalSize(width, height);
std::cout << "Started reading frames\n";
uint64_t frame_count = 0;
//some timers for benchmarking
auto t1 = std::chrono::high_resolution_clock::now();
auto t2 = t1;
auto t3 = t1;
auto t4 = t1;
// Allocate OpenCL input and output buffers
cl::Image2D inputBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, cl::ImageFormat(CL_RGBA, CL_UNORM_INT8), width, height);
cl::Image2D outputBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, cl::ImageFormat(CL_RGBA, CL_UNORM_INT8), width, height);
// Set fixed kernel arguments
rotateKernel.setArg(0, inputBuffer);
rotateKernel.setArg(1, outputBuffer);
rotateKernel.setArg(2, width/2);
rotateKernel.setArg(3, height/2);
cl::array<uint64_t,3> origin = {0, 0, 0};
cl::array<uint64_t,3> region = {(uint32_t)width, (uint32_t)height, 1};
while(true){
static GstClockTime timestamp = 0;
cap >> frame;
t1 = std::chrono::high_resolution_clock::now();
if (stab > 0){
float x,y,a;
st.stabilize(frame,&x,&y,&a);
//avoid stabilization if the correction is too small
//std::cout << x << " " << y << " "<< a << "\n";
if (x < 0.002f && x > -0.002f) x = 0.0f;
if (y < 0.002f && y > -0.002f) y = 0.0f;
if (a < 0.001f && a > -0.001f) a = 0.0f;
gfloat gx = x;
gfloat gy = y;
//clamp rotation angle to 4 degrees
if (a > 0.07) a = 0.07;
if (a < -0.07) a = -0.07;
a= 0.0f;
prevFrame = st.getPrevFrame();
if (prevFrame.empty()) continue;
image = &prevFrame;
if(stab > 2){
t3 = std::chrono::high_resolution_clock::now();
cvtColor(prevFrame,prevFrame,COLOR_BGR2BGRA);
rotateKernel.setArg(4, std::cos(a));
rotateKernel.setArg(5, std::sin(a));
// Run the kernel
queue.enqueueWriteImage(inputBuffer, CL_TRUE, origin, region, 0, 0, (unsigned char*)prevFrame.datastart);
queue.enqueueNDRangeKernel(rotateKernel, cl::NullRange, globalSize, cl::NullRange);
queue.enqueueReadImage(outputBuffer, CL_TRUE, origin, region, 0, 0, (unsigned char*)output.datastart);
cvtColor(output,prevFrame,COLOR_BGRA2BGR);
t4 = std::chrono::high_resolution_clock::now();
}
g_object_set(videocrop, "top", crop_y-(int)(gy*height),
"bottom", crop_y+(int)(gy*height),
"left", crop_x-(int)(gx*width),
"right", crop_x+(int)(gx*width),
NULL);
}
else{
image = &frame;
g_object_set(videocrop, "top", 0,
"bottom", 0,
"left", 0,
"right", 0,
NULL);
}
// Create a new buffer
GstBuffer *buffer;
guint size;
GstFlowReturn ret;
size = image->total() * image->elemSize();
buffer = gst_buffer_new_allocate(NULL, size, NULL);
// Set the buffer data
GstMapInfo map;
gst_buffer_map(buffer, &map, GST_MAP_WRITE);
memcpy(map.data, image->data, size);
gst_buffer_unmap(buffer, &map);
// Set the buffer timestamp and duration
GST_BUFFER_PTS(buffer) = timestamp;
GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale_int(1, GST_SECOND, fps);
timestamp += GST_BUFFER_DURATION(buffer);
// Push the buffer into the appsrc
g_signal_emit_by_name(appsrc, "push-buffer", buffer, &ret);
gst_buffer_unref(buffer);
if (ret != GST_FLOW_OK) {
g_printerr("Error pushing buffer to appsrc\n");
}
t2 = std::chrono::high_resolution_clock::now();
int delay = std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count();
int rot = std::chrono::duration_cast<std::chrono::milliseconds>(t4-t3).count();
if (frame_count % 120 == 0) std::cout << "loop ms: " << delay-rot << "\n" << "rot ms: " << rot << "\n";
frame_count++;
}
}
int main(int argc, char *argv[]) {
gst_init(&argc, &argv);
//std::signal(SIGINT, signalHandler);
if (argc < 7) {
std::cout << "Arguments are v4l2 device, clients for stream 1, clients for stream 2, ts filename and port for JSON RPC listener.\nInput width, input height, stream 1 width, stream 1 height, stream 2 width, stream 2 height, framerate, bitrate 1 kbps, bitrate 2 kbps, stabilization mode.\n";
return -1;
}
const char* device = argv[1];
clients0 = std::string(argv[2]);
clients1 = std::string(argv[3]);
const char* fout = argv[4];
const char* shmout = argv[5];
int port = std::stoi(argv[6]);
if (argc == 17){
std::cout << "Setting extra params\n";
width = std::stoi(argv[7]);
height = std::stoi(argv[8]);
width_t0 = std::stoi(argv[9]);
height_t0 = std::stoi(argv[10]);
width_t1 = std::stoi(argv[11]);
height_t1 = std::stoi(argv[12]);
fps = std::stoi(argv[13]);
bitrate0 = std::stoi(argv[14])*1000;
bitrate1 = std::stoi(argv[15])*1000;
stab = std::stoi(argv[16]);
crop_x = width*CROP_P;
crop_y = height*CROP_P;
}
std::string filename = std::string(shmout);
if (std::filesystem::exists(filename)) {
if (std::filesystem::remove(filename)) {
std::cout << "Pipe deleted successfully: " << filename << std::endl;
}
else {
std::cerr << "Pipe in use?" << filename << std::endl;
}
}
HttpServer httpserver(port);
JSONServer s(httpserver,JSONRPC_SERVER_V2);
s.StartListening();
std::string cvpipeline = "v4l2src device=" + std::string(device) + " io-mode=4 ! "
"image/jpeg,width="+ std::to_string(width) +",height="+ std::to_string(height) +",framerate="+ std::to_string(fps) +"/1 ! "
"queue ! mppjpegdec format=16 ! video/x-raw,format=BGR ! tee name=t t. ! "
"queue ! appsink max-buffers=2 drop=true sync=false t. ! "
"queue ! shmsink socket-path=" + std::string(shmout) + " wait-for-connection=false" ;
cap.open(cvpipeline.c_str(),CAP_GSTREAMER);
if (!cap.isOpened()) {
std::cerr << "Error: Unable to open the camera with GStreamer pipeline." << std::endl;
return -1;
}
// GStreamer pipeline elements
GstElement *pipeline, *videoconvert, *queue0, *glupload, *glcolorconvert0, *gltransform0, *gltransform1, *glcolorconvert1, *gldownload, *enccaps, *queue1, *x264enc, *rtph265pay0, *rtph265pay1, *tee, *queue_h265_0, *queue_h264, *tsmux, *filesink, *h264parse, *queue_h265_1;
GstCaps *caps;
// Create GStreamer pipeline
pipeline = gst_pipeline_new("video-pipeline");
queue0 = gst_element_factory_make("queue", "queue0");
queue1 = gst_element_factory_make("queue", "queue1");
appsrc = gst_element_factory_make("appsrc", "source");
videoconvert = gst_element_factory_make("videoconvert", "videoconvert");
/*
glcolorconvert0 = gst_element_factory_make("glcolorconvert", "glcolorconvert0");
glupload = gst_element_factory_make("glupload", "glupload");
gltransformation0 = gst_element_factory_make("gltransformation", "gltransformation0");
gltransformation1 = gst_element_factory_make("gltransformation", "gltransformation1");
glcolorconvert1 = gst_element_factory_make("glcolorconvert", "glcolorconvert1");
gldownload = gst_element_factory_make("gldownload", "gldownload");
*/
enccaps = gst_element_factory_make("capsfilter", "enccaps");
caps = gst_caps_new_simple("video/x-raw",
"format", G_TYPE_STRING, "I420",
NULL);
g_object_set(enccaps, "caps", caps, NULL);
gst_caps_unref(caps);
videocrop = gst_element_factory_make("videocrop", "videocrop");
g_object_set(videocrop, "top", crop_y,
"bottom", crop_y,
"left", crop_x,
"right", crop_x,
NULL);
x265enc0 = gst_element_factory_make("mpph265enc", "x265-encoder0");
g_object_set(x265enc0, "bps", bitrate0,
"header-mode", 1,
"max-pending", 1,
"rc-mode", 1,
"gop", fps,
"width", width_t0,
"height", height_t0,
NULL);
x265enc1 = gst_element_factory_make("mpph265enc", "x265-encoder1");
g_object_set(x265enc1, "bps", bitrate1,
"header-mode", 1,
"max-pending", 1,
"rc-mode", 1,
"gop", fps,
"width", width_t1,
"height", height_t1,
NULL);
x264enc = gst_element_factory_make("mpph264enc", "x264-encoder");
g_object_set(x264enc, "bps", 20000000,
"bps-min", 10000000,
"bps-max", 30000000,
"header-mode", 1,
"max-pending", 16,
"max-reenc", 3,
"gop", fps*10,
"rc-mode", 0,
"width", width,
"height",height,
NULL);
tsmux = gst_element_factory_make("mpegtsmux", "tsmux");
rtph265pay0 = gst_element_factory_make("rtph265pay", "h265-payloader0");
g_object_set(rtph265pay0, "config-interval", -1,
"aggregate-mode", 1,
"mtu", 768,
NULL);
rtph265pay1 = gst_element_factory_make("rtph265pay", "h265-payloader1");
g_object_set(rtph265pay1, "config-interval", -1,
"aggregate-mode", 1,
"mtu", 1400,
NULL);
filesink = gst_element_factory_make("filesink", "file-sink");
g_object_set(G_OBJECT(filesink), "location", fout, "sync", FALSE, "async", FALSE, NULL);
udpsink0 = gst_element_factory_make("multiudpsink", "udp-sink_0");
g_object_set(G_OBJECT(udpsink0), "clients", clients0.c_str(), "sync", FALSE, "async", FALSE, NULL);
udpsink1 = gst_element_factory_make("multiudpsink", "udp-sink_1");
g_object_set(G_OBJECT(udpsink1), "clients", clients1.c_str(), "sync", FALSE, "async", FALSE, NULL);
// Configure tee elements and queues
tee = gst_element_factory_make("tee", "tee");
queue_h264 = gst_element_factory_make("queue", "queue_h264");
queue_h265_0 = gst_element_factory_make("queue", "queue_h265_0");
queue_h265_1 = gst_element_factory_make("queue", "queue_h265_1");
// Configure the appsrc element
g_object_set(G_OBJECT(appsrc), "caps",
gst_caps_new_simple("video/x-raw",
"format", G_TYPE_STRING, "BGR",
"width", G_TYPE_INT, width,
"height", G_TYPE_INT, height,
"framerate", GST_TYPE_FRACTION, fps, 1,
NULL), NULL);
g_object_set(G_OBJECT(appsrc), "format", GST_FORMAT_TIME, NULL);
g_object_set(G_OBJECT(appsrc), "is-live", TRUE, NULL);
// Set the UDP sink properties
// Build the pipeline
gst_bin_add_many(GST_BIN(pipeline), appsrc, queue0, videocrop, tee,
queue_h265_0, x265enc0, rtph265pay0, udpsink0,
queue_h265_1, x265enc1, rtph265pay1, udpsink1,
queue_h264, x264enc, tsmux, filesink,
NULL);
if (!gst_element_link_many(appsrc, queue0, videocrop, tee, NULL) ||
!gst_element_link_many(queue_h265_0, x265enc0, rtph265pay0, udpsink0, NULL) ||
!gst_element_link_many(queue_h265_1, x265enc1, rtph265pay1, udpsink1, NULL) ||
!gst_element_link_many(queue_h264, x264enc, tsmux, filesink, NULL) ||
!gst_element_link_many(tee, queue_h265_0, NULL) ||
!gst_element_link_many(tee, queue_h265_1, NULL) ||
!gst_element_link_many(tee, queue_h264, NULL)
)
{
g_printerr("GStreamer elements could not be linked.\n");
gst_object_unref(pipeline);
return -1;
}
// Set the pipeline to the playing state
gst_element_set_state(pipeline, GST_STATE_PLAYING);
// Run the main loop
GMainLoop *main_loop = g_main_loop_new(NULL, FALSE);
frames = std::thread(&process_frame);
frames.detach();
g_main_loop_run(main_loop);
// Clean up
gst_element_set_state(pipeline, GST_STATE_NULL);
gst_object_unref(pipeline);
g_main_loop_unref(main_loop);
cap.release();
s.StopListening();
return 0;
}