From 194582931af3b95a5c550a08dc29945840880938 Mon Sep 17 00:00:00 2001 From: fcinfo <34166216+fcinfo@users.noreply.github.com> Date: Wed, 28 Mar 2018 10:31:41 +0800 Subject: [PATCH 1/4] FFmpegFrameGrabber can specify the decoder name and some options --- .../bytedeco/javacv/FFmpegFrameGrabber.java | 2145 ++++++++------- .../bytedeco/javacv/FFmpegFrameRecorder.java | 2304 +++++++++-------- .../org/bytedeco/javacv/FrameGrabber.java | 1416 +++++----- .../org/bytedeco/javacv/FrameRecorder.java | 763 +++--- 4 files changed, 3374 insertions(+), 3254 deletions(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java index b40f3cf0..bc1e52bd 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java @@ -56,7 +56,6 @@ import java.nio.Buffer; import java.nio.ByteBuffer; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -66,6 +65,7 @@ import org.bytedeco.javacpp.Loader; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.PointerPointer; +import org.bytedeco.javacpp.avutil; import static org.bytedeco.javacpp.avcodec.*; import static org.bytedeco.javacpp.avdevice.*; @@ -79,1116 +79,1035 @@ * @author Samuel Audet */ public class FFmpegFrameGrabber extends FrameGrabber { - public static String[] getDeviceDescriptions() throws Exception { - tryLoad(); - throw new UnsupportedOperationException("Device enumeration not support by FFmpeg."); - } - - public static FFmpegFrameGrabber createDefault(File deviceFile) throws Exception { return new FFmpegFrameGrabber(deviceFile); } - public static FFmpegFrameGrabber createDefault(String devicePath) throws Exception { return new FFmpegFrameGrabber(devicePath); } - public static FFmpegFrameGrabber createDefault(int deviceNumber) throws Exception { throw new Exception(FFmpegFrameGrabber.class + " does not support device numbers."); } - - private static Exception loadingException = null; - public static void tryLoad() throws Exception { - if (loadingException != null) { - throw loadingException; - } else { - try { - Loader.load(org.bytedeco.javacpp.avutil.class); - Loader.load(org.bytedeco.javacpp.swresample.class); - Loader.load(org.bytedeco.javacpp.avcodec.class); - Loader.load(org.bytedeco.javacpp.avformat.class); - Loader.load(org.bytedeco.javacpp.swscale.class); - - // Register all formats and codecs - avcodec_register_all(); - av_register_all(); - avformat_network_init(); - - Loader.load(org.bytedeco.javacpp.avdevice.class); - avdevice_register_all(); - } catch (Throwable t) { - if (t instanceof Exception) { - throw loadingException = (Exception)t; - } else { - throw loadingException = new Exception("Failed to load " + FFmpegFrameGrabber.class, t); - } - } - } - } - - static { - try { - tryLoad(); - FFmpegLockCallback.init(); - } catch (Exception ex) { } - } - - public FFmpegFrameGrabber(File file) { - this(file.getAbsolutePath()); - } - public FFmpegFrameGrabber(String filename) { - this.filename = filename; - this.pixelFormat = AV_PIX_FMT_NONE; - this.sampleFormat = AV_SAMPLE_FMT_NONE; - } - public FFmpegFrameGrabber(InputStream inputStream) { - this.inputStream = inputStream; - this.pixelFormat = AV_PIX_FMT_NONE; - this.sampleFormat = AV_SAMPLE_FMT_NONE; - } - public void release() throws Exception { - // synchronized (org.bytedeco.javacpp.avcodec.class) { - releaseUnsafe(); - // } - } - void releaseUnsafe() throws Exception { - if (pkt != null && pkt2 != null) { - if (pkt2.size() > 0) { - av_packet_unref(pkt); - } - pkt = pkt2 = null; - } - - // Free the RGB image - if (image_ptr != null) { - for (int i = 0; i < image_ptr.length; i++) { - av_free(image_ptr[i]); - } - image_ptr = null; - } - if (picture_rgb != null) { - av_frame_free(picture_rgb); - picture_rgb = null; - } - - // Free the native format picture frame - if (picture != null) { - av_frame_free(picture); - picture = null; - } - - // Close the video codec - if (video_c != null) { - avcodec_free_context(video_c); - video_c = null; - } - - // Free the audio samples frame - if (samples_frame != null) { - av_frame_free(samples_frame); - samples_frame = null; - } - - // Close the audio codec - if (audio_c != null) { - avcodec_free_context(audio_c); - audio_c = null; - } - - // Close the video file - if (inputStream == null && oc != null && !oc.isNull()) { - avformat_close_input(oc); - oc = null; - } - - if (img_convert_ctx != null) { - sws_freeContext(img_convert_ctx); - img_convert_ctx = null; - } - - if (samples_ptr_out != null) { - for (int i = 0; i < samples_ptr_out.length; i++) { - av_free(samples_ptr_out[i].position(0)); - } - samples_ptr_out = null; - samples_buf_out = null; - } - - if (samples_convert_ctx != null) { - swr_free(samples_convert_ctx); - samples_convert_ctx = null; - } - - got_frame = null; - frameGrabbed = false; - frame = null; - timestamp = 0; - frameNumber = 0; - - if (inputStream != null) { - try { - if (oc == null) { - // when called a second time - inputStream.close(); - } else { - inputStream.reset(); - } - } catch (IOException ex) { - throw new Exception("Error on InputStream.close(): ", ex); - } finally { - inputStreams.remove(oc); - if (avio != null) { - if (avio.buffer() != null) { - av_free(avio.buffer()); - avio.buffer(null); - } - av_free(avio); - avio = null; - } - if (oc != null) { - avformat_free_context(oc); - oc = null; - } - } - } - } - @Override protected void finalize() throws Throwable { - super.finalize(); - release(); - } - - static Map inputStreams = Collections.synchronizedMap(new HashMap()); - - static class ReadCallback extends Read_packet_Pointer_BytePointer_int { - @Override public int call(Pointer opaque, BytePointer buf, int buf_size) { - try { - byte[] b = new byte[buf_size]; - InputStream is = inputStreams.get(opaque); - int size = is.read(b, 0, buf_size); - if (size < 0) { - return 0; - } else { - buf.put(b, 0, size); - return size; - } - } - catch (Throwable t) { - System.err.println("Error on InputStream.read(): " + t); - return -1; - } - } - } - - static class SeekCallback extends Seek_Pointer_long_int { - @Override public long call(Pointer opaque, long offset, int whence) { - try { - InputStream is = inputStreams.get(opaque); - switch (whence) { - case 0: is.reset(); break; - case 1: break; - default: return -1; - } - long remaining = offset; - while (remaining > 0) { - long skipped = is.skip(remaining); - if (skipped == 0) break; // end of the stream - remaining -= skipped; - } - return 0; - } catch (Throwable t) { - System.err.println("Error on InputStream.reset() or skip(): " + t); - return -1; - } - } - } - - static ReadCallback readCallback = new ReadCallback(); - static SeekCallback seekCallback = new SeekCallback(); - - private InputStream inputStream; - private AVIOContext avio; - private String filename; - private AVFormatContext oc; - private AVStream video_st, audio_st; - private AVCodecContext video_c, audio_c; - private AVFrame picture, picture_rgb; - private BytePointer[] image_ptr; - private Buffer[] image_buf; - private AVFrame samples_frame; - private BytePointer[] samples_ptr; - private Buffer[] samples_buf; - private BytePointer[] samples_ptr_out; - private Buffer[] samples_buf_out; - private AVPacket pkt, pkt2; - private int sizeof_pkt; - private int[] got_frame; - private SwsContext img_convert_ctx; - private SwrContext samples_convert_ctx; - private int samples_channels, samples_format, samples_rate; - private boolean frameGrabbed; - private Frame frame; - - /** - * Is there a video stream? - * @return {@code video_st!=null;} - */ - public boolean hasVideo() { - return video_st!=null; - } - - /** - * Is there an audio stream? - * @return {@code audio_st!=null;} - */ - public boolean hasAudio() { - return audio_st!=null; - } - - @Override public double getGamma() { - // default to a gamma of 2.2 for cheap Webcams, DV cameras, etc. - if (gamma == 0.0) { - return 2.2; - } else { - return gamma; - } - } - - @Override public String getFormat() { - if (oc == null) { - return super.getFormat(); - } else { - return oc.iformat().name().getString(); - } - } - - @Override public int getImageWidth() { - return imageWidth > 0 || video_c == null ? super.getImageWidth() : video_c.width(); - } - - @Override public int getImageHeight() { - return imageHeight > 0 || video_c == null ? super.getImageHeight() : video_c.height(); - } - - @Override public int getAudioChannels() { - return audioChannels > 0 || audio_c == null ? super.getAudioChannels() : audio_c.channels(); - } - - @Override public int getPixelFormat() { - if (imageMode == ImageMode.COLOR || imageMode == ImageMode.GRAY) { - if (pixelFormat == AV_PIX_FMT_NONE) { - return imageMode == ImageMode.COLOR ? AV_PIX_FMT_BGR24 : AV_PIX_FMT_GRAY8; - } else { - return pixelFormat; - } - } else if (video_c != null) { // RAW - return video_c.pix_fmt(); - } else { - return super.getPixelFormat(); - } - } - - @Override public int getVideoCodec() { - return video_c == null ? super.getVideoCodec() : video_c.codec_id(); - } - - @Override public int getVideoBitrate() { - return video_c == null ? super.getVideoBitrate() : (int)video_c.bit_rate(); - } - - @Override public double getAspectRatio() { - if (video_st == null) { - return super.getAspectRatio(); - } else { - AVRational r = av_guess_sample_aspect_ratio(oc, video_st, picture); - double a = (double)r.num() / r.den(); - return a == 0.0 ? 1.0 : a; - } - } - - /** Returns {@link #getVideoFrameRate()} */ - @Override public double getFrameRate() { - return getVideoFrameRate(); - } - - /**Estimation of audio frames per second - * - * @return (double) getSampleRate()) / samples_frame.nb_samples() - * if samples_frame.nb_samples() is not zero, otherwise return 0 - */ - public double getAudioFrameRate() { - if (audio_st == null) { - return 0.0; - } else { - if (samples_frame == null || samples_frame.nb_samples() == 0) { - try { - grabFrame(true, false, false, false); - frameGrabbed = true; - } catch (Exception e) { - return 0.0; - } - } - if (samples_frame != null || samples_frame.nb_samples() != 0) - return ((double) getSampleRate()) / samples_frame.nb_samples(); - else return 0.0; - - } - } - - public double getVideoFrameRate() { - if (video_st == null) { - return super.getFrameRate(); - } else { - AVRational r = video_st.avg_frame_rate(); - if (r.num() == 0 && r.den() == 0) { - r = video_st.r_frame_rate(); - } - return (double)r.num() / r.den(); - } - } - - @Override public int getAudioCodec() { - return audio_c == null ? super.getAudioCodec() : audio_c.codec_id(); - } - - @Override public int getAudioBitrate() { - return audio_c == null ? super.getAudioBitrate() : (int)audio_c.bit_rate(); - } - - @Override public int getSampleFormat() { - if (sampleMode == SampleMode.SHORT || sampleMode == SampleMode.FLOAT) { - if (sampleFormat == AV_SAMPLE_FMT_NONE) { - return sampleMode == SampleMode.SHORT ? AV_SAMPLE_FMT_S16 : AV_SAMPLE_FMT_FLT; - } else { - return sampleFormat; - } - } else if (audio_c != null) { // RAW - return audio_c.sample_fmt(); - } else { - return super.getSampleFormat(); - } - } - - @Override public int getSampleRate() { - return sampleRate > 0 || audio_c == null ? super.getSampleRate() : audio_c.sample_rate(); - } - - @Override public String getMetadata(String key) { - if (oc == null) { - return super.getMetadata(key); - } - AVDictionaryEntry entry = av_dict_get(oc.metadata(), key, null, 0); - return entry == null || entry.value() == null ? null : entry.value().getString(); - } - - @Override public String getVideoMetadata(String key) { - if (video_st == null) { - return super.getVideoMetadata(key); - } - AVDictionaryEntry entry = av_dict_get(video_st.metadata(), key, null, 0); - return entry == null || entry.value() == null ? null : entry.value().getString(); - } - - @Override public String getAudioMetadata(String key) { - if (audio_st == null) { - return super.getAudioMetadata(key); - } - AVDictionaryEntry entry = av_dict_get(audio_st.metadata(), key, null, 0); - return entry == null || entry.value() == null ? null : entry.value().getString(); - } - - /** default override of super.setFrameNumber implies setting - * of a video frame having that number */ - @Override public void setFrameNumber(int frameNumber) throws Exception { - setVideoFrameNumber(frameNumber); - } - - /** if there is video stream tries to seek to video frame with corresponding timestamp - * otherwise sets super.frameNumber only because frameRate==0 if there is no video stream */ - public void setVideoFrameNumber(int frameNumber) throws Exception { - // best guess, AVSEEK_FLAG_FRAME has not been implemented in FFmpeg... - if (hasVideo()) setVideoTimestamp(Math.round(1000000L * frameNumber / getFrameRate())); - else super.frameNumber = frameNumber; - } - - /** if there is audio stream tries to seek to audio frame with corresponding timestamp - * ignoring otherwise */ - public void setAudioFrameNumber(int frameNumber) throws Exception { - // best guess, AVSEEK_FLAG_FRAME has not been implemented in FFmpeg... - if (hasAudio()) setAudioTimestamp(Math.round(1000000L * frameNumber / getAudioFrameRate())); - - } - - /** setTimestamp with disregard of the resulting frame type, video or audio */ - @Override public void setTimestamp(long timestamp) throws Exception { - setTimestamp(timestamp, EnumSet.of(Frame.Type.VIDEO, Frame.Type.AUDIO)); - } - - /** setTimestamp with resulting video frame type */ - public void setVideoTimestamp(long timestamp) throws Exception { - setTimestamp(timestamp, EnumSet.of(Frame.Type.VIDEO)); - } - - /** setTimestamp with resulting audio frame type */ - public void setAudioTimestamp(long timestamp) throws Exception { - setTimestamp(timestamp, EnumSet.of(Frame.Type.AUDIO)); - } - - /** setTimestamp with a priority the resulting frame should be: - * video (frameTypesToSeek contains only Frame.Type.VIDEO), - * audio (frameTypesToSeek contains only Frame.Type.AUDIO), - * or any (frameTypesToSeek contains both) - */ - private void setTimestamp(long timestamp, EnumSet frameTypesToSeek) throws Exception { - int ret; - if (oc == null) { - super.setTimestamp(timestamp); - } else { - timestamp = timestamp * AV_TIME_BASE / 1000000L; - /* add the stream start time */ - if (oc.start_time() != AV_NOPTS_VALUE) { - timestamp += oc.start_time(); - } - if ((ret = avformat_seek_file(oc, -1, Long.MIN_VALUE, timestamp, Long.MAX_VALUE, AVSEEK_FLAG_BACKWARD)) < 0) { - throw new Exception("avformat_seek_file() error " + ret + ": Could not seek file to timestamp " + timestamp + "."); - } - if (video_c != null) { - avcodec_flush_buffers(video_c); - } - if (audio_c != null) { - avcodec_flush_buffers(audio_c); - } - if (pkt2.size() > 0) { - pkt2.size(0); - av_packet_unref(pkt); - } - /* After the call of ffmpeg's avformat_seek_file(...) with the flag set to AVSEEK_FLAG_BACKWARD - * the decoding position should be located before the requested timestamp in a closest position - * from which all the active streams can be decoded successfully. - * The following seeking consists of two stages: - * 1. Grab frames till the frame corresponding to that "closest" position - * (the first frame containing decoded data). - * - * 2. Grab frames till the desired timestamp is reached. The number of steps is restricted - * by doubled estimation of frames between that "closest" position and the desired position. - * - * frameTypesToSeek parameter sets the preferred type of frames to seek. - * It can be chosen from three possible types: VIDEO, AUDIO or ANY. - * The setting means only a preference in the type. That is, if VIDEO or AUDIO is - * specified but the file does not have video or audio stream - ANY type will be used instead. - * - * - * TODO - * Sometimes the ffmpeg's avformat_seek_file(...) function brings us not to a position before - * the desired but few frames after.... What can be a the solution in this case if we really need - * a frame-precision seek? Probably we may try to request even earlier timestamp and look if this - * will bring us before the desired position. - * - */ - - boolean has_video = hasVideo(); - boolean has_audio = hasAudio(); - - if (has_video || has_audio) { - if ((frameTypesToSeek.contains(Frame.Type.VIDEO) && !has_video ) || - (frameTypesToSeek.contains(Frame.Type.AUDIO) && !has_audio )) - frameTypesToSeek = EnumSet.of(Frame.Type.VIDEO, Frame.Type.AUDIO); - - long initialSeekPosition = Long.MIN_VALUE; - long maxSeekSteps = 0; - long count = 0; - Frame seekFrame = null; - - while(count++ < 1000) { //seek to a first frame containing video or audio after avformat_seek_file(...) - seekFrame = grabFrame(true, true, false, false); - if (seekFrame == null) return; //is it better to throw NullPointerException? - EnumSet frameTypes = seekFrame.getTypes(); - frameTypes.retainAll(frameTypesToSeek); - if (!frameTypes.isEmpty()) { - initialSeekPosition = seekFrame.timestamp; - //the position closest to the requested timestamp from which it can be reached by sequential grabFrame calls - break; - } - } - if (has_video && this.getFrameRate() > 0) { - //estimation of video frame duration - double deltaTimeStamp = 1000000.0/this.getFrameRate(); - if (initialSeekPosition < timestamp - deltaTimeStamp/2) - maxSeekSteps = (long)(10*(timestamp - initialSeekPosition)/deltaTimeStamp); - } else if (has_audio && this.getAudioFrameRate() > 0) { - //estimation of audio frame duration - double deltaTimeStamp = 1000000.0/this.getAudioFrameRate(); - if (initialSeekPosition < timestamp - deltaTimeStamp/2) - maxSeekSteps = (long)(10*(timestamp - initialSeekPosition)/deltaTimeStamp); - } else - //zero frameRate - if (initialSeekPosition < timestamp - 1L) maxSeekSteps = 1000; - - count = 0; - while(count < maxSeekSteps) { - seekFrame = grabFrame(true, true, false, false); - if (seekFrame == null) return; //is it better to throw NullPointerException? - EnumSet frameTypes = seekFrame.getTypes(); - frameTypes.retainAll(frameTypesToSeek); - if (!frameTypes.isEmpty()) { - count++; - if (this.timestamp >= timestamp - 1) break; - } - } - - frameGrabbed = true; - } - } - } - - /** Returns {@link #getLengthInVideoFrames()} */ - @Override public int getLengthInFrames() { - // best guess... - return getLengthInVideoFrames(); - } - - @Override public long getLengthInTime() { - return oc.duration() * 1000000L / AV_TIME_BASE; - } - - /** Returns {@code (int) Math.round(getLengthInTime() * getFrameRate() / 1000000L)}, which is an approximation in general. */ - public int getLengthInVideoFrames() { - // best guess... - return (int) Math.round(getLengthInTime() * getFrameRate() / 1000000L); - } - - public int getLengthInAudioFrames() { - // best guess... - double afr = getAudioFrameRate(); - if (afr > 0) return (int) (getLengthInTime() * afr / 1000000L); - else return 0; - } - - public AVFormatContext getFormatContext() { - return oc; - } - - public void start() throws Exception { - // synchronized (org.bytedeco.javacpp.avcodec.class) { - startUnsafe(); - // } - } - void startUnsafe() throws Exception { - int ret; - img_convert_ctx = null; - oc = new AVFormatContext(null); - video_c = null; - audio_c = null; - pkt = new AVPacket(); - pkt2 = new AVPacket(); - sizeof_pkt = pkt.sizeof(); - got_frame = new int[1]; - frameGrabbed = false; - frame = new Frame(); - timestamp = 0; - frameNumber = 0; - - pkt2.size(0); - - // Open video file - AVInputFormat f = null; - if (format != null && format.length() > 0) { - if ((f = av_find_input_format(format)) == null) { - throw new Exception("av_find_input_format() error: Could not find input format \"" + format + "\"."); - } - } - AVDictionary options = new AVDictionary(null); - if (frameRate > 0) { - AVRational r = av_d2q(frameRate, 1001000); - av_dict_set(options, "framerate", r.num() + "/" + r.den(), 0); - } - if (pixelFormat >= 0) { - av_dict_set(options, "pixel_format", av_get_pix_fmt_name(pixelFormat).getString(), 0); - } else if (imageMode != ImageMode.RAW) { - av_dict_set(options, "pixel_format", imageMode == ImageMode.COLOR ? "bgr24" : "gray8", 0); - } - if (imageWidth > 0 && imageHeight > 0) { - av_dict_set(options, "video_size", imageWidth + "x" + imageHeight, 0); - } - if (sampleRate > 0) { - av_dict_set(options, "sample_rate", "" + sampleRate, 0); - } - if (audioChannels > 0) { - av_dict_set(options, "channels", "" + audioChannels, 0); - } - for (Entry e : this.options.entrySet()) { - av_dict_set(options, e.getKey(), e.getValue(), 0); - } - if (inputStream != null) { - if (!inputStream.markSupported()) { - inputStream = new BufferedInputStream(inputStream); - } - inputStream.mark(Integer.MAX_VALUE - 8); // so that the whole input stream is seekable - oc = avformat_alloc_context(); - avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 0, oc, readCallback, null, seekCallback); - oc.pb(avio); - - filename = inputStream.toString(); - inputStreams.put(oc, inputStream); - } - if ((ret = avformat_open_input(oc, filename, f, options)) < 0) { - av_dict_set(options, "pixel_format", null, 0); - if ((ret = avformat_open_input(oc, filename, f, options)) < 0) { - throw new Exception("avformat_open_input() error " + ret + ": Could not open input \"" + filename + "\". (Has setFormat() been called?)"); - } - } - av_dict_free(options); - - oc.max_delay(maxDelay); - - // Retrieve stream information - if ((ret = avformat_find_stream_info(oc, (PointerPointer)null)) < 0) { - throw new Exception("avformat_find_stream_info() error " + ret + ": Could not find stream information."); - } - - if (av_log_get_level() >= AV_LOG_INFO) { - // Dump information about file onto standard error - av_dump_format(oc, 0, filename, 0); - } - - // Find the first video and audio stream, unless the user specified otherwise - video_st = audio_st = null; - AVCodecParameters video_par = null, audio_par = null; - int nb_streams = oc.nb_streams(); - for (int i = 0; i < nb_streams; i++) { - AVStream st = oc.streams(i); - // Get a pointer to the codec context for the video or audio stream - AVCodecParameters par = st.codecpar(); - if (video_st == null && par.codec_type() == AVMEDIA_TYPE_VIDEO && (videoStream < 0 || videoStream == i)) { - video_st = st; - video_par = par; - videoStream = i; - } else if (audio_st == null && par.codec_type() == AVMEDIA_TYPE_AUDIO && (audioStream < 0 || audioStream == i)) { - audio_st = st; - audio_par = par; - audioStream = i; - } - } - if (video_st == null && audio_st == null) { - throw new Exception("Did not find a video or audio stream inside \"" + filename - + "\" for videoStream == " + videoStream + " and audioStream == " + audioStream + "."); - } - - if (video_st != null) { - // Find the decoder for the video stream - AVCodec codec = avcodec_find_decoder(video_par.codec_id()); - if (codec == null) { - throw new Exception("avcodec_find_decoder() error: Unsupported video format or codec not found: " + video_par.codec_id() + "."); - } - - /* Allocate a codec context for the decoder */ - if ((video_c = avcodec_alloc_context3(codec)) == null) { - throw new Exception("avcodec_alloc_context3() error: Could not allocate video decoding context."); - } - - /* copy the stream parameters from the muxer */ - if ((ret = avcodec_parameters_to_context(video_c, video_st.codecpar())) < 0) { - release(); - throw new Exception("avcodec_parameters_to_context() error: Could not copy the video stream parameters."); - } - - options = new AVDictionary(null); - for (Entry e : videoOptions.entrySet()) { - av_dict_set(options, e.getKey(), e.getValue(), 0); - } - // Open video codec - if ((ret = avcodec_open2(video_c, codec, options)) < 0) { - throw new Exception("avcodec_open2() error " + ret + ": Could not open video codec."); - } - av_dict_free(options); - - // Hack to correct wrong frame rates that seem to be generated by some codecs - if (video_c.time_base().num() > 1000 && video_c.time_base().den() == 1) { - video_c.time_base().den(1000); - } - - // Allocate video frame and an AVFrame structure for the RGB image - if ((picture = av_frame_alloc()) == null) { - throw new Exception("av_frame_alloc() error: Could not allocate raw picture frame."); - } - if ((picture_rgb = av_frame_alloc()) == null) { - throw new Exception("av_frame_alloc() error: Could not allocate RGB picture frame."); - } - - initPictureRGB(); - } - - if (audio_st != null) { - // Find the decoder for the audio stream - AVCodec codec = avcodec_find_decoder(audio_par.codec_id()); - if (codec == null) { - throw new Exception("avcodec_find_decoder() error: Unsupported audio format or codec not found: " + audio_par.codec_id() + "."); - } - - /* Allocate a codec context for the decoder */ - if ((audio_c = avcodec_alloc_context3(codec)) == null) { - throw new Exception("avcodec_alloc_context3() error: Could not allocate audio decoding context."); - } - - /* copy the stream parameters from the muxer */ - if ((ret = avcodec_parameters_to_context(audio_c, audio_st.codecpar())) < 0) { - release(); - throw new Exception("avcodec_parameters_to_context() error: Could not copy the audio stream parameters."); - } - - options = new AVDictionary(null); - for (Entry e : audioOptions.entrySet()) { - av_dict_set(options, e.getKey(), e.getValue(), 0); - } - // Open audio codec - if ((ret = avcodec_open2(audio_c, codec, options)) < 0) { - throw new Exception("avcodec_open2() error " + ret + ": Could not open audio codec."); - } - av_dict_free(options); - - // Allocate audio samples frame - if ((samples_frame = av_frame_alloc()) == null) { - throw new Exception("av_frame_alloc() error: Could not allocate audio frame."); - } - - samples_ptr = new BytePointer[] { null }; - samples_buf = new Buffer[] { null }; - } - } - - private void initPictureRGB() { - int width = imageWidth > 0 ? imageWidth : video_c.width(); - int height = imageHeight > 0 ? imageHeight : video_c.height(); - - switch (imageMode) { - case COLOR: - case GRAY: - // If size changes I new allocation is needed -> free the old one. - if (image_ptr != null) { - // First kill all references, then free it. - image_buf = null; - BytePointer[] temp = image_ptr; - image_ptr = null; - av_free(temp[0]); - } - int fmt = getPixelFormat(); - - // Determine required buffer size and allocate buffer - int size = av_image_get_buffer_size(fmt, width, height, 1); - image_ptr = new BytePointer[] { new BytePointer(av_malloc(size)).capacity(size) }; - image_buf = new Buffer[] { image_ptr[0].asBuffer() }; - - // Assign appropriate parts of buffer to image planes in picture_rgb - // Note that picture_rgb is an AVFrame, but AVFrame is a superset of AVPicture - av_image_fill_arrays(new PointerPointer(picture_rgb), picture_rgb.linesize(), image_ptr[0], fmt, width, height, 1); - picture_rgb.format(fmt); - picture_rgb.width(width); - picture_rgb.height(height); - break; - - case RAW: - image_ptr = new BytePointer[] { null }; - image_buf = new Buffer[] { null }; - break; - - default: - assert false; - } - } - - public void stop() throws Exception { - release(); - } - - public void trigger() throws Exception { - if (oc == null || oc.isNull()) { - throw new Exception("Could not trigger: No AVFormatContext. (Has start() been called?)"); - } - if (pkt2.size() > 0) { - pkt2.size(0); - av_packet_unref(pkt); - } - for (int i = 0; i < numBuffers+1; i++) { - if (av_read_frame(oc, pkt) < 0) { - return; - } - av_packet_unref(pkt); - } - } - - private void processImage() throws Exception { - frame.imageWidth = imageWidth > 0 ? imageWidth : video_c.width(); - frame.imageHeight = imageHeight > 0 ? imageHeight : video_c.height(); - frame.imageDepth = Frame.DEPTH_UBYTE; - switch (imageMode) { - case COLOR: - case GRAY: - // Deinterlace Picture - if (deinterlace) { - throw new Exception("Cannot deinterlace: Functionality moved to FFmpegFrameFilter."); - } - - // Has the size changed? - if (frame.imageWidth != picture_rgb.width() || frame.imageHeight != picture_rgb.height()) { - initPictureRGB(); - } - - // Convert the image into BGR or GRAY format that OpenCV uses - img_convert_ctx = sws_getCachedContext(img_convert_ctx, - video_c.width(), video_c.height(), video_c.pix_fmt(), - frame.imageWidth, frame.imageHeight, getPixelFormat(), SWS_BILINEAR, - null, null, (DoublePointer)null); - if (img_convert_ctx == null) { - throw new Exception("sws_getCachedContext() error: Cannot initialize the conversion context."); - } - - // Convert the image from its native format to RGB or GRAY - sws_scale(img_convert_ctx, new PointerPointer(picture), picture.linesize(), 0, - video_c.height(), new PointerPointer(picture_rgb), picture_rgb.linesize()); - frame.imageStride = picture_rgb.linesize(0); - frame.image = image_buf; - break; - - case RAW: - frame.imageStride = picture.linesize(0); - BytePointer ptr = picture.data(0); - if (ptr != null && !ptr.equals(image_ptr[0])) { - image_ptr[0] = ptr.capacity(frame.imageHeight * frame.imageStride); - image_buf[0] = ptr.asBuffer(); - } - frame.image = image_buf; - break; - - default: - assert false; - } - frame.image[0].limit(frame.imageHeight * frame.imageStride); - frame.imageChannels = frame.imageStride / frame.imageWidth; - } - - private void processSamples() throws Exception { - int ret; - - int sample_format = samples_frame.format(); - int planes = av_sample_fmt_is_planar(sample_format) != 0 ? (int)samples_frame.channels() : 1; - int data_size = av_samples_get_buffer_size((IntPointer)null, audio_c.channels(), - samples_frame.nb_samples(), audio_c.sample_fmt(), 1) / planes; - if (samples_buf == null || samples_buf.length != planes) { - samples_ptr = new BytePointer[planes]; - samples_buf = new Buffer[planes]; - } - frame.sampleRate = audio_c.sample_rate(); - frame.audioChannels = audio_c.channels(); - frame.samples = samples_buf; - int sample_size = data_size / av_get_bytes_per_sample(sample_format); - for (int i = 0; i < planes; i++) { - BytePointer p = samples_frame.data(i); - if (!p.equals(samples_ptr[i]) || samples_ptr[i].capacity() < data_size) { - samples_ptr[i] = p.capacity(data_size); - ByteBuffer b = p.asBuffer(); - switch (sample_format) { - case AV_SAMPLE_FMT_U8: - case AV_SAMPLE_FMT_U8P: samples_buf[i] = b; break; - case AV_SAMPLE_FMT_S16: - case AV_SAMPLE_FMT_S16P: samples_buf[i] = b.asShortBuffer(); break; - case AV_SAMPLE_FMT_S32: - case AV_SAMPLE_FMT_S32P: samples_buf[i] = b.asIntBuffer(); break; - case AV_SAMPLE_FMT_FLT: - case AV_SAMPLE_FMT_FLTP: samples_buf[i] = b.asFloatBuffer(); break; - case AV_SAMPLE_FMT_DBL: - case AV_SAMPLE_FMT_DBLP: samples_buf[i] = b.asDoubleBuffer(); break; - default: assert false; - } - } - samples_buf[i].position(0).limit(sample_size); - } - - if (audio_c.channels() != getAudioChannels() || audio_c.sample_fmt() != getSampleFormat() || audio_c.sample_rate() != getSampleRate()) { - if (samples_convert_ctx == null || samples_channels != getAudioChannels() || samples_format != getSampleFormat() || samples_rate != getSampleRate()) { - samples_convert_ctx = swr_alloc_set_opts(samples_convert_ctx, av_get_default_channel_layout(getAudioChannels()), getSampleFormat(), getSampleRate(), - av_get_default_channel_layout(audio_c.channels()), audio_c.sample_fmt(), audio_c.sample_rate(), 0, null); - if (samples_convert_ctx == null) { - throw new Exception("swr_alloc_set_opts() error: Cannot allocate the conversion context."); - } else if ((ret = swr_init(samples_convert_ctx)) < 0) { - throw new Exception("swr_init() error " + ret + ": Cannot initialize the conversion context."); - } - samples_channels = getAudioChannels(); - samples_format = getSampleFormat(); - samples_rate = getSampleRate(); - } - - int sample_size_in = samples_frame.nb_samples(); - int planes_out = av_sample_fmt_is_planar(samples_format) != 0 ? (int)samples_frame.channels() : 1; - int sample_size_out = swr_get_out_samples(samples_convert_ctx, sample_size_in); - int sample_bytes_out = av_get_bytes_per_sample(samples_format); - int buffer_size_out = sample_size_out * sample_bytes_out * (planes_out > 1 ? 1 : samples_channels); - if (samples_buf_out == null || samples_buf.length != planes_out || samples_ptr_out[0].capacity() < buffer_size_out) { - for (int i = 0; samples_ptr_out != null && i < samples_ptr_out.length; i++) { - av_free(samples_ptr_out[i].position(0)); - } - samples_ptr_out = new BytePointer[planes_out]; - samples_buf_out = new Buffer[planes_out]; - - for (int i = 0; i < planes_out; i++) { - samples_ptr_out[i] = new BytePointer(av_malloc(buffer_size_out)).capacity(buffer_size_out); - ByteBuffer b = samples_ptr_out[i].asBuffer(); - switch (samples_format) { - case AV_SAMPLE_FMT_U8: - case AV_SAMPLE_FMT_U8P: samples_buf_out[i] = b; break; - case AV_SAMPLE_FMT_S16: - case AV_SAMPLE_FMT_S16P: samples_buf_out[i] = b.asShortBuffer(); break; - case AV_SAMPLE_FMT_S32: - case AV_SAMPLE_FMT_S32P: samples_buf_out[i] = b.asIntBuffer(); break; - case AV_SAMPLE_FMT_FLT: - case AV_SAMPLE_FMT_FLTP: samples_buf_out[i] = b.asFloatBuffer(); break; - case AV_SAMPLE_FMT_DBL: - case AV_SAMPLE_FMT_DBLP: samples_buf_out[i] = b.asDoubleBuffer(); break; - default: assert false; - } - } - } - frame.sampleRate = samples_rate; - frame.audioChannels = samples_channels; - frame.samples = samples_buf_out; - - if ((ret = swr_convert(samples_convert_ctx, new PointerPointer(samples_ptr_out), sample_size_out, new PointerPointer(samples_ptr), sample_size_in)) < 0) { - throw new Exception("swr_convert() error " + ret + ": Cannot convert audio samples."); - } - for (int i = 0; i < planes_out; i++) { - samples_ptr_out[i].position(0).limit(ret * (planes_out > 1 ? 1 : samples_channels)); - samples_buf_out[i].position(0).limit(ret * (planes_out > 1 ? 1 : samples_channels)); - } - } - } - - public Frame grab() throws Exception { - return grabFrame(true, true, true, false); - } - public Frame grabImage() throws Exception { - return grabFrame(false, true, true, false); - } - public Frame grabSamples() throws Exception { - return grabFrame(true, false, true, false); - } - public Frame grabKeyFrame() throws Exception { - return grabFrame(false, true, true, true); - } - public Frame grabFrame(boolean doAudio, boolean doVideo, boolean doProcessing, boolean keyFrames) throws Exception { - if (oc == null || oc.isNull()) { - throw new Exception("Could not grab: No AVFormatContext. (Has start() been called?)"); - } else if ((!doVideo || video_st == null) && (!doAudio || audio_st == null)) { - return null; - } - boolean videoFrameGrabbed = frameGrabbed && frame.image != null; - boolean audioFrameGrabbed = frameGrabbed && frame.samples != null; - frameGrabbed = false; - frame.keyFrame = false; - frame.imageWidth = 0; - frame.imageHeight = 0; - frame.imageDepth = 0; - frame.imageChannels = 0; - frame.imageStride = 0; - frame.image = null; - frame.sampleRate = 0; - frame.audioChannels = 0; - frame.samples = null; - frame.opaque = null; - if (doVideo && videoFrameGrabbed) { - if (doProcessing) { - processImage(); - } - frame.keyFrame = picture.key_frame() != 0; - frame.opaque = picture; - return frame; - } else if (doAudio && audioFrameGrabbed) { - if (doProcessing) { - processSamples(); - } - frame.keyFrame = samples_frame.key_frame() != 0; - frame.opaque = samples_frame; - return frame; - } - boolean done = false; - while (!done) { - if (pkt2.size() <= 0) { - if (av_read_frame(oc, pkt) < 0) { - if (doVideo && video_st != null) { - // The video codec may have buffered some frames - pkt.stream_index(video_st.index()); - pkt.flags(AV_PKT_FLAG_KEY); - pkt.data(null); - pkt.size(0); - } else { - return null; - } - } - } - - // Is this a packet from the video stream? - if (doVideo && video_st != null && pkt.stream_index() == video_st.index() - && (!keyFrames || pkt.flags() == AV_PKT_FLAG_KEY)) { - // Decode video frame - int len = avcodec_decode_video2(video_c, picture, got_frame, pkt); - - // Did we get a video frame? - if (len >= 0 && got_frame[0] != 0 - && (!keyFrames || picture.pict_type() == AV_PICTURE_TYPE_I)) { - long pts = av_frame_get_best_effort_timestamp(picture); - AVRational time_base = video_st.time_base(); - timestamp = 1000000L * pts * time_base.num() / time_base.den(); - // best guess, AVCodecContext.frame_number = number of decoded frames... - frameNumber = (int)Math.round(timestamp * getFrameRate() / 1000000L); - frame.image = image_buf; - if (doProcessing) { - processImage(); - } - done = true; - frame.timestamp = timestamp; - frame.keyFrame = picture.key_frame() != 0; - frame.opaque = picture; - } else if (pkt.data() == null && pkt.size() == 0) { - return null; - } - } else if (doAudio && audio_st != null && pkt.stream_index() == audio_st.index()) { - if (pkt2.size() <= 0) { - // HashMap is unacceptably slow on Android - // pkt2.put(pkt); - BytePointer.memcpy(pkt2, pkt, sizeof_pkt); - } - av_frame_unref(samples_frame); - // Decode audio frame - int len = avcodec_decode_audio4(audio_c, samples_frame, got_frame, pkt2); - if (len <= 0) { - // On error, trash the whole packet - pkt2.size(0); - } else { - pkt2.data(pkt2.data().position(len)); - pkt2.size(pkt2.size() - len); - if (got_frame[0] != 0) { - long pts = av_frame_get_best_effort_timestamp(samples_frame); - AVRational time_base = audio_st.time_base(); - timestamp = 1000000L * pts * time_base.num() / time_base.den(); - frame.samples = samples_buf; - /* if a frame has been decoded, output it */ - if (doProcessing) { - processSamples(); - } - done = true; - frame.timestamp = timestamp; - frame.keyFrame = samples_frame.key_frame() != 0; - frame.opaque = samples_frame; - } - } - } - - if (pkt2.size() <= 0) { - // Free the packet that was allocated by av_read_frame - av_packet_unref(pkt); - } - } - return frame; - } - - public AVPacket grabPacket() throws Exception { - if (oc == null || oc.isNull()) { - throw new Exception("Could not trigger: No AVFormatContext. (Has start() been called?)"); - } - - // Return the next frame of a stream. - if (av_read_frame(oc, pkt) < 0) { - return null; - } - - return pkt; - } + public static String[] getDeviceDescriptions() throws Exception { + tryLoad(); + throw new UnsupportedOperationException("Device enumeration not support by FFmpeg."); + } + + public static FFmpegFrameGrabber createDefault(File deviceFile) throws Exception { + return new FFmpegFrameGrabber(deviceFile); + } + + public static FFmpegFrameGrabber createDefault(String devicePath) throws Exception { + return new FFmpegFrameGrabber(devicePath); + } + + public static FFmpegFrameGrabber createDefault(String devicePath, int pixelFormat) throws Exception { + return new FFmpegFrameGrabber(devicePath, pixelFormat); + } + + public static FFmpegFrameGrabber createDefault(int deviceNumber) throws Exception { + throw new Exception(FFmpegFrameGrabber.class + " does not support device numbers."); + } + + private static Exception loadingException = null; + + public static void tryLoad() throws Exception { + if (loadingException != null) { + throw loadingException; + } else { + try { + Loader.load(org.bytedeco.javacpp.avutil.class); + Loader.load(org.bytedeco.javacpp.swresample.class); + Loader.load(org.bytedeco.javacpp.avcodec.class); + Loader.load(org.bytedeco.javacpp.avformat.class); + Loader.load(org.bytedeco.javacpp.swscale.class); + + // Register all formats and codecs + avcodec_register_all(); + av_register_all(); + avformat_network_init(); + + Loader.load(org.bytedeco.javacpp.avdevice.class); + avdevice_register_all(); + } catch (Throwable t) { + if (t instanceof Exception) { + throw loadingException = (Exception) t; + } else { + throw loadingException = new Exception("Failed to load " + FFmpegFrameGrabber.class, t); + } + } + } + } + + static { + try { + tryLoad(); + FFmpegLockCallback.init(); + } catch (Exception ex) { + } + } + + public FFmpegFrameGrabber(File file) { + this(file.getAbsolutePath()); + } + + public FFmpegFrameGrabber(String filename) { + this.filename = filename; + this.pixelFormat = avutil.AV_PIX_FMT_NONE; + this.sampleFormat = AV_SAMPLE_FMT_NONE; + } + + public FFmpegFrameGrabber(String filename, int pixelFormat) { + this.filename = filename; + this.pixelFormat = pixelFormat; + this.sampleFormat = AV_SAMPLE_FMT_NONE; + } + + public FFmpegFrameGrabber(InputStream inputStream) { + this.inputStream = inputStream; + this.pixelFormat = AV_PIX_FMT_NONE; + this.sampleFormat = AV_SAMPLE_FMT_NONE; + } + + public void release() throws Exception { + // synchronized (org.bytedeco.javacpp.avcodec.class) { + releaseUnsafe(); + // } + } + + void releaseUnsafe() throws Exception { + if (pkt != null && pkt2 != null) { + if (pkt2.size() > 0) { + av_packet_unref(pkt); + } + pkt = pkt2 = null; + } + + // Free the RGB image + if (image_ptr != null) { + for (int i = 0; i < image_ptr.length; i++) { + av_free(image_ptr[i]); + } + image_ptr = null; + } + if (picture_rgb != null) { + av_frame_free(picture_rgb); + picture_rgb = null; + } + + // Free the native format picture frame + if (picture != null) { + av_frame_free(picture); + picture = null; + } + + // Close the video codec + if (video_c != null) { + avcodec_free_context(video_c); + video_c = null; + } + + // Free the audio samples frame + if (samples_frame != null) { + av_frame_free(samples_frame); + samples_frame = null; + } + + // Close the audio codec + if (audio_c != null) { + avcodec_free_context(audio_c); + audio_c = null; + } + + // Close the video file + if (inputStream == null && oc != null && !oc.isNull()) { + avformat_close_input(oc); + oc = null; + } + + if (img_convert_ctx != null) { + sws_freeContext(img_convert_ctx); + img_convert_ctx = null; + } + + if (samples_ptr_out != null) { + for (int i = 0; i < samples_ptr_out.length; i++) { + av_free(samples_ptr_out[i].position(0)); + } + samples_ptr_out = null; + samples_buf_out = null; + } + + if (samples_convert_ctx != null) { + swr_free(samples_convert_ctx); + samples_convert_ctx = null; + } + + got_frame = null; + frameGrabbed = false; + frame = null; + timestamp = 0; + frameNumber = 0; + + if (inputStream != null) { + try { + if (oc == null) { + // when called a second time + inputStream.close(); + } else { + inputStream.reset(); + } + } catch (IOException ex) { + throw new Exception("Error on InputStream.close(): ", ex); + } finally { + inputStreams.remove(oc); + if (avio != null) { + if (avio.buffer() != null) { + av_free(avio.buffer()); + avio.buffer(null); + } + av_free(avio); + avio = null; + } + if (oc != null) { + avformat_free_context(oc); + oc = null; + } + } + } + } + + @Override + protected void finalize() throws Throwable { + super.finalize(); + release(); + } + + static Map inputStreams = Collections.synchronizedMap(new HashMap()); + + public static class ReadCallback extends Read_packet_Pointer_BytePointer_int { + @Override + public int call(Pointer opaque, BytePointer buf, int buf_size) { + try { + byte[] b = new byte[buf_size]; + InputStream is = inputStreams.get(opaque); + int size = is.read(b, 0, buf_size); + if (size < 0) { + return 0; + } else { + buf.put(b, 0, size); + return size; + } + } catch (Throwable t) { + System.err.println("Error on InputStream.read(): " + t); + return -1; + } + } + } + + public static class SeekCallback extends Seek_Pointer_long_int { + @Override + public long call(Pointer opaque, long offset, int whence) { + try { + InputStream is = inputStreams.get(opaque); + switch (whence) { + case 0: + is.reset(); + break; + case 1: + break; + default: + return -1; + } + long remaining = offset; + while (remaining > 0) { + long skipped = is.skip(remaining); + if (skipped == 0) + break; // end of the stream + remaining -= skipped; + } + return 0; + } catch (Throwable t) { + System.err.println("Error on InputStream.reset() or skip(): " + t); + return -1; + } + } + } + + static ReadCallback readCallback = new ReadCallback(); + static SeekCallback seekCallback = new SeekCallback(); + + private InputStream inputStream; + private AVIOContext avio; + private String filename; + private AVFormatContext oc; + private AVStream video_st, audio_st; + private AVCodecContext video_c, audio_c; + private AVFrame picture, picture_rgb; + private BytePointer[] image_ptr; + private Buffer[] image_buf; + private AVFrame samples_frame; + private BytePointer[] samples_ptr; + private Buffer[] samples_buf; + private BytePointer[] samples_ptr_out; + private Buffer[] samples_buf_out; + private AVPacket pkt, pkt2; + private int sizeof_pkt; + private int[] got_frame; + private SwsContext img_convert_ctx; + private SwrContext samples_convert_ctx; + private int samples_channels, samples_format, samples_rate; + private boolean frameGrabbed; + private Frame frame; + + @Override + public double getGamma() { + // default to a gamma of 2.2 for cheap Webcams, DV cameras, etc. + if (gamma == 0.0) { + return 2.2; + } else { + return gamma; + } + } + + @Override + public String getFormat() { + if (oc == null) { + return super.getFormat(); + } else { + return oc.iformat().name().getString(); + } + } + + @Override + public int getImageWidth() { + return imageWidth > 0 || video_c == null ? super.getImageWidth() : video_c.width(); + } + + @Override + public int getImageHeight() { + return imageHeight > 0 || video_c == null ? super.getImageHeight() : video_c.height(); + } + + @Override + public int getAudioChannels() { + return audioChannels > 0 || audio_c == null ? super.getAudioChannels() : audio_c.channels(); + } + + @Override + public int getPixelFormat() { + if (imageMode == ImageMode.COLOR || imageMode == ImageMode.GRAY) { + if (pixelFormat == AV_PIX_FMT_NONE) { + return imageMode == ImageMode.COLOR ? AV_PIX_FMT_BGR24 : AV_PIX_FMT_GRAY8; + } else { + return pixelFormat; + } + } else if (video_c != null) { // RAW + return video_c.pix_fmt(); + } else { + return super.getPixelFormat(); + } + } + + @Override + public int getVideoCodec() { + return video_c == null ? super.getVideoCodec() : video_c.codec_id(); + } + + @Override + public int getVideoBitrate() { + return video_c == null ? super.getVideoBitrate() : (int) video_c.bit_rate(); + } + + @Override + public double getAspectRatio() { + if (video_st == null) { + return super.getAspectRatio(); + } else { + AVRational r = av_guess_sample_aspect_ratio(oc, video_st, picture); + double a = (double) r.num() / r.den(); + return a == 0.0 ? 1.0 : a; + } + } + + @Override + public double getFrameRate() { + if (video_st == null) { + return super.getFrameRate(); + } else { + AVRational r = video_st.avg_frame_rate(); + if (r.num() == 0 && r.den() == 0) { + r = video_st.r_frame_rate(); + } + return (double) r.num() / r.den(); + } + } + + @Override + public int getAudioCodec() { + return audio_c == null ? super.getAudioCodec() : audio_c.codec_id(); + } + + @Override + public int getAudioBitrate() { + return audio_c == null ? super.getAudioBitrate() : (int) audio_c.bit_rate(); + } + + @Override + public int getSampleFormat() { + if (sampleMode == SampleMode.SHORT || sampleMode == SampleMode.FLOAT) { + if (sampleFormat == AV_SAMPLE_FMT_NONE) { + return sampleMode == SampleMode.SHORT ? AV_SAMPLE_FMT_S16 : AV_SAMPLE_FMT_FLT; + } else { + return sampleFormat; + } + } else if (audio_c != null) { // RAW + return audio_c.sample_fmt(); + } else { + return super.getSampleFormat(); + } + } + + @Override + public int getSampleRate() { + return sampleRate > 0 || audio_c == null ? super.getSampleRate() : audio_c.sample_rate(); + } + + @Override + public String getMetadata(String key) { + if (oc == null) { + return super.getMetadata(key); + } + AVDictionaryEntry entry = av_dict_get(oc.metadata(), key, null, 0); + return entry == null || entry.value() == null ? null : entry.value().getString(); + } + + @Override + public String getVideoMetadata(String key) { + if (video_st == null) { + return super.getVideoMetadata(key); + } + AVDictionaryEntry entry = av_dict_get(video_st.metadata(), key, null, 0); + return entry == null || entry.value() == null ? null : entry.value().getString(); + } + + @Override + public String getAudioMetadata(String key) { + if (audio_st == null) { + return super.getAudioMetadata(key); + } + AVDictionaryEntry entry = av_dict_get(audio_st.metadata(), key, null, 0); + return entry == null || entry.value() == null ? null : entry.value().getString(); + } + + @Override + public void setFrameNumber(int frameNumber) throws Exception { + // best guess, AVSEEK_FLAG_FRAME has not been implemented in FFmpeg... + setTimestamp(Math.round(1000000L * frameNumber / getFrameRate())); + } + + @Override + public void setTimestamp(long timestamp) throws Exception { + int ret; + if (oc == null) { + super.setTimestamp(timestamp); + } else { + timestamp = timestamp * AV_TIME_BASE / 1000000L; + /* add the stream start time */ + if (oc.start_time() != AV_NOPTS_VALUE) { + timestamp += oc.start_time(); + } + if ((ret = avformat_seek_file(oc, -1, Long.MIN_VALUE, timestamp, Long.MAX_VALUE, AVSEEK_FLAG_BACKWARD)) < 0) { + throw new Exception(ret, "avformat_seek_file() error " + ret + ": Could not seek file to timestamp " + timestamp + "."); + } + if (video_c != null) { + avcodec_flush_buffers(video_c); + } + if (audio_c != null) { + avcodec_flush_buffers(audio_c); + } + if (pkt2.size() > 0) { + pkt2.size(0); + av_packet_unref(pkt); + } + /* + * comparing to timestamp +/- 1 avoids rouding issues for framerates which are no proper divisors of 1000000, e.g. where av_frame_get_best_effort_timestamp in grabFrame sets this.timestamp to ...666 and the given timestamp has been rounded to ...667 (or vice versa) + */ + int count = 0; // prevent infinite loops with corrupted files + while (this.timestamp > timestamp + 1 && grabFrame(true, true, false, false) != null && count++ < 1000) { + // flush frames if seeking backwards + } + count = 0; + while (this.timestamp < timestamp - 1 && grabFrame(true, true, false, false) != null && count++ < 1000) { + // decode up to the desired frame + } + if (video_c != null) { + frameGrabbed = true; + } + } + } + + /** Returns {@code getLengthInTime() * getFrameRate() / 1000000L)}, which is an approximation in general. */ + @Override + public int getLengthInFrames() { + // best guess... + return (int) (getLengthInTime() * getFrameRate() / 1000000L); + } + + @Override + public long getLengthInTime() { + return oc.duration() * 1000000L / AV_TIME_BASE; + } + + public AVFormatContext getFormatContext() { + + return oc; + } + + public void start() throws Exception { + // synchronized (org.bytedeco.javacpp.avcodec.class) { + startUnsafe(); + // } + } + + void startUnsafe() throws Exception { + int ret; + img_convert_ctx = null; + oc = new AVFormatContext(null); + video_c = null; + audio_c = null; + pkt = new AVPacket(); + pkt2 = new AVPacket(); + sizeof_pkt = pkt.sizeof(); + got_frame = new int[1]; + frameGrabbed = false; + frame = new Frame(); + timestamp = 0; + frameNumber = 0; + + pkt2.size(0); + + // Open video file + AVInputFormat f = null; + if (format != null && format.length() > 0) { + if ((f = av_find_input_format(format)) == null) { + throw new Exception("av_find_input_format() error: Could not find input format \"" + format + "\"."); + } + } + AVDictionary options = new AVDictionary(null); + if (frameRate > 0) { + AVRational r = av_d2q(frameRate, 1001000); + av_dict_set(options, "framerate", r.num() + "/" + r.den(), 0); + } + + if (pixelFormat >= 0) { + av_dict_set(options, "pixel_format", av_get_pix_fmt_name(pixelFormat).getString(), 0); + } else if (imageMode != ImageMode.RAW) { + av_dict_set(options, "pixel_format", imageMode == ImageMode.COLOR ? "bgr24" : "gray8", 0); + } + + if (imageWidth > 0 && imageHeight > 0) { + av_dict_set(options, "video_size", imageWidth + "x" + imageHeight, 0); + } + if (sampleRate > 0) { + av_dict_set(options, "sample_rate", "" + sampleRate, 0); + } + if (audioChannels > 0) { + av_dict_set(options, "channels", "" + audioChannels, 0); + } + for (Entry e : this.options.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + if (inputStream != null) { + if (!inputStream.markSupported()) { + inputStream = new BufferedInputStream(inputStream); + } + inputStream.mark(Integer.MAX_VALUE - 8); // so that the whole input stream is seekable + oc = avformat_alloc_context(); + avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 0, oc, readCallback, null, seekCallback); + oc.pb(avio); + + filename = inputStream.toString(); + inputStreams.put(oc, inputStream); + } + if ((ret = avformat_open_input(oc, filename, f, options)) < 0) { + av_dict_set(options, "pixel_format", null, 0); + if ((ret = avformat_open_input(oc, filename, f, options)) < 0) { + throw new Exception(ret, "avformat_open_input() error " + ret + ": Could not open input \"" + filename + "\". (Has setFormat() been called?)"); + } + } + av_dict_free(options); + + // Retrieve stream information + if ((ret = avformat_find_stream_info(oc, (PointerPointer) null)) < 0) { + throw new Exception(ret, "avformat_find_stream_info() error " + ret + ": Could not find stream information."); + } + + if (av_log_get_level() >= AV_LOG_INFO) { + // Dump information about file onto standard error + av_dump_format(oc, 0, filename, 0); + } + + // Find the first video and audio stream, unless the user specified otherwise + video_st = audio_st = null; + AVCodecParameters video_par = null, audio_par = null; + int nb_streams = oc.nb_streams(); + for (int i = 0; i < nb_streams; i++) { + AVStream st = oc.streams(i); + // Get a pointer to the codec context for the video or audio stream + AVCodecParameters par = st.codecpar(); + if (video_st == null && par.codec_type() == AVMEDIA_TYPE_VIDEO && (videoStream < 0 || videoStream == i)) { + video_st = st; + video_par = par; + } else if (audio_st == null && par.codec_type() == AVMEDIA_TYPE_AUDIO && (audioStream < 0 || audioStream == i)) { + audio_st = st; + audio_par = par; + } + } + if (video_st == null && audio_st == null) { + throw new Exception("Did not find a video or audio stream inside \"" + filename + "\" for videoStream == " + videoStream + " and audioStream == " + audioStream + "."); + } + + if (video_st != null) { + // Find the decoder for the video stream + AVCodec codec = null; + if (videoCodecName != null && videoCodecName.length() > 0) + codec = avcodec_find_decoder_by_name(videoCodecName); + if (codec == null) + codec = avcodec_find_decoder(video_par.codec_id()); + if (codec == null) { + throw new Exception("avcodec_find_decoder() error: Unsupported video format or codec not found: " + video_par.codec_id() + "."); + } + + /* Allocate a codec context for the decoder */ + if ((video_c = avcodec_alloc_context3(codec)) == null) { + throw new Exception("avcodec_alloc_context3() error: Could not allocate video decoding context."); + } + + if (hwaccelName != null) { + AVHWAccel hwaccel = null; + while ((hwaccel = av_hwaccel_next(hwaccel)) != null) { + System.out.println("hwaccel: " + hwaccel.name().getString()); + if (hwaccelName.equalsIgnoreCase(hwaccel.name().getString())) { + video_c.hwaccel(hwaccel); + break; + } + } + } + + /* copy the stream parameters from the muxer */ + if ((ret = avcodec_parameters_to_context(video_c, video_st.codecpar())) < 0) { + release(); + throw new Exception(ret, "avcodec_parameters_to_context() error: Could not copy the video stream parameters."); + } + + options = new AVDictionary(null); + for (Entry e : videoOptions.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + // Open video codec + if ((ret = avcodec_open2(video_c, codec, options)) < 0) { + throw new Exception(ret, "avcodec_open2() error " + ret + ": Could not open video codec."); + } + av_dict_free(options); + + // Hack to correct wrong frame rates that seem to be generated by some codecs + if (video_c.time_base().num() > 1000 && video_c.time_base().den() == 1) { + video_c.time_base().den(1000); + } + + // Allocate video frame and an AVFrame structure for the RGB image + if ((picture = av_frame_alloc()) == null) { + throw new Exception("av_frame_alloc() error: Could not allocate raw picture frame."); + } + if ((picture_rgb = av_frame_alloc()) == null) { + throw new Exception("av_frame_alloc() error: Could not allocate RGB picture frame."); + } + + initPictureRGB(); + } + + if (audio_st != null) { + // Find the decoder for the audio stream + AVCodec codec = avcodec_find_decoder(audio_par.codec_id()); + if (codec == null) { + throw new Exception("avcodec_find_decoder() error: Unsupported audio format or codec not found: " + audio_par.codec_id() + "."); + } + + /* Allocate a codec context for the decoder */ + if ((audio_c = avcodec_alloc_context3(codec)) == null) { + throw new Exception("avcodec_alloc_context3() error: Could not allocate audio decoding context."); + } + + /* copy the stream parameters from the muxer */ + if ((ret = avcodec_parameters_to_context(audio_c, audio_st.codecpar())) < 0) { + release(); + throw new Exception(ret, "avcodec_parameters_to_context() error: Could not copy the audio stream parameters."); + } + + options = new AVDictionary(null); + for (Entry e : audioOptions.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + // Open audio codec + if ((ret = avcodec_open2(audio_c, codec, options)) < 0) { + throw new Exception(ret, "avcodec_open2() error " + ret + ": Could not open audio codec."); + } + av_dict_free(options); + + // Allocate audio samples frame + if ((samples_frame = av_frame_alloc()) == null) { + throw new Exception("av_frame_alloc() error: Could not allocate audio frame."); + } + } + } + + private void initPictureRGB() { + int width = imageWidth > 0 ? imageWidth : video_c.width(); + int height = imageHeight > 0 ? imageHeight : video_c.height(); + + switch (imageMode) { + case COLOR: + case GRAY: + // If size changes I new allocation is needed -> free the old one. + if (image_ptr != null) { + // First kill all references, then free it. + image_buf = null; + BytePointer[] temp = image_ptr; + image_ptr = null; + av_free(temp[0]); + } + int fmt = getPixelFormat(); + + // Determine required buffer size and allocate buffer + int size = av_image_get_buffer_size(fmt, width, height, 1); + image_ptr = new BytePointer[] { new BytePointer(av_malloc(size)).capacity(size) }; + image_buf = new Buffer[] { image_ptr[0].asBuffer() }; + + // Assign appropriate parts of buffer to image planes in picture_rgb + // Note that picture_rgb is an AVFrame, but AVFrame is a superset of AVPicture + av_image_fill_arrays(new PointerPointer(picture_rgb), picture_rgb.linesize(), image_ptr[0], fmt, width, height, 1); + picture_rgb.format(fmt); + picture_rgb.width(width); + picture_rgb.height(height); + break; + + case RAW: + image_ptr = new BytePointer[] { null }; + image_buf = new Buffer[] { null }; + break; + + default: + assert false; + } + } + + public void stop() throws Exception { + release(); + } + + public void trigger() throws Exception { + if (oc == null || oc.isNull()) { + throw new Exception("Could not trigger: No AVFormatContext. (Has start() been called?)"); + } + if (pkt2.size() > 0) { + pkt2.size(0); + av_packet_unref(pkt); + } + for (int i = 0; i < numBuffers + 1; i++) { + if (av_read_frame(oc, pkt) < 0) { + return; + } + av_packet_unref(pkt); + } + } + + private void processImage() throws Exception { + frame.imageWidth = imageWidth > 0 ? imageWidth : video_c.width(); + frame.imageHeight = imageHeight > 0 ? imageHeight : video_c.height(); + frame.imageDepth = Frame.DEPTH_UBYTE; + switch (imageMode) { + case COLOR: + case GRAY: + // Deinterlace Picture + if (deinterlace) { + throw new Exception("Cannot deinterlace: Functionality moved to FFmpegFrameFilter."); + } + + // Has the size changed? + if (frame.imageWidth != picture_rgb.width() || frame.imageHeight != picture_rgb.height()) { + initPictureRGB(); + } + + // Convert the image into BGR or GRAY format that OpenCV uses + img_convert_ctx = sws_getCachedContext(img_convert_ctx, video_c.width(), video_c.height(), video_c.pix_fmt(), // + frame.imageWidth, frame.imageHeight, getPixelFormat(), SWS_BILINEAR, // + null, null, (DoublePointer) null); + if (img_convert_ctx == null) { + throw new Exception("sws_getCachedContext() error: Cannot initialize the conversion context."); + } + + // Convert the image from its native format to RGB or GRAY + sws_scale(img_convert_ctx, new PointerPointer(picture), picture.linesize(), 0, video_c.height(),// + new PointerPointer(picture_rgb), picture_rgb.linesize()); + frame.imageStride = picture_rgb.linesize(0); + frame.image = image_buf; + break; + + case RAW: + frame.imageStride = picture.linesize(0); + BytePointer ptr = picture.data(0); + if (ptr != null && !ptr.equals(image_ptr[0])) { + image_ptr[0] = ptr.capacity(frame.imageHeight * frame.imageStride); + image_buf[0] = ptr.asBuffer(); + } + frame.image = image_buf; + break; + + default: + assert false; + } + frame.image[0].limit(frame.imageHeight * frame.imageStride); + frame.imageChannels = frame.imageStride / frame.imageWidth; + } + + private void processSamples() throws Exception { + int ret; + + int sample_format = samples_frame.format(); + int planes = av_sample_fmt_is_planar(sample_format) != 0 ? (int) samples_frame.channels() : 1; + int data_size = av_samples_get_buffer_size((IntPointer) null, audio_c.channels(), samples_frame.nb_samples(), audio_c.sample_fmt(), 1) / planes; + if (samples_buf == null || samples_buf.length != planes) { + samples_ptr = new BytePointer[planes]; + samples_buf = new Buffer[planes]; + } + frame.sampleRate = audio_c.sample_rate(); + frame.audioChannels = audio_c.channels(); + frame.samples = samples_buf; + int sample_size = data_size / av_get_bytes_per_sample(sample_format); + for (int i = 0; i < planes; i++) { + BytePointer p = samples_frame.data(i); + if (!p.equals(samples_ptr[i]) || samples_ptr[i].capacity() < data_size) { + samples_ptr[i] = p.capacity(data_size); + ByteBuffer b = p.asBuffer(); + switch (sample_format) { + case AV_SAMPLE_FMT_U8: + case AV_SAMPLE_FMT_U8P: + samples_buf[i] = b; + break; + case AV_SAMPLE_FMT_S16: + case AV_SAMPLE_FMT_S16P: + samples_buf[i] = b.asShortBuffer(); + break; + case AV_SAMPLE_FMT_S32: + case AV_SAMPLE_FMT_S32P: + samples_buf[i] = b.asIntBuffer(); + break; + case AV_SAMPLE_FMT_FLT: + case AV_SAMPLE_FMT_FLTP: + samples_buf[i] = b.asFloatBuffer(); + break; + case AV_SAMPLE_FMT_DBL: + case AV_SAMPLE_FMT_DBLP: + samples_buf[i] = b.asDoubleBuffer(); + break; + default: + assert false; + } + } + samples_buf[i].position(0).limit(sample_size); + } + + if (audio_c.channels() != getAudioChannels() || audio_c.sample_fmt() != getSampleFormat() || audio_c.sample_rate() != getSampleRate()) { + if (samples_convert_ctx == null || samples_channels != getAudioChannels() || samples_format != getSampleFormat() || samples_rate != getSampleRate()) { + samples_convert_ctx = swr_alloc_set_opts(samples_convert_ctx, av_get_default_channel_layout(getAudioChannels()), getSampleFormat(), getSampleRate(), av_get_default_channel_layout(audio_c.channels()), audio_c.sample_fmt(), audio_c.sample_rate(), 0, null); + if (samples_convert_ctx == null) { + throw new Exception("swr_alloc_set_opts() error: Cannot allocate the conversion context."); + } else if ((ret = swr_init(samples_convert_ctx)) < 0) { + throw new Exception(ret, "swr_init() error " + ret + ": Cannot initialize the conversion context."); + } + samples_channels = getAudioChannels(); + samples_format = getSampleFormat(); + samples_rate = getSampleRate(); + } + + int sample_size_in = samples_frame.nb_samples(); + int planes_out = av_sample_fmt_is_planar(samples_format) != 0 ? (int) samples_frame.channels() : 1; + int sample_size_out = swr_get_out_samples(samples_convert_ctx, sample_size_in); + int sample_bytes_out = av_get_bytes_per_sample(samples_format); + int buffer_size_out = sample_size_out * sample_bytes_out * (planes_out > 1 ? 1 : samples_channels); + if (samples_buf_out == null || samples_buf.length != planes_out || samples_ptr_out[0].capacity() < buffer_size_out) { + for (int i = 0; samples_ptr_out != null && i < samples_ptr_out.length; i++) { + av_free(samples_ptr_out[i].position(0)); + } + samples_ptr_out = new BytePointer[planes_out]; + samples_buf_out = new Buffer[planes_out]; + + for (int i = 0; i < planes_out; i++) { + samples_ptr_out[i] = new BytePointer(av_malloc(buffer_size_out)).capacity(buffer_size_out); + ByteBuffer b = samples_ptr_out[i].asBuffer(); + switch (samples_format) { + case AV_SAMPLE_FMT_U8: + case AV_SAMPLE_FMT_U8P: + samples_buf_out[i] = b; + break; + case AV_SAMPLE_FMT_S16: + case AV_SAMPLE_FMT_S16P: + samples_buf_out[i] = b.asShortBuffer(); + break; + case AV_SAMPLE_FMT_S32: + case AV_SAMPLE_FMT_S32P: + samples_buf_out[i] = b.asIntBuffer(); + break; + case AV_SAMPLE_FMT_FLT: + case AV_SAMPLE_FMT_FLTP: + samples_buf_out[i] = b.asFloatBuffer(); + break; + case AV_SAMPLE_FMT_DBL: + case AV_SAMPLE_FMT_DBLP: + samples_buf_out[i] = b.asDoubleBuffer(); + break; + default: + assert false; + } + } + } + frame.sampleRate = samples_rate; + frame.audioChannels = samples_channels; + frame.samples = samples_buf_out; + + if ((ret = swr_convert(samples_convert_ctx, new PointerPointer(samples_ptr_out), sample_size_out, new PointerPointer(samples_ptr), sample_size_in)) < 0) { + throw new Exception(ret, "swr_convert() error " + ret + ": Cannot convert audio samples."); + } + for (int i = 0; i < planes_out; i++) { + samples_ptr_out[i].position(0).limit(ret * (planes_out > 1 ? 1 : samples_channels)); + samples_buf_out[i].position(0).limit(ret * (planes_out > 1 ? 1 : samples_channels)); + } + } + } + + public Frame grab() throws Exception { + return grabFrame(true, true, true, false); + } + + public Frame grabImage() throws Exception { + return grabFrame(false, true, true, false); + } + + public Frame grabSamples() throws Exception { + return grabFrame(true, false, true, false); + } + + public Frame grabKeyFrame() throws Exception { + return grabFrame(false, true, true, true); + } + + public Frame grabFrame(boolean doAudio, boolean doVideo, boolean processImage, boolean keyFrames) throws Exception { + if (oc == null || oc.isNull()) { + throw new Exception("Could not grab: No AVFormatContext. (Has start() been called?)"); + } else if ((!doVideo || video_st == null) && (!doAudio || audio_st == null)) { + return null; + } + frame.keyFrame = false; + frame.imageWidth = 0; + frame.imageHeight = 0; + frame.imageDepth = 0; + frame.imageChannels = 0; + frame.imageStride = 0; + frame.image = null; + frame.sampleRate = 0; + frame.audioChannels = 0; + frame.samples = null; + frame.opaque = null; + if (doVideo && frameGrabbed) { + frameGrabbed = false; + if (processImage) { + processImage(); + } + frame.keyFrame = picture.key_frame() != 0; + frame.image = image_buf; + frame.opaque = picture; + return frame; + } + boolean done = false; + while (!done) { + if (pkt2.size() <= 0) { + if (av_read_frame(oc, pkt) < 0) { + if (doVideo && video_st != null) { + // The video codec may have buffered some frames + pkt.stream_index(video_st.index()); + pkt.flags(AV_PKT_FLAG_KEY); + pkt.data(null); + pkt.size(0); + } else { + return null; + } + } + } + + // Is this a packet from the video stream? + if (doVideo && video_st != null && pkt.stream_index() == video_st.index() && (!keyFrames || pkt.flags() == AV_PKT_FLAG_KEY)) { + // Decode video frame + int len = avcodec_decode_video2(video_c, picture, got_frame, pkt); + + // Did we get a video frame? + if (len >= 0 && got_frame[0] != 0 && (!keyFrames || picture.pict_type() == AV_PICTURE_TYPE_I)) { + long pts = av_frame_get_best_effort_timestamp(picture); + AVRational time_base = video_st.time_base(); + timestamp = 1000000L * pts * time_base.num() / time_base.den(); + // best guess, AVCodecContext.frame_number = number of decoded frames... + frameNumber = (int) (timestamp * getFrameRate() / 1000000L); + if (processImage) { + processImage(); + } + done = true; + frame.timestamp = timestamp; + frame.keyFrame = picture.key_frame() != 0; + frame.image = image_buf; + frame.opaque = picture; + } else if (pkt.data() == null && pkt.size() == 0) { + return null; + } + } else if (doAudio && audio_st != null && pkt.stream_index() == audio_st.index()) { + if (pkt2.size() <= 0) { + // HashMap is unacceptably slow on Android + // pkt2.put(pkt); + BytePointer.memcpy(pkt2, pkt, sizeof_pkt); + } + av_frame_unref(samples_frame); + // Decode audio frame + int len = avcodec_decode_audio4(audio_c, samples_frame, got_frame, pkt2); + if (len <= 0) { + // On error, trash the whole packet + pkt2.size(0); + } else { + pkt2.data(pkt2.data().position(len)); + pkt2.size(pkt2.size() - len); + if (got_frame[0] != 0) { + long pts = av_frame_get_best_effort_timestamp(samples_frame); + AVRational time_base = audio_st.time_base(); + timestamp = 1000000L * pts * time_base.num() / time_base.den(); + /* if a frame has been decoded, output it */ + processSamples(); + done = true; + frame.timestamp = timestamp; + frame.keyFrame = samples_frame.key_frame() != 0; + frame.opaque = samples_frame; + } + } + } + + if (pkt2.size() <= 0) { + // Free the packet that was allocated by av_read_frame + av_packet_unref(pkt); + } + } + return frame; + } + + public AVPacket grabPacket() throws Exception { + + if (oc == null || oc.isNull()) { + throw new Exception("Could not trigger: No AVFormatContext. (Has start() been called?)"); + } + + // Return the next frame of a stream. + if (av_read_frame(oc, pkt) < 0) { + return null; + } + + return pkt; + + } } diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index b89d0201..faec740c 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -72,6 +72,10 @@ import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.PointerPointer; import org.bytedeco.javacpp.ShortPointer; +import org.bytedeco.javacpp.avcodec; +import org.bytedeco.javacpp.avformat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.bytedeco.javacpp.avcodec.*; import static org.bytedeco.javacpp.avdevice.*; @@ -85,1133 +89,1175 @@ * @author Samuel Audet */ public class FFmpegFrameRecorder extends FrameRecorder { - public static FFmpegFrameRecorder createDefault(File f, int w, int h) throws Exception { return new FFmpegFrameRecorder(f, w, h); } - public static FFmpegFrameRecorder createDefault(String f, int w, int h) throws Exception { return new FFmpegFrameRecorder(f, w, h); } - - private static Exception loadingException = null; - public static void tryLoad() throws Exception { - if (loadingException != null) { - throw loadingException; - } else { - try { - Loader.load(org.bytedeco.javacpp.avutil.class); - Loader.load(org.bytedeco.javacpp.swresample.class); - Loader.load(org.bytedeco.javacpp.avcodec.class); - Loader.load(org.bytedeco.javacpp.avformat.class); - Loader.load(org.bytedeco.javacpp.swscale.class); - - /* initialize libavcodec, and register all codecs and formats */ - avcodec_register_all(); - av_register_all(); - avformat_network_init(); - - Loader.load(org.bytedeco.javacpp.avdevice.class); - avdevice_register_all(); - } catch (Throwable t) { - if (t instanceof Exception) { - throw loadingException = (Exception)t; - } else { - throw loadingException = new Exception("Failed to load " + FFmpegFrameRecorder.class, t); - } - } - } - } - - static { - try { - tryLoad(); - FFmpegLockCallback.init(); - } catch (Exception ex) { } - } - - public FFmpegFrameRecorder(File file, int audioChannels) { - this(file, 0, 0, audioChannels); - } - public FFmpegFrameRecorder(String filename, int audioChannels) { - this(filename, 0, 0, audioChannels); - } - public FFmpegFrameRecorder(File file, int imageWidth, int imageHeight) { - this(file, imageWidth, imageHeight, 0); - } - public FFmpegFrameRecorder(String filename, int imageWidth, int imageHeight) { - this(filename, imageWidth, imageHeight, 0); - } - public FFmpegFrameRecorder(File file, int imageWidth, int imageHeight, int audioChannels) { - this(file.getAbsolutePath(), imageWidth, imageHeight, audioChannels); - } - public FFmpegFrameRecorder(String filename, int imageWidth, int imageHeight, int audioChannels) { - this.filename = filename; - this.imageWidth = imageWidth; - this.imageHeight = imageHeight; - this.audioChannels = audioChannels; - - this.pixelFormat = AV_PIX_FMT_NONE; - this.videoCodec = AV_CODEC_ID_NONE; - this.videoBitrate = 400000; - this.frameRate = 30; - - this.sampleFormat = AV_SAMPLE_FMT_NONE; - this.audioCodec = AV_CODEC_ID_NONE; - this.audioBitrate = 64000; - this.sampleRate = 44100; - - this.interleaved = true; - - this.video_pkt = new AVPacket(); - this.audio_pkt = new AVPacket(); - } - public FFmpegFrameRecorder(OutputStream outputStream, int audioChannels) { - this(outputStream.toString(), audioChannels); - this.outputStream = outputStream; - } - public FFmpegFrameRecorder(OutputStream outputStream, int imageWidth, int imageHeight) { - this(outputStream.toString(), imageWidth, imageHeight); - this.outputStream = outputStream; - } - public FFmpegFrameRecorder(OutputStream outputStream, int imageWidth, int imageHeight, int audioChannels) { - this(outputStream.toString(), imageWidth, imageHeight, audioChannels); - this.outputStream = outputStream; - } - public void release() throws Exception { - // synchronized (org.bytedeco.javacpp.avcodec.class) { - releaseUnsafe(); - // } - } - void releaseUnsafe() throws Exception { - /* close each codec */ - if (video_c != null) { - avcodec_free_context(video_c); - video_c = null; - } - if (audio_c != null) { - avcodec_free_context(audio_c); - audio_c = null; - } - if (picture_buf != null) { - av_free(picture_buf); - picture_buf = null; - } - if (picture != null) { - av_frame_free(picture); - picture = null; - } - if (tmp_picture != null) { - av_frame_free(tmp_picture); - tmp_picture = null; - } - if (video_outbuf != null) { - av_free(video_outbuf); - video_outbuf = null; - } - if (frame != null) { - av_frame_free(frame); - frame = null; - } - if (samples_out != null) { - for (int i = 0; i < samples_out.length; i++) { - av_free(samples_out[i].position(0)); - } - samples_out = null; - } - if (audio_outbuf != null) { - av_free(audio_outbuf); - audio_outbuf = null; - } - if (video_st != null && video_st.metadata() != null) { - av_dict_free(video_st.metadata()); - video_st.metadata(null); - } - if (audio_st != null && audio_st.metadata() != null) { - av_dict_free(audio_st.metadata()); - audio_st.metadata(null); - } - video_st = null; - audio_st = null; - filename = null; - - AVFormatContext outputStreamKey = oc; - if (oc != null && !oc.isNull()) { - if (outputStream == null && (oformat.flags() & AVFMT_NOFILE) == 0) { - /* close the output file */ - avio_close(oc.pb()); - } - - /* free the streams */ - int nb_streams = oc.nb_streams(); - for(int i = 0; i < nb_streams; i++) { - av_free(oc.streams(i).codec()); - av_free(oc.streams(i)); - } - - /* free metadata */ - if (oc.metadata() != null) { - av_dict_free(oc.metadata()); - oc.metadata(null); - } - - /* free the stream */ - av_free(oc); - oc = null; - } - - if (img_convert_ctx != null) { - sws_freeContext(img_convert_ctx); - img_convert_ctx = null; - } - - if (samples_convert_ctx != null) { - swr_free(samples_convert_ctx); - samples_convert_ctx = null; - } - - if (outputStream != null) { - try { - outputStream.close(); - } catch (IOException ex) { - throw new Exception("Error on OutputStream.close(): ", ex); - } finally { - outputStream = null; - outputStreams.remove(outputStreamKey); - if (avio != null) { - if (avio.buffer() != null) { - av_free(avio.buffer()); - avio.buffer(null); - } - av_free(avio); - avio = null; - } - } - } - } - @Override protected void finalize() throws Throwable { - super.finalize(); - release(); - } - - static Map outputStreams = Collections.synchronizedMap(new HashMap()); - - static class WriteCallback extends Write_packet_Pointer_BytePointer_int { - @Override public int call(Pointer opaque, BytePointer buf, int buf_size) { - try { - byte[] b = new byte[buf_size]; - OutputStream os = outputStreams.get(opaque); - buf.get(b, 0, buf_size); - os.write(b, 0, buf_size); - return buf_size; - } - catch (Throwable t) { - System.err.println("Error on OutputStream.write(): " + t); - return -1; - } - } - } - - static WriteCallback writeCallback = new WriteCallback(); - - private OutputStream outputStream; - private AVIOContext avio; - private String filename; - private AVFrame picture, tmp_picture; - private BytePointer picture_buf; - private BytePointer video_outbuf; - private int video_outbuf_size; - private AVFrame frame; - private Pointer[] samples_in; - private BytePointer[] samples_out; - private PointerPointer samples_in_ptr; - private PointerPointer samples_out_ptr; - private BytePointer audio_outbuf; - private int audio_outbuf_size; - private int audio_input_frame_size; - private AVOutputFormat oformat; - private AVFormatContext oc; - private AVCodec video_codec, audio_codec; - private AVCodecContext video_c, audio_c; - private AVStream video_st, audio_st; - private SwsContext img_convert_ctx; - private SwrContext samples_convert_ctx; - private int samples_channels, samples_format, samples_rate; - private AVPacket video_pkt, audio_pkt; - private int[] got_video_packet, got_audio_packet; - private AVFormatContext ifmt_ctx; - - @Override public int getFrameNumber() { - return picture == null ? super.getFrameNumber() : (int)picture.pts(); - } - @Override public void setFrameNumber(int frameNumber) { - if (picture == null) { super.setFrameNumber(frameNumber); } else { picture.pts(frameNumber); } - } - - // best guess for timestamp in microseconds... - @Override public long getTimestamp() { - return Math.round(getFrameNumber() * 1000000L / getFrameRate()); - } - @Override public void setTimestamp(long timestamp) { - setFrameNumber((int)Math.round(timestamp * getFrameRate() / 1000000L)); - } - - public void start(AVFormatContext ifmt_ctx) throws Exception { - this.ifmt_ctx = ifmt_ctx; - start(); - } - - public void start() throws Exception { - // synchronized (org.bytedeco.javacpp.avcodec.class) { - startUnsafe(); - // } - } - - void startUnsafe() throws Exception { - int ret; - picture = null; - tmp_picture = null; - picture_buf = null; - frame = null; - video_outbuf = null; - audio_outbuf = null; - oc = new AVFormatContext(null); - video_c = null; - audio_c = null; - video_st = null; - audio_st = null; - got_video_packet = new int[1]; - got_audio_packet = new int[1]; - - /* auto detect the output format from the name. */ - String format_name = format == null || format.length() == 0 ? null : format; - if ((oformat = av_guess_format(format_name, filename, null)) == null) { - int proto = filename.indexOf("://"); - if (proto > 0) { - format_name = filename.substring(0, proto); - } - if ((oformat = av_guess_format(format_name, filename, null)) == null) { - throw new Exception("av_guess_format() error: Could not guess output format for \"" + filename + "\" and " + format + " format."); - } - } - format_name = oformat.name().getString(); - - /* allocate the output media context */ - if (avformat_alloc_output_context2(oc, null, format_name, filename) < 0) { - throw new Exception("avformat_alloc_context2() error:\tCould not allocate format context"); - } - - if (outputStream != null) { - avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 1, oc, null, writeCallback, null); - oc.pb(avio); - - filename = outputStream.toString(); - outputStreams.put(oc, outputStream); - } - oc.oformat(oformat); - oc.filename().putString(filename); - oc.max_delay(maxDelay); - - /* add the audio and video streams using the format codecs - and initialize the codecs */ - AVStream inpVideoStream = null, inpAudioStream = null; - if (ifmt_ctx != null) { - // get input video and audio stream indices from ifmt_ctx - for (int idx = 0; idx < ifmt_ctx.nb_streams(); idx++) { - AVStream inputStream = ifmt_ctx.streams(idx); - if (inputStream.codec().codec_type() == AVMEDIA_TYPE_VIDEO) { - inpVideoStream = inputStream; - videoCodec = inpVideoStream.codec().codec_id(); - if (inpVideoStream.r_frame_rate().num() != AV_NOPTS_VALUE && inpVideoStream.r_frame_rate().den() != 0) { - frameRate = (inpVideoStream.r_frame_rate().num()) / (inpVideoStream.r_frame_rate().den()); - } - - } else if (inputStream.codec().codec_type() == AVMEDIA_TYPE_AUDIO) { - inpAudioStream = inputStream; - audioCodec = inpAudioStream.codec().codec_id(); - } - } - } - - if (imageWidth > 0 && imageHeight > 0) { - if (videoCodec != AV_CODEC_ID_NONE) { - oformat.video_codec(videoCodec); - } else if ("flv".equals(format_name)) { - oformat.video_codec(AV_CODEC_ID_FLV1); - } else if ("mp4".equals(format_name)) { - oformat.video_codec(AV_CODEC_ID_MPEG4); - } else if ("3gp".equals(format_name)) { - oformat.video_codec(AV_CODEC_ID_H263); - } else if ("avi".equals(format_name)) { - oformat.video_codec(AV_CODEC_ID_HUFFYUV); - } - - /* find the video encoder */ - if ((video_codec = avcodec_find_encoder_by_name(videoCodecName)) == null && - (video_codec = avcodec_find_encoder(oformat.video_codec())) == null) { - release(); - throw new Exception("avcodec_find_encoder() error: Video codec not found."); - } - oformat.video_codec(video_codec.id()); - - AVRational frame_rate = av_d2q(frameRate, 1001000); - AVRational supported_framerates = video_codec.supported_framerates(); - if (supported_framerates != null) { - int idx = av_find_nearest_q_idx(frame_rate, supported_framerates); - frame_rate = supported_framerates.position(idx); - } - - /* add a video output stream */ - if ((video_st = avformat_new_stream(oc, null)) == null) { - release(); - throw new Exception("avformat_new_stream() error: Could not allocate video stream."); - } - - if ((video_c = avcodec_alloc_context3(video_codec)) == null) { - release(); - throw new Exception("avcodec_alloc_context3() error: Could not allocate video encoding context."); - } - - if (inpVideoStream != null) { - if ((ret = avcodec_copy_context(video_st.codec(), inpVideoStream.codec())) < 0) { - release(); - throw new Exception("avcodec_copy_context() error:\tFailed to copy context from input to output stream codec context"); - } - - videoBitrate = (int) inpVideoStream.codec().bit_rate(); - pixelFormat = inpVideoStream.codec().pix_fmt(); - aspectRatio = inpVideoStream.codec().sample_aspect_ratio().den() / inpVideoStream.codec().sample_aspect_ratio().den() * 1.d; - videoQuality = inpVideoStream.codec().global_quality(); - video_c.codec_tag(0); - } - - video_c.codec_id(oformat.video_codec()); - video_c.codec_type(AVMEDIA_TYPE_VIDEO); - - - /* put sample parameters */ - video_c.bit_rate(videoBitrate); - /* resolution must be a multiple of two. Scale height to maintain the aspect ratio. */ - if (imageWidth % 2 == 1) { - int roundedWidth = imageWidth + 1; - imageHeight = (roundedWidth * imageHeight + imageWidth / 2) / imageWidth; - imageWidth = roundedWidth; - } - video_c.width(imageWidth); - video_c.height(imageHeight); - if (aspectRatio > 0) { - AVRational r = av_d2q(aspectRatio, 255); - video_c.sample_aspect_ratio(r); - video_st.sample_aspect_ratio(r); - } - /* time base: this is the fundamental unit of time (in seconds) in terms - of which frame timestamps are represented. for fixed-fps content, - timebase should be 1/framerate and timestamp increments should be - identically 1. */ - video_c.time_base(av_inv_q(frame_rate)); - video_st.time_base(av_inv_q(frame_rate)); - if (gopSize >= 0) { - video_c.gop_size(gopSize); /* emit one intra frame every gopSize frames at most */ - } - if (videoQuality >= 0) { - video_c.flags(video_c.flags() | CODEC_FLAG_QSCALE); - video_c.global_quality((int)Math.round(FF_QP2LAMBDA * videoQuality)); - } - - if (pixelFormat != AV_PIX_FMT_NONE) { - video_c.pix_fmt(pixelFormat); - } else if (video_c.codec_id() == AV_CODEC_ID_RAWVIDEO || video_c.codec_id() == AV_CODEC_ID_PNG || - video_c.codec_id() == AV_CODEC_ID_HUFFYUV || video_c.codec_id() == AV_CODEC_ID_FFV1) { - video_c.pix_fmt(AV_PIX_FMT_RGB32); // appropriate for common lossless formats - } else if (video_c.codec_id() == AV_CODEC_ID_JPEGLS) { - video_c.pix_fmt(AV_PIX_FMT_BGR24); - } else if (video_c.codec_id() == AV_CODEC_ID_MJPEG || video_c.codec_id() == AV_CODEC_ID_MJPEGB) { - video_c.pix_fmt(AV_PIX_FMT_YUVJ420P); - } else { - video_c.pix_fmt(AV_PIX_FMT_YUV420P); // lossy, but works with about everything - } - - if (video_c.codec_id() == AV_CODEC_ID_MPEG2VIDEO) { - /* just for testing, we also add B frames */ - video_c.max_b_frames(2); - } else if (video_c.codec_id() == AV_CODEC_ID_MPEG1VIDEO) { - /* Needed to avoid using macroblocks in which some coeffs overflow. - This does not happen with normal video, it just happens here as - the motion of the chroma plane does not match the luma plane. */ - video_c.mb_decision(2); - } else if (video_c.codec_id() == AV_CODEC_ID_H263) { - // H.263 does not support any other resolution than the following - if (imageWidth <= 128 && imageHeight <= 96) { - video_c.width(128).height(96); - } else if (imageWidth <= 176 && imageHeight <= 144) { - video_c.width(176).height(144); - } else if (imageWidth <= 352 && imageHeight <= 288) { - video_c.width(352).height(288); - } else if (imageWidth <= 704 && imageHeight <= 576) { - video_c.width(704).height(576); - } else { - video_c.width(1408).height(1152); - } - } else if (video_c.codec_id() == AV_CODEC_ID_H264) { - // default to constrained baseline to produce content that plays back on anything, - // without any significant tradeoffs for most use cases - video_c.profile(AVCodecContext.FF_PROFILE_H264_CONSTRAINED_BASELINE); - } - - // some formats want stream headers to be separate - if ((oformat.flags() & AVFMT_GLOBALHEADER) != 0) { - video_c.flags(video_c.flags() | CODEC_FLAG_GLOBAL_HEADER); - } - - if ((video_codec.capabilities() & CODEC_CAP_EXPERIMENTAL) != 0) { - video_c.strict_std_compliance(AVCodecContext.FF_COMPLIANCE_EXPERIMENTAL); - } - - if (maxBFrames >= 0) { - video_c.max_b_frames(maxBFrames); - video_c.has_b_frames(maxBFrames == 0 ? 0 : 1); - } - - if (trellis >= 0) { - video_c.trellis(trellis); - } - } - - /* - * add an audio output stream - */ - if (audioChannels > 0 && audioBitrate > 0 && sampleRate > 0) { - if (audioCodec != AV_CODEC_ID_NONE) { - oformat.audio_codec(audioCodec); - } else if ("flv".equals(format_name) || "mp4".equals(format_name) || "3gp".equals(format_name)) { - oformat.audio_codec(AV_CODEC_ID_AAC); - } else if ("avi".equals(format_name)) { - oformat.audio_codec(AV_CODEC_ID_PCM_S16LE); - } - - /* find the audio encoder */ - if ((audio_codec = avcodec_find_encoder_by_name(audioCodecName)) == null && - (audio_codec = avcodec_find_encoder(oformat.audio_codec())) == null) { - release(); - throw new Exception("avcodec_find_encoder() error: Audio codec not found."); - } - oformat.audio_codec(audio_codec.id()); - - if ((audio_st = avformat_new_stream(oc, null)) == null) { - release(); - throw new Exception("avformat_new_stream() error: Could not allocate audio stream."); - } - - if ((audio_c = avcodec_alloc_context3(audio_codec)) == null) { - release(); - throw new Exception("avcodec_alloc_context3() error: Could not allocate audio encoding context."); - } - - if(inpAudioStream != null && audioChannels > 0){ - if ((ret = avcodec_copy_context(audio_st.codec(), inpAudioStream.codec())) < 0) { - throw new Exception("avcodec_copy_context() error:\tFailed to copy context from input audio to output audio stream codec context\n"); - } - - audioBitrate = (int) inpAudioStream.codec().bit_rate(); - sampleRate = inpAudioStream.codec().sample_rate(); - audioChannels = inpAudioStream.codec().channels(); - sampleFormat = inpAudioStream.codec().sample_fmt(); - audioQuality = inpAudioStream.codec().global_quality(); - audio_c.codec_tag(0); - audio_st.pts(inpAudioStream.pts()); - audio_st.duration(inpAudioStream.duration()); - audio_st.time_base().num(inpAudioStream.time_base().num()); - audio_st.time_base().den(inpAudioStream.time_base().den()); - } - - audio_c.codec_id(oformat.audio_codec()); - audio_c.codec_type(AVMEDIA_TYPE_AUDIO); - - - /* put sample parameters */ - audio_c.bit_rate(audioBitrate); - audio_c.sample_rate(sampleRate); - audio_c.channels(audioChannels); - audio_c.channel_layout(av_get_default_channel_layout(audioChannels)); - if (sampleFormat != AV_SAMPLE_FMT_NONE) { - audio_c.sample_fmt(sampleFormat); - } else { - // use AV_SAMPLE_FMT_S16 by default, if available - audio_c.sample_fmt(AV_SAMPLE_FMT_FLTP); - IntPointer formats = audio_c.codec().sample_fmts(); - for (int i = 0; formats.get(i) != -1; i++) { - if (formats.get(i) == AV_SAMPLE_FMT_S16) { - audio_c.sample_fmt(AV_SAMPLE_FMT_S16); - break; - } - } - } - audio_c.time_base().num(1).den(sampleRate); - audio_st.time_base().num(1).den(sampleRate); - switch (audio_c.sample_fmt()) { - case AV_SAMPLE_FMT_U8: - case AV_SAMPLE_FMT_U8P: audio_c.bits_per_raw_sample(8); break; - case AV_SAMPLE_FMT_S16: - case AV_SAMPLE_FMT_S16P: audio_c.bits_per_raw_sample(16); break; - case AV_SAMPLE_FMT_S32: - case AV_SAMPLE_FMT_S32P: audio_c.bits_per_raw_sample(32); break; - case AV_SAMPLE_FMT_FLT: - case AV_SAMPLE_FMT_FLTP: audio_c.bits_per_raw_sample(32); break; - case AV_SAMPLE_FMT_DBL: - case AV_SAMPLE_FMT_DBLP: audio_c.bits_per_raw_sample(64); break; - default: assert false; - } - if (audioQuality >= 0) { - audio_c.flags(audio_c.flags() | CODEC_FLAG_QSCALE); - audio_c.global_quality((int)Math.round(FF_QP2LAMBDA * audioQuality)); - } - - // some formats want stream headers to be separate - if ((oformat.flags() & AVFMT_GLOBALHEADER) != 0) { - audio_c.flags(audio_c.flags() | CODEC_FLAG_GLOBAL_HEADER); - } - - if ((audio_codec.capabilities() & CODEC_CAP_EXPERIMENTAL) != 0) { - audio_c.strict_std_compliance(AVCodecContext.FF_COMPLIANCE_EXPERIMENTAL); - } - } - - /* now that all the parameters are set, we can open the audio and - video codecs and allocate the necessary encode buffers */ - if (video_st != null && inpVideoStream == null) { - AVDictionary options = new AVDictionary(null); - if (videoQuality >= 0) { - av_dict_set(options, "crf", "" + videoQuality, 0); - } - for (Entry e : videoOptions.entrySet()) { - av_dict_set(options, e.getKey(), e.getValue(), 0); - } - /* open the codec */ - if ((ret = avcodec_open2(video_c, video_codec, options)) < 0) { - release(); - av_dict_free(options); - throw new Exception("avcodec_open2() error " + ret + ": Could not open video codec."); - } - av_dict_free(options); - - video_outbuf = null; - if ((oformat.flags() & AVFMT_RAWPICTURE) == 0) { - /* allocate output buffer */ - /* XXX: API change will be done */ - /* buffers passed into lav* can be allocated any way you prefer, - as long as they're aligned enough for the architecture, and - they're freed appropriately (such as using av_free for buffers - allocated with av_malloc) */ - video_outbuf_size = Math.max(256 * 1024, 8 * video_c.width() * video_c.height()); // a la ffmpeg.c - video_outbuf = new BytePointer(av_malloc(video_outbuf_size)); - } - - /* allocate the encoded raw picture */ - if ((picture = av_frame_alloc()) == null) { - release(); - throw new Exception("av_frame_alloc() error: Could not allocate picture."); - } - picture.pts(0); // magic required by libx264 - - int size = av_image_get_buffer_size(video_c.pix_fmt(), video_c.width(), video_c.height(), 1); - if ((picture_buf = new BytePointer(av_malloc(size))).isNull()) { - release(); - throw new Exception("av_malloc() error: Could not allocate picture buffer."); - } - - /* if the output format is not equal to the image format, then a temporary - picture is needed too. It is then converted to the required output format */ - if ((tmp_picture = av_frame_alloc()) == null) { - release(); - throw new Exception("av_frame_alloc() error: Could not allocate temporary picture."); - } - - /* copy the stream parameters to the muxer */ - if ((ret = avcodec_parameters_from_context(video_st.codecpar(), video_c)) < 0) { - release(); - throw new Exception("avcodec_parameters_from_context() error: Could not copy the video stream parameters."); - } - - AVDictionary metadata = new AVDictionary(null); - for (Entry e : videoMetadata.entrySet()) { - av_dict_set(metadata, e.getKey(), e.getValue(), 0); - } - video_st.metadata(metadata); - } - - if (audio_st != null && inpAudioStream == null) { - AVDictionary options = new AVDictionary(null); - if (audioQuality >= 0) { - av_dict_set(options, "crf", "" + audioQuality, 0); - } - for (Entry e : audioOptions.entrySet()) { - av_dict_set(options, e.getKey(), e.getValue(), 0); - } - /* open the codec */ - if ((ret = avcodec_open2(audio_c, audio_codec, options)) < 0) { - release(); - av_dict_free(options); - throw new Exception("avcodec_open2() error " + ret + ": Could not open audio codec."); - } - av_dict_free(options); - - audio_outbuf_size = 256 * 1024; - audio_outbuf = new BytePointer(av_malloc(audio_outbuf_size)); - - /* ugly hack for PCM codecs (will be removed ASAP with new PCM - support to compute the input frame size in samples */ - if (audio_c.frame_size() <= 1) { - audio_outbuf_size = AV_INPUT_BUFFER_MIN_SIZE; - audio_input_frame_size = audio_outbuf_size / audio_c.channels(); - switch (audio_c.codec_id()) { - case AV_CODEC_ID_PCM_S16LE: - case AV_CODEC_ID_PCM_S16BE: - case AV_CODEC_ID_PCM_U16LE: - case AV_CODEC_ID_PCM_U16BE: - audio_input_frame_size >>= 1; - break; - default: - break; - } - } else { - audio_input_frame_size = audio_c.frame_size(); - } - //int bufferSize = audio_input_frame_size * audio_c.bits_per_raw_sample()/8 * audio_c.channels(); - int planes = av_sample_fmt_is_planar(audio_c.sample_fmt()) != 0 ? (int)audio_c.channels() : 1; - int data_size = av_samples_get_buffer_size((IntPointer)null, audio_c.channels(), - audio_input_frame_size, audio_c.sample_fmt(), 1) / planes; - samples_out = new BytePointer[planes]; - for (int i = 0; i < samples_out.length; i++) { - samples_out[i] = new BytePointer(av_malloc(data_size)).capacity(data_size); - } - samples_in = new Pointer[AVFrame.AV_NUM_DATA_POINTERS]; - samples_in_ptr = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS); - samples_out_ptr = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS); - - /* allocate the audio frame */ - if ((frame = av_frame_alloc()) == null) { - release(); - throw new Exception("av_frame_alloc() error: Could not allocate audio frame."); - } - frame.pts(0); // magic required by libvorbis and webm - - /* copy the stream parameters to the muxer */ - if ((ret = avcodec_parameters_from_context(audio_st.codecpar(), audio_c)) < 0) { - release(); - throw new Exception("avcodec_parameters_from_context() error: Could not copy the audio stream parameters."); - } - - AVDictionary metadata = new AVDictionary(null); - for (Entry e : audioMetadata.entrySet()) { - av_dict_set(metadata, e.getKey(), e.getValue(), 0); - } - audio_st.metadata(metadata); - } - - AVDictionary options = new AVDictionary(null); - for (Entry e : this.options.entrySet()) { - av_dict_set(options, e.getKey(), e.getValue(), 0); - } - - /* open the output file, if needed */ - if (outputStream == null && (oformat.flags() & AVFMT_NOFILE) == 0) { - AVIOContext pb = new AVIOContext(null); - if ((ret = avio_open2(pb, filename, AVIO_FLAG_WRITE, null, options)) < 0) { - release(); - av_dict_free(options); - throw new Exception("avio_open2 error() error " + ret + ": Could not open '" + filename + "'"); - } - oc.pb(pb); - } - - AVDictionary metadata = new AVDictionary(null); - for (Entry e : this.metadata.entrySet()) { - av_dict_set(metadata, e.getKey(), e.getValue(), 0); - } - /* write the stream header, if any */ - avformat_write_header(oc.metadata(metadata), options); - av_dict_free(options); - - if (av_log_get_level() >= AV_LOG_INFO) { - av_dump_format(oc, 0, filename, 1); - } - } - - public void stop() throws Exception { - if (oc != null) { - try { - synchronized (oc) { - /* flush all the buffers */ - while (video_st != null && ifmt_ctx == null && recordImage(0, 0, 0, 0, 0, AV_PIX_FMT_NONE, (Buffer[])null)); - while (audio_st != null && ifmt_ctx == null && recordSamples(0, 0, (Buffer[])null)); - - if (interleaved && video_st != null && audio_st != null) { - av_interleaved_write_frame(oc, null); - } else { - av_write_frame(oc, null); - } - - /* write the trailer, if any */ - av_write_trailer(oc); - } - } finally { - release(); - } - } - } - - @Override public void record(Frame frame) throws Exception { - record(frame, AV_PIX_FMT_NONE); - } - public void record(Frame frame, int pixelFormat) throws Exception { - if (frame == null || (frame.image == null && frame.samples == null)) { - recordImage(0, 0, 0, 0, 0, pixelFormat, (Buffer[])null); - } else { - if (frame.image != null) { - frame.keyFrame = recordImage(frame.imageWidth, frame.imageHeight, frame.imageDepth, - frame.imageChannels, frame.imageStride, pixelFormat, frame.image); - } - if (frame.samples != null) { - frame.keyFrame = recordSamples(frame.sampleRate, frame.audioChannels, frame.samples); - } - } - } - - public boolean recordImage(int width, int height, int depth, int channels, int stride, int pixelFormat, Buffer ... image) throws Exception { - if (video_st == null) { - throw new Exception("No video output stream (Is imageWidth > 0 && imageHeight > 0 and has start() been called?)"); - } - int ret; - - if (image == null || image.length == 0) { - /* no more frame to compress. The codec has a latency of a few - frames if using B frames, so we get the last frames by - passing the same picture again */ - } else { - int step = stride * Math.abs(depth) / 8; - BytePointer data = image[0] instanceof ByteBuffer - ? new BytePointer((ByteBuffer)image[0].position(0)) - : new BytePointer(new Pointer(image[0].position(0))); - - if (pixelFormat == AV_PIX_FMT_NONE) { - if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 3) { - pixelFormat = AV_PIX_FMT_BGR24; - } else if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 1) { - pixelFormat = AV_PIX_FMT_GRAY8; - } else if ((depth == Frame.DEPTH_USHORT || depth == Frame.DEPTH_SHORT) && channels == 1) { - pixelFormat = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN) ? - AV_PIX_FMT_GRAY16BE : AV_PIX_FMT_GRAY16LE; - } else if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 4) { - pixelFormat = AV_PIX_FMT_RGBA; - } else if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 2) { - pixelFormat = AV_PIX_FMT_NV21; // Android's camera capture format - } else { - throw new Exception("Could not guess pixel format of image: depth=" + depth + ", channels=" + channels); - } - } - - if (pixelFormat == AV_PIX_FMT_NV21) { - step = width; - } - - if (video_c.pix_fmt() != pixelFormat || video_c.width() != width || video_c.height() != height) { - /* convert to the codec pixel format if needed */ - img_convert_ctx = sws_getCachedContext(img_convert_ctx, width, height, pixelFormat, - video_c.width(), video_c.height(), video_c.pix_fmt(), SWS_BILINEAR, - null, null, (DoublePointer)null); - if (img_convert_ctx == null) { - throw new Exception("sws_getCachedContext() error: Cannot initialize the conversion context."); - } - av_image_fill_arrays(new PointerPointer(tmp_picture), tmp_picture.linesize(), data, pixelFormat, width, height, 1); - av_image_fill_arrays(new PointerPointer(picture), picture.linesize(), picture_buf, video_c.pix_fmt(), video_c.width(), video_c.height(), 1); - tmp_picture.linesize(0, step); - tmp_picture.format(pixelFormat); - tmp_picture.width(width); - tmp_picture.height(height); - picture.format(video_c.pix_fmt()); - picture.width(video_c.width()); - picture.height(video_c.height()); - sws_scale(img_convert_ctx, new PointerPointer(tmp_picture), tmp_picture.linesize(), - 0, height, new PointerPointer(picture), picture.linesize()); - } else { - av_image_fill_arrays(new PointerPointer(picture), picture.linesize(), data, pixelFormat, width, height, 1); - picture.linesize(0, step); - picture.format(pixelFormat); - picture.width(width); - picture.height(height); - } - } - - if ((oformat.flags() & AVFMT_RAWPICTURE) != 0) { - if (image == null || image.length == 0) { - return false; - } - /* raw video case. The API may change slightly in the future for that? */ - av_init_packet(video_pkt); - video_pkt.flags(video_pkt.flags() | AV_PKT_FLAG_KEY); - video_pkt.stream_index(video_st.index()); - video_pkt.data(new BytePointer(picture)); - video_pkt.size(Loader.sizeof(AVFrame.class)); - } else { - /* encode the image */ - av_init_packet(video_pkt); - video_pkt.data(video_outbuf); - video_pkt.size(video_outbuf_size); - picture.quality(video_c.global_quality()); - if ((ret = avcodec_encode_video2(video_c, video_pkt, image == null || image.length == 0 ? null : picture, got_video_packet)) < 0) { - throw new Exception("avcodec_encode_video2() error " + ret + ": Could not encode video packet."); - } - picture.pts(picture.pts() + 1); // magic required by libx264 - - /* if zero size, it means the image was buffered */ - if (got_video_packet[0] != 0) { - if (video_pkt.pts() != AV_NOPTS_VALUE) { - video_pkt.pts(av_rescale_q(video_pkt.pts(), video_c.time_base(), video_st.time_base())); - } - if (video_pkt.dts() != AV_NOPTS_VALUE) { - video_pkt.dts(av_rescale_q(video_pkt.dts(), video_c.time_base(), video_st.time_base())); - } - video_pkt.stream_index(video_st.index()); - } else { - return false; - } - } - - writePacket(AVMEDIA_TYPE_VIDEO, video_pkt); - return image != null ? (video_pkt.flags() & AV_PKT_FLAG_KEY) != 0 : got_video_packet[0] != 0; - } - - public boolean recordSamples(Buffer ... samples) throws Exception { - return recordSamples(0, 0, samples); - } - public boolean recordSamples(int sampleRate, int audioChannels, Buffer ... samples) throws Exception { - if (audio_st == null) { - throw new Exception("No audio output stream (Is audioChannels > 0 and has start() been called?)"); - } - - if (samples == null) { - // Typically samples_out[0].limit() is double the audio_input_frame_size --> sampleDivisor = 2 - double sampleDivisor = Math.floor((int)Math.min(samples_out[0].limit(), Integer.MAX_VALUE) / audio_input_frame_size); - writeSamples((int)Math.floor((int)samples_out[0].position() / sampleDivisor)); - return record((AVFrame)null); - } - - int ret; - - if (sampleRate <= 0) { - sampleRate = audio_c.sample_rate(); - } - if (audioChannels <= 0) { - audioChannels = audio_c.channels(); - } - int inputSize = samples != null ? samples[0].limit() - samples[0].position() : 0; - int inputFormat = samples_format; - int inputChannels = samples != null && samples.length > 1 ? 1 : audioChannels; - int inputDepth = 0; - int outputFormat = audio_c.sample_fmt(); - int outputChannels = samples_out.length > 1 ? 1 : audio_c.channels(); - int outputDepth = av_get_bytes_per_sample(outputFormat); - if (samples != null && samples[0] instanceof ByteBuffer) { - inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_U8P : AV_SAMPLE_FMT_U8; - inputDepth = 1; - for (int i = 0; i < samples.length; i++) { - ByteBuffer b = (ByteBuffer)samples[i]; - if (samples_in[i] instanceof BytePointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { - ((BytePointer)samples_in[i]).position(0).put(b.array(), b.position(), inputSize); - } else { - samples_in[i] = new BytePointer(b); - } - } - } else if (samples != null && samples[0] instanceof ShortBuffer) { - inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_S16P : AV_SAMPLE_FMT_S16; - inputDepth = 2; - for (int i = 0; i < samples.length; i++) { - ShortBuffer b = (ShortBuffer)samples[i]; - if (samples_in[i] instanceof ShortPointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { - ((ShortPointer)samples_in[i]).position(0).put(b.array(), samples[i].position(), inputSize); - } else { - samples_in[i] = new ShortPointer(b); - } - } - } else if (samples != null && samples[0] instanceof IntBuffer) { - inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_S32P : AV_SAMPLE_FMT_S32; - inputDepth = 4; - for (int i = 0; i < samples.length; i++) { - IntBuffer b = (IntBuffer)samples[i]; - if (samples_in[i] instanceof IntPointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { - ((IntPointer)samples_in[i]).position(0).put(b.array(), samples[i].position(), inputSize); - } else { - samples_in[i] = new IntPointer(b); - } - } - } else if (samples != null && samples[0] instanceof FloatBuffer) { - inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_FLTP : AV_SAMPLE_FMT_FLT; - inputDepth = 4; - for (int i = 0; i < samples.length; i++) { - FloatBuffer b = (FloatBuffer)samples[i]; - if (samples_in[i] instanceof FloatPointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { - ((FloatPointer)samples_in[i]).position(0).put(b.array(), b.position(), inputSize); - } else { - samples_in[i] = new FloatPointer(b); - } - } - } else if (samples != null && samples[0] instanceof DoubleBuffer) { - inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_DBLP : AV_SAMPLE_FMT_DBL; - inputDepth = 8; - for (int i = 0; i < samples.length; i++) { - DoubleBuffer b = (DoubleBuffer)samples[i]; - if (samples_in[i] instanceof DoublePointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { - ((DoublePointer)samples_in[i]).position(0).put(b.array(), b.position(), inputSize); - } else { - samples_in[i] = new DoublePointer(b); - } - } - } else if (samples != null) { - throw new Exception("Audio samples Buffer has unsupported type: " + samples); - } - - if (samples_convert_ctx == null || samples_channels != audioChannels || samples_format != inputFormat || samples_rate != sampleRate) { - samples_convert_ctx = swr_alloc_set_opts(samples_convert_ctx, audio_c.channel_layout(), outputFormat, audio_c.sample_rate(), - av_get_default_channel_layout(audioChannels), inputFormat, sampleRate, 0, null); - if (samples_convert_ctx == null) { - throw new Exception("swr_alloc_set_opts() error: Cannot allocate the conversion context."); - } else if ((ret = swr_init(samples_convert_ctx)) < 0) { - throw new Exception("swr_init() error " + ret + ": Cannot initialize the conversion context."); - } - samples_channels = audioChannels; - samples_format = inputFormat; - samples_rate = sampleRate; - } - - for (int i = 0; samples != null && i < samples.length; i++) { - samples_in[i].position(samples_in[i].position() * inputDepth). - limit((samples_in[i].position() + inputSize) * inputDepth); - } - while (true) { - int inputCount = (int)Math.min(samples != null ? (samples_in[0].limit() - samples_in[0].position()) / (inputChannels * inputDepth) : 0, Integer.MAX_VALUE); - int outputCount = (int)Math.min((samples_out[0].limit() - samples_out[0].position()) / (outputChannels * outputDepth), Integer.MAX_VALUE); - inputCount = Math.min(inputCount, (outputCount * sampleRate + audio_c.sample_rate() - 1) / audio_c.sample_rate()); - for (int i = 0; samples != null && i < samples.length; i++) { - samples_in_ptr.put(i, samples_in[i]); - } - for (int i = 0; i < samples_out.length; i++) { - samples_out_ptr.put(i, samples_out[i]); - } - if ((ret = swr_convert(samples_convert_ctx, samples_out_ptr, outputCount, samples_in_ptr, inputCount)) < 0) { - throw new Exception("swr_convert() error " + ret + ": Cannot convert audio samples."); - } else if (ret == 0) { - break; - } - for (int i = 0; samples != null && i < samples.length; i++) { - samples_in[i].position(samples_in[i].position() + inputCount * inputChannels * inputDepth); - } - for (int i = 0; i < samples_out.length; i++) { - samples_out[i].position(samples_out[i].position() + ret * outputChannels * outputDepth); - } - - if (samples == null || samples_out[0].position() >= samples_out[0].limit()) { - writeSamples(audio_input_frame_size); - } - } - return samples != null ? frame.key_frame() != 0 : record((AVFrame)null); - } - - private void writeSamples(int nb_samples) throws Exception { - if (samples_out == null || samples_out.length == 0) { - return; - } - - frame.nb_samples(nb_samples); - avcodec_fill_audio_frame(frame, audio_c.channels(), audio_c.sample_fmt(), samples_out[0], (int)samples_out[0].position(), 0); - for (int i = 0; i < samples_out.length; i++) { - int linesize = 0; - if (samples_out[0].position() > 0 && samples_out[0].position() < samples_out[0].limit()) { - linesize = (int)samples_out[i].position(); - } else { - linesize = (int)Math.min(samples_out[i].limit(), Integer.MAX_VALUE); - } - - frame.data(i, samples_out[i].position(0)); - frame.linesize(i, linesize); - } - frame.quality(audio_c.global_quality()); - record(frame); - } - - boolean record(AVFrame frame) throws Exception { - int ret; - - av_init_packet(audio_pkt); - audio_pkt.data(audio_outbuf); - audio_pkt.size(audio_outbuf_size); - if ((ret = avcodec_encode_audio2(audio_c, audio_pkt, frame, got_audio_packet)) < 0) { - throw new Exception("avcodec_encode_audio2() error " + ret + ": Could not encode audio packet."); - } - if (frame != null) { - frame.pts(frame.pts() + frame.nb_samples()); // magic required by libvorbis and webm - } - if (got_audio_packet[0] != 0) { - if (audio_pkt.pts() != AV_NOPTS_VALUE) { - audio_pkt.pts(av_rescale_q(audio_pkt.pts(), audio_c.time_base(), audio_st.time_base())); - } - if (audio_pkt.dts() != AV_NOPTS_VALUE) { - audio_pkt.dts(av_rescale_q(audio_pkt.dts(), audio_c.time_base(), audio_st.time_base())); - } - audio_pkt.flags(audio_pkt.flags() | AV_PKT_FLAG_KEY); - audio_pkt.stream_index(audio_st.index()); - } else { - return false; - } - - /* write the compressed frame in the media file */ - writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); - - return true; - } - - private void writePacket(int mediaType, AVPacket avPacket) throws Exception { - - AVStream avStream = (mediaType == AVMEDIA_TYPE_VIDEO) ? audio_st : (mediaType == AVMEDIA_TYPE_AUDIO) ? video_st : null; - String mediaTypeStr = (mediaType == AVMEDIA_TYPE_VIDEO) ? "video" : (mediaType == AVMEDIA_TYPE_AUDIO) ? "audio" : "unsupported media stream type"; - - synchronized (oc) { - int ret; - if (interleaved && avStream != null) { - if ((ret = av_interleaved_write_frame(oc, avPacket)) < 0) { - throw new Exception("av_interleaved_write_frame() error " + ret + " while writing interleaved " + mediaTypeStr + " packet."); - } - } else { - if ((ret = av_write_frame(oc, avPacket)) < 0) { - throw new Exception("av_write_frame() error " + ret + " while writing " + mediaTypeStr + " packet."); - } - } - } - } - - public boolean recordPacket(AVPacket pkt) throws Exception { - - if (pkt == null) { - return false; - } - - AVStream in_stream = ifmt_ctx.streams(pkt.stream_index()); - - pkt.dts(AV_NOPTS_VALUE); - pkt.pts(AV_NOPTS_VALUE); - pkt.pos(-1); - - if (in_stream.codec().codec_type() == AVMEDIA_TYPE_VIDEO && video_st != null) { - - pkt.stream_index(video_st.index()); - pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.codec().time_base(), video_st.codec().time_base())); - - writePacket(AVMEDIA_TYPE_VIDEO, pkt); - - } else if (in_stream.codec().codec_type() == AVMEDIA_TYPE_AUDIO && audio_st != null && (audioChannels > 0)) { - - pkt.stream_index(audio_st.index()); - pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.codec().time_base(), audio_st.codec().time_base())); - - writePacket(AVMEDIA_TYPE_AUDIO, pkt); - } - - av_free_packet(pkt); - - return true; - } - + public static FFmpegFrameRecorder createDefault(File f, int w, int h) throws Exception { + return new FFmpegFrameRecorder(f, w, h); + } + + public static FFmpegFrameRecorder createDefault(String f, int w, int h) throws Exception { + return new FFmpegFrameRecorder(f, w, h); + } + + public static Logger logger = LoggerFactory.getLogger(FFmpegFrameRecorder.class); + + private static Exception loadingException = null; + + public static void tryLoad() throws Exception { + if (loadingException != null) { + throw loadingException; + } else { + try { + Loader.load(org.bytedeco.javacpp.avutil.class); + Loader.load(org.bytedeco.javacpp.swresample.class); + Loader.load(org.bytedeco.javacpp.avcodec.class); + Loader.load(org.bytedeco.javacpp.avformat.class); + Loader.load(org.bytedeco.javacpp.swscale.class); + + /* initialize libavcodec, and register all codecs and formats */ + avcodec_register_all(); + av_register_all(); + avformat_network_init(); + + Loader.load(org.bytedeco.javacpp.avdevice.class); + avdevice_register_all(); + } catch (Throwable t) { + if (t instanceof Exception) { + throw loadingException = (Exception) t; + } else { + throw loadingException = new Exception("Failed to load " + FFmpegFrameRecorder.class, t); + } + } + } + } + + static { + try { + tryLoad(); + FFmpegLockCallback.init(); + } catch (Exception ex) { + } + } + + public FFmpegFrameRecorder(File file, int audioChannels) { + this(file, 0, 0, audioChannels); + } + + public FFmpegFrameRecorder(String filename, int audioChannels) { + this(filename, 0, 0, audioChannels); + } + + public FFmpegFrameRecorder(File file, int imageWidth, int imageHeight) { + this(file, imageWidth, imageHeight, 0); + } + + public FFmpegFrameRecorder(String filename, int imageWidth, int imageHeight) { + this(filename, imageWidth, imageHeight, 0); + } + + public FFmpegFrameRecorder(File file, int imageWidth, int imageHeight, int audioChannels) { + this(file.getAbsolutePath(), imageWidth, imageHeight, audioChannels); + } + + public FFmpegFrameRecorder(String filename, int imageWidth, int imageHeight, int audioChannels) { + this.filename = filename; + this.imageWidth = imageWidth; + this.imageHeight = imageHeight; + this.audioChannels = audioChannels; + + this.pixelFormat = AV_PIX_FMT_NONE; + this.videoCodec = AV_CODEC_ID_NONE; + this.videoBitrate = 400000; + this.frameRate = 30; + + this.sampleFormat = AV_SAMPLE_FMT_NONE; + this.audioCodec = AV_CODEC_ID_NONE; + this.audioBitrate = 64000; + this.sampleRate = 44100; + + this.interleaved = true; + + this.video_pkt = new AVPacket(); + this.audio_pkt = new AVPacket(); + } + + public FFmpegFrameRecorder(OutputStream outputStream, int audioChannels) { + this(outputStream.toString(), audioChannels); + this.outputStream = outputStream; + } + + public FFmpegFrameRecorder(OutputStream outputStream, int imageWidth, int imageHeight) { + this(outputStream.toString(), imageWidth, imageHeight); + this.outputStream = outputStream; + } + + public FFmpegFrameRecorder(OutputStream outputStream, int imageWidth, int imageHeight, int audioChannels) { + this(outputStream.toString(), imageWidth, imageHeight, audioChannels); + this.outputStream = outputStream; + } + + public void release() throws Exception { + // synchronized (org.bytedeco.javacpp.avcodec.class) { + releaseUnsafe(); + // } + } + + void releaseUnsafe() throws Exception { + /* close each codec */ + if (video_c != null) { + avcodec_free_context(video_c); + video_c = null; + } + if (audio_c != null) { + avcodec_free_context(audio_c); + audio_c = null; + } + if (picture_buf != null) { + av_free(picture_buf); + picture_buf = null; + } + if (picture != null) { + av_frame_free(picture); + picture = null; + } + if (tmp_picture != null) { + av_frame_free(tmp_picture); + tmp_picture = null; + } + if (video_outbuf != null) { + av_free(video_outbuf); + video_outbuf = null; + } + if (frame != null) { + av_frame_free(frame); + frame = null; + } + if (samples_out != null) { + for (int i = 0; i < samples_out.length; i++) { + av_free(samples_out[i].position(0)); + } + samples_out = null; + } + if (audio_outbuf != null) { + av_free(audio_outbuf); + audio_outbuf = null; + } + if (video_st != null && video_st.metadata() != null) { + av_dict_free(video_st.metadata()); + video_st.metadata(null); + } + if (audio_st != null && audio_st.metadata() != null) { + av_dict_free(audio_st.metadata()); + audio_st.metadata(null); + } + video_st = null; + audio_st = null; + filename = null; + + AVFormatContext outputStreamKey = oc; + if (oc != null && !oc.isNull()) { + if (outputStream == null && (oformat.flags() & AVFMT_NOFILE) == 0) { + /* close the output file */ + avio_close(oc.pb()); + } + + /* free the streams */ + int nb_streams = oc.nb_streams(); + for (int i = 0; i < nb_streams; i++) { + av_free(oc.streams(i).codecpar()); + av_free(oc.streams(i)); + } + + /* free metadata */ + if (oc.metadata() != null) { + av_dict_free(oc.metadata()); + oc.metadata(null); + } + + /* free the stream */ + av_free(oc); + oc = null; + } + + if (img_convert_ctx != null) { + sws_freeContext(img_convert_ctx); + img_convert_ctx = null; + } + + if (samples_convert_ctx != null) { + swr_free(samples_convert_ctx); + samples_convert_ctx = null; + } + + if (outputStream != null) { + try { + outputStream.close(); + } catch (IOException ex) { + throw new Exception("Error on OutputStream.close(): ", ex); + } finally { + outputStream = null; + // outputStreams.remove(outputStreamKey); + if (avio != null) { + if (avio.buffer() != null) { + av_free(avio.buffer()); + avio.buffer(null); + } + av_free(avio); + avio = null; + } + } + } + + if (writeCallback != null) { + writeCallback.close(); + writeCallback = null; + } + } + + @Override + protected void finalize() throws Throwable { + super.finalize(); + release(); + } + + // static Map outputStreams = Collections.synchronizedMap(new HashMap()); + + private class WriteCallback extends Write_packet_Pointer_BytePointer_int { + private FFmpegFrameRecorder recorder; + + public WriteCallback(FFmpegFrameRecorder recorder) { + this.recorder = recorder; + } + + @Override + public int call(Pointer opaque, BytePointer buf, int buf_size) { + try { + byte[] b = new byte[buf_size]; + OutputStream os = recorder.outputStream; + if (os == null) { + logger.error("OutputStream is null at WriteCallback"); + return -1; + } + buf.get(b, 0, buf_size); + os.write(b, 0, buf_size); + return buf_size; + } catch (Throwable t) { + System.err.println("Error on OutputStream.write(): " + t); + logger.error("Error on OutputStream.write(): " + t); + return -1; + } + } + + @Override + public void close(){ + super.close(); + this.recorder = null; + } + } + + private WriteCallback writeCallback = new WriteCallback(this); + + private OutputStream outputStream; + private AVIOContext avio; + private String filename; + private AVFrame picture, tmp_picture; + private BytePointer picture_buf; + private BytePointer video_outbuf; + private int video_outbuf_size; + private AVFrame frame; + private Pointer[] samples_in; + private BytePointer[] samples_out; + private PointerPointer samples_in_ptr; + private PointerPointer samples_out_ptr; + private BytePointer audio_outbuf; + private int audio_outbuf_size; + private int audio_input_frame_size; + private AVOutputFormat oformat; + private AVFormatContext oc; + private AVCodec video_codec, audio_codec; + private AVCodecContext video_c, audio_c; + private AVStream video_st, audio_st; + private SwsContext img_convert_ctx; + private SwrContext samples_convert_ctx; + private int samples_channels, samples_format, samples_rate; + private AVPacket video_pkt, audio_pkt; + private int[] got_video_packet, got_audio_packet; + private AVFormatContext ifmt_ctx; + + @Override + public int getFrameNumber() { + return picture == null ? super.getFrameNumber() : (int) picture.pts(); + } + + @Override + public void setFrameNumber(int frameNumber) { + if (picture == null) { + super.setFrameNumber(frameNumber); + } else { + picture.pts(frameNumber); + } + } + + // best guess for timestamp in microseconds... + @Override + public long getTimestamp() { + return Math.round(getFrameNumber() * 1000000L / getFrameRate()); + } + + @Override + public void setTimestamp(long timestamp) { + setFrameNumber((int) Math.round(timestamp * getFrameRate() / 1000000L)); + } + + public void start(AVFormatContext ifmt_ctx) throws Exception { + this.ifmt_ctx = ifmt_ctx; + start(); + } + + public void start() throws Exception { + // synchronized (org.bytedeco.javacpp.avcodec.class) { + startUnsafe(); + // } + } + + void startUnsafe() throws Exception { + int ret; + picture = null; + tmp_picture = null; + picture_buf = null; + frame = null; + video_outbuf = null; + audio_outbuf = null; + oc = new AVFormatContext(null); + video_c = null; + audio_c = null; + video_st = null; + audio_st = null; + got_video_packet = new int[1]; + got_audio_packet = new int[1]; + + /* auto detect the output format from the name. */ + String format_name = format == null || format.length() == 0 ? null : format; + if ((oformat = av_guess_format(format_name, filename, null)) == null) { + int proto = filename.indexOf("://"); + if (proto > 0) { + format_name = filename.substring(0, proto); + } + if ((oformat = av_guess_format(format_name, filename, null)) == null) { + throw new Exception("av_guess_format() error: Could not guess output format for \"" + filename + "\" and " + format + " format."); + } + } + format_name = oformat.name().getString(); + if ((oformat.flags() & avformat.AVFMT_TS_NONSTRICT) != 0) + oformat.flags(oformat.flags() - avformat.AVFMT_TS_NONSTRICT); + + /* allocate the output media context */ + if (avformat_alloc_output_context2(oc, null, format_name, filename) < 0) { + throw new Exception("avformat_alloc_context2() error:\tCould not allocate format context"); + } + + if (outputStream != null) { + avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 1, oc, null, writeCallback, null);// + oc.pb(avio); + + filename = outputStream.toString(); + // outputStreams.put(oc, outputStream); + } + oc.oformat(oformat); + oc.filename().putString(filename); + + /* + * add the audio and video streams using the format codecs and initialize the codecs + */ + AVStream inpVideoStream = null, inpAudioStream = null; + if (ifmt_ctx != null) { + // get input video and audio stream indices from ifmt_ctx + for (int idx = 0; idx < ifmt_ctx.nb_streams(); idx++) { + AVStream inputStream = ifmt_ctx.streams(idx); + if (inputStream.codecpar().codec_type() == AVMEDIA_TYPE_VIDEO) { + inpVideoStream = inputStream; + videoCodec = inpVideoStream.codecpar().codec_id(); + if (inpVideoStream.r_frame_rate().num() != AV_NOPTS_VALUE && inpVideoStream.r_frame_rate().den() != 0) { + frameRate = (inpVideoStream.r_frame_rate().num()) / (inpVideoStream.r_frame_rate().den()); + } + + } else if (inputStream.codecpar().codec_type() == AVMEDIA_TYPE_AUDIO) { + inpAudioStream = inputStream; + audioCodec = inpAudioStream.codecpar().codec_id(); + } + } + } + + if (imageWidth > 0 && imageHeight > 0) { + if (videoCodec != AV_CODEC_ID_NONE) { + oformat.video_codec(videoCodec); + } else if ("flv".equals(format_name)) { + oformat.video_codec(AV_CODEC_ID_FLV1); + } else if ("mp4".equals(format_name)) { + oformat.video_codec(AV_CODEC_ID_MPEG4); + } else if ("3gp".equals(format_name)) { + oformat.video_codec(AV_CODEC_ID_H263); + } else if ("avi".equals(format_name)) { + oformat.video_codec(AV_CODEC_ID_HUFFYUV); + } + + /* find the video encoder */ + if ((video_codec = avcodec_find_encoder_by_name(videoCodecName)) == null && (video_codec = avcodec_find_encoder(oformat.video_codec())) == null) { + release(); + throw new Exception("avcodec_find_encoder() error: Video codec not found."); + } + oformat.video_codec(video_codec.id()); + + AVRational frame_rate = av_d2q(frameRate, 1001000); + AVRational supported_framerates = video_codec.supported_framerates(); + if (supported_framerates != null) { + int idx = av_find_nearest_q_idx(frame_rate, supported_framerates); + frame_rate = supported_framerates.position(idx); + } + + /* add a video output stream */ + if ((video_st = avformat_new_stream(oc, null)) == null) { + release(); + throw new Exception("avformat_new_stream() error: Could not allocate video stream."); + } + + if ((video_c = avcodec_alloc_context3(video_codec)) == null) { + release(); + throw new Exception("avcodec_alloc_context3() error: Could not allocate video encoding context."); + } + + if (inpVideoStream != null) { + if ((ret = avcodec_parameters_copy(video_st.codecpar(), inpVideoStream.codecpar())) < 0) { + release(); + throw new Exception(ret, "avcodec_copy_context() error:\tFailed to copy context from input to output stream codec context"); + } + + videoBitrate = (int) inpVideoStream.codecpar().bit_rate(); + pixelFormat = inpVideoStream.codecpar().format(); + aspectRatio = inpVideoStream.codecpar().sample_aspect_ratio().den() / inpVideoStream.codecpar().sample_aspect_ratio().den() * 1.d; + videoQuality = inpVideoStream.codec().global_quality(); + video_c.codec_tag(0); + } + + video_c.codec_id(oformat.video_codec()); + video_c.codec_type(AVMEDIA_TYPE_VIDEO); + for (Entry item : encContextOptions.entrySet()) + av_opt_set(video_c.priv_data(), item.getKey(), item.getValue(), 0); + + /* put sample parameters */ + video_c.bit_rate(videoBitrate); + /* resolution must be a multiple of two. Scale height to maintain the aspect ratio. */ + if (imageWidth % 2 == 1) { + int roundedWidth = imageWidth + 1; + imageHeight = (roundedWidth * imageHeight + imageWidth / 2) / imageWidth; + imageWidth = roundedWidth; + } + video_c.width(imageWidth); + video_c.height(imageHeight); + if (aspectRatio > 0) { + AVRational r = av_d2q(aspectRatio, 255); + video_c.sample_aspect_ratio(r); + video_st.sample_aspect_ratio(r); + } + /* + * time base: this is the fundamental unit of time (in seconds) in terms of which frame timestamps are represented. for fixed-fps content, timebase should be 1/framerate and timestamp increments should be identically 1. + */ + video_c.time_base(av_inv_q(frame_rate)); + video_st.time_base(av_inv_q(frame_rate)); + if (gopSize >= 0) { + video_c.gop_size(gopSize); /* emit one intra frame every gopSize frames at most */ + } + if (videoQuality >= 0) { + video_c.flags(video_c.flags() | CODEC_FLAG_QSCALE); + video_c.global_quality((int) Math.round(FF_QP2LAMBDA * videoQuality)); + } + + if (pixelFormat != AV_PIX_FMT_NONE) { + video_c.pix_fmt(pixelFormat); + } else if (video_c.codec_id() == AV_CODEC_ID_RAWVIDEO || video_c.codec_id() == AV_CODEC_ID_PNG || video_c.codec_id() == AV_CODEC_ID_HUFFYUV || video_c.codec_id() == AV_CODEC_ID_FFV1) { + video_c.pix_fmt(AV_PIX_FMT_RGB32); // appropriate for common lossless formats + } else if (video_c.codec_id() == AV_CODEC_ID_JPEGLS) { + video_c.pix_fmt(AV_PIX_FMT_BGR24); + } else if (video_c.codec_id() == AV_CODEC_ID_MJPEG || video_c.codec_id() == AV_CODEC_ID_MJPEGB) { + video_c.pix_fmt(AV_PIX_FMT_YUVJ420P); + } else { + video_c.pix_fmt(AV_PIX_FMT_YUV420P); // lossy, but works with about everything + } + + if (video_c.codec_id() == AV_CODEC_ID_MPEG2VIDEO) { + /* just for testing, we also add B frames */ + video_c.max_b_frames(2); + } else if (video_c.codec_id() == AV_CODEC_ID_MPEG1VIDEO) { + /* + * Needed to avoid using macroblocks in which some coeffs overflow. This does not happen with normal video, it just happens here as the motion of the chroma plane does not match the luma plane. + */ + video_c.mb_decision(2); + } else if (video_c.codec_id() == AV_CODEC_ID_H263) { + // H.263 does not support any other resolution than the following + if (imageWidth <= 128 && imageHeight <= 96) { + video_c.width(128).height(96); + } else if (imageWidth <= 176 && imageHeight <= 144) { + video_c.width(176).height(144); + } else if (imageWidth <= 352 && imageHeight <= 288) { + video_c.width(352).height(288); + } else if (imageWidth <= 704 && imageHeight <= 576) { + video_c.width(704).height(576); + } else { + video_c.width(1408).height(1152); + } + } else if (video_c.codec_id() == AV_CODEC_ID_H264) { + // default to constrained baseline to produce content that plays back on anything, + // without any significant tradeoffs for most use cases + video_c.profile(AVCodecContext.FF_PROFILE_H264_CONSTRAINED_BASELINE); + } + + // some formats want stream headers to be separate + if ((oformat.flags() & AVFMT_GLOBALHEADER) != 0) { + video_c.flags(video_c.flags() | CODEC_FLAG_GLOBAL_HEADER); + } + + if ((video_codec.capabilities() & CODEC_CAP_EXPERIMENTAL) != 0) { + video_c.strict_std_compliance(AVCodecContext.FF_COMPLIANCE_EXPERIMENTAL); + } + } + + /* + * add an audio output stream + */ + if (audioChannels > 0 && audioBitrate > 0 && sampleRate > 0) { + if (audioCodec != AV_CODEC_ID_NONE) { + oformat.audio_codec(audioCodec); + } else if ("flv".equals(format_name) || "mp4".equals(format_name) || "3gp".equals(format_name)) { + oformat.audio_codec(AV_CODEC_ID_AAC); + } else if ("avi".equals(format_name)) { + oformat.audio_codec(AV_CODEC_ID_PCM_S16LE); + } + + /* find the audio encoder */ + if ((audio_codec = avcodec_find_encoder_by_name(audioCodecName)) == null && (audio_codec = avcodec_find_encoder(oformat.audio_codec())) == null) { + release(); + throw new Exception("avcodec_find_encoder() error: Audio codec not found."); + } + oformat.audio_codec(audio_codec.id()); + + if ((audio_st = avformat_new_stream(oc, null)) == null) { + release(); + throw new Exception("avformat_new_stream() error: Could not allocate audio stream."); + } + + if ((audio_c = avcodec_alloc_context3(audio_codec)) == null) { + release(); + throw new Exception("avcodec_alloc_context3() error: Could not allocate audio encoding context."); + } + + if (inpAudioStream != null && audioChannels > 0) { + if ((ret = avcodec_parameters_copy(audio_st.codecpar(), inpAudioStream.codecpar())) < 0) { + throw new Exception(ret, "avcodec_copy_context() error:\tFailed to copy context from input audio to output audio stream codec context\n"); + } + + audioBitrate = (int) inpAudioStream.codecpar().bit_rate(); + sampleRate = inpAudioStream.codecpar().sample_rate(); + audioChannels = inpAudioStream.codecpar().channels(); + sampleFormat = inpAudioStream.codecpar().format(); + audioQuality = inpAudioStream.codec().global_quality(); + audio_c.codec_tag(0); + audio_st.pts(inpAudioStream.pts()); + audio_st.duration(inpAudioStream.duration()); + audio_st.time_base().num(inpAudioStream.time_base().num()); + audio_st.time_base().den(inpAudioStream.time_base().den()); + } + + audio_c.codec_id(oformat.audio_codec()); + audio_c.codec_type(AVMEDIA_TYPE_AUDIO); + + /* put sample parameters */ + audio_c.bit_rate(audioBitrate); + audio_c.sample_rate(sampleRate); + audio_c.channels(audioChannels); + audio_c.channel_layout(av_get_default_channel_layout(audioChannels)); + if (sampleFormat != AV_SAMPLE_FMT_NONE) { + audio_c.sample_fmt(sampleFormat); + } else { + // use AV_SAMPLE_FMT_S16 by default, if available + audio_c.sample_fmt(AV_SAMPLE_FMT_FLTP); + IntPointer formats = audio_c.codec().sample_fmts(); + for (int i = 0; formats.get(i) != -1; i++) { + if (formats.get(i) == AV_SAMPLE_FMT_S16) { + audio_c.sample_fmt(AV_SAMPLE_FMT_S16); + break; + } + } + } + audio_c.time_base().num(1).den(sampleRate); + audio_st.time_base().num(1).den(sampleRate); + switch (audio_c.sample_fmt()) { + case AV_SAMPLE_FMT_U8: + case AV_SAMPLE_FMT_U8P: + audio_c.bits_per_raw_sample(8); + break; + case AV_SAMPLE_FMT_S16: + case AV_SAMPLE_FMT_S16P: + audio_c.bits_per_raw_sample(16); + break; + case AV_SAMPLE_FMT_S32: + case AV_SAMPLE_FMT_S32P: + audio_c.bits_per_raw_sample(32); + break; + case AV_SAMPLE_FMT_FLT: + case AV_SAMPLE_FMT_FLTP: + audio_c.bits_per_raw_sample(32); + break; + case AV_SAMPLE_FMT_DBL: + case AV_SAMPLE_FMT_DBLP: + audio_c.bits_per_raw_sample(64); + break; + default: + assert false; + } + if (audioQuality >= 0) { + audio_c.flags(audio_c.flags() | CODEC_FLAG_QSCALE); + audio_c.global_quality((int) Math.round(FF_QP2LAMBDA * audioQuality)); + } + + // some formats want stream headers to be separate + if ((oformat.flags() & AVFMT_GLOBALHEADER) != 0) { + audio_c.flags(audio_c.flags() | CODEC_FLAG_GLOBAL_HEADER); + } + + if ((audio_codec.capabilities() & CODEC_CAP_EXPERIMENTAL) != 0) { + audio_c.strict_std_compliance(AVCodecContext.FF_COMPLIANCE_EXPERIMENTAL); + } + } + + /* + * now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers + */ + if (video_st != null && inpVideoStream == null) { + AVDictionary options = new AVDictionary(null); + if (videoQuality >= 0) { + av_dict_set(options, "crf", "" + videoQuality, 0); + } + for (Entry e : videoOptions.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + /* open the codec */ + if ((ret = avcodec_open2(video_c, video_codec, options)) < 0) { + release(); + av_dict_free(options); + throw new Exception(ret, "avcodec_open2() error " + ret + ": Could not open video codec."); + } + av_dict_free(options); + + video_outbuf = null; + if ((oformat.flags() & AVFMT_RAWPICTURE) == 0) { + /* allocate output buffer */ + /* XXX: API change will be done */ + /* + * buffers passed into lav* can be allocated any way you prefer, as long as they're aligned enough for the architecture, and they're freed appropriately (such as using av_free for buffers allocated with av_malloc) + */ + video_outbuf_size = Math.max(256 * 1024, 8 * video_c.width() * video_c.height()); // a la ffmpeg.c + video_outbuf = new BytePointer(av_malloc(video_outbuf_size)); + } + + /* allocate the encoded raw picture */ + if ((picture = av_frame_alloc()) == null) { + release(); + throw new Exception("av_frame_alloc() error: Could not allocate picture."); + } + picture.pts(0); // magic required by libx264 + + int size = av_image_get_buffer_size(video_c.pix_fmt(), video_c.width(), video_c.height(), 1); + if ((picture_buf = new BytePointer(av_malloc(size))).isNull()) { + release(); + throw new Exception("av_malloc() error: Could not allocate picture buffer."); + } + + /* + * if the output format is not equal to the image format, then a temporary picture is needed too. It is then converted to the required output format + */ + if ((tmp_picture = av_frame_alloc()) == null) { + release(); + throw new Exception("av_frame_alloc() error: Could not allocate temporary picture."); + } + + /* copy the stream parameters to the muxer */ + if ((ret = avcodec_parameters_from_context(video_st.codecpar(), video_c)) < 0) { + release(); + throw new Exception(ret, "avcodec_parameters_from_context() error: Could not copy the video stream parameters."); + } + + AVDictionary metadata = new AVDictionary(null); + for (Entry e : videoMetadata.entrySet()) { + av_dict_set(metadata, e.getKey(), e.getValue(), 0); + } + video_st.metadata(metadata); + } + + if (audio_st != null && inpAudioStream == null) { + AVDictionary options = new AVDictionary(null); + if (audioQuality >= 0) { + av_dict_set(options, "crf", "" + audioQuality, 0); + } + for (Entry e : audioOptions.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + /* open the codec */ + if ((ret = avcodec_open2(audio_c, audio_codec, options)) < 0) { + release(); + av_dict_free(options); + throw new Exception(ret, "avcodec_open2() error " + ret + ": Could not open audio codec."); + } + av_dict_free(options); + + audio_outbuf_size = 256 * 1024; + audio_outbuf = new BytePointer(av_malloc(audio_outbuf_size)); + + /* + * ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples + */ + if (audio_c.frame_size() <= 1) { + audio_outbuf_size = AV_INPUT_BUFFER_MIN_SIZE; + audio_input_frame_size = audio_outbuf_size / audio_c.channels(); + switch (audio_c.codec_id()) { + case AV_CODEC_ID_PCM_S16LE: + case AV_CODEC_ID_PCM_S16BE: + case AV_CODEC_ID_PCM_U16LE: + case AV_CODEC_ID_PCM_U16BE: + audio_input_frame_size >>= 1; + break; + default: + break; + } + } else { + audio_input_frame_size = audio_c.frame_size(); + } + // int bufferSize = audio_input_frame_size * audio_c.bits_per_raw_sample()/8 * audio_c.channels(); + int planes = av_sample_fmt_is_planar(audio_c.sample_fmt()) != 0 ? (int) audio_c.channels() : 1; + int data_size = av_samples_get_buffer_size((IntPointer) null, audio_c.channels(), audio_input_frame_size, audio_c.sample_fmt(), 1) / planes; + samples_out = new BytePointer[planes]; + for (int i = 0; i < samples_out.length; i++) { + samples_out[i] = new BytePointer(av_malloc(data_size)).capacity(data_size); + } + samples_in = new Pointer[AVFrame.AV_NUM_DATA_POINTERS]; + samples_in_ptr = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS); + samples_out_ptr = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS); + + /* allocate the audio frame */ + if ((frame = av_frame_alloc()) == null) { + release(); + throw new Exception("av_frame_alloc() error: Could not allocate audio frame."); + } + frame.pts(0); // magic required by libvorbis and webm + + /* copy the stream parameters to the muxer */ + if ((ret = avcodec_parameters_from_context(audio_st.codecpar(), audio_c)) < 0) { + release(); + throw new Exception(ret, "avcodec_parameters_from_context() error: Could not copy the audio stream parameters."); + } + + AVDictionary metadata = new AVDictionary(null); + for (Entry e : audioMetadata.entrySet()) { + av_dict_set(metadata, e.getKey(), e.getValue(), 0); + } + audio_st.metadata(metadata); + } + + AVDictionary options = new AVDictionary(null); + for (Entry e : this.options.entrySet()) { + av_dict_set(options, e.getKey(), e.getValue(), 0); + } + + /* open the output file, if needed */ + if (outputStream == null && (oformat.flags() & AVFMT_NOFILE) == 0) { + AVIOContext pb = new AVIOContext(null); + if ((ret = avio_open2(pb, filename, AVIO_FLAG_WRITE, null, options)) < 0) { + release(); + av_dict_free(options); + throw new Exception(ret, "avio_open2 error() error " + ret + ": Could not open '" + filename + "'"); + } + oc.pb(pb); + } + + AVDictionary metadata = new AVDictionary(null); + for (Entry e : this.metadata.entrySet()) { + av_dict_set(metadata, e.getKey(), e.getValue(), 0); + } + /* write the stream header, if any */ + avformat_write_header(oc.metadata(metadata), options); + av_dict_free(options); + + if (av_log_get_level() >= AV_LOG_INFO) { + av_dump_format(oc, 0, filename, 1); + } + } + + public void stop() throws Exception { + if (oc != null) { + try { + synchronized (oc) { + /* flush all the buffers */ + while (video_st != null && ifmt_ctx == null && recordImage(0, 0, 0, 0, 0, AV_PIX_FMT_NONE, (Buffer[]) null)) + ; + while (audio_st != null && ifmt_ctx == null && recordSamples(0, 0, (Buffer[]) null)) + ; + + if (interleaved && video_st != null && audio_st != null) { + av_interleaved_write_frame(oc, null); + } else { + av_write_frame(oc, null); + } + + /* write the trailer, if any */ + av_write_trailer(oc); + } + } finally { + release(); + } + } + } + + @Override + public void record(Frame frame) throws Exception { + record(frame, AV_PIX_FMT_NONE); + } + + public void record(Frame frame, int pixelFormat) throws Exception { + if (frame == null || (frame.image == null && frame.samples == null)) { + recordImage(0, 0, 0, 0, 0, pixelFormat, (Buffer[]) null); + } else { + if (frame.image != null) { + frame.keyFrame = recordImage(frame.imageWidth, frame.imageHeight, frame.imageDepth, frame.imageChannels, frame.imageStride, pixelFormat, frame.image); + } + if (frame.samples != null) { + frame.keyFrame = recordSamples(frame.sampleRate, frame.audioChannels, frame.samples); + } + } + } + + public boolean recordImage(int width, int height, int depth, int channels, int stride, int pixelFormat, Buffer... image) throws Exception { + if (video_st == null) { + throw new Exception("No video output stream (Is imageWidth > 0 && imageHeight > 0 and has start() been called?)"); + } + int ret; + + if (image == null || image.length == 0) { + /* + * no more frame to compress. The codec has a latency of a few frames if using B frames, so we get the last frames by passing the same picture again + */ + } else { + int step = stride * Math.abs(depth) / 8; + BytePointer data = image[0] instanceof ByteBuffer ? new BytePointer((ByteBuffer) image[0].position(0)) : new BytePointer(new Pointer(image[0].position(0))); + + if (pixelFormat == AV_PIX_FMT_NONE) { + if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 3) { + pixelFormat = AV_PIX_FMT_BGR24; + } else if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 1) { + pixelFormat = AV_PIX_FMT_GRAY8; + } else if ((depth == Frame.DEPTH_USHORT || depth == Frame.DEPTH_SHORT) && channels == 1) { + pixelFormat = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN) ? AV_PIX_FMT_GRAY16BE : AV_PIX_FMT_GRAY16LE; + } else if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 4) { + pixelFormat = AV_PIX_FMT_RGBA; + } else if ((depth == Frame.DEPTH_UBYTE || depth == Frame.DEPTH_BYTE) && channels == 2) { + pixelFormat = AV_PIX_FMT_NV21; // Android's camera capture format + } else { + throw new Exception("Could not guess pixel format of image: depth=" + depth + ", channels=" + channels); + } + } + + if (pixelFormat == AV_PIX_FMT_NV21) { + step = width; + } + + if (video_c.pix_fmt() != pixelFormat || video_c.width() != width || video_c.height() != height) { + /* convert to the codec pixel format if needed */ + img_convert_ctx = sws_getCachedContext(img_convert_ctx, width, height, pixelFormat, video_c.width(), video_c.height(), video_c.pix_fmt(), SWS_BILINEAR, null, null, (DoublePointer) null); + if (img_convert_ctx == null) { + throw new Exception("sws_getCachedContext() error: Cannot initialize the conversion context."); + } + av_image_fill_arrays(new PointerPointer(tmp_picture), tmp_picture.linesize(), data, pixelFormat, width, height, 1); + av_image_fill_arrays(new PointerPointer(picture), picture.linesize(), picture_buf, video_c.pix_fmt(), video_c.width(), video_c.height(), 1); + tmp_picture.linesize(0, step); + tmp_picture.format(pixelFormat); + tmp_picture.width(width); + tmp_picture.height(height); + picture.format(video_c.pix_fmt()); + picture.width(video_c.width()); + picture.height(video_c.height()); + sws_scale(img_convert_ctx, new PointerPointer(tmp_picture), tmp_picture.linesize(), 0, height, new PointerPointer(picture), picture.linesize()); + } else { + av_image_fill_arrays(new PointerPointer(picture), picture.linesize(), data, pixelFormat, width, height, 1); + picture.linesize(0, step); + picture.format(pixelFormat); + picture.width(width); + picture.height(height); + } + } + + if ((oformat.flags() & AVFMT_RAWPICTURE) != 0) { + if (image == null || image.length == 0) { + return false; + } + /* raw video case. The API may change slightly in the future for that? */ + av_init_packet(video_pkt); + video_pkt.flags(video_pkt.flags() | AV_PKT_FLAG_KEY); + video_pkt.stream_index(video_st.index()); + video_pkt.data(new BytePointer(picture)); + video_pkt.size(Loader.sizeof(AVFrame.class)); + } else { + /* encode the image */ + av_init_packet(video_pkt); + video_pkt.data(video_outbuf); + video_pkt.size(video_outbuf_size); + picture.quality(video_c.global_quality()); + if ((ret = avcodec_encode_video2(video_c, video_pkt, image == null || image.length == 0 ? null : picture, got_video_packet)) < 0) { + throw new Exception(ret, "avcodec_encode_video2() error " + ret + ": Could not encode video packet."); + } + picture.pts(picture.pts() + 1); // magic required by libx264 + + /* if zero size, it means the image was buffered */ + if (got_video_packet[0] != 0) { + if (video_pkt.pts() != AV_NOPTS_VALUE) { + video_pkt.pts(av_rescale_q(video_pkt.pts(), video_c.time_base(), video_st.time_base())); + } + if (video_pkt.dts() != AV_NOPTS_VALUE) { + video_pkt.dts(av_rescale_q(video_pkt.dts(), video_c.time_base(), video_st.time_base())); + } + video_pkt.stream_index(video_st.index()); + } else { + return false; + } + } + + writePacket(AVMEDIA_TYPE_VIDEO, video_pkt); + return image != null ? (video_pkt.flags() & AV_PKT_FLAG_KEY) != 0 : got_video_packet[0] != 0; + } + + public boolean recordSamples(Buffer... samples) throws Exception { + return recordSamples(0, 0, samples); + } + + public boolean recordSamples(int sampleRate, int audioChannels, Buffer... samples) throws Exception { + if (audio_st == null) { + throw new Exception("No audio output stream (Is audioChannels > 0 and has start() been called?)"); + } + int ret; + + if (sampleRate <= 0) { + sampleRate = audio_c.sample_rate(); + } + if (audioChannels <= 0) { + audioChannels = audio_c.channels(); + } + int inputSize = samples != null ? samples[0].limit() - samples[0].position() : 0; + int inputFormat = samples_format; + int inputChannels = samples != null && samples.length > 1 ? 1 : audioChannels; + int inputDepth = 0; + int outputFormat = audio_c.sample_fmt(); + int outputChannels = samples_out.length > 1 ? 1 : audio_c.channels(); + int outputDepth = av_get_bytes_per_sample(outputFormat); + if (samples != null && samples[0] instanceof ByteBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_U8P : AV_SAMPLE_FMT_U8; + inputDepth = 1; + for (int i = 0; i < samples.length; i++) { + ByteBuffer b = (ByteBuffer) samples[i]; + if (samples_in[i] instanceof BytePointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((BytePointer) samples_in[i]).position(0).put(b.array(), b.position(), inputSize); + } else { + samples_in[i] = new BytePointer(b); + } + } + } else if (samples != null && samples[0] instanceof ShortBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_S16P : AV_SAMPLE_FMT_S16; + inputDepth = 2; + for (int i = 0; i < samples.length; i++) { + ShortBuffer b = (ShortBuffer) samples[i]; + if (samples_in[i] instanceof ShortPointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((ShortPointer) samples_in[i]).position(0).put(b.array(), samples[i].position(), inputSize); + } else { + samples_in[i] = new ShortPointer(b); + } + } + } else if (samples != null && samples[0] instanceof IntBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_S32P : AV_SAMPLE_FMT_S32; + inputDepth = 4; + for (int i = 0; i < samples.length; i++) { + IntBuffer b = (IntBuffer) samples[i]; + if (samples_in[i] instanceof IntPointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((IntPointer) samples_in[i]).position(0).put(b.array(), samples[i].position(), inputSize); + } else { + samples_in[i] = new IntPointer(b); + } + } + } else if (samples != null && samples[0] instanceof FloatBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_FLTP : AV_SAMPLE_FMT_FLT; + inputDepth = 4; + for (int i = 0; i < samples.length; i++) { + FloatBuffer b = (FloatBuffer) samples[i]; + if (samples_in[i] instanceof FloatPointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((FloatPointer) samples_in[i]).position(0).put(b.array(), b.position(), inputSize); + } else { + samples_in[i] = new FloatPointer(b); + } + } + } else if (samples != null && samples[0] instanceof DoubleBuffer) { + inputFormat = samples.length > 1 ? AV_SAMPLE_FMT_DBLP : AV_SAMPLE_FMT_DBL; + inputDepth = 8; + for (int i = 0; i < samples.length; i++) { + DoubleBuffer b = (DoubleBuffer) samples[i]; + if (samples_in[i] instanceof DoublePointer && samples_in[i].capacity() >= inputSize && b.hasArray()) { + ((DoublePointer) samples_in[i]).position(0).put(b.array(), b.position(), inputSize); + } else { + samples_in[i] = new DoublePointer(b); + } + } + } else if (samples != null) { + throw new Exception("Audio samples Buffer has unsupported type: " + samples); + } + + if (samples_convert_ctx == null || samples_channels != audioChannels || samples_format != inputFormat || samples_rate != sampleRate) { + samples_convert_ctx = swr_alloc_set_opts(samples_convert_ctx, audio_c.channel_layout(), outputFormat, audio_c.sample_rate(), av_get_default_channel_layout(audioChannels), inputFormat, sampleRate, 0, null); + if (samples_convert_ctx == null) { + throw new Exception("swr_alloc_set_opts() error: Cannot allocate the conversion context."); + } else if ((ret = swr_init(samples_convert_ctx)) < 0) { + throw new Exception("swr_init() error " + ret + ": Cannot initialize the conversion context."); + } + samples_channels = audioChannels; + samples_format = inputFormat; + samples_rate = sampleRate; + } + + for (int i = 0; samples != null && i < samples.length; i++) { + samples_in[i].position(samples_in[i].position() * inputDepth).limit((samples_in[i].position() + inputSize) * inputDepth); + } + while (true) { + int inputCount = (int) Math.min(samples != null ? (samples_in[0].limit() - samples_in[0].position()) / (inputChannels * inputDepth) : 0, Integer.MAX_VALUE); + int outputCount = (int) Math.min((samples_out[0].limit() - samples_out[0].position()) / (outputChannels * outputDepth), Integer.MAX_VALUE); + inputCount = Math.min(inputCount, (outputCount * sampleRate + audio_c.sample_rate() - 1) / audio_c.sample_rate()); + for (int i = 0; samples != null && i < samples.length; i++) { + samples_in_ptr.put(i, samples_in[i]); + } + for (int i = 0; i < samples_out.length; i++) { + samples_out_ptr.put(i, samples_out[i]); + } + if ((ret = swr_convert(samples_convert_ctx, samples_out_ptr, outputCount, samples_in_ptr, inputCount)) < 0) { + throw new Exception(ret, "swr_convert() error " + ret + ": Cannot convert audio samples."); + } else if (ret == 0) { + break; + } + for (int i = 0; samples != null && i < samples.length; i++) { + samples_in[i].position(samples_in[i].position() + inputCount * inputChannels * inputDepth); + } + for (int i = 0; i < samples_out.length; i++) { + samples_out[i].position(samples_out[i].position() + ret * outputChannels * outputDepth); + } + + if (samples == null || samples_out[0].position() >= samples_out[0].limit()) { + frame.nb_samples(audio_input_frame_size); + avcodec_fill_audio_frame(frame, audio_c.channels(), outputFormat, samples_out[0], (int) Math.min(samples_out[0].limit(), Integer.MAX_VALUE), 0); + for (int i = 0; i < samples_out.length; i++) { + frame.data(i, samples_out[i].position(0)); + frame.linesize(i, (int) Math.min(samples_out[i].limit(), Integer.MAX_VALUE)); + } + frame.quality(audio_c.global_quality()); + record(frame); + } + } + return samples != null ? frame.key_frame() != 0 : record((AVFrame) null); + } + + boolean record(AVFrame frame) throws Exception { + int ret; + + av_init_packet(audio_pkt); + audio_pkt.data(audio_outbuf); + audio_pkt.size(audio_outbuf_size); + if ((ret = avcodec_encode_audio2(audio_c, audio_pkt, frame, got_audio_packet)) < 0) { + throw new Exception(ret, "avcodec_encode_audio2() error " + ret + ": Could not encode audio packet."); + } + if (frame != null) { + frame.pts(frame.pts() + frame.nb_samples()); // magic required by libvorbis and webm + } + if (got_audio_packet[0] != 0) { + if (audio_pkt.pts() != AV_NOPTS_VALUE) { + audio_pkt.pts(av_rescale_q(audio_pkt.pts(), audio_c.time_base(), audio_st.time_base())); + } + if (audio_pkt.dts() != AV_NOPTS_VALUE) { + audio_pkt.dts(av_rescale_q(audio_pkt.dts(), audio_c.time_base(), audio_st.time_base())); + } + audio_pkt.flags(audio_pkt.flags() | AV_PKT_FLAG_KEY); + audio_pkt.stream_index(audio_st.index()); + } else { + return false; + } + + /* write the compressed frame in the media file */ + writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); + + return true; + } + + private void writePacket(int mediaType, AVPacket avPacket) throws Exception { + AVStream avStream = null; + if (mediaType == AVMEDIA_TYPE_VIDEO) + avStream = audio_st; + else if (mediaType == AVMEDIA_TYPE_AUDIO) + avStream = video_st; + String mediaTypeStr = (mediaType == AVMEDIA_TYPE_VIDEO) ? "video" : (mediaType == AVMEDIA_TYPE_AUDIO) ? "audio" : "unsupported media stream type"; + + synchronized (oc) { + int ret; + if (interleaved && avStream != null) { + if ((ret = av_interleaved_write_frame(oc, avPacket)) < 0) { + throw new Exception(ret, "av_interleaved_write_frame() error " + ret + " while writing interleaved " + mediaTypeStr + " packet."); + } + } else { + if ((ret = av_write_frame(oc, avPacket)) < 0) { + throw new Exception(ret, "av_write_frame() error " + ret + " while writing " + mediaTypeStr + " packet."); + } + } + } + } + + public boolean recordPacket(AVPacket pkt) throws Exception { + if (pkt == null) { + return false; + } + + AVStream in_stream = ifmt_ctx.streams(pkt.stream_index()); + // pkt.pts(AV_NOPTS_VALUE); + // pkt.dts(AV_NOPTS_VALUE); + pkt.pos(-1); + try { + if (in_stream.codecpar().codec_type() == AVMEDIA_TYPE_VIDEO && video_st != null) { + pkt.stream_index(video_st.index()); + pkt.pts(av_rescale_q_rnd(pkt.pts(), in_stream.time_base(), video_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); + pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), video_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); + pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), video_st.time_base())); + if (video_st.cur_dts() >= pkt.dts()) + pkt.dts(video_st.cur_dts() + pkt.duration()); + if (pkt.pts() < pkt.dts()) + pkt.pts(pkt.dts()); + writePacket(AVMEDIA_TYPE_VIDEO, pkt); + } else if (in_stream.codecpar().codec_type() == AVMEDIA_TYPE_AUDIO && audio_st != null && (audioChannels > 0)) { + pkt.stream_index(audio_st.index()); + pkt.pts(av_rescale_q_rnd(pkt.pts(), in_stream.time_base(), audio_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); + pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), audio_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); + pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), audio_st.time_base())); + if (audio_st.cur_dts() >= pkt.dts()) + pkt.dts(audio_st.cur_dts() + pkt.duration()); + if (pkt.pts() < pkt.dts()) + pkt.pts(pkt.dts()); + writePacket(AVMEDIA_TYPE_AUDIO, pkt); + } + } finally { + av_packet_unref(pkt); + } + return true; + } + + public OutputStream getOutputStream() { + return this.outputStream; + } + + public void setInputAVFormatContext(AVFormatContext ctx) { + this.ifmt_ctx = ctx; + } } diff --git a/src/main/java/org/bytedeco/javacv/FrameGrabber.java b/src/main/java/org/bytedeco/javacv/FrameGrabber.java index f21509ef..f23ff0bd 100644 --- a/src/main/java/org/bytedeco/javacv/FrameGrabber.java +++ b/src/main/java/org/bytedeco/javacv/FrameGrabber.java @@ -44,663 +44,761 @@ */ public abstract class FrameGrabber implements Closeable { - public static final List list = new LinkedList(Arrays.asList(new String[] { - "DC1394", "FlyCapture", "FlyCapture2", "OpenKinect", "OpenKinect2", "RealSense", "PS3Eye", "VideoInput", "OpenCV", "FFmpeg", "IPCamera" })); - public static void init() { - for (String name : list) { - try { - Class c = get(name); - c.getMethod("tryLoad").invoke(null); - } catch (Throwable t) { - continue; - } - } - } - public static Class getDefault() { - // select first frame grabber that can load and that may have some cameras.. - for (String name : list) { - try { - Class c = get(name); - c.getMethod("tryLoad").invoke(null); - boolean mayContainCameras = false; - try { - String[] s = (String[])c.getMethod("getDeviceDescriptions").invoke(null); - if (s.length > 0) { - mayContainCameras = true; - } - } catch (Throwable t) { - if (t.getCause() instanceof UnsupportedOperationException) { - mayContainCameras = true; - } - } - if (mayContainCameras) { - return c; - } - } catch (Throwable t) { - continue; - } - } - return null; - } - public static Class get(String className) throws Exception { - className = FrameGrabber.class.getPackage().getName() + "." + className; - try { - return Class.forName(className).asSubclass(FrameGrabber.class); - } catch (ClassNotFoundException e) { - String className2 = className + "FrameGrabber"; - try { - return Class.forName(className2).asSubclass(FrameGrabber.class); - } catch (ClassNotFoundException ex) { - throw new Exception("Could not get FrameGrabber class for " + className + " or " + className2, e); - } - } - } - - public static FrameGrabber create(Class c, Class p, Object o) throws Exception { - Throwable cause = null; - try { - return c.getConstructor(p).newInstance(o); - } catch (InstantiationException ex) { - cause = ex; - } catch (IllegalAccessException ex) { - cause = ex; - } catch (IllegalArgumentException ex) { - cause = ex; - } catch (NoSuchMethodException ex) { - cause = ex; - } catch (InvocationTargetException ex) { - cause = ex.getCause(); - } - throw new Exception("Could not create new " + c.getSimpleName() + "(" + o + ")", cause); - } - - public static FrameGrabber createDefault(File deviceFile) throws Exception { - return create(getDefault(), File.class, deviceFile); - } - public static FrameGrabber createDefault(String devicePath) throws Exception { - return create(getDefault(), String.class, devicePath); - } - public static FrameGrabber createDefault(int deviceNumber) throws Exception { - try { - return create(getDefault(), int.class, deviceNumber); - } catch (Exception ex) { - return create(getDefault(), Integer.class, deviceNumber); - } - } - - public static FrameGrabber create(String className, File deviceFile) throws Exception { - return create(get(className), File.class, deviceFile); - } - public static FrameGrabber create(String className, String devicePath) throws Exception { - return create(get(className), String.class, devicePath); - } - public static FrameGrabber create(String className, int deviceNumber) throws Exception { - try { - return create(get(className), int.class, deviceNumber); - } catch (Exception ex) { - return create(get(className), Integer.class, deviceNumber); - } - } - - public static class PropertyEditor extends PropertyEditorSupport { - @Override public String getAsText() { - Class c = (Class)getValue(); - return c == null ? "null" : c.getSimpleName().split("FrameGrabber")[0]; - } - @Override public void setAsText(String s) { - if (s == null) { - setValue(null); - } - try { - setValue(get(s)); - } catch (Exception ex) { - throw new IllegalArgumentException(ex); - } - } - @Override public String[] getTags() { - return list.toArray(new String[list.size()]); - } - } - - - public static enum ImageMode { - COLOR, GRAY, RAW - } - - public static enum SampleMode { - SHORT, FLOAT, RAW - } - - public static final long - SENSOR_PATTERN_RGGB = 0, - SENSOR_PATTERN_GBRG = (1L << 32), - SENSOR_PATTERN_GRBG = 1, - SENSOR_PATTERN_BGGR = (1L << 32) | 1; - - protected int videoStream = -1, audioStream = -1; - protected String format = null; - protected int imageWidth = 0, imageHeight = 0, audioChannels = 0; - protected ImageMode imageMode = ImageMode.COLOR; - protected long sensorPattern = -1L; - protected int pixelFormat = -1, videoCodec, videoBitrate = 0; - protected double aspectRatio = 0, frameRate = 0; - protected SampleMode sampleMode = SampleMode.SHORT; - protected int sampleFormat = -1, audioCodec, audioBitrate = 0, sampleRate = 0; - protected boolean triggerMode = false; - protected int bpp = 0; - protected int timeout = 10000; - protected int numBuffers = 4; - protected double gamma = 0.0; - protected boolean deinterlace = false; - protected Map options = new HashMap(); - protected Map videoOptions = new HashMap(); - protected Map audioOptions = new HashMap(); - protected Map metadata = new HashMap(); - protected Map videoMetadata = new HashMap(); - protected Map audioMetadata = new HashMap(); - protected int frameNumber = 0; - protected long timestamp = 0; - protected int maxDelay = -1; - - public int getVideoStream() { - return videoStream; - } - public void setVideoStream(int videoStream) { - this.videoStream = videoStream; - } - - public int getAudioStream() { - return audioStream; - } - public void setAudioStream(int audioStream) { - this.audioStream = audioStream; - } - - public String getFormat() { - return format; - } - public void setFormat(String format) { - this.format = format; - } - - public int getImageWidth() { - return imageWidth; - } - public void setImageWidth(int imageWidth) { - this.imageWidth = imageWidth; - } - - public int getImageHeight() { - return imageHeight; - } - public void setImageHeight(int imageHeight) { - this.imageHeight = imageHeight; - } - - public int getAudioChannels() { - return audioChannels; - } - public void setAudioChannels(int audioChannels) { - this.audioChannels = audioChannels; - } - - public ImageMode getImageMode() { - return imageMode; - } - public void setImageMode(ImageMode imageMode) { - this.imageMode = imageMode; - } - - public long getSensorPattern() { - return sensorPattern; - } - public void setSensorPattern(long sensorPattern) { - this.sensorPattern = sensorPattern; - } - - public int getPixelFormat() { - return pixelFormat; - } - public void setPixelFormat(int pixelFormat) { - this.pixelFormat = pixelFormat; - } - - public int getVideoCodec() { - return videoCodec; - } - public void setVideoCodec(int videoCodec) { - this.videoCodec = videoCodec; - } - - public int getVideoBitrate() { - return videoBitrate; - } - public void setVideoBitrate(int videoBitrate) { - this.videoBitrate = videoBitrate; - } - - public double getAspectRatio() { - return aspectRatio; - } - public void setAspectRatio(double aspectRatio) { - this.aspectRatio = aspectRatio; - } - - public double getFrameRate() { - return frameRate; - } - public void setFrameRate(double frameRate) { - this.frameRate = frameRate; - } - - public int getAudioCodec() { - return audioCodec; - } - public void setAudioCodec(int audioCodec) { - this.audioCodec = audioCodec; - } - - public int getAudioBitrate() { - return audioBitrate; - } - public void setAudioBitrate(int audioBitrate) { - this.audioBitrate = audioBitrate; - } - - public SampleMode getSampleMode() { - return sampleMode; - } - public void setSampleMode(SampleMode samplesMode) { - this.sampleMode = samplesMode; - } - - public int getSampleFormat() { - return sampleFormat; - } - public void setSampleFormat(int sampleFormat) { - this.sampleFormat = sampleFormat; - } - - public int getSampleRate() { - return sampleRate; - } - public void setSampleRate(int sampleRate) { - this.sampleRate = sampleRate; - } - - public boolean isTriggerMode() { - return triggerMode; - } - public void setTriggerMode(boolean triggerMode) { - this.triggerMode = triggerMode; - } - - public int getBitsPerPixel() { - return bpp; - } - public void setBitsPerPixel(int bitsPerPixel) { - this.bpp = bitsPerPixel; - } - - public int getTimeout() { - return timeout; - } - public void setTimeout(int timeout) { - this.timeout = timeout; - } - - public int getNumBuffers() { - return numBuffers; - } - public void setNumBuffers(int numBuffers) { - this.numBuffers = numBuffers; - } - - public double getGamma() { - return gamma; - } - public void setGamma(double gamma) { - this.gamma = gamma; - } - - public boolean isDeinterlace() { - return deinterlace; - } - public void setDeinterlace(boolean deinterlace) { - this.deinterlace = deinterlace; - } - - public Map getOptions() { - return options; - } - public void setOptions(Map options) { - this.options = options; - } - - public Map getVideoOptions() { - return videoOptions; - } - public void setVideoOptions(Map options) { - this.videoOptions = options; - } - - public Map getAudioOptions() { - return audioOptions; - } - public void setAudioOptions(Map options) { - this.audioOptions = options; - } - - public Map getMetadata() { - return metadata; - } - public void setMetadata(Map metadata) { - this.metadata = metadata; - } - - public Map getVideoMetadata() { - return videoMetadata; - } - public void setVideoMetadata(Map metadata) { - this.videoMetadata = metadata; - } - - public Map getAudioMetadata() { - return audioMetadata; - } - public void setAudioMetadata(Map metadata) { - this.audioMetadata = metadata; - } - - public String getOption(String key) { - return options.get(key); - } - public void setOption(String key, String value) { - options.put(key, value); - } - - public String getVideoOption(String key) { - return videoOptions.get(key); - } - public void setVideoOption(String key, String value) { - videoOptions.put(key, value); - } - - public String getAudioOption(String key) { - return audioOptions.get(key); - } - public void setAudioOption(String key, String value) { - audioOptions.put(key, value); - } - - public String getMetadata(String key) { - return metadata.get(key); - } - public void setMetadata(String key, String value) { - metadata.put(key, value); - } - - public String getVideoMetadata(String key) { - return videoMetadata.get(key); - } - public void setVideoMetadata(String key, String value) { - videoMetadata.put(key, value); - } - - public String getAudioMetadata(String key) { - return audioMetadata.get(key); - } - public void setAudioMetadata(String key, String value) { - audioMetadata.put(key, value); - } - - public int getFrameNumber() { - return frameNumber; - } - public void setFrameNumber(int frameNumber) throws Exception { - this.frameNumber = frameNumber; - } - - public long getTimestamp() { - return timestamp; - } - public void setTimestamp(long timestamp) throws Exception { - this.timestamp = timestamp; - } - - public int getMaxDelay() { - return maxDelay; - } - public void setMaxDelay(int maxDelay) { - this.maxDelay = maxDelay; - } - - public int getLengthInFrames() { - return 0; - } - public long getLengthInTime() { - return 0; - } - - public static class Exception extends IOException { - public Exception(String message) { super(message); } - public Exception(String message, Throwable cause) { super(message, cause); } - } - - public abstract void start() throws Exception; - public abstract void stop() throws Exception; - public abstract void trigger() throws Exception; - - @Override public void close() throws Exception { - stop(); - release(); - } - - /** - * Each call to grab stores the new image in the memory address for the previously returned frame.
- * IE.
- * - * grabber.grab() == grabber.grab() - * - *
- * This means that if you need to cache images returned from grab you should {@link Frame#clone()} the - * returned frame as the next call to grab will overwrite your existing image's memory. - *
- * Why?
- * Using this method instead of allocating a new buffer every time a frame - * is grabbed improves performance by reducing the frequency of garbage collections. - * Almost no additional heap space is typically allocated per frame. - * - * @return The frame returned from the grabber - * @throws Exception If there is a problem grabbing the frame. - */ - public abstract Frame grab() throws Exception; - public Frame grabFrame() throws Exception { return grab(); } - public abstract void release() throws Exception; - - public void restart() throws Exception { - stop(); - start(); - } - public void flush() throws Exception { - for (int i = 0; i < numBuffers+1; i++) { - grab(); - } - } - - private ExecutorService executor = Executors.newSingleThreadExecutor(); - private Future future = null; - private Frame delayedFrame = null; - private long delayedTime = 0; - public void delayedGrab(final long delayTime) { - delayedFrame = null; - delayedTime = 0; - final long start = System.nanoTime()/1000; - if (future != null && !future.isDone()) { - return; - } - future = executor.submit(new Callable() { public Void call() throws Exception { - do { - delayedFrame = grab(); - delayedTime = System.nanoTime()/1000 - start; - } while (delayedTime < delayTime); - return null; - }}); - } - public long getDelayedTime() throws InterruptedException, ExecutionException { - if (future == null) { - return 0; - } - future.get(); - return delayedTime; - } - public Frame getDelayedFrame() throws InterruptedException, ExecutionException { - if (future == null) { - return null; - } - future.get(); - return delayedFrame; - } - - public static class Array { - // declared protected to force users to use createArray(), which - // can be overridden without changing the calling code... - protected Array(FrameGrabber[] frameGrabbers) { - setFrameGrabbers(frameGrabbers); - } - - private Frame[] grabbedFrames = null; - private long[] latencies = null; - private long[] bestLatencies = null; - private long lastNewestTimestamp = 0; - private long bestInterval = Long.MAX_VALUE; - - protected FrameGrabber[] frameGrabbers = null; - public FrameGrabber[] getFrameGrabbers() { - return frameGrabbers; - } - public void setFrameGrabbers(FrameGrabber[] frameGrabbers) { - this.frameGrabbers = frameGrabbers; - grabbedFrames = new Frame[frameGrabbers.length]; - latencies = new long[frameGrabbers.length]; - bestLatencies = null; - lastNewestTimestamp = 0; - } - public int size() { - return frameGrabbers.length; - } - - public void start() throws Exception { - for (FrameGrabber f : frameGrabbers) { - f.start(); - } - } - public void stop() throws Exception { - for (FrameGrabber f : frameGrabbers) { - f.stop(); - } - } - // should be overriden to implement a broadcast trigger... - public void trigger() throws Exception { - for (FrameGrabber f : frameGrabbers) { - if (f.isTriggerMode()) { - f.trigger(); - } - } - } - // should be overriden to implement a broadcast grab... - public Frame[] grab() throws Exception { - if (frameGrabbers.length == 1) { - grabbedFrames[0] = frameGrabbers[0].grab(); - return grabbedFrames; - } - - // assume we sometimes get perfectly synchronized images, - // so save the best latencies we find as the perfectly - // synchronized case, so we know what to aim for in - // cases of missing/dropped frames ... - long newestTimestamp = 0; - boolean unsynchronized = false; - for (int i = 0; i < frameGrabbers.length; i++) { - grabbedFrames[i] = frameGrabbers[i].grab(); - if (grabbedFrames[i] != null) { - newestTimestamp = Math.max(newestTimestamp, frameGrabbers[i].getTimestamp()); - } - if (frameGrabbers[i].getClass() != frameGrabbers[(i + 1) % frameGrabbers.length].getClass()) { - // assume we can't synchronize different types of cameras with each other - unsynchronized = true; - } - } - if (unsynchronized) { - return grabbedFrames; - } - for (int i = 0; i < frameGrabbers.length; i++) { - if (grabbedFrames[i] != null) { - latencies[i] = newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp()); - } - } - if (bestLatencies == null) { - bestLatencies = Arrays.copyOf(latencies, latencies.length); - } else { - int sum1 = 0, sum2 = 0; - for (int i = 0; i < frameGrabbers.length; i++) { - sum1 += latencies[i]; - sum2 += bestLatencies[i]; - } - if (sum1 < sum2) { - bestLatencies = Arrays.copyOf(latencies, latencies.length); - } - } - - // we cannot have latencies higher than the time between frames.. - // or something too close to it anyway... 90% is good? - bestInterval = Math.min(bestInterval, newestTimestamp-lastNewestTimestamp); - for (int i = 0; i < bestLatencies.length; i++) { - bestLatencies[i] = Math.min(bestLatencies[i], bestInterval*9/10); - } - - // try to synchronize by attempting to land within 10% of - // the bestLatencies looking up to 2 frames ahead ... - for (int j = 0; j < 2; j++) { - for (int i = 0; i < frameGrabbers.length; i++) { - if (frameGrabbers[i].isTriggerMode() || grabbedFrames[i] == null) { - continue; - } - int latency = (int)(newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp())); - while (latency-bestLatencies[i] > 0.1*bestLatencies[i]) { - grabbedFrames[i] = frameGrabbers[i].grab(); - if (grabbedFrames[i] == null) { - break; - } - latency = (int)(newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp())); - if (latency < 0) { - // woops, a camera seems to have dropped a frame somewhere... - // bump up the newestTimestamp - newestTimestamp = Math.max(0, frameGrabbers[i].getTimestamp()); - break; - } - } - } - } - -//for (int i = 0; i < frameGrabbers.length; i++) { -// long latency = newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp()); -// System.out.print(bestLatencies[i] + " " + latency + " "); -//} -//System.out.println(" " + bestInterval); - - lastNewestTimestamp = newestTimestamp; - - return grabbedFrames; - } - public void release() throws Exception { - for (FrameGrabber f : frameGrabbers) { - f.release(); - } - } - } - - public Array createArray(FrameGrabber[] frameGrabbers) { - return new Array(frameGrabbers); - } + public static final List list = new LinkedList(Arrays.asList(new String[] { "DC1394", "FlyCapture", "FlyCapture2", "OpenKinect", "OpenKinect2", "RealSense", "PS3Eye", "VideoInput", "OpenCV", "FFmpeg", "IPCamera" })); + + public static void init() { + for (String name : list) { + try { + Class c = get(name); + c.getMethod("tryLoad").invoke(null); + } catch (Throwable t) { + continue; + } + } + } + + public static Class getDefault() { + // select first frame grabber that can load and that may have some cameras.. + for (String name : list) { + try { + Class c = get(name); + c.getMethod("tryLoad").invoke(null); + boolean mayContainCameras = false; + try { + String[] s = (String[]) c.getMethod("getDeviceDescriptions").invoke(null); + if (s.length > 0) { + mayContainCameras = true; + } + } catch (Throwable t) { + if (t.getCause() instanceof UnsupportedOperationException) { + mayContainCameras = true; + } + } + if (mayContainCameras) { + return c; + } + } catch (Throwable t) { + continue; + } + } + return null; + } + + public static Class get(String className) throws Exception { + className = FrameGrabber.class.getPackage().getName() + "." + className; + try { + return Class.forName(className).asSubclass(FrameGrabber.class); + } catch (ClassNotFoundException e) { + String className2 = className + "FrameGrabber"; + try { + return Class.forName(className2).asSubclass(FrameGrabber.class); + } catch (ClassNotFoundException ex) { + throw new Exception("Could not get FrameGrabber class for " + className + " or " + className2, e); + } + } + } + + public static FrameGrabber create(Class c, Class p, Object o) throws Exception { + Throwable cause = null; + try { + return c.getConstructor(p).newInstance(o); + } catch (InstantiationException ex) { + cause = ex; + } catch (IllegalAccessException ex) { + cause = ex; + } catch (IllegalArgumentException ex) { + cause = ex; + } catch (NoSuchMethodException ex) { + cause = ex; + } catch (InvocationTargetException ex) { + cause = ex.getCause(); + } + throw new Exception("Could not create new " + c.getSimpleName() + "(" + o + ")", cause); + } + + public static FrameGrabber createDefault(File deviceFile) throws Exception { + return create(getDefault(), File.class, deviceFile); + } + + public static FrameGrabber createDefault(String devicePath) throws Exception { + return create(getDefault(), String.class, devicePath); + } + + public static FrameGrabber createDefault(int deviceNumber) throws Exception { + try { + return create(getDefault(), int.class, deviceNumber); + } catch (Exception ex) { + return create(getDefault(), Integer.class, deviceNumber); + } + } + + public static FrameGrabber create(String className, File deviceFile) throws Exception { + return create(get(className), File.class, deviceFile); + } + + public static FrameGrabber create(String className, String devicePath) throws Exception { + return create(get(className), String.class, devicePath); + } + + public static FrameGrabber create(String className, int deviceNumber) throws Exception { + try { + return create(get(className), int.class, deviceNumber); + } catch (Exception ex) { + return create(get(className), Integer.class, deviceNumber); + } + } + + public static class PropertyEditor extends PropertyEditorSupport { + @Override + public String getAsText() { + Class c = (Class) getValue(); + return c == null ? "null" : c.getSimpleName().split("FrameGrabber")[0]; + } + + @Override + public void setAsText(String s) { + if (s == null) { + setValue(null); + } + try { + setValue(get(s)); + } catch (Exception ex) { + throw new IllegalArgumentException(ex); + } + } + + @Override + public String[] getTags() { + return list.toArray(new String[list.size()]); + } + } + + public static enum ImageMode { + COLOR, GRAY, RAW + } + + public static enum SampleMode { + SHORT, FLOAT, RAW + } + + public static final long SENSOR_PATTERN_RGGB = 0, SENSOR_PATTERN_GBRG = (1L << 32), SENSOR_PATTERN_GRBG = 1, SENSOR_PATTERN_BGGR = (1L << 32) | 1; + + protected int videoStream = -1, audioStream = -1; + protected String format = null, videoCodecName = null, audioCodecName = null, hwaccelName = null;; + protected int imageWidth = 0, imageHeight = 0, audioChannels = 0; + protected ImageMode imageMode = ImageMode.COLOR; + protected long sensorPattern = -1L; + protected int pixelFormat = -1, videoCodec, videoBitrate = 0; + protected double aspectRatio = 0, frameRate = 0; + protected SampleMode sampleMode = SampleMode.SHORT; + protected int sampleFormat = -1, audioCodec, audioBitrate = 0, sampleRate = 0; + protected boolean triggerMode = false; + protected int bpp = 0; + protected int timeout = 10000; + protected int numBuffers = 4; + protected double gamma = 0.0; + protected boolean deinterlace = false; + protected Map options = new HashMap(); + protected Map videoOptions = new HashMap(); + protected Map audioOptions = new HashMap(); + protected Map metadata = new HashMap(); + protected Map videoMetadata = new HashMap(); + protected Map audioMetadata = new HashMap(); + protected int frameNumber = 0; + protected long timestamp = 0; + + public int getVideoStream() { + return videoStream; + } + + public void setVideoStream(int videoStream) { + this.videoStream = videoStream; + } + + public int getAudioStream() { + return audioStream; + } + + public void setAudioStream(int audioStream) { + this.audioStream = audioStream; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public String getVideoCodecName() { + return videoCodecName; + } + + public void setVideoCodecName(String videoCodecName) { + this.videoCodecName = videoCodecName; + } + + public String getAudioCodecName() { + return audioCodecName; + } + + public void setAudioCodecName(String audioCodecName) { + this.audioCodecName = audioCodecName; + } + + public String getHwaccelName() { + return hwaccelName; + } + + public void setHwaccelName(String hwaccelName) { + this.hwaccelName = hwaccelName; + } + + public int getImageWidth() { + return imageWidth; + } + + public void setImageWidth(int imageWidth) { + this.imageWidth = imageWidth; + } + + public int getImageHeight() { + return imageHeight; + } + + public void setImageHeight(int imageHeight) { + this.imageHeight = imageHeight; + } + + public int getAudioChannels() { + return audioChannels; + } + + public void setAudioChannels(int audioChannels) { + this.audioChannels = audioChannels; + } + + public ImageMode getImageMode() { + return imageMode; + } + + public void setImageMode(ImageMode imageMode) { + this.imageMode = imageMode; + } + + public long getSensorPattern() { + return sensorPattern; + } + + public void setSensorPattern(long sensorPattern) { + this.sensorPattern = sensorPattern; + } + + public int getPixelFormat() { + return pixelFormat; + } + + public void setPixelFormat(int pixelFormat) { + this.pixelFormat = pixelFormat; + } + + public int getVideoCodec() { + return videoCodec; + } + + public void setVideoCodec(int videoCodec) { + this.videoCodec = videoCodec; + } + + public int getVideoBitrate() { + return videoBitrate; + } + + public void setVideoBitrate(int videoBitrate) { + this.videoBitrate = videoBitrate; + } + + public double getAspectRatio() { + return aspectRatio; + } + + public void setAspectRatio(double aspectRatio) { + this.aspectRatio = aspectRatio; + } + + public double getFrameRate() { + return frameRate; + } + + public void setFrameRate(double frameRate) { + this.frameRate = frameRate; + } + + public int getAudioCodec() { + return audioCodec; + } + + public void setAudioCodec(int audioCodec) { + this.audioCodec = audioCodec; + } + + public int getAudioBitrate() { + return audioBitrate; + } + + public void setAudioBitrate(int audioBitrate) { + this.audioBitrate = audioBitrate; + } + + public SampleMode getSampleMode() { + return sampleMode; + } + + public void setSampleMode(SampleMode samplesMode) { + this.sampleMode = samplesMode; + } + + public int getSampleFormat() { + return sampleFormat; + } + + public void setSampleFormat(int sampleFormat) { + this.sampleFormat = sampleFormat; + } + + public int getSampleRate() { + return sampleRate; + } + + public void setSampleRate(int sampleRate) { + this.sampleRate = sampleRate; + } + + public boolean isTriggerMode() { + return triggerMode; + } + + public void setTriggerMode(boolean triggerMode) { + this.triggerMode = triggerMode; + } + + public int getBitsPerPixel() { + return bpp; + } + + public void setBitsPerPixel(int bitsPerPixel) { + this.bpp = bitsPerPixel; + } + + public int getTimeout() { + return timeout; + } + + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + public int getNumBuffers() { + return numBuffers; + } + + public void setNumBuffers(int numBuffers) { + this.numBuffers = numBuffers; + } + + public double getGamma() { + return gamma; + } + + public void setGamma(double gamma) { + this.gamma = gamma; + } + + public boolean isDeinterlace() { + return deinterlace; + } + + public void setDeinterlace(boolean deinterlace) { + this.deinterlace = deinterlace; + } + + public Map getOptions() { + return options; + } + + public void setOptions(Map options) { + this.options = options; + } + + public Map getVideoOptions() { + return videoOptions; + } + + public void setVideoOptions(Map options) { + this.videoOptions = options; + } + + public Map getAudioOptions() { + return audioOptions; + } + + public void setAudioOptions(Map options) { + this.audioOptions = options; + } + + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public Map getVideoMetadata() { + return videoMetadata; + } + + public void setVideoMetadata(Map metadata) { + this.videoMetadata = metadata; + } + + public Map getAudioMetadata() { + return audioMetadata; + } + + public void setAudioMetadata(Map metadata) { + this.audioMetadata = metadata; + } + + public String getOption(String key) { + return options.get(key); + } + + public void setOption(String key, String value) { + options.put(key, value); + } + + public String getVideoOption(String key) { + return videoOptions.get(key); + } + + public void setVideoOption(String key, String value) { + videoOptions.put(key, value); + } + + public String getAudioOption(String key) { + return audioOptions.get(key); + } + + public void setAudioOption(String key, String value) { + audioOptions.put(key, value); + } + + public String getMetadata(String key) { + return metadata.get(key); + } + + public void setMetadata(String key, String value) { + metadata.put(key, value); + } + + public String getVideoMetadata(String key) { + return videoMetadata.get(key); + } + + public void setVideoMetadata(String key, String value) { + videoMetadata.put(key, value); + } + + public String getAudioMetadata(String key) { + return audioMetadata.get(key); + } + + public void setAudioMetadata(String key, String value) { + audioMetadata.put(key, value); + } + + public int getFrameNumber() { + return frameNumber; + } + + public void setFrameNumber(int frameNumber) throws Exception { + this.frameNumber = frameNumber; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) throws Exception { + this.timestamp = timestamp; + } + + public int getLengthInFrames() { + return 0; + } + + public long getLengthInTime() { + return 0; + } + + public static class Exception extends IOException { + private int errorCode = -1; + + public Exception(String message) { + super(message); + } + + public Exception(int errorCode, String message) { + super(message); + this.errorCode = errorCode; + } + + public Exception(String message, Throwable cause) { + super(message, cause); + } + + public Exception(int errorCode, String message, Throwable cause) { + super(message, cause); + this.errorCode = errorCode; + } + + public int getErrorCode() { + return this.errorCode; + } + } + + public abstract void start() throws Exception; + + public abstract void stop() throws Exception; + + public abstract void trigger() throws Exception; + + @Override + public void close() throws Exception { + stop(); + release(); + } + + /** + * Each call to grab stores the new image in the memory address for the previously returned frame.
+ * IE.
+ * + * grabber.grab() == grabber.grab() + *
+ * This means that if you need to cache images returned from grab you should {@link Frame#clone()} the returned frame as the next call to grab will overwrite your existing image's memory.
+ * Why?
+ * Using this method instead of allocating a new buffer every time a frame is grabbed improves performance by reducing the frequency of garbage collections. Almost no additional heap space is typically allocated per frame. + * + * @return The frame returned from the grabber + * @throws Exception + * If there is a problem grabbing the frame. + */ + public abstract Frame grab() throws Exception; + + public Frame grabFrame() throws Exception { + return grab(); + } + + public abstract void release() throws Exception; + + public void restart() throws Exception { + stop(); + start(); + } + + public void flush() throws Exception { + for (int i = 0; i < numBuffers + 1; i++) { + grab(); + } + } + + private ExecutorService executor = Executors.newSingleThreadExecutor(); + private Future future = null; + private Frame delayedFrame = null; + private long delayedTime = 0; + + public void delayedGrab(final long delayTime) { + delayedFrame = null; + delayedTime = 0; + final long start = System.nanoTime() / 1000; + if (future != null && !future.isDone()) { + return; + } + future = executor.submit(new Callable() { + public Void call() throws Exception { + do { + delayedFrame = grab(); + delayedTime = System.nanoTime() / 1000 - start; + } while (delayedTime < delayTime); + return null; + } + }); + } + + public long getDelayedTime() throws InterruptedException, ExecutionException { + if (future == null) { + return 0; + } + future.get(); + return delayedTime; + } + + public Frame getDelayedFrame() throws InterruptedException, ExecutionException { + if (future == null) { + return null; + } + future.get(); + return delayedFrame; + } + + public static class Array { + // declared protected to force users to use createArray(), which + // can be overridden without changing the calling code... + protected Array(FrameGrabber[] frameGrabbers) { + setFrameGrabbers(frameGrabbers); + } + + private Frame[] grabbedFrames = null; + private long[] latencies = null; + private long[] bestLatencies = null; + private long lastNewestTimestamp = 0; + private long bestInterval = Long.MAX_VALUE; + + protected FrameGrabber[] frameGrabbers = null; + + public FrameGrabber[] getFrameGrabbers() { + return frameGrabbers; + } + + public void setFrameGrabbers(FrameGrabber[] frameGrabbers) { + this.frameGrabbers = frameGrabbers; + grabbedFrames = new Frame[frameGrabbers.length]; + latencies = new long[frameGrabbers.length]; + bestLatencies = null; + lastNewestTimestamp = 0; + } + + public int size() { + return frameGrabbers.length; + } + + public void start() throws Exception { + for (FrameGrabber f : frameGrabbers) { + f.start(); + } + } + + public void stop() throws Exception { + for (FrameGrabber f : frameGrabbers) { + f.stop(); + } + } + + // should be overriden to implement a broadcast trigger... + public void trigger() throws Exception { + for (FrameGrabber f : frameGrabbers) { + if (f.isTriggerMode()) { + f.trigger(); + } + } + } + + // should be overriden to implement a broadcast grab... + public Frame[] grab() throws Exception { + if (frameGrabbers.length == 1) { + grabbedFrames[0] = frameGrabbers[0].grab(); + return grabbedFrames; + } + + // assume we sometimes get perfectly synchronized images, + // so save the best latencies we find as the perfectly + // synchronized case, so we know what to aim for in + // cases of missing/dropped frames ... + long newestTimestamp = 0; + boolean unsynchronized = false; + for (int i = 0; i < frameGrabbers.length; i++) { + grabbedFrames[i] = frameGrabbers[i].grab(); + if (grabbedFrames[i] != null) { + newestTimestamp = Math.max(newestTimestamp, frameGrabbers[i].getTimestamp()); + } + if (frameGrabbers[i].getClass() != frameGrabbers[(i + 1) % frameGrabbers.length].getClass()) { + // assume we can't synchronize different types of cameras with each other + unsynchronized = true; + } + } + if (unsynchronized) { + return grabbedFrames; + } + for (int i = 0; i < frameGrabbers.length; i++) { + if (grabbedFrames[i] != null) { + latencies[i] = newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp()); + } + } + if (bestLatencies == null) { + bestLatencies = Arrays.copyOf(latencies, latencies.length); + } else { + int sum1 = 0, sum2 = 0; + for (int i = 0; i < frameGrabbers.length; i++) { + sum1 += latencies[i]; + sum2 += bestLatencies[i]; + } + if (sum1 < sum2) { + bestLatencies = Arrays.copyOf(latencies, latencies.length); + } + } + + // we cannot have latencies higher than the time between frames.. + // or something too close to it anyway... 90% is good? + bestInterval = Math.min(bestInterval, newestTimestamp - lastNewestTimestamp); + for (int i = 0; i < bestLatencies.length; i++) { + bestLatencies[i] = Math.min(bestLatencies[i], bestInterval * 9 / 10); + } + + // try to synchronize by attempting to land within 10% of + // the bestLatencies looking up to 2 frames ahead ... + for (int j = 0; j < 2; j++) { + for (int i = 0; i < frameGrabbers.length; i++) { + if (frameGrabbers[i].isTriggerMode() || grabbedFrames[i] == null) { + continue; + } + int latency = (int) (newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp())); + while (latency - bestLatencies[i] > 0.1 * bestLatencies[i]) { + grabbedFrames[i] = frameGrabbers[i].grab(); + if (grabbedFrames[i] == null) { + break; + } + latency = (int) (newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp())); + if (latency < 0) { + // woops, a camera seems to have dropped a frame somewhere... + // bump up the newestTimestamp + newestTimestamp = Math.max(0, frameGrabbers[i].getTimestamp()); + break; + } + } + } + } + + // for (int i = 0; i < frameGrabbers.length; i++) { + // long latency = newestTimestamp - Math.max(0, frameGrabbers[i].getTimestamp()); + // System.out.print(bestLatencies[i] + " " + latency + " "); + // } + // System.out.println(" " + bestInterval); + + lastNewestTimestamp = newestTimestamp; + + return grabbedFrames; + } + + public void release() throws Exception { + for (FrameGrabber f : frameGrabbers) { + f.release(); + } + } + } + + public Array createArray(FrameGrabber[] frameGrabbers) { + return new Array(frameGrabbers); + } } diff --git a/src/main/java/org/bytedeco/javacv/FrameRecorder.java b/src/main/java/org/bytedeco/javacv/FrameRecorder.java index 950fcbdc..9c5e25bd 100644 --- a/src/main/java/org/bytedeco/javacv/FrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FrameRecorder.java @@ -38,357 +38,414 @@ */ public abstract class FrameRecorder implements Closeable { - public static final List list = new LinkedList(Arrays.asList(new String[] { "FFmpeg", "OpenCV" })); - public static void init() { - for (String name : list) { - try { - Class c = get(name); - c.getMethod("tryLoad").invoke(null); - } catch (Throwable t) { } - } - } - public static Class getDefault() { - // select first frame recorder that can load.. - for (String name : list) { - try { - Class c = get(name); - c.getMethod("tryLoad").invoke(null); - return c; - } catch (Throwable t) { } - } - return null; - } - public static Class get(String className) throws Exception { - className = FrameRecorder.class.getPackage().getName() + "." + className; - try { - return Class.forName(className).asSubclass(FrameRecorder.class); - } catch (ClassNotFoundException e) { - String className2 = className + "FrameRecorder"; - try { - return Class.forName(className2).asSubclass(FrameRecorder.class); - } catch (ClassNotFoundException ex) { - throw new Exception("Could not get FrameRecorder class for " + className + " or " + className2, e); - } - } - } - - public static FrameRecorder create(Class c, Class p, Object o, int w, int h) throws Exception { - Throwable cause = null; - try { - return (FrameRecorder)c.getConstructor(p, int.class, int.class).newInstance(o, w, h); - } catch (InstantiationException ex) { - cause = ex; - } catch (IllegalAccessException ex) { - cause = ex; - } catch (IllegalArgumentException ex) { - cause = ex; - } catch (NoSuchMethodException ex) { - cause = ex; - } catch (InvocationTargetException ex) { - cause = ex.getCause(); - } - throw new Exception("Could not create new " + c.getSimpleName() + "(" + o + ", " + w + ", " + h + ")", cause); - } - - public static FrameRecorder createDefault(File file, int width, int height) throws Exception { - return create(getDefault(), File.class, file, width, height); - } - public static FrameRecorder createDefault(String filename, int width, int height) throws Exception { - return create(getDefault(), String.class, filename, width, height); - } - - public static FrameRecorder create(String className, File file, int width, int height) throws Exception { - return create(get(className), File.class, file, width, height); - } - public static FrameRecorder create(String className, String filename, int width, int height) throws Exception { - return create(get(className), String.class, filename, width, height); - } - - protected String format, videoCodecName, audioCodecName; - protected int imageWidth, imageHeight, audioChannels; - protected int pixelFormat, videoCodec, videoBitrate, gopSize = -1; - protected double aspectRatio, frameRate, videoQuality = -1; - protected int sampleFormat, audioCodec, audioBitrate, sampleRate; - protected double audioQuality = -1; - protected boolean interleaved; - protected Map options = new HashMap(); - protected Map videoOptions = new HashMap(); - protected Map audioOptions = new HashMap(); - protected Map metadata = new HashMap(); - protected Map videoMetadata = new HashMap(); - protected Map audioMetadata = new HashMap(); - protected int frameNumber = 0; - protected long timestamp = 0; - protected int maxBFrames = -1; - protected int trellis = -1; - protected int maxDelay = -1; - - public String getFormat() { - return format; - } - public void setFormat(String format) { - this.format = format; - } - - public String getVideoCodecName() { - return videoCodecName; - } - public void setVideoCodecName(String videoCodecName) { - this.videoCodecName = videoCodecName; - } - - public String getAudioCodecName() { - return audioCodecName; - } - public void setAudioCodecName(String audioCodecName) { - this.audioCodecName = audioCodecName; - } - - public int getImageWidth() { - return imageWidth; - } - public void setImageWidth(int imageWidth) { - this.imageWidth = imageWidth; - } - - public int getImageHeight() { - return imageHeight; - } - public void setImageHeight(int imageHeight) { - this.imageHeight = imageHeight; - } - - public int getAudioChannels() { - return audioChannels; - } - public void setAudioChannels(int audioChannels) { - this.audioChannels = audioChannels; - } - - public int getPixelFormat() { - return pixelFormat; - } - public void setPixelFormat(int pixelFormat) { - this.pixelFormat = pixelFormat; - } - - public int getVideoCodec() { - return videoCodec; - } - public void setVideoCodec(int videoCodec) { - this.videoCodec = videoCodec; - } - - public int getVideoBitrate() { - return videoBitrate; - } - public void setVideoBitrate(int videoBitrate) { - this.videoBitrate = videoBitrate; - } - - public int getGopSize() { - return gopSize; - } - public void setGopSize(int gopSize) { - this.gopSize = gopSize; - } - - public double getAspectRatio() { - return aspectRatio; - } - public void setAspectRatio(double aspectRatio) { - this.aspectRatio = aspectRatio; - } - - public double getFrameRate() { - return frameRate; - } - public void setFrameRate(double frameRate) { - this.frameRate = frameRate; - } - - public double getVideoQuality() { - return videoQuality; - } - public void setVideoQuality(double videoQuality) { - this.videoQuality = videoQuality; - } - - public int getSampleFormat() { - return sampleFormat; - } - public void setSampleFormat(int sampleFormat) { - this.sampleFormat = sampleFormat; - } - - public int getAudioCodec() { - return audioCodec; - } - public void setAudioCodec(int audioCodec) { - this.audioCodec = audioCodec; - } - - public int getAudioBitrate() { - return audioBitrate; - } - public void setAudioBitrate(int audioBitrate) { - this.audioBitrate = audioBitrate; - } - - public int getSampleRate() { - return sampleRate; - } - public void setSampleRate(int sampleRate) { - this.sampleRate = sampleRate; - } - - public double getAudioQuality() { - return audioQuality; - } - public void setAudioQuality(double audioQuality) { - this.audioQuality = audioQuality; - } - - public boolean isInterleaved() { - return interleaved; - } - public void setInterleaved(boolean interleaved) { - this.interleaved = interleaved; - } - - public Map getOptions() { - return options; - } - public void setOptions(Map options) { - this.options = options; - } - - public Map getVideoOptions() { - return videoOptions; - } - public void setVideoOptions(Map options) { - this.videoOptions = options; - } - - public Map getAudioOptions() { - return audioOptions; - } - public void setAudioOptions(Map options) { - this.audioOptions = options; - } - - public Map getMetadata() { - return metadata; - } - public void setMetadata(Map metadata) { - this.metadata = metadata; - } - - public Map getVideoMetadata() { - return videoMetadata; - } - public void setVideoMetadata(Map metadata) { - this.videoMetadata = metadata; - } - - public Map getAudioMetadata() { - return audioMetadata; - } - public void setAudioMetadata(Map metadata) { - this.audioMetadata = metadata; - } - - public String getOption(String key) { - return options.get(key); - } - public void setOption(String key, String value) { - options.put(key, value); - } - - public String getVideoOption(String key) { - return videoOptions.get(key); - } - public void setVideoOption(String key, String value) { - videoOptions.put(key, value); - } - - public String getAudioOption(String key) { - return audioOptions.get(key); - } - public void setAudioOption(String key, String value) { - audioOptions.put(key, value); - } - - public String getMetadata(String key) { - return metadata.get(key); - } - public void setMetadata(String key, String value) { - metadata.put(key, value); - } - - public String getVideoMetadata(String key) { - return videoMetadata.get(key); - } - public void setVideoMetadata(String key, String value) { - videoMetadata.put(key, value); - } - - public String getAudioMetadata(String key) { - return audioMetadata.get(key); - } - public void setAudioMetadata(String key, String value) { - audioMetadata.put(key, value); - } - - public int getFrameNumber() { - return frameNumber; - } - public void setFrameNumber(int frameNumber) { - this.frameNumber = frameNumber; - } - - public long getTimestamp() { - return timestamp; - } - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - public int getMaxBFrames() { - return maxBFrames; - } - public void setMaxBFrames(int maxBFrames) { - this.maxBFrames = maxBFrames; - } - - public int getTrellis() { - return trellis; - } - - public void setTrellis(int trellis) { - this.trellis = trellis; - } - - public int getMaxDelay() { - return maxDelay; - } - - public void setMaxDelay(int maxDelay) { - this.maxDelay = maxDelay; - } - - public static class Exception extends IOException { - public Exception(String message) { super(message); } - public Exception(String message, Throwable cause) { super(message, cause); } - } - - public abstract void start() throws Exception; - public abstract void stop() throws Exception; - public abstract void record(Frame frame) throws Exception; - public abstract void release() throws Exception; - - @Override public void close() throws Exception { - stop(); - release(); - } + public static final List list = new LinkedList(Arrays.asList(new String[] { "FFmpeg", "OpenCV" })); + + public static void init() { + for (String name : list) { + try { + Class c = get(name); + c.getMethod("tryLoad").invoke(null); + } catch (Throwable t) { + } + } + } + + public static Class getDefault() { + // select first frame recorder that can load.. + for (String name : list) { + try { + Class c = get(name); + c.getMethod("tryLoad").invoke(null); + return c; + } catch (Throwable t) { + } + } + return null; + } + + public static Class get(String className) throws Exception { + className = FrameRecorder.class.getPackage().getName() + "." + className; + try { + return Class.forName(className).asSubclass(FrameRecorder.class); + } catch (ClassNotFoundException e) { + String className2 = className + "FrameRecorder"; + try { + return Class.forName(className2).asSubclass(FrameRecorder.class); + } catch (ClassNotFoundException ex) { + throw new Exception("Could not get FrameRecorder class for " + className + " or " + className2, e); + } + } + } + + public static FrameRecorder create(Class c, Class p, Object o, int w, int h) throws Exception { + Throwable cause = null; + try { + return (FrameRecorder) c.getConstructor(p, int.class, int.class).newInstance(o, w, h); + } catch (InstantiationException ex) { + cause = ex; + } catch (IllegalAccessException ex) { + cause = ex; + } catch (IllegalArgumentException ex) { + cause = ex; + } catch (NoSuchMethodException ex) { + cause = ex; + } catch (InvocationTargetException ex) { + cause = ex.getCause(); + } + throw new Exception("Could not create new " + c.getSimpleName() + "(" + o + ", " + w + ", " + h + ")", cause); + } + + public static FrameRecorder createDefault(File file, int width, int height) throws Exception { + return create(getDefault(), File.class, file, width, height); + } + + public static FrameRecorder createDefault(String filename, int width, int height) throws Exception { + return create(getDefault(), String.class, filename, width, height); + } + + public static FrameRecorder create(String className, File file, int width, int height) throws Exception { + return create(get(className), File.class, file, width, height); + } + + public static FrameRecorder create(String className, String filename, int width, int height) throws Exception { + return create(get(className), String.class, filename, width, height); + } + + protected String format, videoCodecName, audioCodecName; + protected int imageWidth, imageHeight, audioChannels; + protected int pixelFormat, videoCodec, videoBitrate, gopSize = -1; + protected double aspectRatio, frameRate, videoQuality = -1; + protected int sampleFormat, audioCodec, audioBitrate, sampleRate; + protected double audioQuality = -1; + protected boolean interleaved; + protected Map options = new HashMap(); + protected Map videoOptions = new HashMap(); + protected Map audioOptions = new HashMap(); + protected Map metadata = new HashMap(); + protected Map videoMetadata = new HashMap(); + protected Map audioMetadata = new HashMap(); + protected Map encContextOptions = new HashMap<>(); + protected int frameNumber = 0; + protected long timestamp = 0; + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public String getVideoCodecName() { + return videoCodecName; + } + + public void setVideoCodecName(String videoCodecName) { + this.videoCodecName = videoCodecName; + } + + public String getAudioCodecName() { + return audioCodecName; + } + + public void setAudioCodecName(String audioCodecName) { + this.audioCodecName = audioCodecName; + } + + public int getImageWidth() { + return imageWidth; + } + + public void setImageWidth(int imageWidth) { + this.imageWidth = imageWidth; + } + + public int getImageHeight() { + return imageHeight; + } + + public void setImageHeight(int imageHeight) { + this.imageHeight = imageHeight; + } + + public int getAudioChannels() { + return audioChannels; + } + + public void setAudioChannels(int audioChannels) { + this.audioChannels = audioChannels; + } + + public int getPixelFormat() { + return pixelFormat; + } + + public void setPixelFormat(int pixelFormat) { + this.pixelFormat = pixelFormat; + } + + public int getVideoCodec() { + return videoCodec; + } + + public void setVideoCodec(int videoCodec) { + this.videoCodec = videoCodec; + } + + public int getVideoBitrate() { + return videoBitrate; + } + + public void setVideoBitrate(int videoBitrate) { + this.videoBitrate = videoBitrate; + } + + public int getGopSize() { + return gopSize; + } + + public void setGopSize(int gopSize) { + this.gopSize = gopSize; + } + + public double getAspectRatio() { + return aspectRatio; + } + + public void setAspectRatio(double aspectRatio) { + this.aspectRatio = aspectRatio; + } + + public double getFrameRate() { + return frameRate; + } + + public void setFrameRate(double frameRate) { + this.frameRate = frameRate; + } + + public double getVideoQuality() { + return videoQuality; + } + + public void setVideoQuality(double videoQuality) { + this.videoQuality = videoQuality; + } + + public int getSampleFormat() { + return sampleFormat; + } + + public void setSampleFormat(int sampleFormat) { + this.sampleFormat = sampleFormat; + } + + public int getAudioCodec() { + return audioCodec; + } + + public void setAudioCodec(int audioCodec) { + this.audioCodec = audioCodec; + } + + public int getAudioBitrate() { + return audioBitrate; + } + + public void setAudioBitrate(int audioBitrate) { + this.audioBitrate = audioBitrate; + } + + public int getSampleRate() { + return sampleRate; + } + + public void setSampleRate(int sampleRate) { + this.sampleRate = sampleRate; + } + + public double getAudioQuality() { + return audioQuality; + } + + public void setAudioQuality(double audioQuality) { + this.audioQuality = audioQuality; + } + + public boolean isInterleaved() { + return interleaved; + } + + public void setInterleaved(boolean interleaved) { + this.interleaved = interleaved; + } + + public Map getOptions() { + return options; + } + + public void setOptions(Map options) { + this.options = options; + } + + public Map getVideoOptions() { + return videoOptions; + } + + public void setVideoOptions(Map options) { + this.videoOptions = options; + } + + public Map getAudioOptions() { + return audioOptions; + } + + public void setAudioOptions(Map options) { + this.audioOptions = options; + } + + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public Map getVideoMetadata() { + return videoMetadata; + } + + public void setVideoMetadata(Map metadata) { + this.videoMetadata = metadata; + } + + public Map getAudioMetadata() { + return audioMetadata; + } + + public void setAudioMetadata(Map metadata) { + this.audioMetadata = metadata; + } + + public void setEncContextOption(String key, String value) { + encContextOptions.put(key, value); + } + + public String getEncContextOption(String key) { + return encContextOptions.get(key); + } + + public String getOption(String key) { + return options.get(key); + } + + public void setOption(String key, String value) { + options.put(key, value); + } + + public String getVideoOption(String key) { + return videoOptions.get(key); + } + + public void setVideoOption(String key, String value) { + videoOptions.put(key, value); + } + + public String getAudioOption(String key) { + return audioOptions.get(key); + } + + public void setAudioOption(String key, String value) { + audioOptions.put(key, value); + } + + public String getMetadata(String key) { + return metadata.get(key); + } + + public void setMetadata(String key, String value) { + metadata.put(key, value); + } + + public String getVideoMetadata(String key) { + return videoMetadata.get(key); + } + + public void setVideoMetadata(String key, String value) { + videoMetadata.put(key, value); + } + + public String getAudioMetadata(String key) { + return audioMetadata.get(key); + } + + public void setAudioMetadata(String key, String value) { + audioMetadata.put(key, value); + } + + public int getFrameNumber() { + return frameNumber; + } + + public void setFrameNumber(int frameNumber) { + this.frameNumber = frameNumber; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public static class Exception extends IOException { + private static final long serialVersionUID = 1L; + private int errorCode = 0;// unknown + + public Exception(String message) { + super(message); + } + + public Exception(int errorCode, String message) { + super(message); + this.errorCode = errorCode; + } + + public Exception(String message, Throwable cause) { + super(message, cause); + } + + public Exception(int errorCode, String message, Throwable cause) { + super(message, cause); + this.errorCode = errorCode; + } + + public int getErrorCode() { + return errorCode; + } + + @Override + public String toString() { + if (errorCode != -1) + return String.format("%d:%s", errorCode, super.toString()); + else + return super.toString(); + } + } + + public abstract void start() throws Exception; + + public abstract void stop() throws Exception; + + public abstract void record(Frame frame) throws Exception; + + public abstract void release() throws Exception; + + @Override + public void close() throws Exception { + stop(); + release(); + } } From 537b4bd43896ea9b273c7e749a68b4b015612546 Mon Sep 17 00:00:00 2001 From: fcinfo <34166216+fcinfo@users.noreply.github.com> Date: Wed, 28 Mar 2018 11:43:57 +0800 Subject: [PATCH 2/4] lost bad packet --- .../java/org/bytedeco/javacv/FFmpegFrameRecorder.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index faec740c..075ac6cc 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -316,9 +316,7 @@ void releaseUnsafe() throws Exception { protected void finalize() throws Throwable { super.finalize(); release(); - } - - // static Map outputStreams = Collections.synchronizedMap(new HashMap()); + } private class WriteCallback extends Write_packet_Pointer_BytePointer_int { private FFmpegFrameRecorder recorder; @@ -1231,8 +1229,8 @@ public boolean recordPacket(AVPacket pkt) throws Exception { pkt.pts(av_rescale_q_rnd(pkt.pts(), in_stream.time_base(), video_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), video_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), video_st.time_base())); - if (video_st.cur_dts() >= pkt.dts()) - pkt.dts(video_st.cur_dts() + pkt.duration()); + if (video_st.cur_dts() >= pkt.dts()) + return true; if (pkt.pts() < pkt.dts()) pkt.pts(pkt.dts()); writePacket(AVMEDIA_TYPE_VIDEO, pkt); @@ -1242,7 +1240,7 @@ public boolean recordPacket(AVPacket pkt) throws Exception { pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), audio_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), audio_st.time_base())); if (audio_st.cur_dts() >= pkt.dts()) - pkt.dts(audio_st.cur_dts() + pkt.duration()); + return true; if (pkt.pts() < pkt.dts()) pkt.pts(pkt.dts()); writePacket(AVMEDIA_TYPE_AUDIO, pkt); From 7f41eb0f44ad00fdbbb1de1136c8056b3def1f3f Mon Sep 17 00:00:00 2001 From: fcinfo <34166216+fcinfo@users.noreply.github.com> Date: Thu, 29 Mar 2018 09:42:38 +0800 Subject: [PATCH 3/4] OutputStream in class WriteCallback is null bugfix: OutputStream in class WriteCallback is null --- .../bytedeco/javacv/FFmpegFrameRecorder.java | 252 +++++++++++++----- 1 file changed, 189 insertions(+), 63 deletions(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index 075ac6cc..e0ca3f8f 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -50,6 +50,117 @@ package org.bytedeco.javacv; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_AAC; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_FFV1; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_FLV1; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_H263; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_H264; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_HUFFYUV; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_JPEGLS; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_MJPEG; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_MJPEGB; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_MPEG1VIDEO; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_MPEG2VIDEO; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_MPEG4; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_NONE; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_PCM_S16BE; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_PCM_S16LE; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_PCM_U16BE; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_PCM_U16LE; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_PNG; +import static org.bytedeco.javacpp.avcodec.AV_CODEC_ID_RAWVIDEO; +import static org.bytedeco.javacpp.avcodec.AV_INPUT_BUFFER_MIN_SIZE; +import static org.bytedeco.javacpp.avcodec.AV_PKT_FLAG_KEY; +import static org.bytedeco.javacpp.avcodec.CODEC_CAP_EXPERIMENTAL; +import static org.bytedeco.javacpp.avcodec.CODEC_FLAG_GLOBAL_HEADER; +import static org.bytedeco.javacpp.avcodec.CODEC_FLAG_QSCALE; +import static org.bytedeco.javacpp.avcodec.av_init_packet; +import static org.bytedeco.javacpp.avcodec.av_packet_unref; +import static org.bytedeco.javacpp.avcodec.avcodec_alloc_context3; +import static org.bytedeco.javacpp.avcodec.avcodec_encode_audio2; +import static org.bytedeco.javacpp.avcodec.avcodec_encode_video2; +import static org.bytedeco.javacpp.avcodec.avcodec_fill_audio_frame; +import static org.bytedeco.javacpp.avcodec.avcodec_find_encoder; +import static org.bytedeco.javacpp.avcodec.avcodec_find_encoder_by_name; +import static org.bytedeco.javacpp.avcodec.avcodec_free_context; +import static org.bytedeco.javacpp.avcodec.avcodec_open2; +import static org.bytedeco.javacpp.avcodec.avcodec_parameters_copy; +import static org.bytedeco.javacpp.avcodec.avcodec_parameters_from_context; +import static org.bytedeco.javacpp.avcodec.avcodec_register_all; +import static org.bytedeco.javacpp.avdevice.avdevice_register_all; +import static org.bytedeco.javacpp.avformat.AVFMT_GLOBALHEADER; +import static org.bytedeco.javacpp.avformat.AVFMT_NOFILE; +import static org.bytedeco.javacpp.avformat.AVFMT_RAWPICTURE; +import static org.bytedeco.javacpp.avformat.AVIO_FLAG_WRITE; +import static org.bytedeco.javacpp.avformat.av_dump_format; +import static org.bytedeco.javacpp.avformat.av_guess_format; +import static org.bytedeco.javacpp.avformat.av_interleaved_write_frame; +import static org.bytedeco.javacpp.avformat.av_register_all; +import static org.bytedeco.javacpp.avformat.av_write_frame; +import static org.bytedeco.javacpp.avformat.av_write_trailer; +import static org.bytedeco.javacpp.avformat.avformat_alloc_output_context2; +import static org.bytedeco.javacpp.avformat.avformat_network_init; +import static org.bytedeco.javacpp.avformat.avformat_new_stream; +import static org.bytedeco.javacpp.avformat.avformat_write_header; +import static org.bytedeco.javacpp.avformat.avio_alloc_context; +import static org.bytedeco.javacpp.avformat.avio_close; +import static org.bytedeco.javacpp.avformat.avio_open2; +import static org.bytedeco.javacpp.avutil.AVMEDIA_TYPE_AUDIO; +import static org.bytedeco.javacpp.avutil.AVMEDIA_TYPE_VIDEO; +import static org.bytedeco.javacpp.avutil.AV_LOG_INFO; +import static org.bytedeco.javacpp.avutil.AV_NOPTS_VALUE; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_BGR24; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_GRAY16BE; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_GRAY16LE; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_GRAY8; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_NONE; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_NV21; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_RGB32; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_RGBA; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_YUV420P; +import static org.bytedeco.javacpp.avutil.AV_PIX_FMT_YUVJ420P; +import static org.bytedeco.javacpp.avutil.AV_ROUND_NEAR_INF; +import static org.bytedeco.javacpp.avutil.AV_ROUND_PASS_MINMAX; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_DBL; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_DBLP; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_FLT; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_FLTP; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_NONE; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_S16; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_S16P; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_S32; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_S32P; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_U8; +import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_U8P; +import static org.bytedeco.javacpp.avutil.FF_QP2LAMBDA; +import static org.bytedeco.javacpp.avutil.av_d2q; +import static org.bytedeco.javacpp.avutil.av_dict_free; +import static org.bytedeco.javacpp.avutil.av_dict_set; +import static org.bytedeco.javacpp.avutil.av_find_nearest_q_idx; +import static org.bytedeco.javacpp.avutil.av_frame_alloc; +import static org.bytedeco.javacpp.avutil.av_frame_free; +import static org.bytedeco.javacpp.avutil.av_free; +import static org.bytedeco.javacpp.avutil.av_get_bytes_per_sample; +import static org.bytedeco.javacpp.avutil.av_get_default_channel_layout; +import static org.bytedeco.javacpp.avutil.av_image_fill_arrays; +import static org.bytedeco.javacpp.avutil.av_image_get_buffer_size; +import static org.bytedeco.javacpp.avutil.av_inv_q; +import static org.bytedeco.javacpp.avutil.av_log_get_level; +import static org.bytedeco.javacpp.avutil.av_malloc; +import static org.bytedeco.javacpp.avutil.av_opt_set; +import static org.bytedeco.javacpp.avutil.av_rescale_q; +import static org.bytedeco.javacpp.avutil.av_rescale_q_rnd; +import static org.bytedeco.javacpp.avutil.av_sample_fmt_is_planar; +import static org.bytedeco.javacpp.avutil.av_samples_get_buffer_size; +import static org.bytedeco.javacpp.swresample.swr_alloc_set_opts; +import static org.bytedeco.javacpp.swresample.swr_convert; +import static org.bytedeco.javacpp.swresample.swr_free; +import static org.bytedeco.javacpp.swresample.swr_init; +import static org.bytedeco.javacpp.swscale.SWS_BILINEAR; +import static org.bytedeco.javacpp.swscale.sws_freeContext; +import static org.bytedeco.javacpp.swscale.sws_getCachedContext; +import static org.bytedeco.javacpp.swscale.sws_scale; + import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -60,10 +171,9 @@ import java.nio.FloatBuffer; import java.nio.IntBuffer; import java.nio.ShortBuffer; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + import org.bytedeco.javacpp.BytePointer; import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.FloatPointer; @@ -72,18 +182,22 @@ import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.PointerPointer; import org.bytedeco.javacpp.ShortPointer; -import org.bytedeco.javacpp.avcodec; -import org.bytedeco.javacpp.avformat; +import org.bytedeco.javacpp.avcodec.AVCodec; +import org.bytedeco.javacpp.avcodec.AVCodecContext; +import org.bytedeco.javacpp.avcodec.AVPacket; +import org.bytedeco.javacpp.avformat.AVFormatContext; +import org.bytedeco.javacpp.avformat.AVIOContext; +import org.bytedeco.javacpp.avformat.AVOutputFormat; +import org.bytedeco.javacpp.avformat.AVStream; +import org.bytedeco.javacpp.avformat.Write_packet_Pointer_BytePointer_int; +import org.bytedeco.javacpp.avutil.AVDictionary; +import org.bytedeco.javacpp.avutil.AVFrame; +import org.bytedeco.javacpp.avutil.AVRational; +import org.bytedeco.javacpp.swresample.SwrContext; +import org.bytedeco.javacpp.swscale.SwsContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.bytedeco.javacpp.avcodec.*; -import static org.bytedeco.javacpp.avdevice.*; -import static org.bytedeco.javacpp.avformat.*; -import static org.bytedeco.javacpp.avutil.*; -import static org.bytedeco.javacpp.swresample.*; -import static org.bytedeco.javacpp.swscale.*; - /** * * @author Samuel Audet @@ -189,11 +303,17 @@ public FFmpegFrameRecorder(OutputStream outputStream, int imageWidth, int imageH this.outputStream = outputStream; } - public FFmpegFrameRecorder(OutputStream outputStream, int imageWidth, int imageHeight, int audioChannels) { + public FFmpegFrameRecorder(String name, OutputStream outputStream, int imageWidth, int imageHeight, int audioChannels) { this(outputStream.toString(), imageWidth, imageHeight, audioChannels); + this.name = name; this.outputStream = outputStream; } + private String name = "noname"; + public String getName() { + return this.name; + } + public void release() throws Exception { // synchronized (org.bytedeco.javacpp.avcodec.class) { releaseUnsafe(); @@ -201,6 +321,7 @@ public void release() throws Exception { } void releaseUnsafe() throws Exception { + this.releaseing = true; /* close each codec */ if (video_c != null) { avcodec_free_context(video_c); @@ -252,7 +373,6 @@ void releaseUnsafe() throws Exception { audio_st = null; filename = null; - AVFormatContext outputStreamKey = oc; if (oc != null && !oc.isNull()) { if (outputStream == null && (oformat.flags() & AVFMT_NOFILE) == 0) { /* close the output file */ @@ -304,55 +424,15 @@ void releaseUnsafe() throws Exception { avio = null; } } - } - - if (writeCallback != null) { - writeCallback.close(); - writeCallback = null; - } + } } @Override protected void finalize() throws Throwable { super.finalize(); release(); - } - - private class WriteCallback extends Write_packet_Pointer_BytePointer_int { - private FFmpegFrameRecorder recorder; - - public WriteCallback(FFmpegFrameRecorder recorder) { - this.recorder = recorder; - } - - @Override - public int call(Pointer opaque, BytePointer buf, int buf_size) { - try { - byte[] b = new byte[buf_size]; - OutputStream os = recorder.outputStream; - if (os == null) { - logger.error("OutputStream is null at WriteCallback"); - return -1; - } - buf.get(b, 0, buf_size); - os.write(b, 0, buf_size); - return buf_size; - } catch (Throwable t) { - System.err.println("Error on OutputStream.write(): " + t); - logger.error("Error on OutputStream.write(): " + t); - return -1; - } - } - - @Override - public void close(){ - super.close(); - this.recorder = null; - } } - private WriteCallback writeCallback = new WriteCallback(this); - private OutputStream outputStream; private AVIOContext avio; private String filename; @@ -379,12 +459,17 @@ public void close(){ private AVPacket video_pkt, audio_pkt; private int[] got_video_packet, got_audio_packet; private AVFormatContext ifmt_ctx; + private boolean releaseing = false; @Override public int getFrameNumber() { return picture == null ? super.getFrameNumber() : (int) picture.pts(); } + public boolean isReleaseing() { + return this.releaseing; + } + @Override public void setFrameNumber(int frameNumber) { if (picture == null) { @@ -444,8 +529,6 @@ void startUnsafe() throws Exception { } } format_name = oformat.name().getString(); - if ((oformat.flags() & avformat.AVFMT_TS_NONSTRICT) != 0) - oformat.flags(oformat.flags() - avformat.AVFMT_TS_NONSTRICT); /* allocate the output media context */ if (avformat_alloc_output_context2(oc, null, format_name, filename) < 0) { @@ -453,11 +536,10 @@ void startUnsafe() throws Exception { } if (outputStream != null) { - avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 1, oc, null, writeCallback, null);// + avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 1, oc, null, new WriteCallback(this), null);// oc.pb(avio); filename = outputStream.toString(); - // outputStreams.put(oc, outputStream); } oc.oformat(oformat); oc.filename().putString(filename); @@ -536,7 +618,7 @@ void startUnsafe() throws Exception { } video_c.codec_id(oformat.video_codec()); - video_c.codec_type(AVMEDIA_TYPE_VIDEO); + video_c.codec_type(AVMEDIA_TYPE_VIDEO); for (Entry item : encContextOptions.entrySet()) av_opt_set(video_c.priv_data(), item.getKey(), item.getValue(), 0); @@ -1229,8 +1311,11 @@ public boolean recordPacket(AVPacket pkt) throws Exception { pkt.pts(av_rescale_q_rnd(pkt.pts(), in_stream.time_base(), video_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), video_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), video_st.time_base())); - if (video_st.cur_dts() >= pkt.dts()) - return true; + if (video_st.cur_dts() >= pkt.dts()) + { + pkt.dts(video_st.cur_dts()+pkt.duration()); + //return true; + } if (pkt.pts() < pkt.dts()) pkt.pts(pkt.dts()); writePacket(AVMEDIA_TYPE_VIDEO, pkt); @@ -1240,7 +1325,8 @@ public boolean recordPacket(AVPacket pkt) throws Exception { pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), audio_st.time_base(), AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), audio_st.time_base())); if (audio_st.cur_dts() >= pkt.dts()) - return true; + //return true; + pkt.dts(audio_st.cur_dts()+pkt.duration()); if (pkt.pts() < pkt.dts()) pkt.pts(pkt.dts()); writePacket(AVMEDIA_TYPE_AUDIO, pkt); @@ -1259,3 +1345,43 @@ public void setInputAVFormatContext(AVFormatContext ctx) { this.ifmt_ctx = ctx; } } + +class WriteCallback extends Write_packet_Pointer_BytePointer_int { + private static final Logger logger = LoggerFactory.getLogger(WriteCallback.class); + private FFmpegFrameRecorder recorder; + + public WriteCallback(FFmpegFrameRecorder recorder) { + this.recorder = recorder; + } + + @Override + public int call(Pointer opaque, BytePointer buf, int buf_size) { + if (recorder.isReleaseing()) { + logger.error("recorder has stopped!"); + return 0; + } + try { + byte[] b = new byte[buf_size]; + while (recorder.getOutputStream() == null) { + logger.info(recorder.getName() + ":OutputStream is null "); + TimeUnit.MICROSECONDS.sleep(100); + } + if (recorder.getOutputStream() == null) { + logger.error(Thread.currentThread().getName() + ":OutputStream is null at WriteCallback!buffer size:" + buf_size + ", why?"); + return -1; + } + buf.get(b, 0, buf_size); + recorder.getOutputStream().write(b, 0, buf_size); + return buf_size; + } catch (Throwable t) { + logger.error("Error on OutputStream.write(): " + t); + return -1; + } + } + + @Override + public void close() { + super.close(); + this.recorder = null; + } +} From 945e7a43937089f8c25cde6cff820c505fb927fa Mon Sep 17 00:00:00 2001 From: fcinfo <34166216+fcinfo@users.noreply.github.com> Date: Thu, 29 Mar 2018 13:48:26 +0800 Subject: [PATCH 4/4] Release WriteCallback instance at FFmpegFrameRecorder release --- .../java/org/bytedeco/javacv/FFmpegFrameRecorder.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index e0ca3f8f..f62a53c2 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -406,6 +406,11 @@ void releaseUnsafe() throws Exception { swr_free(samples_convert_ctx); samples_convert_ctx = null; } + + if (writeCallback != null) { + writeCallback.close(); + writeCallback = null; + } if (outputStream != null) { try { @@ -459,6 +464,7 @@ protected void finalize() throws Throwable { private AVPacket video_pkt, audio_pkt; private int[] got_video_packet, got_audio_packet; private AVFormatContext ifmt_ctx; + private WriteCallback writeCallback; private boolean releaseing = false; @Override @@ -536,7 +542,8 @@ void startUnsafe() throws Exception { } if (outputStream != null) { - avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 1, oc, null, new WriteCallback(this), null);// + this.writeCallback = new WriteCallback(this); + avio = avio_alloc_context(new BytePointer(av_malloc(4096)), 4096, 1, oc, null, this.writeCallback, null);// oc.pb(avio); filename = outputStream.toString();