From eabd02f5bd7c50f9e0df1a1669ff06fbef6f3e0b Mon Sep 17 00:00:00 2001 From: hanguk0726 Date: Sat, 25 Mar 2023 12:02:46 +0900 Subject: [PATCH] Support audio track _ _ chore chore dependency version update --- Cargo.toml | 3 +- minimp4-sys/Cargo.toml | 4 +- src/enc.rs | 200 +++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 194 ++++++++++++++++++++++++++++++++++----- src/main.rs | 32 ++++--- 5 files changed, 396 insertions(+), 37 deletions(-) create mode 100644 src/enc.rs diff --git a/Cargo.toml b/Cargo.toml index 15de589..135a859 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,4 +13,5 @@ repository = "https://github.com/darkskygit/minimp4.rs" [dependencies] libc = "0.2.66" -minimp4-sys = "0.1.0" +minimp4-sys = {path = "minimp4-sys"} +fdk-aac-sys = "0.4.0" diff --git a/minimp4-sys/Cargo.toml b/minimp4-sys/Cargo.toml index 902374d..bc6e6d1 100644 --- a/minimp4-sys/Cargo.toml +++ b/minimp4-sys/Cargo.toml @@ -13,5 +13,5 @@ repository = "https://github.com/darkskygit/minimp4.rs" categories = ["external-ffi-bindings"] [build-dependencies] -bindgen = "0.52.0" -cc = "1.0.48" +bindgen = "0.64.0" +cc = { version = "1.0.79", features = ["parallel"] } diff --git a/src/enc.rs b/src/enc.rs new file mode 100644 index 0000000..bcd1beb --- /dev/null +++ b/src/enc.rs @@ -0,0 +1,200 @@ +use std::cmp; +use std::fmt::{self, Display, Debug}; +use std::mem::{self, MaybeUninit}; +use std::os::raw::{c_void, c_uint, c_int}; +use std::ptr; + +use fdk_aac_sys as sys; + +pub use sys::AACENC_InfoStruct as InfoStruct; + +//mostly from https://github.com/haileys/fdk-aac-rs but with some changes + +pub struct EncoderError(sys::AACENC_ERROR); + +impl EncoderError { + fn message(&self) -> &'static str { + match self.0 { + sys::AACENC_ERROR_AACENC_OK => "Ok", + sys::AACENC_ERROR_AACENC_INVALID_HANDLE => "Handle passed to function call was invalid.", + sys::AACENC_ERROR_AACENC_MEMORY_ERROR => "Memory allocation failed.", + sys::AACENC_ERROR_AACENC_UNSUPPORTED_PARAMETER => "Parameter not available.", + sys::AACENC_ERROR_AACENC_INVALID_CONFIG => "Configuration not provided.", + sys::AACENC_ERROR_AACENC_INIT_ERROR => "General initialization error.", + sys::AACENC_ERROR_AACENC_INIT_AAC_ERROR => "AAC library initialization error.", + sys::AACENC_ERROR_AACENC_INIT_SBR_ERROR => "SBR library initialization error.", + sys::AACENC_ERROR_AACENC_INIT_TP_ERROR => "Transport library initialization error.", + sys::AACENC_ERROR_AACENC_INIT_META_ERROR => "Meta data library initialization error.", + sys::AACENC_ERROR_AACENC_INIT_MPS_ERROR => "MPS library initialization error.", + sys::AACENC_ERROR_AACENC_ENCODE_ERROR => "The encoding process was interrupted by an unexpected error.", + sys::AACENC_ERROR_AACENC_ENCODE_EOF => "End of file reached.", + _ => "Unknown error", + } + } +} + +impl Debug for EncoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "EncoderError {{ code: {:?}, message: {:?} }}", self.0 as c_int, self.message()) + } +} + +impl Display for EncoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.message()) + } +} + +fn check(e: sys::AACENC_ERROR) -> Result<(), EncoderError> { + if e == sys::AACENC_ERROR_AACENC_OK { + Ok(()) + } else { + Err(EncoderError(e)) + } +} + +struct EncoderHandle { + ptr: sys::HANDLE_AACENCODER, +} + +impl EncoderHandle { + pub fn alloc(max_modules: usize, max_channels: usize) -> Result { + let mut ptr: sys::HANDLE_AACENCODER = ptr::null_mut(); + check(unsafe { + sys::aacEncOpen(&mut ptr as *mut _, max_modules as c_uint, max_channels as c_uint) + })?; + Ok(EncoderHandle { ptr }) + } +} + +impl Drop for EncoderHandle { + fn drop(&mut self) { + unsafe { sys::aacEncClose(&mut self.ptr as *mut _); } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum BitRate { + Cbr(u32), + VbrVeryLow, + VbrLow, + VbrMedium, + VbrHigh, + VbrVeryHigh, +} + +#[derive(Debug, Clone, Copy)] +pub struct EncoderParams { + pub bit_rate: BitRate, + pub sample_rate: u32, + pub channel_count: u32 +} + +pub struct Encoder { + handle: EncoderHandle, +} + +#[derive(Debug)] +pub enum Transport { + Adts, + Raw, +} + +#[derive(Debug)] +pub struct EncodeInfo { + pub input_consumed: usize, + pub output_size: usize, +} + +impl Encoder { + pub fn new(params: EncoderParams) -> Result { + let handle = EncoderHandle::alloc(0, 2 /* hardcode stereo */)?; + assert!(params.channel_count == 1 || params.channel_count == 2); + unsafe { + // hardcode MPEG-4 AAC Low Complexity for now: + check(sys::aacEncoder_SetParam(handle.ptr, sys::AACENC_PARAM_AACENC_AOT, 2))?; + + let bitrate_mode = match params.bit_rate { + BitRate::Cbr(bitrate) => { + check(sys::aacEncoder_SetParam(handle.ptr, sys::AACENC_PARAM_AACENC_BITRATE, bitrate))?; + 0 + } + BitRate::VbrVeryLow => 1, + BitRate::VbrLow => 2, + BitRate::VbrMedium => 3, + BitRate::VbrHigh => 4, + BitRate::VbrVeryHigh => 5, + }; + + check(sys::aacEncoder_SetParam(handle.ptr, sys::AACENC_PARAM_AACENC_BITRATEMODE, bitrate_mode))?; + + check(sys::aacEncoder_SetParam(handle.ptr, sys::AACENC_PARAM_AACENC_SAMPLERATE, params.sample_rate))?; + + check(sys::aacEncoder_SetParam(handle.ptr, sys::AACENC_PARAM_AACENC_TRANSMUX, 0))?; + + // hardcode SBR off for now + check(sys::aacEncoder_SetParam(handle.ptr, sys::AACENC_PARAM_AACENC_SBR_MODE, 0))?; + + check(sys::aacEncoder_SetParam(handle.ptr, sys::AACENC_PARAM_AACENC_CHANNELMODE, params.channel_count))?; + + // call encode once with all null params according to docs + check(sys::aacEncEncode(handle.ptr, ptr::null(), ptr::null(), ptr::null(), ptr::null_mut()))?; + } + + Ok(Encoder { handle }) + } + + pub fn info(&self) -> Result { + let mut info = MaybeUninit::uninit(); + check(unsafe { sys::aacEncInfo(self.handle.ptr, info.as_mut_ptr()) })?; + Ok(unsafe { info.assume_init() }) + } + + pub fn encode(&self, input: &[i16], output: &mut [u8]) -> Result { + let input_len = cmp::min(i32::max_value() as usize, input.len()) as i32; + + let mut input_buf = input.as_ptr() as *mut i16; + let mut input_buf_ident: c_int = sys::AACENC_BufferIdentifier_IN_AUDIO_DATA as c_int; + let mut input_buf_size: c_int = input_len as c_int; + let mut input_buf_el_size: c_int = mem::size_of::() as c_int; + let input_desc = sys::AACENC_BufDesc { + numBufs: 1, + bufs: &mut input_buf as *mut _ as *mut *mut c_void, + bufferIdentifiers: &mut input_buf_ident as *mut c_int, + bufSizes: &mut input_buf_size as *mut c_int, + bufElSizes: &mut input_buf_el_size as *mut c_int, + }; + + let mut output_buf = output.as_mut_ptr(); + let mut output_buf_ident: c_int = sys::AACENC_BufferIdentifier_OUT_BITSTREAM_DATA as c_int; + let mut output_buf_size: c_int = output.len() as c_int; + let mut output_buf_el_size: c_int = mem::size_of::() as c_int; + let output_desc = sys::AACENC_BufDesc { + numBufs: 1, + bufs: &mut output_buf as *mut _ as *mut *mut c_void, + bufferIdentifiers: &mut output_buf_ident as *mut _, + bufSizes: &mut output_buf_size as *mut _, + bufElSizes: &mut output_buf_el_size as *mut _, + }; + + let in_args = sys::AACENC_InArgs { + numInSamples: input_len, + numAncBytes: 0, + }; + + let mut out_args = unsafe { mem::zeroed() }; + + check(unsafe { sys::aacEncEncode(self.handle.ptr, &input_desc, &output_desc, &in_args, &mut out_args) })?; + + Ok(EncodeInfo { + output_size: out_args.numOutBytes as usize, + input_consumed: out_args.numInSamples as usize, + }) + } +} + +impl Debug for Encoder { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Encoder {{ handle: {:?} }}", self.handle.ptr) + } +} diff --git a/src/lib.rs b/src/lib.rs index 5c59436..8091b57 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,30 +1,28 @@ use libc::malloc; use minimp4_sys::{ - mp4_h26x_write_init, mp4_h26x_writer_t, MP4E_close, MP4E_mux_t, MP4E_open, - MP4E_set_text_comment, + mp4_h26x_write_init, mp4_h26x_write_nal, mp4_h26x_writer_t, track_media_kind_t_e_audio, + MP4E_add_track, MP4E_close, MP4E_mux_t, MP4E_open, MP4E_put_sample, MP4E_set_dsi, + MP4E_set_text_comment, MP4E_track_t, MP4E_track_t__bindgen_ty_1, + MP4E_track_t__bindgen_ty_1__bindgen_ty_1, MP4E_SAMPLE_RANDOM_ACCESS, + MP4_OBJECT_TYPE_AUDIO_ISO_IEC_14496_3, }; use std::convert::TryInto; use std::ffi::CString; use std::io::{Seek, SeekFrom, Write}; use std::mem::size_of; -use std::os::raw::{c_int, c_void}; +use std::os::raw::c_void; use std::ptr::null_mut; use std::slice::from_raw_parts; -extern "C" { - fn write_mp4( - mp4wr: *const mp4_h26x_writer_t, - fps: c_int, - data: *const c_void, - data_size: c_int, - ); -} +use crate::enc::{BitRate, Encoder, EncoderParams}; +pub mod enc; pub struct Mp4Muxer { writer: W, muxer: *mut MP4E_mux_t, muxer_writer: *mut mp4_h26x_writer_t, str_buffer: Vec, + encoder_params: Option, } impl Mp4Muxer { @@ -35,6 +33,7 @@ impl Mp4Muxer { muxer: null_mut(), muxer_writer: malloc(size_of::()) as *mut mp4_h26x_writer_t, str_buffer: Vec::new(), + encoder_params: None, } } } @@ -52,24 +51,33 @@ impl Mp4Muxer { width, height, if is_hevc { 1 } else { 0 }, - self.str_buffer.last().unwrap().as_ptr(), ); } } + pub fn init_audio(&mut self, bit_rate: u32, sample_rate: u32, channel_count: u32) { + self.encoder_params = Some(EncoderParams { + bit_rate: BitRate::Cbr(bit_rate), + sample_rate, + channel_count, + }); + } pub fn write_video(&self, data: &[u8]) { - self.write_video_with_fps(data, 60) + self.write_video_with_fps(data, 60); + } + + pub fn write_video_with_audio(&self, data: &[u8], fps: u32, pcm: &[u8]) { + assert!(self.encoder_params.is_some()); + let mp4wr = unsafe { self.muxer_writer.as_mut().unwrap() }; + let fps = fps.try_into().unwrap(); + let encoder_params = self.encoder_params.clone().unwrap(); + write_mp4_with_audio(mp4wr, fps, data, pcm, encoder_params) } pub fn write_video_with_fps(&self, data: &[u8], fps: u32) { - unsafe { - write_mp4( - self.muxer_writer, - fps as c_int, - data.as_ptr() as *const c_void, - data.len().try_into().unwrap(), - ); - } + let mp4wr = unsafe { self.muxer_writer.as_mut().unwrap() }; + let fps = fps.try_into().unwrap(); + write_mp4(mp4wr, fps, data); } pub fn write_comment(&mut self, comment: &str) { @@ -103,3 +111,147 @@ impl Mp4Muxer { } } } + +fn get_nal_size(buf: &mut [u8], size: usize) -> usize { + let mut pos = 3; + while size - pos > 3 { + if buf[pos] == 0 && buf[pos + 1] == 0 && buf[pos + 2] == 1 { + return pos; + } + if buf[pos] == 0 && buf[pos + 1] == 0 && buf[pos + 2] == 0 && buf[pos + 3] == 1 { + return pos; + } + pos += 1; + } + size +} + +fn write_mp4(mp4wr: &mut mp4_h26x_writer_t, fps: i32, data: &[u8]) { + let mut data_size = data.len(); + let mut data_ptr = data.as_ptr(); + + while data_size > 0 { + let buf = unsafe { std::slice::from_raw_parts_mut(data_ptr as *mut u8, data_size) }; + let nal_size = get_nal_size(buf, data_size); + if nal_size < 4 { + data_ptr = unsafe { data_ptr.add(1) }; + data_size -= 1; + continue; + } + unsafe { mp4_h26x_write_nal(mp4wr, data_ptr, nal_size as i32, (90000 / fps) as u32) }; + data_ptr = unsafe { data_ptr.add(nal_size) }; + data_size -= nal_size; + } +} + +fn write_mp4_with_audio( + mp4wr: &mut mp4_h26x_writer_t, + fps: i32, + data: &[u8], + pcm: &[u8], + encoder_params: EncoderParams, +) { + let mut data_size = data.len(); + let mut data_ptr = data.as_ptr(); + + let sample_rate = encoder_params.sample_rate; + let channel_count = encoder_params.channel_count; + + let encoder = Encoder::new(encoder_params).unwrap(); + let info = encoder.info().unwrap(); + + let language: [u8; 4] = [0x75, 0x6e, 0x64, 0x00]; // und\0 + let tr: MP4E_track_t = MP4E_track_t { + object_type_indication: MP4_OBJECT_TYPE_AUDIO_ISO_IEC_14496_3, + language: language, + track_media_kind: track_media_kind_t_e_audio, + time_scale: 90000, + default_duration: 0, + u: MP4E_track_t__bindgen_ty_1 { + a: MP4E_track_t__bindgen_ty_1__bindgen_ty_1 { + channelcount: channel_count, + }, + }, + }; + + let mux = mp4wr.mux; + let audio_track_id = unsafe { MP4E_add_track(mux, &tr) }; + + unsafe { + MP4E_set_dsi( + mux, + audio_track_id, + info.confBuf.as_ptr() as *const c_void, + info.confSize.try_into().unwrap(), + ) + }; + + let length: u64 = if channel_count == 1 { 1024 } else { 2048 }; + let mut input_buffer = vec![0i16; length as usize]; + let mut output_buffer = vec![0u8; length as usize]; + + let pcm_size: u64 = pcm.len().try_into().unwrap(); + + let mut sample: u64 = 0; + let mut total_samples: u64 = pcm_size ; + let mut ts: u64 = 0; + let mut ats: u64 = 0; + + let in_args_num_in_samples = length; + let mut pcm_ptr = pcm.as_ptr(); + + while data_size > 0 { + let buf = unsafe { std::slice::from_raw_parts_mut(data_ptr as *mut u8, data_size) }; + let nal_size = get_nal_size(buf, data_size); + if nal_size < 4 { + data_ptr = unsafe { data_ptr.add(1) }; + data_size -= 1; + continue; + } + unsafe { mp4_h26x_write_nal(mp4wr, data_ptr, nal_size as i32, (90000 / fps) as u32) }; + data_ptr = unsafe { data_ptr.add(nal_size) }; + data_size -= nal_size; + + ts += 90000 / fps as u64; + while ats < ts { + let bytes_to_read = std::cmp::min(total_samples, in_args_num_in_samples as u64); + let bytes_read = bytes_to_read * 2; // 2 bytes per i16 + let pcm_buf = unsafe { + std::slice::from_raw_parts(pcm_ptr as *const i16, bytes_to_read.try_into().unwrap()) + }; + // Copy PCM data into input buffer + input_buffer[..bytes_to_read as usize].copy_from_slice(pcm_buf); + pcm_ptr = unsafe { pcm_ptr.add(bytes_read.try_into().unwrap()) }; + + // if total_samples < in_args_num_in_samples as u64 { + // total_samples = pcm_size ; + // } + + // Encode audio data using AAC encoder + match encoder.encode(&input_buffer[..bytes_to_read as usize], &mut output_buffer) { + Ok(encoding_info) => { + // Write encoded audio data to output buffer + let buf = &output_buffer[..encoding_info.output_size]; + sample += 1024; + // total_samples -= bytes_to_read; + ats = sample * 90000 / sample_rate as u64; + unsafe { + MP4E_put_sample( + mux, + audio_track_id, + buf.as_ptr() as *mut c_void, + encoding_info.output_size.try_into().unwrap(), + (1024 * 90000 / sample_rate as usize).try_into().unwrap(), + MP4E_SAMPLE_RANDOM_ACCESS.try_into().unwrap(), + ) + }; + + } + Err(e) => { + println!("encode error:{}", e); + break; + } + } + } + } +} diff --git a/src/main.rs b/src/main.rs index 5b2179e..c7cf4b6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,9 @@ -mod lib; - -use lib::Mp4Muxer; use std::fs::File; -use std::io::{self, Cursor, Read}; +use std::io::{self, Cursor, Read, Seek, SeekFrom}; use std::path::Path; +use minimp4::Mp4Muxer; + fn read_file>(path: P) -> io::Result> { let mut buf = vec![]; File::open(path)?.read_to_end(&mut buf)?; @@ -12,16 +11,23 @@ fn read_file>(path: P) -> io::Result> { } fn main() { - let buffer = Cursor::new(vec![]); - let mut mp4muxer = Mp4Muxer::new(buffer); - if let Ok(h264) = read_file("1.264") { - mp4muxer.init_video(1920, 1200, false, "h264 stream"); - mp4muxer.write_video(&h264); - } - if let Ok(h265) = read_file("1.265") { - mp4muxer.init_video(1920, 1200, false, "h265 stream"); - mp4muxer.write_video(&h265); + let mut buffer = Cursor::new(vec![]); + let mut mp4muxer = Mp4Muxer::new(&mut buffer); + if let Ok(h264) = read_file("input.264") { + let pcm = read_file("input.pcm").unwrap(); + mp4muxer.init_video(1280, 720, false, "h264 stream"); + // mp4muxer.write_video_with_fps(&h264, 28); + mp4muxer.init_audio(128000,8000, 1); + mp4muxer.write_video_with_audio(&h264, 30, &pcm); } + // if let Ok(h265) = read_file("1.265") { + // mp4muxer.init_video(1920, 1200, false, "h265 stream"); + // mp4muxer.write_video(&h265); + // } mp4muxer.write_comment("test comment"); mp4muxer.close(); + buffer.seek(SeekFrom::Start(0)).unwrap(); + let mut video_bytes = Vec::new(); + buffer.read_to_end(&mut video_bytes).unwrap(); + std::fs::write( Path::new("output.mp4"), video_bytes).unwrap(); }