Android中音视频合成的几种方案详析

前言

最近工作中遇到了音视频处理的需求，Android下音视频合成，在当前调研方案中主要有三大类方法：MediaMux硬解码，mp4parser，FFmepg。三种方法均可实现，但是也有不同的局限和问题，先将实现和问题记录于此，便于之后的总结学习。下面话不多说了，来一起看看详细的介绍吧。

方法一（Fail）

利用MediaMux实现音视频的合成。

效果：可以实现音视频的合并，利用Android原生的VideoView和SurfaceView播放正常，大部分的播放器也播放正常，但是，但是，在上传Youtube就会出现问题：音频不连续，分析主要是上传Youtube时会被再次的压缩，可能在压缩的过程中出现音频的帧率出现问题。

分析：在MediaCodec.BufferInfo的处理中，时间戳presentationTimeUs出现问题，导致Youtube的压缩造成音频的紊乱。

public static void muxVideoAndAudio(String videoPath, String audioPath, String muxPath) { try {  MediaExtractor videoExtractor = new MediaExtractor();  videoExtractor.setDataSource(videoPath);  MediaFormat videoFormat = null;  int videoTrackIndex = -1;  int videoTrackCount = videoExtractor.getTrackCount();  for (int i = 0; i < videoTrackCount; i++) {  videoFormat = videoExtractor.getTrackFormat(i);  String mimeType = videoFormat.getString(MediaFormat.KEY_MIME);  if (mimeType.startsWith("video/")) {   videoTrackIndex = i;   break;  }  }  MediaExtractor audioExtractor = new MediaExtractor();  audioExtractor.setDataSource(audioPath);  MediaFormat audioFormat = null;  int audioTrackIndex = -1;  int audioTrackCount = audioExtractor.getTrackCount();  for (int i = 0; i < audioTrackCount; i++) {  audioFormat = audioExtractor.getTrackFormat(i);  String mimeType = audioFormat.getString(MediaFormat.KEY_MIME);  if (mimeType.startsWith("audio/")) {   audioTrackIndex = i;   break;  }  }  videoExtractor.selectTrack(videoTrackIndex);  audioExtractor.selectTrack(audioTrackIndex);  MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();  MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();  MediaMuxer mediaMuxer = new MediaMuxer(muxPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);  int writeVideoTrackIndex = mediaMuxer.addTrack(videoFormat);  int writeAudioTrackIndex = mediaMuxer.addTrack(audioFormat);  mediaMuxer.start();  ByteBuffer byteBuffer = ByteBuffer.allocate(500 * 1024);  long sampleTime = 0;  {  videoExtractor.readSampleData(byteBuffer, 0);  if (videoExtractor.getSampleFlags() == MediaExtractor.SAMPLE_FLAG_SYNC) {   videoExtractor.advance();  }  videoExtractor.readSampleData(byteBuffer, 0);  long secondTime = videoExtractor.getSampleTime();  videoExtractor.advance();  long thirdTime = videoExtractor.getSampleTime();  sampleTime = Math.abs(thirdTime - secondTime);  }  videoExtractor.unselectTrack(videoTrackIndex);  videoExtractor.selectTrack(videoTrackIndex);  while (true) {  int readVideoSampleSize = videoExtractor.readSampleData(byteBuffer, 0);  if (readVideoSampleSize < 0) {   break;  }  videoBufferInfo.size = readVideoSampleSize;  videoBufferInfo.presentationTimeUs += sampleTime;  videoBufferInfo.offset = 0;  //noinspection WrongConstant  videoBufferInfo.flags = MediaCodec.BUFFER_FLAG_SYNC_FRAME;//videoExtractor.getSampleFlags()  mediaMuxer.writeSampleData(writeVideoTrackIndex, byteBuffer, videoBufferInfo);  videoExtractor.advance();  }  while (true) {  int readAudioSampleSize = audioExtractor.readSampleData(byteBuffer, 0);  if (readAudioSampleSize < 0) {   break;  }  audioBufferInfo.size = readAudioSampleSize;  audioBufferInfo.presentationTimeUs += sampleTime;  audioBufferInfo.offset = 0;  //noinspection WrongConstant  audioBufferInfo.flags = MediaCodec.BUFFER_FLAG_SYNC_FRAME;// videoExtractor.getSampleFlags()  mediaMuxer.writeSampleData(writeAudioTrackIndex, byteBuffer, audioBufferInfo);  audioExtractor.advance();  }  mediaMuxer.stop();  mediaMuxer.release();  videoExtractor.release();  audioExtractor.release(); } catch (IOException e) {  e.printStackTrace(); } }

方法二（Success）

public static void muxVideoAudio(String videoFilePath, String audioFilePath, String outputFile) { try {  MediaExtractor videoExtractor = new MediaExtractor();  videoExtractor.setDataSource(videoFilePath);  MediaExtractor audioExtractor = new MediaExtractor();  audioExtractor.setDataSource(audioFilePath);  MediaMuxer muxer = new MediaMuxer(outputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);  videoExtractor.selectTrack(0);  MediaFormat videoFormat = videoExtractor.getTrackFormat(0);  int videoTrack = muxer.addTrack(videoFormat);  audioExtractor.selectTrack(0);  MediaFormat audioFormat = audioExtractor.getTrackFormat(0);  int audioTrack = muxer.addTrack(audioFormat);  LogUtil.d(TAG, "Video Format " + videoFormat.toString());  LogUtil.d(TAG, "Audio Format " + audioFormat.toString());  boolean sawEOS = false;  int frameCount = 0;  int offset = 100;  int sampleSize = 256 * 1024;  ByteBuffer videoBuf = ByteBuffer.allocate(sampleSize);  ByteBuffer audioBuf = ByteBuffer.allocate(sampleSize);  MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();  MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();  videoExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);  audioExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);  muxer.start();  while (!sawEOS) {  videoBufferInfo.offset = offset;  videoBufferInfo.size = videoExtractor.readSampleData(videoBuf, offset);  if (videoBufferInfo.size < 0 || audioBufferInfo.size < 0) {   sawEOS = true;   videoBufferInfo.size = 0;  } else {   videoBufferInfo.presentationTimeUs = videoExtractor.getSampleTime();   //noinspection WrongConstant   videoBufferInfo.flags = videoExtractor.getSampleFlags();   muxer.writeSampleData(videoTrack, videoBuf, videoBufferInfo);   videoExtractor.advance();   frameCount++;  }  }    boolean sawEOS2 = false;  int frameCount2 = 0;  while (!sawEOS2) {  frameCount2++;  audioBufferInfo.offset = offset;  audioBufferInfo.size = audioExtractor.readSampleData(audioBuf, offset);  if (videoBufferInfo.size < 0 || audioBufferInfo.size < 0) {   sawEOS2 = true;   audioBufferInfo.size = 0;  } else {   audioBufferInfo.presentationTimeUs = audioExtractor.getSampleTime();   //noinspection WrongConstant   audioBufferInfo.flags = audioExtractor.getSampleFlags();   muxer.writeSampleData(audioTrack, audioBuf, audioBufferInfo);   audioExtractor.advance();  }  }  muxer.stop();  muxer.release();  LogUtil.d(TAG,"Output: "+outputFile); } catch (IOException e) {  LogUtil.d(TAG, "Mixer Error 1 " + e.getMessage()); } catch (Exception e) {  LogUtil.d(TAG, "Mixer Error 2 " + e.getMessage()); } }

方法三

利用mp4parser实现

mp4parser是一个视频处理的开源工具箱，由于mp4parser里的方法都依靠工具箱里的一些内容，所以需要将这些内容打包成jar包，放到自己的工程里，才能对mp4parser的方法进行调用。

compile “com.googlecode.mp4parser:isoparser:1.1.21”

问题：上传Youtube压缩后，视频数据丢失严重，大部分就只剩下一秒钟的时长，相当于把视频变成图片了，��

 public boolean mux(String videoFile, String audioFile, final String outputFile) { if (isStopMux) {  return false; } Movie video; try {  video = MovieCreator.build(videoFile); } catch (RuntimeException e) {  e.printStackTrace();  return false; } catch (IOException e) {  e.printStackTrace();  return false; } Movie audio; try {  audio = MovieCreator.build(audioFile); } catch (IOException e) {  e.printStackTrace();  return false; } catch (NullPointerException e) {  e.printStackTrace();  return false; } Track audioTrack = audio.getTracks().get(0); video.addTrack(audioTrack); Container out = new DefaultMp4Builder().build(video); FileOutputStream fos; try {  fos = new FileOutputStream(outputFile); } catch (FileNotFoundException e) {  e.printStackTrace();  return false; } BufferedWritableFileByteChannel byteBufferByteChannel = new  BufferedWritableFileByteChannel(fos); try {  out.writeContainer(byteBufferByteChannel);  byteBufferByteChannel.close();  fos.close();  if (isStopMux) {  return false;  }  runOnUiThread(new Runnable() {  @Override  public void run() {   mCustomeProgressDialog.setProgress(100);   goShareActivity(outputFile);//   FileUtils.insertMediaDB(AddAudiosActivity.this,outputFile);//  }  }); } catch (IOException e) {  e.printStackTrace();  if (mCustomeProgressDialog.isShowing()) {  mCustomeProgressDialog.dismiss();  }  ToastUtil.showShort(getString(R.string.process_failed));  return false; } return true; } private static class BufferedWritableFileByteChannel implements WritableByteChannel { private static final int BUFFER_CAPACITY = 2000000; private boolean isOpen = true; private final OutputStream outputStream; private final ByteBuffer byteBuffer; private final byte[] rawBuffer = new byte[BUFFER_CAPACITY]; private BufferedWritableFileByteChannel(OutputStream outputStream) {  this.outputStream = outputStream;  this.byteBuffer = ByteBuffer.wrap(rawBuffer); } @Override public int write(ByteBuffer inputBuffer) throws IOException {  int inputBytes = inputBuffer.remaining();  if (inputBytes > byteBuffer.remaining()) {  dumpToFile();  byteBuffer.clear();  if (inputBytes > byteBuffer.remaining()) {   throw new BufferOverflowException();  }  }  byteBuffer.put(inputBuffer);  return inputBytes; } @Override public boolean isOpen() {  return isOpen; } @Override public void close() throws IOException {  dumpToFile();  isOpen = false; } private void dumpToFile() {  try {  outputStream.write(rawBuffer, 0, byteBuffer.position());  } catch (IOException e) {  throw new RuntimeException(e);  } } }

方法四

利用FFmpeg大法

FFmpeg 由于其丰富的 codec 插件，详细的文档说明，并且与其调试复杂量大的编解码代码(是的，用 MediaCodec 实现起来十分��嗦和繁琐)还是不如调试一行 ffmpeg 命令来的简单。

Merge Video /Audio and retain both audios

可以实现，兼容性强，但由于是软解码，合并速度很慢，忍受不了，而相应的FFmpeg优化还不太了解，�濉�….

总结

以上就是这篇文章的全部内容了，希望本文的内容对大家的学习或者工作具有一定的参考学习价值，如果有疑问大家可以留言交流，谢谢大家对脚本之家的支持。

更多相关文章

随机推荐