From fb9a53d96449dbe8053e4ec35e40ebd6f884b5ae Mon Sep 17 00:00:00 2001 From: mudar Date: Fri, 20 Dec 2019 15:37:27 -0500 Subject: [PATCH 01/13] Add support for trim, related to issue #37 - New component TrimDataSource, wrapping DataSource to be trimmed. - MediaExtractorDataSource is an abstract class to limit visibility of MediaExtractor to package - Updates to Engine to replace selectAudio/transcode/selectVideo/transcode sequence by selectAudio/selectVideo/transcode/transcode --- .../transcoder/engine/Engine.java | 9 +- .../transcoder/source/DefaultDataSource.java | 8 +- .../source/MediaExtractorDataSource.java | 10 + .../transcoder/source/TrimDataSource.java | 177 ++++++++++++++++++ 4 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/source/MediaExtractorDataSource.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index c4511f1a..4fa35061 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -364,10 +364,13 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce // Now step for transcoders that are not completed. audioCompleted = isCompleted(TrackType.AUDIO); videoCompleted = isCompleted(TrackType.VIDEO); - if (!audioCompleted) { + if (!audioCompleted && !videoCompleted) { + final TrackTranscoder videoTranscoder = getCurrentTrackTranscoder(TrackType.VIDEO, options); + final TrackTranscoder audioTranscoder = getCurrentTrackTranscoder(TrackType.AUDIO, options); + stepped |= videoTranscoder.transcode(forceVideoEos) | audioTranscoder.transcode(forceAudioEos); + } else if (!audioCompleted) { stepped |= getCurrentTrackTranscoder(TrackType.AUDIO, options).transcode(forceAudioEos); - } - if (!videoCompleted) { + } else if (!videoCompleted) { stepped |= getCurrentTrackTranscoder(TrackType.VIDEO, options).transcode(forceVideoEos); } if (++loopCount % PROGRESS_INTERVAL_STEPS == 0) { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java index e9140dca..0b333bb0 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java @@ -18,7 +18,7 @@ /** * A DataSource implementation that uses Android's Media APIs. */ -public abstract class DefaultDataSource implements DataSource { +public abstract class DefaultDataSource extends MediaExtractorDataSource { private final static String TAG = DefaultDataSource.class.getSimpleName(); private final static Logger LOG = new Logger(TAG); @@ -214,4 +214,10 @@ public void rewind() { mMetadata = new MediaMetadataRetriever(); mMetadataApplied = false; } + + @Override + protected MediaExtractor requireExtractor() { + ensureExtractor(); + return mExtractor; + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/MediaExtractorDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/MediaExtractorDataSource.java new file mode 100644 index 00000000..31bae1c6 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/MediaExtractorDataSource.java @@ -0,0 +1,10 @@ +package com.otaliastudios.transcoder.source; + +import android.media.MediaExtractor; + +/** + * DataSource that allows access to its MediaExtractor. + */ +abstract class MediaExtractorDataSource implements DataSource { + abstract protected MediaExtractor requireExtractor(); +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java new file mode 100644 index 00000000..b6be47e4 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -0,0 +1,177 @@ +package com.otaliastudios.transcoder.source; + + +import android.media.MediaExtractor; +import android.media.MediaFormat; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import com.otaliastudios.transcoder.engine.TrackType; +import com.otaliastudios.transcoder.internal.Logger; + +import org.jetbrains.annotations.Contract; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +/** + * A {@link DataSource} wrapper that trims source at both ends. + */ +public class TrimDataSource implements DataSource { + private static final String TAG = "TrimDataSource"; + private static final Logger LOG = new Logger(TAG); + private static final int UNKNOWN = -1; + + @NonNull + private MediaExtractorDataSource source; + private long trimStartUs; + private long trimDurationUs; + private boolean isVideoTrackReady = false; + private boolean hasSelectedVideoTrack = false; + + public TrimDataSource(@NonNull MediaExtractorDataSource source, long trimStartMillis, long trimEndMillis) { + this.source = source; + this.trimStartUs = MILLISECONDS.toMicros(trimStartMillis); + final long trimEndUs = MILLISECONDS.toMicros(trimEndMillis); + this.trimDurationUs = computeTrimDuration(source.getDurationUs(), trimStartUs, trimEndUs); + } + + @Contract(pure = true) + private static long computeTrimDuration(long duration, long trimStart, long trimEnd) { + if (duration == UNKNOWN) { + return UNKNOWN; + } else { + final long result = duration - trimStart - trimEnd; + return result >= 0 ? result : UNKNOWN; + } + } + + @Override + public int getOrientation() { + return source.getOrientation(); + } + + @Nullable + @Override + public double[] getLocation() { + return source.getLocation(); + } + + @Override + public long getDurationUs() { + return trimDurationUs; + } + + @Nullable + @Override + public MediaFormat getTrackFormat(@NonNull TrackType type) { + final MediaFormat trackFormat = source.getTrackFormat(type); + if (trackFormat != null) { + trackFormat.setLong(MediaFormat.KEY_DURATION, trimDurationUs); + } + return trackFormat; + } + + @Override + public void selectTrack(@NonNull TrackType type) { + if (trimStartUs > 0) { + switch (type) { + case AUDIO: + if (hasTrack(TrackType.VIDEO) && !hasSelectedVideoTrack) { + selectAndSeekVideoTrack(); + } + source.selectTrack(TrackType.AUDIO); + break; + case VIDEO: + if (!hasSelectedVideoTrack) { + selectAndSeekVideoTrack(); + } + break; + } + } else { + source.selectTrack(type); + } + } + + private boolean hasTrack(@NonNull TrackType type) { + return source.getTrackFormat(type) != null; + } + + private void selectAndSeekVideoTrack() { + source.selectTrack(TrackType.VIDEO); + source.requireExtractor().seekTo(trimStartUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + hasSelectedVideoTrack = true; + } + + /** + * Check if trim operation was completed successfully for selected track. + * We apply the seek operation for the video track only, so all audio frames are skipped + * until MediaExtractor reaches the first video key frame. + */ + private boolean isTrackReady(@NonNull TrackType type) { + if (isVideoTrackReady) { + return true; + } + final MediaExtractor extractor = source.requireExtractor(); + if (type == TrackType.VIDEO) { + final boolean isKeyFrame = (extractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; + if (isKeyFrame) { + final long originalTrimStartUs = trimStartUs; + trimStartUs = extractor.getSampleTime(); + trimDurationUs += originalTrimStartUs - trimStartUs; + LOG.v("First video key frame is at " + trimStartUs + ", actual duration will be " + trimDurationUs); + isVideoTrackReady = true; + return true; + } + } + extractor.advance(); + return false; + } + + @Override + public boolean canReadTrack(@NonNull TrackType type) { + boolean canRead = source.canReadTrack(type); + + if (canRead) { + return isTrackReady(type); + } else { + return false; + } + } + + @Override + public void readTrack(@NonNull Chunk chunk) { + source.readTrack(chunk); + chunk.timestampUs -= trimStartUs; + } + + @Override + public long getReadUs() { + return source.getReadUs(); + } + + @Override + public boolean isDrained() { + return source.isDrained(); + } + + @Override + public void releaseTrack(@NonNull TrackType type) { + switch (type) { + case AUDIO: + hasSelectedVideoTrack = false; + break; + case VIDEO: + isVideoTrackReady = false; + break; + } + source.releaseTrack(type); + } + + @Override + public void rewind() { + hasSelectedVideoTrack = false; + isVideoTrackReady = false; + source.rewind(); + } +} From 4c1a3afb13079e960f176617fe2a16e101a9b6b7 Mon Sep 17 00:00:00 2001 From: mudar Date: Fri, 20 Dec 2019 15:45:26 -0500 Subject: [PATCH 02/13] Added support for TrimDataSource into demo app Using 2 editText fields, default value is zero. --- .../transcoder/demo/TranscoderActivity.java | 84 +++++++++++++++++-- .../main/res/layout/activity_transcoder.xml | 49 +++++++++++ 2 files changed, 126 insertions(+), 7 deletions(-) diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index 478fd41e..cb2fd9f5 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -3,10 +3,12 @@ import android.annotation.SuppressLint; import android.content.ClipData; import android.content.Intent; -import android.media.MediaMuxer; import android.net.Uri; import android.os.Bundle; import android.os.SystemClock; +import android.text.Editable; +import android.text.TextWatcher; +import android.widget.EditText; import android.widget.ProgressBar; import android.widget.RadioGroup; import android.widget.TextView; @@ -20,6 +22,8 @@ import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.sink.DataSink; import com.otaliastudios.transcoder.sink.DefaultDataSink; +import com.otaliastudios.transcoder.source.TrimDataSource; +import com.otaliastudios.transcoder.source.UriDataSource; import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy; import com.otaliastudios.transcoder.strategy.RemoveTrackStrategy; @@ -61,6 +65,8 @@ public class TranscoderActivity extends AppCompatActivity implements private ProgressBar mProgressView; private TextView mButtonView; + private EditText mTrimStartView; + private EditText mTrimEndView; private TextView mAudioReplaceView; private boolean mIsTranscoding; @@ -74,6 +80,52 @@ public class TranscoderActivity extends AppCompatActivity implements private long mTranscodeStartTime; private TrackStrategy mTranscodeVideoStrategy; private TrackStrategy mTranscodeAudioStrategy; + private long mTrimStartMillis = 0; + private long mTrimEndMillis = 0; + + private TextWatcher mTrimStartTextWatcher = new TextWatcher() { + @Override + public void beforeTextChanged(CharSequence s, int start, int count, int after) { + } + + @Override + public void onTextChanged(CharSequence s, int start, int before, int count) { + } + + @Override + public void afterTextChanged(Editable s) { + if (s.length() > 0) { + try { + mTrimStartMillis = Long.valueOf(s.toString()) * 1000; + } catch (NumberFormatException e) { + mTrimStartMillis = 0; + LOG.w("Failed to read trimStart value."); + } + } + } + }; + private TextWatcher mTrimEndTextWatcher = new TextWatcher() { + @Override + public void beforeTextChanged(CharSequence s, int start, int count, int after) { + } + + @Override + public void onTextChanged(CharSequence s, int start, int before, int count) { + } + + @Override + public void afterTextChanged(Editable s) { + if (s.length() > 0) { + try { + mTrimEndMillis = Long.valueOf(s.toString()) * 1000; + } catch (NumberFormatException e) { + mTrimEndMillis = 0; + LOG.w("Failed to read trimEnd value."); + } + } + } + }; + @SuppressLint("SetTextI18n") @Override @@ -97,6 +149,8 @@ protected void onCreate(Bundle savedInstanceState) { mProgressView = findViewById(R.id.progress); mProgressView.setMax(PROGRESS_BAR_MAX); + mTrimStartView = findViewById(R.id.trim_start); + mTrimEndView = findViewById(R.id.trim_end); mAudioReplaceView = findViewById(R.id.replace_info); mAudioChannelsGroup = findViewById(R.id.channels); @@ -113,6 +167,8 @@ protected void onCreate(Bundle savedInstanceState) { mVideoResolutionGroup.setOnCheckedChangeListener(this); mVideoAspectGroup.setOnCheckedChangeListener(this); mAudioSampleRateGroup.setOnCheckedChangeListener(this); + mTrimStartView.addTextChangedListener(mTrimStartTextWatcher); + mTrimEndView.addTextChangedListener(mTrimEndTextWatcher); syncParameters(); mAudioReplaceGroup.setOnCheckedChangeListener((group, checkedId) -> { @@ -257,13 +313,27 @@ private void transcode() { DataSink sink = new DefaultDataSink(mTranscodeOutputFile.getAbsolutePath()); TranscoderOptions.Builder builder = Transcoder.into(sink); if (mAudioReplacementUri == null) { - if (mTranscodeInputUri1 != null) builder.addDataSource(this, mTranscodeInputUri1); - if (mTranscodeInputUri2 != null) builder.addDataSource(this, mTranscodeInputUri2); - if (mTranscodeInputUri3 != null) builder.addDataSource(this, mTranscodeInputUri3); + if (mTrimStartMillis > 0 || mTrimEndMillis > 0) { + if (mTranscodeInputUri1 != null) builder.addDataSource(new TrimDataSource(new UriDataSource(this, mTranscodeInputUri1), mTrimStartMillis, mTrimEndMillis)); + if (mTranscodeInputUri2 != null) builder.addDataSource(new TrimDataSource(new UriDataSource(this, mTranscodeInputUri2), mTrimStartMillis, mTrimEndMillis)); + if (mTranscodeInputUri3 != null) builder.addDataSource(new TrimDataSource(new UriDataSource(this, mTranscodeInputUri3), mTrimStartMillis, mTrimEndMillis)); + } + else { + if (mTranscodeInputUri1 != null) builder.addDataSource(this, mTranscodeInputUri1); + if (mTranscodeInputUri2 != null) builder.addDataSource(this, mTranscodeInputUri2); + if (mTranscodeInputUri3 != null) builder.addDataSource(this, mTranscodeInputUri3); + } } else { - if (mTranscodeInputUri1 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri1); - if (mTranscodeInputUri2 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri2); - if (mTranscodeInputUri3 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri3); + if (mTrimStartMillis > 0 || mTrimEndMillis > 0) { + if (mTranscodeInputUri1 != null) builder.addDataSource(TrackType.VIDEO, new TrimDataSource(new UriDataSource(this, mTranscodeInputUri1), mTrimStartMillis, mTrimEndMillis)); + if (mTranscodeInputUri2 != null) builder.addDataSource(TrackType.VIDEO, new TrimDataSource(new UriDataSource(this, mTranscodeInputUri2), mTrimStartMillis, mTrimEndMillis)); + if (mTranscodeInputUri3 != null) builder.addDataSource(TrackType.VIDEO, new TrimDataSource(new UriDataSource(this, mTranscodeInputUri3), mTrimStartMillis, mTrimEndMillis)); + } + else { + if (mTranscodeInputUri1 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri1); + if (mTranscodeInputUri2 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri2); + if (mTranscodeInputUri3 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri3); + } builder.addDataSource(TrackType.AUDIO, this, mAudioReplacementUri); } mTranscodeFuture = builder.setListener(this) diff --git a/demo/src/main/res/layout/activity_transcoder.xml b/demo/src/main/res/layout/activity_transcoder.xml index e142c221..9cbcafb7 100644 --- a/demo/src/main/res/layout/activity_transcoder.xml +++ b/demo/src/main/res/layout/activity_transcoder.xml @@ -285,6 +285,55 @@ android:layout_height="wrap_content" /> + + + + + + + + + + + + + + Date: Fri, 20 Dec 2019 15:47:47 -0500 Subject: [PATCH 03/13] TranscoderOptions Builder support for TrimDataSource Builder can add directly trim values for UriDataSource --- .../transcoder/demo/TranscoderActivity.java | 14 ++++++-------- .../transcoder/TranscoderOptions.java | 13 +++++++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index cb2fd9f5..690df694 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -22,8 +22,6 @@ import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.sink.DataSink; import com.otaliastudios.transcoder.sink.DefaultDataSink; -import com.otaliastudios.transcoder.source.TrimDataSource; -import com.otaliastudios.transcoder.source.UriDataSource; import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy; import com.otaliastudios.transcoder.strategy.RemoveTrackStrategy; @@ -314,9 +312,9 @@ private void transcode() { TranscoderOptions.Builder builder = Transcoder.into(sink); if (mAudioReplacementUri == null) { if (mTrimStartMillis > 0 || mTrimEndMillis > 0) { - if (mTranscodeInputUri1 != null) builder.addDataSource(new TrimDataSource(new UriDataSource(this, mTranscodeInputUri1), mTrimStartMillis, mTrimEndMillis)); - if (mTranscodeInputUri2 != null) builder.addDataSource(new TrimDataSource(new UriDataSource(this, mTranscodeInputUri2), mTrimStartMillis, mTrimEndMillis)); - if (mTranscodeInputUri3 != null) builder.addDataSource(new TrimDataSource(new UriDataSource(this, mTranscodeInputUri3), mTrimStartMillis, mTrimEndMillis)); + if (mTranscodeInputUri1 != null) builder.addDataSource(this, mTranscodeInputUri1, mTrimStartMillis, mTrimEndMillis); + if (mTranscodeInputUri2 != null) builder.addDataSource(this, mTranscodeInputUri2, mTrimStartMillis, mTrimEndMillis); + if (mTranscodeInputUri3 != null) builder.addDataSource(this, mTranscodeInputUri3, mTrimStartMillis, mTrimEndMillis); } else { if (mTranscodeInputUri1 != null) builder.addDataSource(this, mTranscodeInputUri1); @@ -325,9 +323,9 @@ private void transcode() { } } else { if (mTrimStartMillis > 0 || mTrimEndMillis > 0) { - if (mTranscodeInputUri1 != null) builder.addDataSource(TrackType.VIDEO, new TrimDataSource(new UriDataSource(this, mTranscodeInputUri1), mTrimStartMillis, mTrimEndMillis)); - if (mTranscodeInputUri2 != null) builder.addDataSource(TrackType.VIDEO, new TrimDataSource(new UriDataSource(this, mTranscodeInputUri2), mTrimStartMillis, mTrimEndMillis)); - if (mTranscodeInputUri3 != null) builder.addDataSource(TrackType.VIDEO, new TrimDataSource(new UriDataSource(this, mTranscodeInputUri3), mTrimStartMillis, mTrimEndMillis)); + if (mTranscodeInputUri1 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri1, mTrimStartMillis, mTrimEndMillis); + if (mTranscodeInputUri2 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri2, mTrimStartMillis, mTrimEndMillis); + if (mTranscodeInputUri3 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri3, mTrimStartMillis, mTrimEndMillis); } else { if (mTranscodeInputUri1 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri1); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index 8ead0cfb..98e2f632 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -13,6 +13,7 @@ import com.otaliastudios.transcoder.source.DataSource; import com.otaliastudios.transcoder.source.FileDescriptorDataSource; import com.otaliastudios.transcoder.source.FilePathDataSource; +import com.otaliastudios.transcoder.source.TrimDataSource; import com.otaliastudios.transcoder.source.UriDataSource; import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; import com.otaliastudios.transcoder.strategy.DefaultVideoStrategies; @@ -174,12 +175,24 @@ public Builder addDataSource(@NonNull Context context, @NonNull Uri uri) { return addDataSource(new UriDataSource(context, uri)); } + @NonNull + @SuppressWarnings({"unused", "UnusedReturnValue"}) + public Builder addDataSource(@NonNull Context context, @NonNull Uri uri, long trimStartMs, long trimEndMs) { + return addDataSource(new TrimDataSource(new UriDataSource(context, uri), trimStartMs, trimEndMs)); + } + @NonNull @SuppressWarnings({"unused", "UnusedReturnValue"}) public Builder addDataSource(@NonNull TrackType type, @NonNull Context context, @NonNull Uri uri) { return addDataSource(type, new UriDataSource(context, uri)); } + @NonNull + @SuppressWarnings({"unused", "UnusedReturnValue"}) + public Builder addDataSource(@NonNull TrackType type, @NonNull Context context, @NonNull Uri uri, long trimStartMs, long trimEndMs) { + return addDataSource(type, new TrimDataSource(new UriDataSource(context, uri), trimStartMs, trimEndMs)); + } + /** * Sets the audio output strategy. If absent, this defaults to * {@link com.otaliastudios.transcoder.strategy.DefaultAudioStrategy}. From 8d8ea9369b492f66dd84762baacbd9f56cb7d61e Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Sat, 21 Dec 2019 11:30:51 -0500 Subject: [PATCH 04/13] TrimDataSource updates following PR code review - fixed case where video track is absent - throw exceptions for invalid trim values --- .../transcoder/demo/TranscoderActivity.java | 28 ++++---- .../transcoder/TranscoderOptions.java | 8 +-- .../transcoder/source/TrimDataSource.java | 64 +++++++++---------- 3 files changed, 47 insertions(+), 53 deletions(-) diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index 690df694..40b7f995 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -78,8 +78,8 @@ public class TranscoderActivity extends AppCompatActivity implements private long mTranscodeStartTime; private TrackStrategy mTranscodeVideoStrategy; private TrackStrategy mTranscodeAudioStrategy; - private long mTrimStartMillis = 0; - private long mTrimEndMillis = 0; + private long mTrimStartUs = 0; + private long mTrimEndUs = 0; private TextWatcher mTrimStartTextWatcher = new TextWatcher() { @Override @@ -94,9 +94,9 @@ public void onTextChanged(CharSequence s, int start, int before, int count) { public void afterTextChanged(Editable s) { if (s.length() > 0) { try { - mTrimStartMillis = Long.valueOf(s.toString()) * 1000; + mTrimStartUs = Long.valueOf(s.toString()) * 1000000; } catch (NumberFormatException e) { - mTrimStartMillis = 0; + mTrimStartUs = 0; LOG.w("Failed to read trimStart value."); } } @@ -115,9 +115,9 @@ public void onTextChanged(CharSequence s, int start, int before, int count) { public void afterTextChanged(Editable s) { if (s.length() > 0) { try { - mTrimEndMillis = Long.valueOf(s.toString()) * 1000; + mTrimEndUs = Long.valueOf(s.toString()) * 1000000; } catch (NumberFormatException e) { - mTrimEndMillis = 0; + mTrimEndUs = 0; LOG.w("Failed to read trimEnd value."); } } @@ -311,10 +311,10 @@ private void transcode() { DataSink sink = new DefaultDataSink(mTranscodeOutputFile.getAbsolutePath()); TranscoderOptions.Builder builder = Transcoder.into(sink); if (mAudioReplacementUri == null) { - if (mTrimStartMillis > 0 || mTrimEndMillis > 0) { - if (mTranscodeInputUri1 != null) builder.addDataSource(this, mTranscodeInputUri1, mTrimStartMillis, mTrimEndMillis); - if (mTranscodeInputUri2 != null) builder.addDataSource(this, mTranscodeInputUri2, mTrimStartMillis, mTrimEndMillis); - if (mTranscodeInputUri3 != null) builder.addDataSource(this, mTranscodeInputUri3, mTrimStartMillis, mTrimEndMillis); + if (mTrimStartUs > 0 || mTrimEndUs > 0) { + if (mTranscodeInputUri1 != null) builder.addDataSource(this, mTranscodeInputUri1, mTrimStartUs, mTrimEndUs); + if (mTranscodeInputUri2 != null) builder.addDataSource(this, mTranscodeInputUri2, mTrimStartUs, mTrimEndUs); + if (mTranscodeInputUri3 != null) builder.addDataSource(this, mTranscodeInputUri3, mTrimStartUs, mTrimEndUs); } else { if (mTranscodeInputUri1 != null) builder.addDataSource(this, mTranscodeInputUri1); @@ -322,10 +322,10 @@ private void transcode() { if (mTranscodeInputUri3 != null) builder.addDataSource(this, mTranscodeInputUri3); } } else { - if (mTrimStartMillis > 0 || mTrimEndMillis > 0) { - if (mTranscodeInputUri1 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri1, mTrimStartMillis, mTrimEndMillis); - if (mTranscodeInputUri2 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri2, mTrimStartMillis, mTrimEndMillis); - if (mTranscodeInputUri3 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri3, mTrimStartMillis, mTrimEndMillis); + if (mTrimStartUs > 0 || mTrimEndUs > 0) { + if (mTranscodeInputUri1 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri1, mTrimStartUs, mTrimEndUs); + if (mTranscodeInputUri2 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri2, mTrimStartUs, mTrimEndUs); + if (mTranscodeInputUri3 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri3, mTrimStartUs, mTrimEndUs); } else { if (mTranscodeInputUri1 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri1); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index 98e2f632..cd3a506f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -177,8 +177,8 @@ public Builder addDataSource(@NonNull Context context, @NonNull Uri uri) { @NonNull @SuppressWarnings({"unused", "UnusedReturnValue"}) - public Builder addDataSource(@NonNull Context context, @NonNull Uri uri, long trimStartMs, long trimEndMs) { - return addDataSource(new TrimDataSource(new UriDataSource(context, uri), trimStartMs, trimEndMs)); + public Builder addDataSource(@NonNull Context context, @NonNull Uri uri, long trimStartUs, long trimEndUs) { + return addDataSource(new TrimDataSource(new UriDataSource(context, uri), trimStartUs, trimEndUs)); } @NonNull @@ -189,8 +189,8 @@ public Builder addDataSource(@NonNull TrackType type, @NonNull Context context, @NonNull @SuppressWarnings({"unused", "UnusedReturnValue"}) - public Builder addDataSource(@NonNull TrackType type, @NonNull Context context, @NonNull Uri uri, long trimStartMs, long trimEndMs) { - return addDataSource(type, new TrimDataSource(new UriDataSource(context, uri), trimStartMs, trimEndMs)); + public Builder addDataSource(@NonNull TrackType type, @NonNull Context context, @NonNull Uri uri, long trimStartUs, long trimEndUs) { + return addDataSource(type, new TrimDataSource(new UriDataSource(context, uri), trimStartUs, trimEndUs)); } /** diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index b6be47e4..6dc3af35 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -12,38 +12,36 @@ import org.jetbrains.annotations.Contract; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - /** * A {@link DataSource} wrapper that trims source at both ends. */ public class TrimDataSource implements DataSource { private static final String TAG = "TrimDataSource"; private static final Logger LOG = new Logger(TAG); - private static final int UNKNOWN = -1; - + private final boolean hasVideoTrack; @NonNull private MediaExtractorDataSource source; private long trimStartUs; private long trimDurationUs; - private boolean isVideoTrackReady = false; + private boolean isSeekTrackReady = false; private boolean hasSelectedVideoTrack = false; - public TrimDataSource(@NonNull MediaExtractorDataSource source, long trimStartMillis, long trimEndMillis) { + public TrimDataSource(@NonNull MediaExtractorDataSource source, long trimStartUs, long trimEndUs) { + if (trimStartUs < 0 || trimEndUs < 0) { + throw new IllegalArgumentException("Trim values cannot be negative."); + } this.source = source; - this.trimStartUs = MILLISECONDS.toMicros(trimStartMillis); - final long trimEndUs = MILLISECONDS.toMicros(trimEndMillis); + this.trimStartUs = trimStartUs; this.trimDurationUs = computeTrimDuration(source.getDurationUs(), trimStartUs, trimEndUs); + this.hasVideoTrack = source.getTrackFormat(TrackType.VIDEO) != null; } @Contract(pure = true) private static long computeTrimDuration(long duration, long trimStart, long trimEnd) { - if (duration == UNKNOWN) { - return UNKNOWN; - } else { - final long result = duration - trimStart - trimEnd; - return result >= 0 ? result : UNKNOWN; + if (trimStart + trimEnd > duration) { + throw new IllegalArgumentException("Trim values cannot be greater than media duration."); } + return duration - trimStart - trimEnd; } @Override @@ -77,7 +75,7 @@ public void selectTrack(@NonNull TrackType type) { if (trimStartUs > 0) { switch (type) { case AUDIO: - if (hasTrack(TrackType.VIDEO) && !hasSelectedVideoTrack) { + if (hasVideoTrack && !hasSelectedVideoTrack) { selectAndSeekVideoTrack(); } source.selectTrack(TrackType.AUDIO); @@ -93,10 +91,6 @@ public void selectTrack(@NonNull TrackType type) { } } - private boolean hasTrack(@NonNull TrackType type) { - return source.getTrackFormat(type) != null; - } - private void selectAndSeekVideoTrack() { source.selectTrack(TrackType.VIDEO); source.requireExtractor().seekTo(trimStartUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); @@ -107,25 +101,25 @@ private void selectAndSeekVideoTrack() { * Check if trim operation was completed successfully for selected track. * We apply the seek operation for the video track only, so all audio frames are skipped * until MediaExtractor reaches the first video key frame. + * In the case there's no video track, audio frames are skipped until extractor reaches trimStartUs. */ private boolean isTrackReady(@NonNull TrackType type) { - if (isVideoTrackReady) { - return true; - } final MediaExtractor extractor = source.requireExtractor(); + final long timestampUs = extractor.getSampleTime(); if (type == TrackType.VIDEO) { - final boolean isKeyFrame = (extractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; - if (isKeyFrame) { - final long originalTrimStartUs = trimStartUs; - trimStartUs = extractor.getSampleTime(); - trimDurationUs += originalTrimStartUs - trimStartUs; - LOG.v("First video key frame is at " + trimStartUs + ", actual duration will be " + trimDurationUs); - isVideoTrackReady = true; - return true; - } + isSeekTrackReady = (extractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; + } else if (type == TrackType.AUDIO && !hasVideoTrack) { + isSeekTrackReady = timestampUs >= trimStartUs; + } + + if (isSeekTrackReady) { + trimDurationUs += trimStartUs - timestampUs; + trimStartUs = timestampUs; + LOG.v("First " + type + " key frame is at " + trimStartUs + ", actual duration will be " + trimDurationUs); + } else { + extractor.advance(); } - extractor.advance(); - return false; + return isSeekTrackReady; } @Override @@ -133,7 +127,7 @@ public boolean canReadTrack(@NonNull TrackType type) { boolean canRead = source.canReadTrack(type); if (canRead) { - return isTrackReady(type); + return isSeekTrackReady || isTrackReady(type); } else { return false; } @@ -162,7 +156,7 @@ public void releaseTrack(@NonNull TrackType type) { hasSelectedVideoTrack = false; break; case VIDEO: - isVideoTrackReady = false; + isSeekTrackReady = false; break; } source.releaseTrack(type); @@ -171,7 +165,7 @@ public void releaseTrack(@NonNull TrackType type) { @Override public void rewind() { hasSelectedVideoTrack = false; - isVideoTrackReady = false; + isSeekTrackReady = false; source.rewind(); } } From 52583ad3f68f273e8d580c22686359d7de0b67b7 Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Sun, 22 Dec 2019 10:06:35 -0500 Subject: [PATCH 05/13] Removed unnecessary changes to Engine In the original sequence selectAudio / transcodeAudio / selectVideo / transcodeVideo the first step (selectAudio) is intercepted by selectVideo + seekVideo and the 3rd step (selectVideo) is skipped. So it becomes selectVideo / seekVideo / selectAudio / transcodeAudio / transcodeVideo - Also added throws IllegalArgumentException to TrimDataSource --- .../java/com/otaliastudios/transcoder/engine/Engine.java | 9 +++------ .../otaliastudios/transcoder/source/TrimDataSource.java | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index 4fa35061..c4511f1a 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -364,13 +364,10 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce // Now step for transcoders that are not completed. audioCompleted = isCompleted(TrackType.AUDIO); videoCompleted = isCompleted(TrackType.VIDEO); - if (!audioCompleted && !videoCompleted) { - final TrackTranscoder videoTranscoder = getCurrentTrackTranscoder(TrackType.VIDEO, options); - final TrackTranscoder audioTranscoder = getCurrentTrackTranscoder(TrackType.AUDIO, options); - stepped |= videoTranscoder.transcode(forceVideoEos) | audioTranscoder.transcode(forceAudioEos); - } else if (!audioCompleted) { + if (!audioCompleted) { stepped |= getCurrentTrackTranscoder(TrackType.AUDIO, options).transcode(forceAudioEos); - } else if (!videoCompleted) { + } + if (!videoCompleted) { stepped |= getCurrentTrackTranscoder(TrackType.VIDEO, options).transcode(forceVideoEos); } if (++loopCount % PROGRESS_INTERVAL_STEPS == 0) { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index 6dc3af35..80b940fe 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -26,7 +26,7 @@ public class TrimDataSource implements DataSource { private boolean isSeekTrackReady = false; private boolean hasSelectedVideoTrack = false; - public TrimDataSource(@NonNull MediaExtractorDataSource source, long trimStartUs, long trimEndUs) { + public TrimDataSource(@NonNull MediaExtractorDataSource source, long trimStartUs, long trimEndUs) throws IllegalArgumentException { if (trimStartUs < 0 || trimEndUs < 0) { throw new IllegalArgumentException("Trim values cannot be negative."); } @@ -37,7 +37,7 @@ public TrimDataSource(@NonNull MediaExtractorDataSource source, long trimStartUs } @Contract(pure = true) - private static long computeTrimDuration(long duration, long trimStart, long trimEnd) { + private static long computeTrimDuration(long duration, long trimStart, long trimEnd) throws IllegalArgumentException { if (trimStart + trimEnd > duration) { throw new IllegalArgumentException("Trim values cannot be greater than media duration."); } From 7ddc093991fdfebf8d0bad6f688fc97fda6cbc3c Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Mon, 23 Dec 2019 13:17:58 -0500 Subject: [PATCH 06/13] Moved seekTo() from selectTrack() to canReadTrack() Cleaner simplified code :) The extractor needs a second call to seekTo() after reaching a video keyframe, to obtain better values for audio track. Otherwise, too many audio frames can be lost, causing visible off-sync. --- .../transcoder/source/TrimDataSource.java | 98 ++++++++----------- 1 file changed, 40 insertions(+), 58 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index 80b940fe..8f5b2e4b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -18,13 +18,12 @@ public class TrimDataSource implements DataSource { private static final String TAG = "TrimDataSource"; private static final Logger LOG = new Logger(TAG); - private final boolean hasVideoTrack; @NonNull private MediaExtractorDataSource source; private long trimStartUs; private long trimDurationUs; - private boolean isSeekTrackReady = false; - private boolean hasSelectedVideoTrack = false; + private boolean isAudioTrackReady; + private boolean isVideoTrackReady; public TrimDataSource(@NonNull MediaExtractorDataSource source, long trimStartUs, long trimEndUs) throws IllegalArgumentException { if (trimStartUs < 0 || trimEndUs < 0) { @@ -33,7 +32,8 @@ public TrimDataSource(@NonNull MediaExtractorDataSource source, long trimStartUs this.source = source; this.trimStartUs = trimStartUs; this.trimDurationUs = computeTrimDuration(source.getDurationUs(), trimStartUs, trimEndUs); - this.hasVideoTrack = source.getTrackFormat(TrackType.VIDEO) != null; + this.isAudioTrackReady = !hasTrack(TrackType.AUDIO) || trimStartUs == 0; + this.isVideoTrackReady = !hasTrack(TrackType.VIDEO) || trimStartUs == 0; } @Contract(pure = true) @@ -70,67 +70,49 @@ public MediaFormat getTrackFormat(@NonNull TrackType type) { return trackFormat; } + private boolean hasTrack(@NonNull TrackType type) { + return source.getTrackFormat(type) != null; + } + @Override public void selectTrack(@NonNull TrackType type) { - if (trimStartUs > 0) { + source.selectTrack(type); + } + + @Override + public boolean canReadTrack(@NonNull TrackType type) { + if (source.canReadTrack(type)) { + if (isAudioTrackReady && isVideoTrackReady) { + return true; + } + final MediaExtractor extractor = source.requireExtractor(); switch (type) { case AUDIO: - if (hasVideoTrack && !hasSelectedVideoTrack) { - selectAndSeekVideoTrack(); + if (!isAudioTrackReady) { + extractor.seekTo(trimStartUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC); + updateTrimValues(extractor.getSampleTime()); + isAudioTrackReady = true; } - source.selectTrack(TrackType.AUDIO); - break; + return isVideoTrackReady; case VIDEO: - if (!hasSelectedVideoTrack) { - selectAndSeekVideoTrack(); + if (!isVideoTrackReady) { + extractor.seekTo(trimStartUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC); + updateTrimValues(extractor.getSampleTime()); + isVideoTrackReady = true; + if (isAudioTrackReady) { + // Seeking a second time helps the extractor with Audio sampleTime issues + extractor.seekTo(trimStartUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC); + } } - break; + return isAudioTrackReady; } - } else { - source.selectTrack(type); - } - } - - private void selectAndSeekVideoTrack() { - source.selectTrack(TrackType.VIDEO); - source.requireExtractor().seekTo(trimStartUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); - hasSelectedVideoTrack = true; - } - - /** - * Check if trim operation was completed successfully for selected track. - * We apply the seek operation for the video track only, so all audio frames are skipped - * until MediaExtractor reaches the first video key frame. - * In the case there's no video track, audio frames are skipped until extractor reaches trimStartUs. - */ - private boolean isTrackReady(@NonNull TrackType type) { - final MediaExtractor extractor = source.requireExtractor(); - final long timestampUs = extractor.getSampleTime(); - if (type == TrackType.VIDEO) { - isSeekTrackReady = (extractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; - } else if (type == TrackType.AUDIO && !hasVideoTrack) { - isSeekTrackReady = timestampUs >= trimStartUs; - } - - if (isSeekTrackReady) { - trimDurationUs += trimStartUs - timestampUs; - trimStartUs = timestampUs; - LOG.v("First " + type + " key frame is at " + trimStartUs + ", actual duration will be " + trimDurationUs); - } else { - extractor.advance(); } - return isSeekTrackReady; + return false; } - @Override - public boolean canReadTrack(@NonNull TrackType type) { - boolean canRead = source.canReadTrack(type); - - if (canRead) { - return isSeekTrackReady || isTrackReady(type); - } else { - return false; - } + private void updateTrimValues(long timestampUs) { + trimDurationUs += trimStartUs - timestampUs; + trimStartUs = timestampUs; } @Override @@ -153,10 +135,10 @@ public boolean isDrained() { public void releaseTrack(@NonNull TrackType type) { switch (type) { case AUDIO: - hasSelectedVideoTrack = false; + isAudioTrackReady = false; break; case VIDEO: - isSeekTrackReady = false; + isVideoTrackReady = false; break; } source.releaseTrack(type); @@ -164,8 +146,8 @@ public void releaseTrack(@NonNull TrackType type) { @Override public void rewind() { - hasSelectedVideoTrack = false; - isSeekTrackReady = false; + isAudioTrackReady = false; + isVideoTrackReady = false; source.rewind(); } } From d596091aabce1379962155954a005cc84b46bf77 Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Mon, 23 Dec 2019 17:43:19 -0500 Subject: [PATCH 07/13] Removed MediaExtractorDataSource Replaced by adding seekTo() to DataSource interface --- .../transcoder/source/DataSource.java | 8 +++++++ .../transcoder/source/DefaultDataSource.java | 15 ++++++------- .../source/MediaExtractorDataSource.java | 10 --------- .../transcoder/source/TrimDataSource.java | 21 +++++++++++-------- 4 files changed, 28 insertions(+), 26 deletions(-) delete mode 100644 lib/src/main/java/com/otaliastudios/transcoder/source/MediaExtractorDataSource.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java index e7eb3dd9..47773dd3 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java @@ -54,6 +54,14 @@ public interface DataSource { */ void selectTrack(@NonNull TrackType type); + /** + * Moves all selected tracks near the specified time position. + * + * @param timestampUs requested time + * @return the new presentation time in microseconds + */ + long seekTo(long timestampUs); + /** * Returns true if we can read the given track at this point. * If true if returned, source should expect a {@link #readTrack(Chunk)} call. diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java index 0b333bb0..1ed18edb 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java @@ -18,7 +18,7 @@ /** * A DataSource implementation that uses Android's Media APIs. */ -public abstract class DefaultDataSource extends MediaExtractorDataSource { +public abstract class DefaultDataSource implements DataSource { private final static String TAG = DefaultDataSource.class.getSimpleName(); private final static Logger LOG = new Logger(TAG); @@ -63,6 +63,13 @@ public void selectTrack(@NonNull TrackType type) { mExtractor.selectTrack(mIndex.require(type)); } + @Override + public long seekTo(long timestampUs) { + ensureExtractor(); + mExtractor.seekTo(timestampUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC); + return mExtractor.getSampleTime(); + } + @Override public boolean isDrained() { ensureExtractor(); @@ -214,10 +221,4 @@ public void rewind() { mMetadata = new MediaMetadataRetriever(); mMetadataApplied = false; } - - @Override - protected MediaExtractor requireExtractor() { - ensureExtractor(); - return mExtractor; - } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/MediaExtractorDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/MediaExtractorDataSource.java deleted file mode 100644 index 31bae1c6..00000000 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/MediaExtractorDataSource.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.otaliastudios.transcoder.source; - -import android.media.MediaExtractor; - -/** - * DataSource that allows access to its MediaExtractor. - */ -abstract class MediaExtractorDataSource implements DataSource { - abstract protected MediaExtractor requireExtractor(); -} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index 8f5b2e4b..177a4f01 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -1,7 +1,6 @@ package com.otaliastudios.transcoder.source; -import android.media.MediaExtractor; import android.media.MediaFormat; import androidx.annotation.NonNull; @@ -19,13 +18,13 @@ public class TrimDataSource implements DataSource { private static final String TAG = "TrimDataSource"; private static final Logger LOG = new Logger(TAG); @NonNull - private MediaExtractorDataSource source; + private DataSource source; private long trimStartUs; private long trimDurationUs; private boolean isAudioTrackReady; private boolean isVideoTrackReady; - public TrimDataSource(@NonNull MediaExtractorDataSource source, long trimStartUs, long trimEndUs) throws IllegalArgumentException { + public TrimDataSource(@NonNull DataSource source, long trimStartUs, long trimEndUs) throws IllegalArgumentException { if (trimStartUs < 0 || trimEndUs < 0) { throw new IllegalArgumentException("Trim values cannot be negative."); } @@ -79,29 +78,33 @@ public void selectTrack(@NonNull TrackType type) { source.selectTrack(type); } + @Override + public long seekTo(long timestampUs) { + return source.seekTo(timestampUs); + } + @Override public boolean canReadTrack(@NonNull TrackType type) { if (source.canReadTrack(type)) { if (isAudioTrackReady && isVideoTrackReady) { return true; } - final MediaExtractor extractor = source.requireExtractor(); switch (type) { case AUDIO: if (!isAudioTrackReady) { - extractor.seekTo(trimStartUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC); - updateTrimValues(extractor.getSampleTime()); + final long sampleTimeUs = seekTo(trimStartUs); + updateTrimValues(sampleTimeUs); isAudioTrackReady = true; } return isVideoTrackReady; case VIDEO: if (!isVideoTrackReady) { - extractor.seekTo(trimStartUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC); - updateTrimValues(extractor.getSampleTime()); + final long sampleTimeUs = seekTo(trimStartUs); + updateTrimValues(sampleTimeUs); isVideoTrackReady = true; if (isAudioTrackReady) { // Seeking a second time helps the extractor with Audio sampleTime issues - extractor.seekTo(trimStartUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC); + seekTo(trimStartUs); } } return isAudioTrackReady; From a9c4975149a32869c19f9704501ba5fe4b5b0fa1 Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Mon, 23 Dec 2019 17:44:31 -0500 Subject: [PATCH 08/13] Removed timestamp adjustment The rest of the lib already assumes that timestamps start from arbitrary values. --- .../java/com/otaliastudios/transcoder/source/TrimDataSource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index 177a4f01..0173a8c1 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -121,7 +121,6 @@ private void updateTrimValues(long timestampUs) { @Override public void readTrack(@NonNull Chunk chunk) { source.readTrack(chunk); - chunk.timestampUs -= trimStartUs; } @Override From 8f45d8c62c3e2af55abc08ef982a79752c8f0c4c Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Mon, 23 Dec 2019 17:47:28 -0500 Subject: [PATCH 09/13] Use TrackTypeMap for readyTracks flags replacing two booleans --- .../transcoder/source/TrimDataSource.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index 0173a8c1..49f7e1a2 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -8,6 +8,7 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.internal.Logger; +import com.otaliastudios.transcoder.internal.TrackTypeMap; import org.jetbrains.annotations.Contract; @@ -21,8 +22,7 @@ public class TrimDataSource implements DataSource { private DataSource source; private long trimStartUs; private long trimDurationUs; - private boolean isAudioTrackReady; - private boolean isVideoTrackReady; + private TrackTypeMap readyTracks; public TrimDataSource(@NonNull DataSource source, long trimStartUs, long trimEndUs) throws IllegalArgumentException { if (trimStartUs < 0 || trimEndUs < 0) { @@ -31,8 +31,8 @@ public TrimDataSource(@NonNull DataSource source, long trimStartUs, long trimEnd this.source = source; this.trimStartUs = trimStartUs; this.trimDurationUs = computeTrimDuration(source.getDurationUs(), trimStartUs, trimEndUs); - this.isAudioTrackReady = !hasTrack(TrackType.AUDIO) || trimStartUs == 0; - this.isVideoTrackReady = !hasTrack(TrackType.VIDEO) || trimStartUs == 0; + this.readyTracks = new TrackTypeMap<>(!hasTrack(TrackType.VIDEO) || trimStartUs == 0, + !hasTrack(TrackType.AUDIO) || trimStartUs == 0); } @Contract(pure = true) @@ -86,28 +86,28 @@ public long seekTo(long timestampUs) { @Override public boolean canReadTrack(@NonNull TrackType type) { if (source.canReadTrack(type)) { - if (isAudioTrackReady && isVideoTrackReady) { + if (readyTracks.requireAudio() && readyTracks.requireVideo()) { return true; } switch (type) { case AUDIO: - if (!isAudioTrackReady) { + if (!readyTracks.requireAudio()) { final long sampleTimeUs = seekTo(trimStartUs); updateTrimValues(sampleTimeUs); - isAudioTrackReady = true; + readyTracks.setAudio(true); } - return isVideoTrackReady; + return readyTracks.requireVideo(); case VIDEO: - if (!isVideoTrackReady) { + if (!readyTracks.requireVideo()) { final long sampleTimeUs = seekTo(trimStartUs); updateTrimValues(sampleTimeUs); - isVideoTrackReady = true; - if (isAudioTrackReady) { + readyTracks.setVideo(true); + if (readyTracks.requireAudio()) { // Seeking a second time helps the extractor with Audio sampleTime issues seekTo(trimStartUs); } } - return isAudioTrackReady; + return readyTracks.requireAudio(); } } return false; @@ -137,10 +137,10 @@ public boolean isDrained() { public void releaseTrack(@NonNull TrackType type) { switch (type) { case AUDIO: - isAudioTrackReady = false; + readyTracks.setAudio(false); break; case VIDEO: - isVideoTrackReady = false; + readyTracks.setVideo(false); break; } source.releaseTrack(type); @@ -148,8 +148,8 @@ public void releaseTrack(@NonNull TrackType type) { @Override public void rewind() { - isAudioTrackReady = false; - isVideoTrackReady = false; + readyTracks.setAudio(false); + readyTracks.setVideo(false); source.rewind(); } } From b382f346d0e159e536a30a43a3ef2d553a87907c Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Mon, 23 Dec 2019 18:15:47 -0500 Subject: [PATCH 10/13] Handle trimEnd in isDrained() Stop reading when readUs is past duration. This removes the need to manually define KEY_DURATION in the mediaFormat --- .../otaliastudios/transcoder/source/TrimDataSource.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index 49f7e1a2..dd209c72 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -62,11 +62,7 @@ public long getDurationUs() { @Nullable @Override public MediaFormat getTrackFormat(@NonNull TrackType type) { - final MediaFormat trackFormat = source.getTrackFormat(type); - if (trackFormat != null) { - trackFormat.setLong(MediaFormat.KEY_DURATION, trimDurationUs); - } - return trackFormat; + return source.getTrackFormat(type); } private boolean hasTrack(@NonNull TrackType type) { @@ -130,7 +126,7 @@ public long getReadUs() { @Override public boolean isDrained() { - return source.isDrained(); + return source.isDrained() || getReadUs() >= getDurationUs(); } @Override From 421f8af2bca89a26e0332a0e52e6c0d7dbce3883 Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Mon, 23 Dec 2019 18:27:07 -0500 Subject: [PATCH 11/13] Fix seek vs canRead order to avoid possible bug where seekTo lands on a different track. Ex: The upstream source might be on AUDIO position, but if you seek later, the next position might VIDEO instead. --- .../otaliastudios/transcoder/source/TrimDataSource.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index dd209c72..5b02ac05 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -81,10 +81,7 @@ public long seekTo(long timestampUs) { @Override public boolean canReadTrack(@NonNull TrackType type) { - if (source.canReadTrack(type)) { - if (readyTracks.requireAudio() && readyTracks.requireVideo()) { - return true; - } + if (!readyTracks.requireAudio() || !readyTracks.requireVideo()) { switch (type) { case AUDIO: if (!readyTracks.requireAudio()) { @@ -106,7 +103,7 @@ public boolean canReadTrack(@NonNull TrackType type) { return readyTracks.requireAudio(); } } - return false; + return source.canReadTrack(type); } private void updateTrimValues(long timestampUs) { From 82de65eef5c16980ea0807ab3a5fe6732b0c7f6b Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Mon, 6 Jan 2020 18:28:58 -0500 Subject: [PATCH 12/13] Apply seekTo() once per selected track - Updates to Engine to replace selectAudio/transcode/selectVideo/transcode sequence by selectAudio/selectVideo/transcode/transcode - remove unnecessary hasTrack() - seekTo() is applied in canReadTrack(), once per selected track. This now works because all track selection operations are done before the first call to canReadTrack(). - When 2 tracks are selected, seekTo() is called twice and this helps the extractor with Audio sampleTime issues. --- .../transcoder/engine/Engine.java | 9 ++-- .../transcoder/source/TrimDataSource.java | 48 ++++++------------- 2 files changed, 20 insertions(+), 37 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index c4511f1a..4fa35061 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -364,10 +364,13 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce // Now step for transcoders that are not completed. audioCompleted = isCompleted(TrackType.AUDIO); videoCompleted = isCompleted(TrackType.VIDEO); - if (!audioCompleted) { + if (!audioCompleted && !videoCompleted) { + final TrackTranscoder videoTranscoder = getCurrentTrackTranscoder(TrackType.VIDEO, options); + final TrackTranscoder audioTranscoder = getCurrentTrackTranscoder(TrackType.AUDIO, options); + stepped |= videoTranscoder.transcode(forceVideoEos) | audioTranscoder.transcode(forceAudioEos); + } else if (!audioCompleted) { stepped |= getCurrentTrackTranscoder(TrackType.AUDIO, options).transcode(forceAudioEos); - } - if (!videoCompleted) { + } else if (!videoCompleted) { stepped |= getCurrentTrackTranscoder(TrackType.VIDEO, options).transcode(forceVideoEos); } if (++loopCount % PROGRESS_INTERVAL_STEPS == 0) { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index 5b02ac05..7aa42185 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -8,21 +8,23 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.internal.Logger; -import com.otaliastudios.transcoder.internal.TrackTypeMap; import org.jetbrains.annotations.Contract; +import java.util.HashSet; + /** * A {@link DataSource} wrapper that trims source at both ends. */ public class TrimDataSource implements DataSource { private static final String TAG = "TrimDataSource"; private static final Logger LOG = new Logger(TAG); + private final HashSet selectedTracks = new HashSet<>(); @NonNull private DataSource source; private long trimStartUs; private long trimDurationUs; - private TrackTypeMap readyTracks; + private boolean didSeekTracks = false; public TrimDataSource(@NonNull DataSource source, long trimStartUs, long trimEndUs) throws IllegalArgumentException { if (trimStartUs < 0 || trimEndUs < 0) { @@ -31,8 +33,6 @@ public TrimDataSource(@NonNull DataSource source, long trimStartUs, long trimEnd this.source = source; this.trimStartUs = trimStartUs; this.trimDurationUs = computeTrimDuration(source.getDurationUs(), trimStartUs, trimEndUs); - this.readyTracks = new TrackTypeMap<>(!hasTrack(TrackType.VIDEO) || trimStartUs == 0, - !hasTrack(TrackType.AUDIO) || trimStartUs == 0); } @Contract(pure = true) @@ -72,6 +72,7 @@ private boolean hasTrack(@NonNull TrackType type) { @Override public void selectTrack(@NonNull TrackType type) { source.selectTrack(type); + selectedTracks.add(type); } @Override @@ -81,27 +82,13 @@ public long seekTo(long timestampUs) { @Override public boolean canReadTrack(@NonNull TrackType type) { - if (!readyTracks.requireAudio() || !readyTracks.requireVideo()) { - switch (type) { - case AUDIO: - if (!readyTracks.requireAudio()) { - final long sampleTimeUs = seekTo(trimStartUs); - updateTrimValues(sampleTimeUs); - readyTracks.setAudio(true); - } - return readyTracks.requireVideo(); - case VIDEO: - if (!readyTracks.requireVideo()) { - final long sampleTimeUs = seekTo(trimStartUs); - updateTrimValues(sampleTimeUs); - readyTracks.setVideo(true); - if (readyTracks.requireAudio()) { - // Seeking a second time helps the extractor with Audio sampleTime issues - seekTo(trimStartUs); - } - } - return readyTracks.requireAudio(); + if (!didSeekTracks) { + // Seeking once per selected track helps the extractor with Audio sampleTime issues + for (TrackType t : selectedTracks) { + final long sampleTimeUs = seekTo(trimStartUs); + updateTrimValues(sampleTimeUs); } + didSeekTracks = true; } return source.canReadTrack(type); } @@ -128,21 +115,14 @@ public boolean isDrained() { @Override public void releaseTrack(@NonNull TrackType type) { - switch (type) { - case AUDIO: - readyTracks.setAudio(false); - break; - case VIDEO: - readyTracks.setVideo(false); - break; - } + selectedTracks.remove(type); source.releaseTrack(type); } @Override public void rewind() { - readyTracks.setAudio(false); - readyTracks.setVideo(false); + selectedTracks.clear(); + didSeekTracks = false; source.rewind(); } } From 7dcdc9645624ff1cd088adddf0052d1c256a69cb Mon Sep 17 00:00:00 2001 From: Mudar Noufal Date: Tue, 7 Jan 2020 13:33:31 -0500 Subject: [PATCH 13/13] seekBy() replaces seekTo(), selecting all tracks - reverted unnecessary changes to Engine class. Previous changes cannot guarantee that all calls to selectTracks() are done before the first canRead(). Latest bug was with merging multiple trimmed files. - use seekBy() to better handle first extractor timestamp - DefaultDataSource makes sure all available tracks are selected by extractor (without adding to mSelectedTracks array). Then seekTo() is called mutlitple times, using the resulting sampletimeUs for the later calls. --- .../transcoder/engine/Engine.java | 9 +++----- .../transcoder/source/DataSource.java | 6 +++--- .../transcoder/source/DefaultDataSource.java | 15 ++++++++++--- .../transcoder/source/TrimDataSource.java | 21 ++++--------------- 4 files changed, 22 insertions(+), 29 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index 4fa35061..c4511f1a 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -364,13 +364,10 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce // Now step for transcoders that are not completed. audioCompleted = isCompleted(TrackType.AUDIO); videoCompleted = isCompleted(TrackType.VIDEO); - if (!audioCompleted && !videoCompleted) { - final TrackTranscoder videoTranscoder = getCurrentTrackTranscoder(TrackType.VIDEO, options); - final TrackTranscoder audioTranscoder = getCurrentTrackTranscoder(TrackType.AUDIO, options); - stepped |= videoTranscoder.transcode(forceVideoEos) | audioTranscoder.transcode(forceAudioEos); - } else if (!audioCompleted) { + if (!audioCompleted) { stepped |= getCurrentTrackTranscoder(TrackType.AUDIO, options).transcode(forceAudioEos); - } else if (!videoCompleted) { + } + if (!videoCompleted) { stepped |= getCurrentTrackTranscoder(TrackType.VIDEO, options).transcode(forceVideoEos); } if (++loopCount % PROGRESS_INTERVAL_STEPS == 0) { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java index 47773dd3..e8e3675e 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java @@ -55,12 +55,12 @@ public interface DataSource { void selectTrack(@NonNull TrackType type); /** - * Moves all selected tracks near the specified time position. + * Moves all selected tracks forward by the specified duration. * - * @param timestampUs requested time + * @param durationUs requested duration * @return the new presentation time in microseconds */ - long seekTo(long timestampUs); + long seekBy(long durationUs); /** * Returns true if we can read the given track at this point. diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java index 1ed18edb..396999e2 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java @@ -64,10 +64,19 @@ public void selectTrack(@NonNull TrackType type) { } @Override - public long seekTo(long timestampUs) { + public long seekBy(long durationUs) { ensureExtractor(); - mExtractor.seekTo(timestampUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC); - return mExtractor.getSampleTime(); + final int trackCount = mExtractor.getTrackCount(); + for (int i = 0; i < trackCount; i++) { + mExtractor.selectTrack(i); + } + long timestampUs = mExtractor.getSampleTime() + durationUs; + // Seeking once per track helps the extractor with Audio sampleTime issues + for (int i = 0; i < trackCount; i++) { + mExtractor.seekTo(timestampUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC); + timestampUs = mExtractor.getSampleTime(); + } + return timestampUs; } @Override diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index 7aa42185..f5fdc5e5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -11,15 +11,12 @@ import org.jetbrains.annotations.Contract; -import java.util.HashSet; - /** * A {@link DataSource} wrapper that trims source at both ends. */ public class TrimDataSource implements DataSource { private static final String TAG = "TrimDataSource"; private static final Logger LOG = new Logger(TAG); - private final HashSet selectedTracks = new HashSet<>(); @NonNull private DataSource source; private long trimStartUs; @@ -65,29 +62,21 @@ public MediaFormat getTrackFormat(@NonNull TrackType type) { return source.getTrackFormat(type); } - private boolean hasTrack(@NonNull TrackType type) { - return source.getTrackFormat(type) != null; - } - @Override public void selectTrack(@NonNull TrackType type) { source.selectTrack(type); - selectedTracks.add(type); } @Override - public long seekTo(long timestampUs) { - return source.seekTo(timestampUs); + public long seekBy(long durationUs) { + return source.seekBy(durationUs); } @Override public boolean canReadTrack(@NonNull TrackType type) { if (!didSeekTracks) { - // Seeking once per selected track helps the extractor with Audio sampleTime issues - for (TrackType t : selectedTracks) { - final long sampleTimeUs = seekTo(trimStartUs); - updateTrimValues(sampleTimeUs); - } + final long sampleTimeUs = seekBy(trimStartUs); + updateTrimValues(sampleTimeUs); didSeekTracks = true; } return source.canReadTrack(type); @@ -115,13 +104,11 @@ public boolean isDrained() { @Override public void releaseTrack(@NonNull TrackType type) { - selectedTracks.remove(type); source.releaseTrack(type); } @Override public void rewind() { - selectedTracks.clear(); didSeekTracks = false; source.rewind(); }