From 2c3b6d850d9f08aa7fbaba90997a42eed99670df Mon Sep 17 00:00:00 2001 From: Sho Amano Date: Sat, 11 Mar 2017 23:43:58 +0900 Subject: [PATCH] Add RTPH264Packetizer and its test cases This packetizer sends H.264 video stream using RTP frames defined by RFC 6184 and RFC 4571. This commit also includes a temporal change to test RTP stream output. To enable it, change "useRTP" flag to true in SdlSession.java. This is for discussion on proposal SDL-0048. --- .../test/streaming/RTPH264PacketizerTest.java | 699 ++++++++++++++++++ .../SdlConnection/SdlSession.java | 48 +- .../encoder/IEncoderListener.java | 70 ++ .../smartdevicelink/encoder/SdlEncoder.java | 13 +- .../protocol/ProtocolMessage.java | 8 +- .../streaming/RTPH264Packetizer.java | 486 ++++++++++++ 6 files changed, 1306 insertions(+), 18 deletions(-) create mode 100644 sdl_android/src/androidTest/java/com/smartdevicelink/test/streaming/RTPH264PacketizerTest.java create mode 100644 sdl_android/src/main/java/com/smartdevicelink/encoder/IEncoderListener.java create mode 100644 sdl_android/src/main/java/com/smartdevicelink/streaming/RTPH264Packetizer.java diff --git a/sdl_android/src/androidTest/java/com/smartdevicelink/test/streaming/RTPH264PacketizerTest.java b/sdl_android/src/androidTest/java/com/smartdevicelink/test/streaming/RTPH264PacketizerTest.java new file mode 100644 index 0000000000..73d1ea6cdd --- /dev/null +++ b/sdl_android/src/androidTest/java/com/smartdevicelink/test/streaming/RTPH264PacketizerTest.java @@ -0,0 +1,699 @@ +/* + * Copyright (c) 2017, Xevo Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the name of Xevo Inc. nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.smartdevicelink.test.streaming; + +import com.smartdevicelink.SdlConnection.SdlSession; +import com.smartdevicelink.encoder.IEncoderListener; +import com.smartdevicelink.protocol.ProtocolMessage; +import com.smartdevicelink.protocol.enums.SessionType; +import com.smartdevicelink.streaming.IStreamListener; +import com.smartdevicelink.streaming.RTPH264Packetizer; +import com.smartdevicelink.transport.BTTransportConfig; + +import junit.framework.TestCase; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * This class includes a unit test for {@link com.smartdevicelink.streaming.RTPH264Packetizer}. + * + * @author Sho Amano + */ +public class RTPH264PacketizerTest extends TestCase { + + private static final int FRAME_LENGTH_LEN = 2; + private static final int RTP_HEADER_LEN = 12; + + private static final byte WIPRO_VERSION = 0x0B; + private static final byte SESSION_ID = 0x0A; + + private class ByteStreamNALU { + byte[] startCode; + byte[] nalu; + int frameNum; + + ByteStreamNALU(byte[] startCode, byte[] nalu, int frameNum) { + this.startCode = startCode; + this.nalu = nalu; + this.frameNum = frameNum; + } + + byte[] createArray() { + byte[] array = new byte[startCode.length + nalu.length]; + System.arraycopy(startCode, 0, array, 0, startCode.length); + System.arraycopy(nalu, 0, array, startCode.length, nalu.length); + return array; + } + + public int getLength() { + return startCode.length + nalu.length; + } + } + + private static final byte[] START_CODE_3 = {0x00, 0x00, 0x01}; + private static final byte[] START_CODE_4 = {0x00, 0x00, 0x00, 0x01}; + + /* a sample H.264 stream, including 33 frames of 16px white square */ + private final ByteStreamNALU[] SAMPLE_STREAM = new ByteStreamNALU[] { + // SPS + new ByteStreamNALU(START_CODE_4, new byte[]{0x67, 0x42, (byte)0xC0, 0x0A, (byte)0xA6, 0x11, 0x11, (byte)0xE8, + 0x40, 0x00, 0x00, (byte)0xFA, 0x40, 0x00, 0x3A, (byte)0x98, + 0x23, (byte)0xC4, (byte)0x89, (byte)0x84, 0x60}, 0), + // PPS + new ByteStreamNALU(START_CODE_4, new byte[]{0x68, (byte)0xC8, 0x42, 0x0F, 0x13, 0x20}, 0), + // I + new ByteStreamNALU(START_CODE_3, new byte[]{0x65, (byte)0x88, (byte)0x82, 0x07, 0x67, 0x39, 0x31, 0x40, + 0x00, 0x5E, 0x0A, (byte)0xFB, (byte)0xEF, (byte)0xAE, (byte)0xBA, (byte)0xEB, + (byte)0xAE, (byte)0xBA, (byte)0xEB, (byte)0xC0}, 0), + // P + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, 0x1C, 0x0E, (byte)0xCE, 0x71, (byte)0xB0}, 1), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, 0x2A, 0x03, (byte)0xB3, (byte)0x9C, 0x6C}, 2), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, 0x3B, 0x03, (byte)0xB3, (byte)0x9C, 0x6C}, 3), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, 0x49, 0x00, (byte)0xEC, (byte)0xE7, 0x1B}, 4), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, 0x59, 0x40, (byte)0xEC, (byte)0xE7, 0x1B}, 5), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, 0x69, (byte)0x80, (byte)0xEC, (byte)0xE7, 0x1B}, 6), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, 0x79, (byte)0xC0, (byte)0xEC, (byte)0xE7, 0x1B}, 7), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, (byte)0x88, (byte)0x80, 0x3B, 0x39, (byte)0xC6, (byte)0xC0}, 8), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, (byte)0x98, (byte)0x90, 0x3B, 0x39, (byte)0xC6, (byte)0xC0}, 9), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, (byte)0xA8, (byte)0xA0, 0x3B, 0x39, (byte)0xC6, (byte)0xC0}, 10), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, (byte)0xB8, (byte)0xB0, 0x3B, 0x39, (byte)0xC6, (byte)0xC0}, 11), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, (byte)0xC8, (byte)0xC0, 0x3B, 0x39, (byte)0xC6, (byte)0xC0}, 12), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, (byte)0xD8, (byte)0xD0, 0x3B, 0x39, (byte)0xC6, (byte)0xC0}, 13), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, (byte)0xE8, (byte)0xE0, 0x3B, 0x39, (byte)0xC6, (byte)0xC0}, 14), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, (byte)0xF8, (byte)0xF0, 0x3B, 0x39, (byte)0xC6, (byte)0xC0}, 15), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, 0x00, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 16), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, 0x10, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 17), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, 0x20, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 18), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, 0x30, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 19), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, 0x40, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 20), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, 0x50, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 21), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, 0x60, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 22), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, 0x70, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 23), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, (byte)0x80, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 24), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, (byte)0x90, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 25), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, (byte)0xA0, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 26), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, (byte)0xB0, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 27), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, (byte)0xC0, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 28), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9B, (byte)0xD0, 0x1D, (byte)0x9C, (byte)0xE3, 0x60}, 29), + // SPS + new ByteStreamNALU(START_CODE_4, new byte[]{0x67, 0x42, (byte)0xC0, 0x0A, (byte)0xA6, 0x11, 0x11, (byte)0xE8, + 0x40, 0x00, 0x00, (byte)0xFA, 0x40, 0x00, 0x3A, (byte)0x98, + 0x23, (byte)0xC4, (byte)0x89, (byte)0x84, 0x60}, 30), + // PPS + new ByteStreamNALU(START_CODE_4, new byte[]{0x68, (byte)0xC8, 0x42, 0x0F, 0x13, 0x20}, 30), + // I + new ByteStreamNALU(START_CODE_3, new byte[]{0x65, (byte)0x88, (byte)0x81, 0x00, (byte)0x8E, 0x73, (byte)0x93, 0x14, + 0x00, 0x06, (byte)0xA4, 0x2F, (byte)0xBE, (byte)0xFA, (byte)0xEB, (byte)0xAE, + (byte)0xBA, (byte)0xEB, (byte)0xAE, (byte)0xBC}, 30), + // P + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, 0x1C, 0x0D, (byte)0xCE, 0x71, (byte)0xB0}, 31), + new ByteStreamNALU(START_CODE_4, new byte[]{0x41, (byte)0x9A, 0x2A, 0x03, 0x33, (byte)0x9C, 0x6C}, 32), + }; + + /** + * Test for creating Single Frame RTP packets from H.264 byte stream + */ + public void testSingleFrames() { + StreamVerifier verifier = new StreamVerifier(SAMPLE_STREAM); + SdlSession session = createTestSession(); + RTPH264Packetizer packetizer = null; + try { + packetizer = new RTPH264Packetizer(verifier, SessionType.NAV, SESSION_ID, session); + } catch (IOException e) { + fail(); + } + MockEncoder encoder = new MockEncoder(packetizer); + + try { + packetizer.start(); + } catch (IOException e) { + fail(); + } + + encoder.inputByteStream(SAMPLE_STREAM); + try { + Thread.sleep(1000, 0); + } catch (InterruptedException e) {} + + packetizer.stop(); + assertEquals(SAMPLE_STREAM.length, verifier.getPacketCount()); + } + + /** + * Test for creating Single Frame RTP packets then splitting into multiple SDL frames + */ + public void testSingleFramesIntoMultipleMessages() { + StreamVerifier verifier = new StreamVerifier(SAMPLE_STREAM); + SdlSession session = createTestSession(); + RTPH264Packetizer packetizer = null; + try { + packetizer = new RTPH264Packetizer(verifier, SessionType.NAV, SESSION_ID, session); + } catch (IOException e) { + fail(); + } + // use small MTU and make some RTP packets split into multiple SDL frames + packetizer.setMTU(FRAME_LENGTH_LEN + RTP_HEADER_LEN + 16); + MockEncoder encoder = new MockEncoder(packetizer); + + try { + packetizer.start(); + } catch (IOException e) { + fail(); + } + + encoder.inputByteStream(SAMPLE_STREAM); + try { + Thread.sleep(1000, 0); + } catch (InterruptedException e) {} + + packetizer.stop(); + assertEquals(SAMPLE_STREAM.length, verifier.getPacketCount()); + } + + /** + * Test for creating Fragmentation Units from H.264 byte stream + */ + public void testFragmentationUnits() { + ByteStreamNALU[] stream = new ByteStreamNALU[] { + SAMPLE_STREAM[0], SAMPLE_STREAM[1], null, null, null, SAMPLE_STREAM[5] + }; + byte[] fakeNALU1 = new byte[65535 - RTP_HEADER_LEN]; // not fragmented + byte[] fakeNALU2 = new byte[65536 - RTP_HEADER_LEN]; // will be fragmented + byte[] fakeNALU3 = new byte[65537 - RTP_HEADER_LEN]; // ditto + + for (int i = 0; i < fakeNALU1.length; i++) { + fakeNALU1[i] = (byte)(i % 256); + } + for (int i = 0; i < fakeNALU2.length; i++) { + fakeNALU2[i] = (byte)(i % 256); + } + for (int i = 0; i < fakeNALU3.length; i++) { + fakeNALU3[i] = (byte)(i % 256); + } + + stream[2] = new ByteStreamNALU(START_CODE_3, fakeNALU1, 0); + stream[3] = new ByteStreamNALU(START_CODE_4, fakeNALU2, 1); + stream[4] = new ByteStreamNALU(START_CODE_4, fakeNALU3, 2); + + StreamVerifier verifier = new StreamVerifier(stream); + SdlSession session = createTestSession(); + RTPH264Packetizer packetizer = null; + try { + packetizer = new RTPH264Packetizer(verifier, SessionType.NAV, SESSION_ID, session); + } catch (IOException e) { + fail(); + } + MockEncoder encoder = new MockEncoder(packetizer); + + try { + packetizer.start(); + } catch (IOException e) { + fail(); + } + + encoder.inputByteStream(stream); + try { + Thread.sleep(1000, 0); + } catch (InterruptedException e) {} + + packetizer.stop(); + assertEquals(stream.length + 2, verifier.getPacketCount()); + } + + /** + * Test for RTP sequence number gets wrap-around correctly + */ + public void testSequenceNumWrapAround() { + ByteStreamNALU[] stream = new ByteStreamNALU[70000]; + for (int i = 0; i < stream.length; i++) { + stream[i] = new ByteStreamNALU(START_CODE_4, SAMPLE_STREAM[3].nalu, i); + } + + StreamVerifier verifier = new StreamVerifier(stream); + SdlSession session = createTestSession(); + RTPH264Packetizer packetizer = null; + try { + packetizer = new RTPH264Packetizer(verifier, SessionType.NAV, SESSION_ID, session); + } catch (IOException e) { + fail(); + } + MockEncoder encoder = new MockEncoder(packetizer); + + try { + packetizer.start(); + } catch (IOException e) { + fail(); + } + + encoder.inputByteStream(stream); + try { + Thread.sleep(2000, 0); + } catch (InterruptedException e) {} + + packetizer.stop(); + assertEquals(stream.length, verifier.getPacketCount()); + } + + /** + * Test for {@link com.smartdevicelink.streaming.RTPH264Packetizer#setPayloadType(byte)} + */ + public void testSetPayloadType() { + byte pt = (byte)123; + StreamVerifier verifier = new StreamVerifier(SAMPLE_STREAM, pt); + SdlSession session = createTestSession(); + RTPH264Packetizer packetizer = null; + try { + packetizer = new RTPH264Packetizer(verifier, SessionType.NAV, SESSION_ID, session); + } catch (IOException e) { + fail(); + } + packetizer.setPayloadType(pt); + MockEncoder encoder = new MockEncoder(packetizer); + + try { + packetizer.start(); + } catch (IOException e) { + fail(); + } + + encoder.inputByteStream(SAMPLE_STREAM); + try { + Thread.sleep(1000, 0); + } catch (InterruptedException e) {} + + packetizer.stop(); + assertEquals(SAMPLE_STREAM.length, verifier.getPacketCount()); + } + + /** + * Test for {@link com.smartdevicelink.streaming.RTPH264Packetizer#setSSRC(int)} + */ + public void testSetSSRC() { + int ssrc = 0xFEDCBA98; + StreamVerifier verifier = new StreamVerifier(SAMPLE_STREAM); + verifier.setExpectedSSRC(ssrc); + SdlSession session = createTestSession(); + RTPH264Packetizer packetizer = null; + try { + packetizer = new RTPH264Packetizer(verifier, SessionType.NAV, SESSION_ID, session); + } catch (IOException e) { + fail(); + } + packetizer.setSSRC(ssrc); + MockEncoder encoder = new MockEncoder(packetizer); + + try { + packetizer.start(); + } catch (IOException e) { + fail(); + } + + encoder.inputByteStream(SAMPLE_STREAM); + try { + Thread.sleep(1000, 0); + } catch (InterruptedException e) {} + + packetizer.stop(); + assertEquals(SAMPLE_STREAM.length, verifier.getPacketCount()); + } + + /** + * Test for {@link com.smartdevicelink.streaming.RTPH264Packetizer#pause()} and + * {@link com.smartdevicelink.streaming.RTPH264Packetizer#resume()} + */ + public void testPauseResume() { + int index = 0; + // split SAMPLE_STREAM into three parts + ByteStreamNALU[] inputStream1 = new ByteStreamNALU[8]; + ByteStreamNALU[] inputStream2 = new ByteStreamNALU[19]; + ByteStreamNALU[] inputStream3 = new ByteStreamNALU[10]; + for (int i = 0; i < inputStream1.length; i++) { + inputStream1[i] = SAMPLE_STREAM[index++]; + } + for (int i = 0; i < inputStream2.length; i++) { + inputStream2[i] = SAMPLE_STREAM[index++]; + } + for (int i = 0; i < inputStream3.length; i++) { + inputStream3[i] = SAMPLE_STREAM[index++]; + } + + index = 0; + // expected output is "all NAL units in inputStream1" plus "I frame and onwards in inputStream3" + ByteStreamNALU[] expectedStream = new ByteStreamNALU[inputStream1.length + 3]; + for (int i = 0; i < inputStream1.length; i++) { + expectedStream[index++] = inputStream1[i]; + } + expectedStream[index++] = SAMPLE_STREAM[34]; + expectedStream[index++] = SAMPLE_STREAM[35]; + expectedStream[index] = SAMPLE_STREAM[36]; + + StreamVerifier verifier = new StreamVerifier(expectedStream); + SdlSession session = createTestSession(); + RTPH264Packetizer packetizer = null; + try { + packetizer = new RTPH264Packetizer(verifier, SessionType.NAV, SESSION_ID, session); + } catch (IOException e) { + fail(); + } + MockEncoder encoder = new MockEncoder(packetizer); + + try { + packetizer.start(); + } catch (IOException e) { + fail(); + } + + encoder.inputByteStream(inputStream1); + try { + Thread.sleep(1000, 0); + } catch (InterruptedException e) {} + + packetizer.pause(); + + // this input stream should be disposed + encoder.inputByteStream(inputStream2); + try { + Thread.sleep(1000, 0); + } catch (InterruptedException e) {} + + packetizer.resume(); + + // packetizer should resume from a I frame + encoder.inputByteStream(inputStream3); + try { + Thread.sleep(1000, 0); + } catch (InterruptedException e) {} + + packetizer.stop(); + assertEquals(expectedStream.length, verifier.getPacketCount()); + } + + private SdlSession createTestSession() { + return SdlSession.createSession(WIPRO_VERSION, new MockInterfaceBroker(), new BTTransportConfig(true)); + } + + private class StreamVerifier implements IStreamListener { + private static final int STATE_LENGTH = 0; + private static final int STATE_PACKET = 1; + + private ByteStreamNALU[] mStream; + private byte[] mExpectedNALU; + private ByteBuffer mReceiveBuffer; + private int mPacketLen; + private int mState; + private int mNALCount; + private int mTotalPacketCount; + private boolean mFragmented; + private int mOffsetInNALU; + private byte mPayloadType; + private boolean mVerifySSRC; + private int mExpectedSSRC; + private boolean mFirstPacketReceived; + private short mFirstSequenceNum; + private int mFirstTimestamp; + + StreamVerifier(ByteStreamNALU[] stream) { + this(stream, (byte)96); + } + + StreamVerifier(ByteStreamNALU[] stream, byte payloadType) { + mStream = stream; + mReceiveBuffer = ByteBuffer.allocate(256 * 1024); + mReceiveBuffer.order(ByteOrder.BIG_ENDIAN); + mPacketLen = 0; + mState = STATE_LENGTH; + + mNALCount = 0; + mTotalPacketCount = 0; + mFragmented = false; + mOffsetInNALU = 1; // Used when verifying FUs. The first byte is skipped. + + mPayloadType = payloadType; + mVerifySSRC = false; + mExpectedSSRC = 0; + mFirstPacketReceived = false; + mFirstSequenceNum = 0; + mFirstTimestamp = 0; + } + + void setExpectedSSRC(int ssrc) { + mVerifySSRC = true; + mExpectedSSRC = ssrc; + } + + int getPacketCount() { + return mTotalPacketCount; + } + + @Override + public void sendStreamPacket(ProtocolMessage pm) { + mExpectedNALU = mStream[mNALCount].nalu; + // should be same as MockEncoder's configuration (29.97 FPS) + int expectedPTSDelta = mStream[mNALCount].frameNum * 1001 * 3; + boolean isLast = shouldBeLast(); + + verifyProtocolMessage(pm, SESSION_ID); + + mReceiveBuffer.put(pm.getData()); + mReceiveBuffer.flip(); + + if (mState == STATE_LENGTH) { + if (mReceiveBuffer.remaining() >= 2) { + mPacketLen = mReceiveBuffer.getShort() & 0xFFFF; + mState = STATE_PACKET; + } + } + + if (mState == STATE_PACKET) { + if (mReceiveBuffer.remaining() >= mPacketLen) { + byte[] packet = new byte[mPacketLen]; + mReceiveBuffer.get(packet); + + verifyRTPPacket(packet, mPayloadType, expectedPTSDelta, + mVerifySSRC, mExpectedSSRC, isLast); + mFirstPacketReceived = true; + + mState = STATE_LENGTH; + mPacketLen = 0; + mTotalPacketCount++; + } + } + + mReceiveBuffer.compact(); + } + + private void verifyProtocolMessage(ProtocolMessage pm, byte sessionId) { + assertEquals(true, pm != null); + assertEquals(sessionId, pm.getSessionID()); + assertEquals(SessionType.NAV, pm.getSessionType()); + assertEquals(0, pm.getFunctionID()); + assertEquals(0, pm.getCorrID()); + assertEquals(false, pm.getPayloadProtected()); + } + + private void verifyRTPPacket(byte[] packet, byte payloadType, int expectedPTSDelta, + boolean verifySSRC, int expectedSSRC, boolean isLast) { + assertTrue(packet.length > RTP_HEADER_LEN); + verifyRTPHeader(packet, false, isLast, payloadType, (short)(mTotalPacketCount % 65536), + expectedPTSDelta, verifySSRC, expectedSSRC); + + byte type = (byte)(packet[RTP_HEADER_LEN] & 0x1F); + if (type == 28) { // FU-A frame + boolean fuEnd = verifyFUTypeA(packet); + if (fuEnd) { + mNALCount++; + } + } else if (type == 29) { // FU-B frame + fail("Fragmentation Unit B is not supported by this test"); + } else if (type == 24 || type == 25 || type == 26 || type == 27) { + fail("STAP and MTAP are not supported by this test"); + } else { + // Single Frame + verifySingleFrame(packet); + mNALCount++; + } + } + + private void verifyRTPHeader(byte[] packet, + boolean hasPadding, boolean isLast, byte payloadType, + short seqNumDelta, int ptsDelta, boolean checkSSRC, int ssrc) { + int byte0 = packet[0] & 0xFF; + assertEquals((byte)2, (byte)((byte0 >> 6) & 3)); // version + assertEquals((byte)(hasPadding ? 1 : 0), (byte)((byte0 >> 5) & 1)); // padding + assertEquals((byte)0, (byte)((byte0 >> 4) & 1)); // extension + assertEquals((byte)0, (byte)(byte0 & 0xF)); // CSRC count + + int byte1 = packet[1] & 0xFF; + assertEquals((byte)(isLast ? 1 : 0), (byte)((byte1 >> 7) & 1)); // marker + assertEquals(payloadType, (byte)(byte1 & 0x7F)); // Payload Type + + short actualSeq = (short)(((packet[2] & 0xFF) << 8) | (packet[3] & 0xFF)); + if (!mFirstPacketReceived) { + mFirstSequenceNum = actualSeq; + } else { + assertEquals((short)(mFirstSequenceNum + seqNumDelta), actualSeq); + } + + int actualPTS = ((packet[4] & 0xFF) << 24) | ((packet[5] & 0xFF) << 16) | + ((packet[6] & 0xFF) << 8) | (packet[7] & 0xFF); + if (!mFirstPacketReceived) { + mFirstTimestamp = actualPTS; + } else { + // accept calculation error + assertTrue(mFirstTimestamp + ptsDelta - 1 <= actualPTS && + actualPTS <= mFirstTimestamp + ptsDelta + 1); + } + + if (checkSSRC) { + int actualSSRC = ((packet[8] & 0xFF) << 24) | ((packet[9] & 0xFF) << 16) | + ((packet[10] & 0xFF) << 8) | (packet[11] & 0xFF); + assertEquals(ssrc, actualSSRC); + } + } + + private void verifySingleFrame(byte[] packet) { + assertEquals(true, arrayCompare(packet, RTP_HEADER_LEN, packet.length - RTP_HEADER_LEN, + mExpectedNALU, 0, mExpectedNALU.length)); + } + + private boolean verifyFUTypeA(byte[] packet) { + int firstByte = mExpectedNALU[0] & 0xFF; + + int byte0 = packet[RTP_HEADER_LEN] & 0xFF; + assertEquals((byte)((firstByte >> 7) & 1), (byte)((byte0 >> 7) & 1)); // F bit + assertEquals((byte)((firstByte >> 5) & 3), (byte)((byte0 >> 5) & 3)); // NRI + assertEquals((byte)28, (byte)(byte0 & 0x1F)); // Type + + int byte1 = packet[RTP_HEADER_LEN+1] & 0xFF; + boolean isFirstFU = ((byte1 >> 7) & 1) == 1; // Start bit + boolean isLastFU = ((byte1 >> 6) & 1) == 1; // End bit + assertEquals((byte)0, (byte)((byte1 >> 5) & 1)); // Reserved bit + assertEquals((byte)(firstByte & 0x1F), (byte)(byte1 & 0x1F)); // Type + + int len = packet.length - (RTP_HEADER_LEN + 2); + assertEquals(true, arrayCompare(packet, RTP_HEADER_LEN + 2, len, mExpectedNALU, mOffsetInNALU, len)); + mOffsetInNALU += len; + + if (!mFragmented) { + // this should be the first fragmentation unit + assertEquals(true, isFirstFU); + assertEquals(false, isLastFU); + mFragmented = true; + } else { + assertEquals(false, isFirstFU); + if (mExpectedNALU.length == mOffsetInNALU) { + // this is the last fragmentation unit + assertEquals(true, isLastFU); + + mFragmented = false; + mOffsetInNALU = 1; + return true; + } else { + assertEquals(false, isLastFU); + } + } + return false; + } + + private boolean shouldBeLast() { + if (mNALCount + 1 >= mStream.length) { + return true; + } + ByteStreamNALU current = mStream[mNALCount]; + ByteStreamNALU next = mStream[mNALCount + 1]; + if (next.frameNum != current.frameNum) { + return true; + } else { + return false; + } + } + + private boolean arrayCompare(byte[] a1, int start1, int len1, byte[] a2, int start2, int len2) { + assertTrue(start1 + len1 <= a1.length); + assertTrue(start2 + len2 <= a2.length); + + if (len1 != len2) { + return false; + } + + for (int i = 0; i < len1; i++) { + if (a1[start1 + i] != a2[start2 + i]) { + return false; + } + } + return true; + } + } + + private class MockEncoder { + private IEncoderListener mListener; + private int mFPSNum; + private int mFPSDen; + + MockEncoder(IEncoderListener listener) { + mListener = listener; + // 29.97 fps + mFPSNum = 30000; + mFPSDen = 1001; + } + + void inputByteStream(ByteStreamNALU[] stream) { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + + for (int i = 0; i < stream.length; i++) { + ByteStreamNALU bs = stream[i]; + byte[] array = bs.createArray(); + os.write(array, 0, array.length); + + if (i < stream.length - 1) { + ByteStreamNALU next = stream[i + 1]; + if (bs.frameNum == next.frameNum) { + // enqueue it and send at once + continue; + } + } + + long timestampUs = bs.frameNum * 1000L * 1000L * mFPSDen / mFPSNum; + byte[] data = os.toByteArray(); + mListener.onEncoderOutput(IEncoderListener.Format.H264_BYTE_STREAM, data, timestampUs); + os.reset(); + } + + try { + os.close(); + } catch (IOException e) { + } + } + } +} diff --git a/sdl_android/src/main/java/com/smartdevicelink/SdlConnection/SdlSession.java b/sdl_android/src/main/java/com/smartdevicelink/SdlConnection/SdlSession.java index 1a9d051a66..35c17d0aa8 100644 --- a/sdl_android/src/main/java/com/smartdevicelink/SdlConnection/SdlSession.java +++ b/sdl_android/src/main/java/com/smartdevicelink/SdlConnection/SdlSession.java @@ -24,7 +24,9 @@ import com.smartdevicelink.proxy.RPCRequest; import com.smartdevicelink.security.ISecurityInitializedListener; import com.smartdevicelink.security.SdlSecurityBase; +import com.smartdevicelink.streaming.AbstractPacketizer; import com.smartdevicelink.streaming.IStreamListener; +import com.smartdevicelink.streaming.RTPH264Packetizer; import com.smartdevicelink.streaming.StreamPacketizer; import com.smartdevicelink.streaming.StreamRPCPacketizer; import com.smartdevicelink.transport.BaseTransportConfig; @@ -46,7 +48,7 @@ public class SdlSession implements ISdlConnectionListener, IHeartbeatMonitorList private LockScreenManager lockScreenMan = new LockScreenManager(); private SdlSecurityBase sdlSecurity = null; StreamRPCPacketizer mRPCPacketizer = null; - StreamPacketizer mVideoPacketizer = null; + AbstractPacketizer mVideoPacketizer = null; StreamPacketizer mAudioPacketizer = null; SdlEncoder mSdlEncoder = null; private final static int BUFF_READ_SIZE = 1024; @@ -134,8 +136,9 @@ public void close() { public void startStream(InputStream is, SessionType sType, byte rpcSessionID) throws IOException { if (sType.equals(SessionType.NAV)) { - mVideoPacketizer = new StreamPacketizer(this, is, sType, rpcSessionID, this); - mVideoPacketizer.sdlConnection = this.getSdlConnection(); + StreamPacketizer packetizer = new StreamPacketizer(this, is, sType, rpcSessionID, this); + packetizer.sdlConnection = this.getSdlConnection(); + mVideoPacketizer = packetizer; mVideoPacketizer.start(); } else if (sType.equals(SessionType.PCM)) @@ -157,8 +160,9 @@ public OutputStream startStream(SessionType sType, byte rpcSessionID) throws IOE } if (sType.equals(SessionType.NAV)) { - mVideoPacketizer = new StreamPacketizer(this, is, sType, rpcSessionID, this); - mVideoPacketizer.sdlConnection = this.getSdlConnection(); + StreamPacketizer packetizer = new StreamPacketizer(this, is, sType, rpcSessionID, this); + packetizer.sdlConnection = this.getSdlConnection(); + mVideoPacketizer = packetizer; mVideoPacketizer.start(); } else if (sType.equals(SessionType.PCM)) @@ -284,19 +288,35 @@ public boolean resumeVideoStream() public Surface createOpenGLInputSurface(int frameRate, int iFrameInterval, int width, int height, int bitrate, SessionType sType, byte rpcSessionID) { + PipedOutputStream stream = null; + RTPH264Packetizer rtpPacketizer = null; + boolean useRTP = false; + try { - PipedOutputStream stream = (PipedOutputStream) startStream(sType, rpcSessionID); - if (stream == null) return null; - mSdlEncoder = new SdlEncoder(); - mSdlEncoder.setFrameRate(frameRate); - mSdlEncoder.setFrameInterval(iFrameInterval); - mSdlEncoder.setFrameWidth(width); - mSdlEncoder.setFrameHeight(height); - mSdlEncoder.setBitrate(bitrate); - mSdlEncoder.setOutputStream(stream); + if (useRTP) { + rtpPacketizer = new RTPH264Packetizer(this, sType, rpcSessionID, this); + mVideoPacketizer = rtpPacketizer; + mVideoPacketizer.start(); + } else { + stream = (PipedOutputStream) startStream(sType, rpcSessionID); + if (stream == null) return null; + } } catch (IOException e) { return null; } + + mSdlEncoder = new SdlEncoder(); + mSdlEncoder.setFrameRate(frameRate); + mSdlEncoder.setFrameInterval(iFrameInterval); + mSdlEncoder.setFrameWidth(width); + mSdlEncoder.setFrameHeight(height); + mSdlEncoder.setBitrate(bitrate); + + if (useRTP) { + mSdlEncoder.setOutputListener(rtpPacketizer); + } else { + mSdlEncoder.setOutputStream(stream); + } return mSdlEncoder.prepareEncoder(); } diff --git a/sdl_android/src/main/java/com/smartdevicelink/encoder/IEncoderListener.java b/sdl_android/src/main/java/com/smartdevicelink/encoder/IEncoderListener.java new file mode 100644 index 0000000000..ce07afc5dd --- /dev/null +++ b/sdl_android/src/main/java/com/smartdevicelink/encoder/IEncoderListener.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017, Xevo Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the name of Xevo Inc. nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.smartdevicelink.encoder; + +/** + * A listener that receives a chunk of data from an encoder. + */ +public interface IEncoderListener { + /** + * Definitions of data formats. + */ + enum Format { + /** + * H.264 byte stream in Annex B format. "data" contains one or more H.264 NAL units. + */ + H264_BYTE_STREAM, + } + + /** + * Called when a chunk of data is output by the encoder. + * + * @param format The format of the data + * @param data The raw data output by the encoder + * @param presentationTimeUs The presentation timestamp (PTS) of this data in microseconds + */ + void onEncoderOutput(Format format, byte[] data, long presentationTimeUs); + + /** + * Called when a chunk of data is output by the encoder. + * + * @param format The format of the data + * @param data An array containing the raw data output by the encoder + * @param offset Starting offset in 'data' + * @param length Length of the raw data + * @param presentationTimeUs The presentation timestamp (PTS) of this data in microseconds + * + * @throws ArrayIndexOutOfBoundsException When offset does not satisfy {@code 0 <= offset && offset <= data.length} + * or length does not satisfy {@code 0 < length && offset + length <= data.length} + */ + void onEncoderOutput(Format format, byte[] data, int offset, int length, long presentationTimeUs) + throws ArrayIndexOutOfBoundsException; +} diff --git a/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java b/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java index 81e3448407..4554ba666c 100644 --- a/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java +++ b/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java @@ -26,6 +26,7 @@ public class SdlEncoder { // encoder state private MediaCodec mEncoder; private PipedOutputStream mOutputStream; + private IEncoderListener mOutputListener; // allocate one of these up front so we don't need to do it every time private MediaCodec.BufferInfo mBufferInfo; @@ -51,6 +52,9 @@ public void setBitrate(int iVal){ public void setOutputStream(PipedOutputStream mStream){ mOutputStream = mStream; } + public void setOutputListener(IEncoderListener listener) { + mOutputListener = listener; + } public Surface prepareEncoder () { mBufferInfo = new MediaCodec.BufferInfo(); @@ -127,7 +131,7 @@ public void releaseEncoder() { public void drainEncoder(boolean endOfStream) { final int TIMEOUT_USEC = 10000; - if(mEncoder == null || mOutputStream == null) { + if(mEncoder == null || (mOutputStream == null && mOutputListener == null)) { return; } if (endOfStream) { @@ -155,7 +159,12 @@ public void drainEncoder(boolean endOfStream) { mBufferInfo.offset, mBufferInfo.size); try { - mOutputStream.write(dataToWrite, 0, mBufferInfo.size); + if (mOutputStream != null) { + mOutputStream.write(dataToWrite, 0, mBufferInfo.size); + } else if (mOutputListener != null) { + mOutputListener.onEncoderOutput(IEncoderListener.Format.H264_BYTE_STREAM, + dataToWrite, mBufferInfo.presentationTimeUs); + } } catch (Exception e) {} } diff --git a/sdl_android/src/main/java/com/smartdevicelink/protocol/ProtocolMessage.java b/sdl_android/src/main/java/com/smartdevicelink/protocol/ProtocolMessage.java index 99b34aff65..cb5ad3a740 100644 --- a/sdl_android/src/main/java/com/smartdevicelink/protocol/ProtocolMessage.java +++ b/sdl_android/src/main/java/com/smartdevicelink/protocol/ProtocolMessage.java @@ -45,12 +45,16 @@ public void setData(byte[] data) { this._data = data; this._jsonSize = data.length; } - + public void setData(byte[] data, int length) { + setData(data, 0, length); + } + + public void setData(byte[] data, int offset, int length) { if (this._data != null) this._data = null; this._data = new byte[length]; - System.arraycopy(data, 0, this._data, 0, length); + System.arraycopy(data, offset, this._data, 0, length); this._jsonSize = 0; } diff --git a/sdl_android/src/main/java/com/smartdevicelink/streaming/RTPH264Packetizer.java b/sdl_android/src/main/java/com/smartdevicelink/streaming/RTPH264Packetizer.java new file mode 100644 index 0000000000..4b884ff5bd --- /dev/null +++ b/sdl_android/src/main/java/com/smartdevicelink/streaming/RTPH264Packetizer.java @@ -0,0 +1,486 @@ +/* + * Copyright (c) 2017, Xevo Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the name of Xevo Inc. nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.smartdevicelink.streaming; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Random; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import com.smartdevicelink.SdlConnection.SdlConnection; +import com.smartdevicelink.SdlConnection.SdlSession; +import com.smartdevicelink.encoder.IEncoderListener; +import com.smartdevicelink.protocol.ProtocolMessage; +import com.smartdevicelink.protocol.enums.SessionType; + +/* + * Note for testing. + * The RTP stream generated by this packetizer can be tested with GStreamer. Assuming that + * "VideoStreamPort" is configured as 5050 in smartDeviceLink.ini, here is the GStreamer pipeline + * that receives the stream, decode it and render it: + * + * $ gst-launch-1.0 souphttpsrc location=http://127.0.0.1:5050 ! "application/x-rtp-stream" ! rtpstreamdepay ! "application/x-rtp,media=(string)video,clock-rate=90000,encoding-name=(string)H264" ! rtph264depay ! "video/x-h264, stream-format=(string)avc, alignment=(string)au" ! avdec_h264 ! autovideosink sync=false + * + * Note that GStreamer version 1.4 or later is needed for "rtpstreamdepay" component. "souphttpsrc" + * component is used instead of "tcpclientsrc" because SDL core implementation adds a (fake) HTTP + * response header before sending actual video stream. + */ + +/** + * This class receives H.264 byte stream (in Annex-B format), parses it, construct RTP packets + * from it based on RFC 6184, then frame the packets based on RFC 4571. + * The primary purpose of using RTP is to carry timestamp information along with the data. + * + * @author Sho Amano + */ +public class RTPH264Packetizer extends AbstractPacketizer implements IEncoderListener, Runnable { + + // Approximate size of data that mOutputQueue can hold in bytes. + // By adding a buffer, we accept underlying transport being stuck for a short time. By setting + // a limit of the buffer size, we avoid buffer overflows when underlying transport is too slow. + private static final int MAX_QUEUE_SIZE = 256 * 1024; + + private static final int FRAME_LENGTH_LEN = 2; + private static final int MAX_RTP_PACKET_SIZE = 65535; // because length field is two bytes (RFC 4571) + private static final int RTP_HEADER_LEN = 12; + private static final byte DEFAULT_RTP_PAYLOAD_TYPE = 96; + private static final int FU_INDICATOR_LEN = 1; + private static final int FU_HEADER_LEN = 1; + private static final byte TYPE_FU_A = 28; + + // See StreamPacketizer class. We can only use 1024 as the max buffer size if the service is encrypted. + private final static int MAX_DATA_SIZE_FOR_ENCRYPTED_SERVICE = 1024; + + private boolean mServiceProtected; + private Thread mThread; + private BlockingQueue mOutputQueue; + private volatile boolean mPaused; + private boolean mWaitForIDR; + private NALUnitReader mNALUnitReader; + private byte mPayloadType = 0; + private int mSSRC = 0; + private char mSequenceNum = 0; + private int mInitialPTS = 0; + + /** + * Constructor + * + * @param streamListener The listener which this packetizer outputs SDL frames to + * @param serviceType The value of "Service Type" field in SDL frames + * @param sessionID The value of "Session ID" field in SDL frames + * @param session The SdlSession instance that this packetizer belongs to + */ + public RTPH264Packetizer(IStreamListener streamListener, + SessionType serviceType, byte sessionID, SdlSession session) throws IOException { + + super(streamListener, null, serviceType, sessionID, session); + + mServiceProtected = session.isServiceProtected(_serviceType); + + if (bufferSize == 0) { + // fail safe + bufferSize = MAX_DATA_SIZE_FOR_ENCRYPTED_SERVICE; + } + if (mServiceProtected && bufferSize > MAX_DATA_SIZE_FOR_ENCRYPTED_SERVICE) { + bufferSize = MAX_DATA_SIZE_FOR_ENCRYPTED_SERVICE; + } + + mOutputQueue = new LinkedBlockingQueue(MAX_QUEUE_SIZE / bufferSize); + mNALUnitReader = new NALUnitReader(); + mPayloadType = DEFAULT_RTP_PAYLOAD_TYPE; + + Random r = new Random(); + mSSRC = r.nextInt(); + + // initial value of the sequence number and timestamp should be random ([5.1] in RFC3550) + mSequenceNum = (char)r.nextInt(65536); + mInitialPTS = r.nextInt(); + } + + /** + * Sets the Payload Type (PT) of RTP header field. + * + * Use this method if PT needs to be specified. The value should be between 0 and 127. + * Otherwise, a default value (96) is used. + * + * @param type A value indicating the Payload Type + */ + public void setPayloadType(byte type) { + if (type >= 0 && type <= 127) { + mPayloadType = type; + } else { + mPayloadType = DEFAULT_RTP_PAYLOAD_TYPE; + } + } + + /** + * Sets the SSRC of RTP header field. + * + * Use this method if SSRC needs to be specified. Otherwise, a random value is generated and + * used. + * + * @param ssrc An integer value representing SSRC + */ + public void setSSRC(int ssrc) { + mSSRC = ssrc; + } + + /** + * Overwrites MTU value. This is only for testing. + * + * @param mtu maximum payload size of SDL frames + */ + public void setMTU(int mtu) { + if (mtu <= 0) { + mtu = MAX_DATA_SIZE_FOR_ENCRYPTED_SERVICE; + } + bufferSize = mtu; + } + + /** + * Starts this packetizer. + * + * It is recommended that the video encoder is started after the packetizer is started. + */ + @Override + public void start() throws IOException { + if (mThread != null) { + return; + } + + mThread = new Thread(this); + mThread.start(); + } + + /** + * Stops this packetizer. + * + * It is recommended that the video encoder is stopped prior to the packetizer. + */ + @Override + public void stop() { + if (mThread == null) { + return; + } + + mThread.interrupt(); + mThread = null; + + mPaused = false; + mWaitForIDR = false; + mOutputQueue.clear(); + } + + /** + * Pauses this packetizer. + * + * This pauses the packetizer but does not pause the video encoder. + */ + @Override + public void pause() { + mPaused = true; + } + + /** + * Resumes this packetizer. + */ + @Override + public void resume() { + mWaitForIDR = true; + mPaused = false; + } + + /** + * The thread routine. + */ + public void run() { + SdlConnection connection = _session.getSdlConnection(); + + while (mThread != null && !mThread.isInterrupted()) { + ByteBuffer frame; + try { + frame = mOutputQueue.take(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + + while (frame.hasRemaining()) { + int len = frame.remaining() > bufferSize ? bufferSize : frame.remaining(); + + ProtocolMessage pm = new ProtocolMessage(); + pm.setSessionID(_rpcSessionID); + pm.setSessionType(_serviceType); + pm.setFunctionID(0); + pm.setCorrID(0); + pm.setData(frame.array(), frame.arrayOffset() + frame.position(), len); + pm.setPayloadProtected(mServiceProtected); + + _streamListener.sendStreamPacket(pm); + + frame.position(frame.position() + len); + } + } + + // XXX: This is added to sync with StreamPacketizer. Actually it shouldn't be here since + // it's confusing that a packetizer takes care of End Service request. + if (connection != null) { + connection.endService(_serviceType, _rpcSessionID); + } + } + + /** + * Called by the encoder. + * + * @see com.smartdevicelink.encoder.IEncoderListener#onEncoderOutput(Format, byte[], long) + */ + @Override + public void onEncoderOutput(IEncoderListener.Format format, byte[] data, long ptsInUs) { + if (data == null || format != Format.H264_BYTE_STREAM) { + return; + } + + mNALUnitReader.init(data); + onEncoderOutput(mNALUnitReader, ptsInUs); + } + + /** + * Called by the encoder. + * + * @see com.smartdevicelink.encoder.IEncoderListener#onEncoderOutput(Format, byte[], int, int, long) + */ + @Override + public void onEncoderOutput(IEncoderListener.Format format, byte[] data, + int offset, int length, long ptsInUs) throws ArrayIndexOutOfBoundsException { + if (data == null || format != Format.H264_BYTE_STREAM) { + return; + } + + mNALUnitReader.init(data, offset, length); + onEncoderOutput(mNALUnitReader, ptsInUs); + } + + private void onEncoderOutput(NALUnitReader nalUnitReader, long ptsInUs) { + if (mPaused) { + return; + } + + ByteBuffer nalu; + + while ((nalu = nalUnitReader.getNalUnit()) != null) { + if (mWaitForIDR) { + if (isIDR(nalu)) { + mWaitForIDR = false; + } else { + continue; + } + } + outputRTPFrames(nalu, ptsInUs, nalUnitReader.hasConsumedAll()); + } + } + + private boolean outputRTPFrames(ByteBuffer nalu, long ptsInUs, boolean isLast) { + if (RTP_HEADER_LEN + nalu.remaining() > MAX_RTP_PACKET_SIZE) { + // Split into multiple Fragmentation Units ([5.8] in RFC 6184) + byte firstByte = nalu.get(); + boolean firstFragment = true; + boolean lastFragment = false; + + while (nalu.remaining() > 0) { + int payloadLength = MAX_RTP_PACKET_SIZE - (RTP_HEADER_LEN + FU_INDICATOR_LEN + FU_HEADER_LEN); + if (nalu.remaining() <= payloadLength) { + payloadLength = nalu.remaining(); + lastFragment = true; + } + + ByteBuffer frame = allocateRTPFrame(FU_INDICATOR_LEN + FU_HEADER_LEN + payloadLength, + false, isLast, ptsInUs); + // FU indicator + frame.put((byte)((firstByte & 0xE0) | TYPE_FU_A)); + // FU header + frame.put((byte)((firstFragment ? 0x80 : lastFragment ? 0x40 : 0) | (firstByte & 0x1F))); + // FU payload + frame.put(nalu.array(), nalu.position(), payloadLength); + nalu.position(nalu.position() + payloadLength); + frame.flip(); + + try { + mOutputQueue.put(frame); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + + firstFragment = false; + } + } else { + // Use Single NAL Unit Packet ([5.6] in RFC 6184) + ByteBuffer frame = allocateRTPFrame(nalu.remaining(), false, isLast, ptsInUs); + frame.put(nalu); + frame.flip(); + + try { + mOutputQueue.put(frame); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + return true; + } + + private ByteBuffer allocateRTPFrame(int rtpPayloadLen, + boolean hasPadding, boolean isLast, long ptsInUs) { + assert rtpPayloadLen > 0; + assert ptsInUs > 0; + + int packetLength = RTP_HEADER_LEN + rtpPayloadLen; + assert packetLength <= MAX_RTP_PACKET_SIZE; + int ptsIn90kHz = (int)(ptsInUs * 9 / 100) + mInitialPTS; + + ByteBuffer frame = ByteBuffer.allocate(FRAME_LENGTH_LEN + packetLength); + frame.order(ByteOrder.BIG_ENDIAN); + frame.putShort((short)packetLength); + + // Version = 2, Padding = hasPadding, Extension = 0, CSRC count = 0 + frame.put((byte)(0x80 | (hasPadding ? 0x20 : 0))) + // Marker = isLast, Payload type = mPayloadType + .put((byte)((isLast ? 0x80 : 0) | (mPayloadType & 0x7F))) + .putChar(mSequenceNum) + .putInt(ptsIn90kHz) + .putInt(mSSRC); + + assert frame.position() == FRAME_LENGTH_LEN + RTP_HEADER_LEN; + + mSequenceNum++; + return frame; + } + + private static boolean isIDR(ByteBuffer nalu) { + assert nalu != null; + assert nalu.hasRemaining(); + + byte nalUnitType = (byte)(nalu.get(nalu.position()) & 0x1F); + return nalUnitType == 5; + } + + + private static int SKIP_TABLE[] = new int[256]; + static { + // Sunday's quick search algorithm is used to find the start code. + // Prepare the table (SKIP_TABLE[0] = 2, SKIP_TABLE[1] = 1 and other elements will be 4). + byte[] NALU_START_CODE = {0, 0, 1}; + int searchStringLen = NALU_START_CODE.length; + for (int i = 0; i < SKIP_TABLE.length; i++) { + SKIP_TABLE[i] = searchStringLen + 1; + } + for (int i = 0; i < searchStringLen; i++) { + SKIP_TABLE[NALU_START_CODE[i] & 0xFF] = searchStringLen - i; + } + } + + private class NALUnitReader { + private byte[] mData; + private int mOffset; + private int mLimit; + + NALUnitReader() { + } + + void init(byte[] data) { + mData = data; + mOffset = 0; + mLimit = data.length; + } + + void init(byte[] data, int offset, int length) throws ArrayIndexOutOfBoundsException { + if (offset < 0 || offset > data.length || length <= 0 || offset + length > data.length) { + throw new ArrayIndexOutOfBoundsException(); + } + mData = data; + mOffset = offset; + mLimit = offset + length; + } + + ByteBuffer getNalUnit() { + if (hasConsumedAll()) { + return null; + } + + int pos = mOffset; + int start = -1; + + while (mLimit - pos >= 3) { + if (mData[pos] == 0 && mData[pos+1] == 0 && mData[pos+2] == 1) { + if (start != -1) { + // We've found a start code, a NAL unit and then another start code. + mOffset = pos; + // remove 0x00s in front of the start code + while (pos > start && mData[pos-1] == 0) { + pos--; + } + if (pos > start) { + return ByteBuffer.wrap(mData, start, pos - start); + } else { + // No NAL unit between two start codes?! Forget it and search for + // another start code. + pos = mOffset; + } + } + // This is the first start code. + pos += 3; + start = pos; + } else { + try { + pos += SKIP_TABLE[mData[pos+3] & 0xFF]; + } catch (ArrayIndexOutOfBoundsException e) { + break; + } + } + } + + mOffset = mLimit; + if (start != -1 && mLimit > start) { + // We've found a start code and then reached to the end of array. + return ByteBuffer.wrap(mData, start, mLimit - start); + } + // A start code was not found + return null; + } + + boolean hasConsumedAll() { + return (mData == null) || (mLimit - mOffset < 4); + } + } +}