drgreen/android_frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Resilent media time stamp adjustment" into gingerbread

Browse Source
This commit is contained in:
James Dong
2010-10-04 15:35:40 -07:00
committed by Android (Google) Code Review
parent b621e2030d acee8e7131
commit b99f0c7eae
1 changed files with 181 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
media/libstagefright/MPEG4Writer.cpp
View File

@ -40,6 +40,7 @@ namespace android {
static const int64_t kMax32BitFileSize = 0x007fffffffLL;
static const uint8_t kNalUnitTypeSeqParamSet = 0x07;
static const uint8_t kNalUnitTypePicParamSet = 0x08;
static const int64_t kVideoMediaTimeAdjustPeriodTimeUs = 10000000LL; // 10s
class MPEG4Writer::Track {
public:
@ -148,6 +149,28 @@ private:
int64_t mPreviousTrackTimeUs;
int64_t mTrackEveryTimeDurationUs;
// Has the media time adjustment for video started?
bool mIsMediaTimeAdjustmentOn;
// The time stamp when previous media time adjustment period starts
int64_t mPrevMediaTimeAdjustTimestampUs;
// Number of vidoe frames whose time stamp may be adjusted
int64_t mMediaTimeAdjustNumFrames;
// The sample number when previous meida time adjustmnet period starts
int64_t mPrevMediaTimeAdjustSample;
// The total accumulated drift time within a period of
// kVideoMediaTimeAdjustPeriodTimeUs.
int64_t mTotalDriftTimeToAdjustUs;
// The total accumalated drift time since the start of the recording
// excluding the current time adjustment period
int64_t mPrevTotalAccumDriftTimeUs;
// Update the audio track's drift information.
void updateDriftTime(const sp<MetaData>& meta);
// Adjust the time stamp of the video track according to
// the drift time information from the audio track.
void adjustMediaTime(int64_t *timestampUs);
static void *ThreadWrapper(void *me);
status_t threadEntry();
@ -319,7 +342,7 @@ int64_t MPEG4Writer::estimateMoovBoxSize(int32_t bitRate) {
size = MAX_MOOV_BOX_SIZE;
}
LOGI("limits: %lld/%lld bytes/us, bit rate: %d bps and the estimated"
LOGV("limits: %lld/%lld bytes/us, bit rate: %d bps and the estimated"
" moov size %lld bytes",
mMaxFileSizeLimitBytes, mMaxFileDurationLimitUs, bitRate, size);
return factor * size;
@ -346,7 +369,7 @@ status_t MPEG4Writer::start(MetaData *param) {
// If file size is set to be larger than the 32 bit file
// size limit, treat it as an error.
if (mMaxFileSizeLimitBytes > kMax32BitFileSize) {
LOGW("32-bi file size limit (%lld bytes) too big. "
LOGW("32-bit file size limit (%lld bytes) too big. "
"It is changed to %lld bytes",
mMaxFileSizeLimitBytes, kMax32BitFileSize);
mMaxFileSizeLimitBytes = kMax32BitFileSize;
@ -1149,6 +1172,12 @@ status_t MPEG4Writer::Track::start(MetaData *params) {
mNumStscTableEntries = 0;
mNumSttsTableEntries = 0;
mMdatSizeBytes = 0;
mIsMediaTimeAdjustmentOn = false;
mPrevMediaTimeAdjustTimestampUs = 0;
mMediaTimeAdjustNumFrames = 0;
mPrevMediaTimeAdjustSample = 0;
mTotalDriftTimeToAdjustUs = 0;
mPrevTotalAccumDriftTimeUs = 0;
pthread_create(&mThread, &attr, ThreadWrapper, this);
pthread_attr_destroy(&attr);
@ -1437,6 +1466,145 @@ status_t MPEG4Writer::Track::makeAVCCodecSpecificData(
return OK;
}
/*
* The video track's media time adjustment for real-time applications
* is described as follows:
*
* First, the media time adjustment is done for every period of
* kVideoMediaTimeAdjustPeriodTimeUs. kVideoMediaTimeAdjustPeriodTimeUs
* is currently a fixed value chosen heuristically. The value of
* kVideoMediaTimeAdjustPeriodTimeUs should not be very large or very small
* for two considerations: on one hand, a relatively large value
* helps reduce large fluctuation of drift time in the audio encoding
* path; while on the other hand, a relatively small value helps keep
* restoring synchronization in audio/video more frequently. Note for the
* very first period of kVideoMediaTimeAdjustPeriodTimeUs, there is
* no media time adjustment for the video track.
*
* Second, the total accumulated audio track time drift found
* in a period of kVideoMediaTimeAdjustPeriodTimeUs is distributed
* over a stream of incoming video frames. The number of video frames
* affected is determined based on the number of recorded video frames
* within the past kVideoMediaTimeAdjustPeriodTimeUs period.
* We choose to distribute the drift time over only a portion
* (rather than all) of the total number of recorded video frames
* in order to make sure that the video track media time adjustment is
* completed for the current period before the next video track media
* time adjustment period starts. Currently, the portion chosen is a
* half (0.5).
*
* Last, various additional checks are performed to ensure that
* the actual audio encoding path does not have too much drift.
* In particular, 1) we want to limit the average incremental time
* adjustment for each video frame to be less than a threshold
* for a single period of kVideoMediaTimeAdjustPeriodTimeUs.
* Currently, the threshold is set to 5 ms. If the average incremental
* media time adjustment for a video frame is larger than the
* threshold, the audio encoding path has too much time drift.
* 2) We also want to limit the total time drift in the audio
* encoding path to be less than a threshold for a period of
* kVideoMediaTimeAdjustPeriodTimeUs. Currently, the threshold
* is 0.5% of kVideoMediaTimeAdjustPeriodTimeUs. If the time drift of
* the audio encoding path is larger than the threshold, the audio
* encoding path has too much time drift. We treat the large time
* drift of the audio encoding path as errors, since there is no
* way to keep audio/video in synchronization for real-time
* applications if the time drift is too large unless we drop some
* video frames, which has its own problems that we don't want
* to get into for the time being.
*/
void MPEG4Writer::Track::adjustMediaTime(int64_t *timestampUs) {
if (*timestampUs - mPrevMediaTimeAdjustTimestampUs >=
kVideoMediaTimeAdjustPeriodTimeUs) {
LOGV("New media time adjustment period at %lld us", *timestampUs);
mIsMediaTimeAdjustmentOn = true;
mMediaTimeAdjustNumFrames =
(mNumSamples - mPrevMediaTimeAdjustSample) >> 1;
mPrevMediaTimeAdjustTimestampUs = *timestampUs;
mPrevMediaTimeAdjustSample = mNumSamples;
int64_t totalAccumDriftTimeUs = mOwner->getDriftTimeUs();
mTotalDriftTimeToAdjustUs =
totalAccumDriftTimeUs - mPrevTotalAccumDriftTimeUs;
mPrevTotalAccumDriftTimeUs = totalAccumDriftTimeUs;
// Check on incremental adjusted time per frame
int64_t adjustTimePerFrameUs =
mTotalDriftTimeToAdjustUs / mMediaTimeAdjustNumFrames;
if (adjustTimePerFrameUs < 0) {
adjustTimePerFrameUs = -adjustTimePerFrameUs;
}
if (adjustTimePerFrameUs >= 5000) {
LOGE("Adjusted time per video frame is %lld us",
adjustTimePerFrameUs);
CHECK(!"Video frame time adjustment is too large!");
}
// Check on total accumulated time drift within a period of
// kVideoMediaTimeAdjustPeriodTimeUs.
int64_t driftPercentage = (mTotalDriftTimeToAdjustUs * 1000)
/ kVideoMediaTimeAdjustPeriodTimeUs;
if (driftPercentage < 0) {
driftPercentage = -driftPercentage;
}
if (driftPercentage > 5) {
LOGE("Audio track has time drift %lld us over %lld us",
mTotalDriftTimeToAdjustUs,
kVideoMediaTimeAdjustPeriodTimeUs);
CHECK(!"The audio track media time drifts too much!");
}
}
if (mIsMediaTimeAdjustmentOn) {
if (mNumSamples - mPrevMediaTimeAdjustSample <=
mMediaTimeAdjustNumFrames) {
// Do media time incremental adjustment
int64_t incrementalAdjustTimeUs =
(mTotalDriftTimeToAdjustUs *
(mNumSamples - mPrevMediaTimeAdjustSample))
/ mMediaTimeAdjustNumFrames;
*timestampUs +=
(incrementalAdjustTimeUs + mPrevTotalAccumDriftTimeUs);
LOGV("Incremental video frame media time adjustment: %lld us",
(incrementalAdjustTimeUs + mPrevTotalAccumDriftTimeUs));
} else {
// Within the remaining adjustment period,
// no incremental adjustment is needed.
*timestampUs +=
(mTotalDriftTimeToAdjustUs + mPrevTotalAccumDriftTimeUs);
LOGV("Fixed video frame media time adjustment: %lld us",
(mTotalDriftTimeToAdjustUs + mPrevTotalAccumDriftTimeUs));
}
}
}
/*
* Updates the drift time from the audio track so that
* the video track can get the updated drift time information
* from the file writer. The fluctuation of the drift time of the audio
* encoding path is smoothed out with a simple filter by giving a larger
* weight to more recently drift time. The filter coefficients, 0.5 and 0.5,
* are heuristically determined.
*/
void MPEG4Writer::Track::updateDriftTime(const sp<MetaData>& meta) {
int64_t driftTimeUs = 0;
if (meta->findInt64(kKeyDriftTime, &driftTimeUs)) {
int64_t prevDriftTimeUs = mOwner->getDriftTimeUs();
int64_t timeUs = (driftTimeUs + prevDriftTimeUs) >> 1;
mOwner->setDriftTimeUs(timeUs);
}
}
status_t MPEG4Writer::Track::threadEntry() {
int32_t count = 0;
const int64_t interleaveDurationUs = mOwner->interleaveDuration();
@ -1587,24 +1755,16 @@ status_t MPEG4Writer::Track::threadEntry() {
timestampUs -= previousPausedDurationUs;
CHECK(timestampUs >= 0);
if (mIsRealTimeRecording && !mIsAudio) {
// The minor adjustment on the timestamp is heuristic/experimental
// We are adjusting the timestamp to reduce the fluctuation of the duration
// of neighboring samples. This in turn helps reduce the track header size,
// especially, the number of entries in the "stts" box.
if (mNumSamples > 1) {
int64_t currDriftTimeUs = mOwner->getDriftTimeUs();
int64_t durationUs = timestampUs + currDriftTimeUs - lastTimestampUs;
int64_t diffUs = (durationUs > lastDurationUs)
? durationUs - lastDurationUs
: lastDurationUs - durationUs;
if (diffUs <= 5000) { // XXX: Magic number 5ms
timestampUs = lastTimestampUs + lastDurationUs;
} else {
timestampUs += currDriftTimeUs;
}
// Media time adjustment for real-time applications
if (mIsRealTimeRecording) {
if (mIsAudio) {
updateDriftTime(meta_data);
} else {
adjustMediaTime(&timestampUs);
}
}
CHECK(timestampUs >= 0);
if (mNumSamples > 1) {
if (timestampUs <= lastTimestampUs) {
@ -1656,12 +1816,6 @@ status_t MPEG4Writer::Track::threadEntry() {
lastDurationUs = timestampUs - lastTimestampUs;
lastDurationTicks = currDurationTicks;
lastTimestampUs = timestampUs;
if (mIsRealTimeRecording && mIsAudio) {
int64_t driftTimeUs = 0;
if (meta_data->findInt64(kKeyDriftTime, &driftTimeUs)) {
mOwner->setDriftTimeUs(driftTimeUs);
}
}
if (isSync != 0) {
addOneStssTableEntry(mNumSamples);
@ -1735,6 +1889,9 @@ status_t MPEG4Writer::Track::threadEntry() {
mReachedEOS = true;
LOGI("Received total/0-length (%d/%d) buffers and encoded %d frames. - %s",
count, nZeroLengthFrames, mNumSamples, mIsAudio? "audio": "video");
if (mIsAudio) {
LOGI("Audio track drift time: %lld us", mOwner->getDriftTimeUs());
}
if (err == ERROR_END_OF_STREAM) {
return OK;