482 lines
19 KiB
C++
482 lines
19 KiB
C++
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
* You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
#include "MediaStreamGraphImpl.h"
|
|
#include "MediaStreamListener.h"
|
|
#include "mozilla/MathAlgorithms.h"
|
|
#include "mozilla/Unused.h"
|
|
|
|
#include "AudioSegment.h"
|
|
#include "VideoSegment.h"
|
|
#include "nsContentUtils.h"
|
|
#include "nsIAppShell.h"
|
|
#include "nsIObserver.h"
|
|
#include "nsPrintfCString.h"
|
|
#include "nsServiceManagerUtils.h"
|
|
#include "nsWidgetsCID.h"
|
|
#include "prerror.h"
|
|
#include "mozilla/Logging.h"
|
|
#include "mozilla/Attributes.h"
|
|
#include "TrackUnionStream.h"
|
|
#include "ImageContainer.h"
|
|
#include "AudioChannelService.h"
|
|
#include "AudioNodeEngine.h"
|
|
#include "AudioNodeStream.h"
|
|
#include "AudioNodeExternalInputStream.h"
|
|
#include "webaudio/MediaStreamAudioDestinationNode.h"
|
|
#include <algorithm>
|
|
#include "DOMMediaStream.h"
|
|
#include "GeckoProfiler.h"
|
|
#ifdef MOZ_WEBRTC
|
|
#include "AudioOutputObserver.h"
|
|
#endif
|
|
|
|
using namespace mozilla::layers;
|
|
using namespace mozilla::dom;
|
|
using namespace mozilla::gfx;
|
|
|
|
namespace mozilla {
|
|
|
|
#ifdef STREAM_LOG
|
|
#undef STREAM_LOG
|
|
#endif
|
|
|
|
LazyLogModule gTrackUnionStreamLog("TrackUnionStream");
|
|
#define STREAM_LOG(type, msg) MOZ_LOG(gTrackUnionStreamLog, type, msg)
|
|
|
|
TrackUnionStream::TrackUnionStream() :
|
|
ProcessedMediaStream(), mNextAvailableTrackID(1)
|
|
{
|
|
}
|
|
|
|
void TrackUnionStream::RemoveInput(MediaInputPort* aPort)
|
|
{
|
|
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p removing input %p", this, aPort));
|
|
for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) {
|
|
if (mTrackMap[i].mInputPort == aPort) {
|
|
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p removing trackmap entry %d", this, i));
|
|
EndTrack(i);
|
|
nsTArray<RefPtr<DirectMediaStreamTrackListener>> listeners(
|
|
mTrackMap[i].mOwnedDirectListeners);
|
|
for (auto listener : listeners) {
|
|
// Remove listeners while the entry still exists.
|
|
RemoveDirectTrackListenerImpl(listener, mTrackMap[i].mOutputTrackID);
|
|
}
|
|
mTrackMap.RemoveElementAt(i);
|
|
}
|
|
}
|
|
ProcessedMediaStream::RemoveInput(aPort);
|
|
}
|
|
void TrackUnionStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags)
|
|
{
|
|
if (IsFinishedOnGraphThread()) {
|
|
return;
|
|
}
|
|
AutoTArray<bool,8> mappedTracksFinished;
|
|
AutoTArray<bool,8> mappedTracksWithMatchingInputTracks;
|
|
for (uint32_t i = 0; i < mTrackMap.Length(); ++i) {
|
|
mappedTracksFinished.AppendElement(true);
|
|
mappedTracksWithMatchingInputTracks.AppendElement(false);
|
|
}
|
|
bool allFinished = !mInputs.IsEmpty();
|
|
bool allHaveCurrentData = !mInputs.IsEmpty();
|
|
for (uint32_t i = 0; i < mInputs.Length(); ++i) {
|
|
MediaStream* stream = mInputs[i]->GetSource();
|
|
if (!stream->IsFinishedOnGraphThread()) {
|
|
// XXX we really should check whether 'stream' has finished within time aTo,
|
|
// not just that it's finishing when all its queued data eventually runs
|
|
// out.
|
|
allFinished = false;
|
|
}
|
|
if (!stream->HasCurrentData()) {
|
|
allHaveCurrentData = false;
|
|
}
|
|
bool trackAdded = false;
|
|
for (StreamTracks::TrackIter tracks(stream->GetStreamTracks());
|
|
!tracks.IsEnded(); tracks.Next()) {
|
|
bool found = false;
|
|
for (uint32_t j = 0; j < mTrackMap.Length(); ++j) {
|
|
TrackMapEntry* map = &mTrackMap[j];
|
|
if (map->mInputPort == mInputs[i] && map->mInputTrackID == tracks->GetID()) {
|
|
bool trackFinished = false;
|
|
StreamTracks::Track* outputTrack = mTracks.FindTrack(map->mOutputTrackID);
|
|
found = true;
|
|
if (!outputTrack || outputTrack->IsEnded() ||
|
|
!mInputs[i]->PassTrackThrough(tracks->GetID())) {
|
|
trackFinished = true;
|
|
} else {
|
|
CopyTrackData(tracks.get(), j, aFrom, aTo, &trackFinished);
|
|
}
|
|
mappedTracksFinished[j] = trackFinished;
|
|
mappedTracksWithMatchingInputTracks[j] = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!found && mInputs[i]->AllowCreationOf(tracks->GetID())) {
|
|
bool trackFinished = false;
|
|
trackAdded = true;
|
|
uint32_t mapIndex = AddTrack(mInputs[i], tracks.get(), aFrom);
|
|
CopyTrackData(tracks.get(), mapIndex, aFrom, aTo, &trackFinished);
|
|
mappedTracksFinished.AppendElement(trackFinished);
|
|
mappedTracksWithMatchingInputTracks.AppendElement(true);
|
|
}
|
|
}
|
|
if (trackAdded) {
|
|
for (MediaStreamListener* l : mListeners) {
|
|
l->NotifyFinishedTrackCreation(Graph());
|
|
}
|
|
}
|
|
}
|
|
for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) {
|
|
if (mappedTracksFinished[i]) {
|
|
EndTrack(i);
|
|
} else {
|
|
allFinished = false;
|
|
}
|
|
if (!mappedTracksWithMatchingInputTracks[i]) {
|
|
for (auto listener : mTrackMap[i].mOwnedDirectListeners) {
|
|
// Remove listeners while the entry still exists.
|
|
RemoveDirectTrackListenerImpl(listener, mTrackMap[i].mOutputTrackID);
|
|
}
|
|
mTrackMap.RemoveElementAt(i);
|
|
}
|
|
}
|
|
if (allFinished && mAutofinish && (aFlags & ALLOW_FINISH)) {
|
|
// All streams have finished and won't add any more tracks, and
|
|
// all our tracks have actually finished and been removed from our map,
|
|
// so we're finished now.
|
|
FinishOnGraphThread();
|
|
} else {
|
|
mTracks.AdvanceKnownTracksTime(GraphTimeToStreamTimeWithBlocking(aTo));
|
|
}
|
|
if (allHaveCurrentData) {
|
|
// We can make progress if we're not blocked
|
|
mHasCurrentData = true;
|
|
}
|
|
}
|
|
|
|
uint32_t TrackUnionStream::AddTrack(MediaInputPort* aPort, StreamTracks::Track* aTrack,
|
|
GraphTime aFrom)
|
|
{
|
|
STREAM_LOG(LogLevel::Verbose, ("TrackUnionStream %p adding track %d for "
|
|
"input stream %p track %d, desired id %d",
|
|
this, aTrack->GetID(), aPort->GetSource(),
|
|
aTrack->GetID(),
|
|
aPort->GetDestinationTrackId()));
|
|
|
|
TrackID id;
|
|
if (IsTrackIDExplicit(id = aPort->GetDestinationTrackId())) {
|
|
MOZ_ASSERT(id >= mNextAvailableTrackID &&
|
|
mUsedTracks.BinaryIndexOf(id) == mUsedTracks.NoIndex,
|
|
"Desired destination id taken. Only provide a destination ID "
|
|
"if you can assure its availability, or we may not be able "
|
|
"to bind to the correct DOM-side track.");
|
|
#ifdef DEBUG
|
|
for (size_t i = 0; mInputs[i] != aPort; ++i) {
|
|
MOZ_ASSERT(mInputs[i]->GetSourceTrackId() != TRACK_ANY,
|
|
"You are adding a MediaInputPort with a track mapping "
|
|
"while there already exist generic MediaInputPorts for this "
|
|
"destination stream. This can lead to TrackID collisions!");
|
|
}
|
|
#endif
|
|
mUsedTracks.InsertElementSorted(id);
|
|
} else if ((id = aTrack->GetID()) &&
|
|
id > mNextAvailableTrackID &&
|
|
mUsedTracks.BinaryIndexOf(id) == mUsedTracks.NoIndex) {
|
|
// Input id available. Mark it used in mUsedTracks.
|
|
mUsedTracks.InsertElementSorted(id);
|
|
} else {
|
|
// No desired destination id and Input id taken, allocate a new one.
|
|
id = mNextAvailableTrackID;
|
|
|
|
// Update mNextAvailableTrackID and prune any mUsedTracks members it now
|
|
// covers.
|
|
while (1) {
|
|
if (!mUsedTracks.RemoveElementSorted(++mNextAvailableTrackID)) {
|
|
// Not in use. We're done.
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Round up the track start time so the track, if anything, starts a
|
|
// little later than the true time. This means we'll have enough
|
|
// samples in our input stream to go just beyond the destination time.
|
|
StreamTime outputStart = GraphTimeToStreamTimeWithBlocking(aFrom);
|
|
|
|
nsAutoPtr<MediaSegment> segment;
|
|
segment = aTrack->GetSegment()->CreateEmptyClone();
|
|
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
|
|
MediaStreamListener* l = mListeners[j];
|
|
l->NotifyQueuedTrackChanges(Graph(), id, outputStart,
|
|
TrackEventCommand::TRACK_EVENT_CREATED,
|
|
*segment,
|
|
aPort->GetSource(), aTrack->GetID());
|
|
}
|
|
segment->AppendNullData(outputStart);
|
|
StreamTracks::Track* track =
|
|
&mTracks.AddTrack(id, outputStart, segment.forget());
|
|
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p added track %d for input stream %p track %d, start ticks %lld",
|
|
this, track->GetID(), aPort->GetSource(), aTrack->GetID(),
|
|
(long long)outputStart));
|
|
|
|
TrackMapEntry* map = mTrackMap.AppendElement();
|
|
map->mEndOfConsumedInputTicks = 0;
|
|
map->mEndOfLastInputIntervalInInputStream = -1;
|
|
map->mEndOfLastInputIntervalInOutputStream = -1;
|
|
map->mInputPort = aPort;
|
|
map->mInputTrackID = aTrack->GetID();
|
|
map->mOutputTrackID = track->GetID();
|
|
map->mSegment = aTrack->GetSegment()->CreateEmptyClone();
|
|
|
|
for (int32_t i = mPendingDirectTrackListeners.Length() - 1; i >= 0; --i) {
|
|
TrackBound<DirectMediaStreamTrackListener>& bound =
|
|
mPendingDirectTrackListeners[i];
|
|
if (bound.mTrackID != map->mOutputTrackID) {
|
|
continue;
|
|
}
|
|
MediaStream* source = map->mInputPort->GetSource();
|
|
map->mOwnedDirectListeners.AppendElement(bound.mListener);
|
|
DisabledTrackMode currentMode = GetDisabledTrackMode(bound.mTrackID);
|
|
if (currentMode != DisabledTrackMode::ENABLED) {
|
|
bound.mListener->IncreaseDisabled(currentMode);
|
|
}
|
|
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p adding direct listener "
|
|
"%p for track %d. Forwarding to input "
|
|
"stream %p track %d.",
|
|
this, bound.mListener.get(), bound.mTrackID,
|
|
source, map->mInputTrackID));
|
|
source->AddDirectTrackListenerImpl(bound.mListener.forget(),
|
|
map->mInputTrackID);
|
|
mPendingDirectTrackListeners.RemoveElementAt(i);
|
|
}
|
|
|
|
return mTrackMap.Length() - 1;
|
|
}
|
|
|
|
void TrackUnionStream::EndTrack(uint32_t aIndex)
|
|
{
|
|
StreamTracks::Track* outputTrack = mTracks.FindTrack(mTrackMap[aIndex].mOutputTrackID);
|
|
if (!outputTrack || outputTrack->IsEnded())
|
|
return;
|
|
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p ending track %d", this, outputTrack->GetID()));
|
|
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
|
|
MediaStreamListener* l = mListeners[j];
|
|
StreamTime offset = outputTrack->GetSegment()->GetDuration();
|
|
nsAutoPtr<MediaSegment> segment;
|
|
segment = outputTrack->GetSegment()->CreateEmptyClone();
|
|
l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(), offset,
|
|
TrackEventCommand::TRACK_EVENT_ENDED,
|
|
*segment,
|
|
mTrackMap[aIndex].mInputPort->GetSource(),
|
|
mTrackMap[aIndex].mInputTrackID);
|
|
}
|
|
for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
|
|
if (b.mTrackID == outputTrack->GetID()) {
|
|
b.mListener->NotifyEnded();
|
|
}
|
|
}
|
|
outputTrack->SetEnded();
|
|
}
|
|
|
|
void TrackUnionStream::CopyTrackData(StreamTracks::Track* aInputTrack,
|
|
uint32_t aMapIndex, GraphTime aFrom, GraphTime aTo,
|
|
bool* aOutputTrackFinished)
|
|
{
|
|
TrackMapEntry* map = &mTrackMap[aMapIndex];
|
|
StreamTracks::Track* outputTrack = mTracks.FindTrack(map->mOutputTrackID);
|
|
MOZ_ASSERT(outputTrack && !outputTrack->IsEnded(), "Can't copy to ended track");
|
|
|
|
MediaSegment* segment = map->mSegment;
|
|
MediaStream* source = map->mInputPort->GetSource();
|
|
|
|
GraphTime next;
|
|
*aOutputTrackFinished = false;
|
|
for (GraphTime t = aFrom; t < aTo; t = next) {
|
|
MediaInputPort::InputInterval interval = map->mInputPort->GetNextInputInterval(t);
|
|
interval.mEnd = std::min(interval.mEnd, aTo);
|
|
StreamTime inputEnd = source->GraphTimeToStreamTimeWithBlocking(interval.mEnd);
|
|
StreamTime inputTrackEndPoint = STREAM_TIME_MAX;
|
|
|
|
if (aInputTrack->IsEnded() &&
|
|
aInputTrack->GetEnd() <= inputEnd) {
|
|
inputTrackEndPoint = aInputTrack->GetEnd();
|
|
*aOutputTrackFinished = true;
|
|
}
|
|
|
|
if (interval.mStart >= interval.mEnd) {
|
|
break;
|
|
}
|
|
StreamTime ticks = interval.mEnd - interval.mStart;
|
|
next = interval.mEnd;
|
|
|
|
StreamTime outputStart = outputTrack->GetEnd();
|
|
|
|
if (interval.mInputIsBlocked) {
|
|
// Maybe the input track ended?
|
|
segment->AppendNullData(ticks);
|
|
STREAM_LOG(LogLevel::Verbose, ("TrackUnionStream %p appending %lld ticks of null data to track %d",
|
|
this, (long long)ticks, outputTrack->GetID()));
|
|
} else if (InMutedCycle()) {
|
|
segment->AppendNullData(ticks);
|
|
} else {
|
|
if (source->IsSuspended()) {
|
|
segment->AppendNullData(aTo - aFrom);
|
|
} else {
|
|
MOZ_ASSERT(outputTrack->GetEnd() == GraphTimeToStreamTimeWithBlocking(interval.mStart),
|
|
"Samples missing");
|
|
StreamTime inputStart = source->GraphTimeToStreamTimeWithBlocking(interval.mStart);
|
|
segment->AppendSlice(*aInputTrack->GetSegment(),
|
|
std::min(inputTrackEndPoint, inputStart),
|
|
std::min(inputTrackEndPoint, inputEnd));
|
|
}
|
|
}
|
|
ApplyTrackDisabling(outputTrack->GetID(), segment);
|
|
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
|
|
MediaStreamListener* l = mListeners[j];
|
|
// Separate Audio and Video.
|
|
if (segment->GetType() == MediaSegment::AUDIO) {
|
|
l->NotifyQueuedAudioData(Graph(), outputTrack->GetID(),
|
|
outputStart,
|
|
*static_cast<AudioSegment*>(segment),
|
|
map->mInputPort->GetSource(),
|
|
map->mInputTrackID);
|
|
}
|
|
}
|
|
for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
|
|
if (b.mTrackID != outputTrack->GetID()) {
|
|
continue;
|
|
}
|
|
b.mListener->NotifyQueuedChanges(Graph(), outputStart, *segment);
|
|
}
|
|
outputTrack->GetSegment()->AppendFrom(segment);
|
|
}
|
|
}
|
|
|
|
void
|
|
TrackUnionStream::SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode) {
|
|
bool enabled = aMode == DisabledTrackMode::ENABLED;
|
|
for (TrackMapEntry& entry : mTrackMap) {
|
|
if (entry.mOutputTrackID == aTrackID) {
|
|
STREAM_LOG(LogLevel::Info, ("TrackUnionStream %p track %d was explicitly %s",
|
|
this, aTrackID, enabled ? "enabled" : "disabled"));
|
|
for (DirectMediaStreamTrackListener* listener : entry.mOwnedDirectListeners) {
|
|
DisabledTrackMode oldMode = GetDisabledTrackMode(aTrackID);
|
|
bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
|
|
if (!oldEnabled && enabled) {
|
|
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p track %d setting "
|
|
"direct listener enabled",
|
|
this, aTrackID));
|
|
listener->DecreaseDisabled(oldMode);
|
|
} else if (oldEnabled && !enabled) {
|
|
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p track %d setting "
|
|
"direct listener disabled",
|
|
this, aTrackID));
|
|
listener->IncreaseDisabled(aMode);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
MediaStream::SetTrackEnabledImpl(aTrackID, aMode);
|
|
}
|
|
|
|
MediaStream*
|
|
TrackUnionStream::GetInputStreamFor(TrackID aTrackID)
|
|
{
|
|
for (TrackMapEntry& entry : mTrackMap) {
|
|
if (entry.mOutputTrackID == aTrackID && entry.mInputPort) {
|
|
return entry.mInputPort->GetSource();
|
|
}
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
TrackID
|
|
TrackUnionStream::GetInputTrackIDFor(TrackID aTrackID)
|
|
{
|
|
for (TrackMapEntry& entry : mTrackMap) {
|
|
if (entry.mOutputTrackID == aTrackID) {
|
|
return entry.mInputTrackID;
|
|
}
|
|
}
|
|
|
|
return TRACK_NONE;
|
|
}
|
|
|
|
void
|
|
TrackUnionStream::AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
|
|
TrackID aTrackID)
|
|
{
|
|
RefPtr<DirectMediaStreamTrackListener> listener = aListener;
|
|
|
|
for (TrackMapEntry& entry : mTrackMap) {
|
|
if (entry.mOutputTrackID == aTrackID) {
|
|
MediaStream* source = entry.mInputPort->GetSource();
|
|
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p adding direct listener "
|
|
"%p for track %d. Forwarding to input "
|
|
"stream %p track %d.",
|
|
this, listener.get(), aTrackID, source,
|
|
entry.mInputTrackID));
|
|
entry.mOwnedDirectListeners.AppendElement(listener);
|
|
DisabledTrackMode currentMode = GetDisabledTrackMode(aTrackID);
|
|
if (currentMode != DisabledTrackMode::ENABLED) {
|
|
listener->IncreaseDisabled(currentMode);
|
|
}
|
|
source->AddDirectTrackListenerImpl(listener.forget(),
|
|
entry.mInputTrackID);
|
|
return;
|
|
}
|
|
}
|
|
|
|
TrackBound<DirectMediaStreamTrackListener>* bound =
|
|
mPendingDirectTrackListeners.AppendElement();
|
|
bound->mListener = listener.forget();
|
|
bound->mTrackID = aTrackID;
|
|
}
|
|
|
|
void
|
|
TrackUnionStream::RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
|
|
TrackID aTrackID)
|
|
{
|
|
for (TrackMapEntry& entry : mTrackMap) {
|
|
// OutputTrackID is unique to this stream so we only need to do this once.
|
|
if (entry.mOutputTrackID != aTrackID) {
|
|
continue;
|
|
}
|
|
for (size_t i = 0; i < entry.mOwnedDirectListeners.Length(); ++i) {
|
|
if (entry.mOwnedDirectListeners[i] == aListener) {
|
|
STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p removing direct "
|
|
"listener %p for track %d, forwarding "
|
|
"to input stream %p track %d",
|
|
this, aListener, aTrackID,
|
|
entry.mInputPort->GetSource(),
|
|
entry.mInputTrackID));
|
|
DisabledTrackMode currentMode = GetDisabledTrackMode(aTrackID);
|
|
if (currentMode != DisabledTrackMode::ENABLED) {
|
|
// Reset the listener's state.
|
|
aListener->DecreaseDisabled(currentMode);
|
|
}
|
|
entry.mOwnedDirectListeners.RemoveElementAt(i);
|
|
break;
|
|
}
|
|
}
|
|
// Forward to the input
|
|
MediaStream* source = entry.mInputPort->GetSource();
|
|
source->RemoveDirectTrackListenerImpl(aListener, entry.mInputTrackID);
|
|
return;
|
|
}
|
|
|
|
for (size_t i = 0; i < mPendingDirectTrackListeners.Length(); ++i) {
|
|
TrackBound<DirectMediaStreamTrackListener>& bound =
|
|
mPendingDirectTrackListeners[i];
|
|
if (bound.mListener == aListener && bound.mTrackID == aTrackID) {
|
|
mPendingDirectTrackListeners.RemoveElementAt(i);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
} // namespace mozilla
|