Remote Media

You've learned how to handle local media. In this tutorial, you will learn how to handle remote media to capture a remote user's audio and video.

Note

For web, we don't implement remote media because LiveSwitch provides one that works with the major browsers.

Prerequisites

This tutorial requires the Local Media app you have created earlier.

Create Remote Media

Step1. Define Remote Media: Similar to a user's local media, we implement other participants' remote media by extending the RtcRemoteMedia<T> class. The generic type T represents the type of object is used for displaying the video preview of these remote participants. We will create a class named RemoteMedia in a separate file.

Step2.Enable AEC in Remote Media: We've already implemented AEC when we created the local media. We now need to enable it in our remote media so that the remote user's echo can be removed for the local user.

Step3. Play Remote Audio: To play audio from a remote video feed, we implement the following:

The CreateAudioRecorder method that records a user's audio.
The CreateAudioSink method that returns an audio sink for audio playback.
The CreateOpusDecoder method that decodes Opus encoder.

Step4. Play Remote Video: To play remote video, we implement the following:

The CreateVideoRecorder method that records a user's video.
The CreateViewSink method that creates a view object to playback a remote video feed.
The CreateVp8Decoder, CreateVp9Decoder, and CreateH264Decoder methods that decode the VP8, VP9, and H.264 encoders.
The CreateImageConverter method that provides a minor image formatting utility.

Paste the following code inside the HelloWorld namespace of the RemoteMedia.cs file.

public class RemoteMedia : RtcRemoteMedia<System.Windows.Controls.Image>
{
    // Enable AEC
    public RemoteMedia(bool disableAudio, bool disableVideo, AecContext aecContext)
    : base(disableAudio, disableVideo, aecContext)
    {
        Initialize();
    }

    // Remote Audio
    protected override AudioSink CreateAudioRecorder(AudioFormat inputFormat)
    {
        return new FM.LiveSwitch.Matroska.AudioSink(Id + "-remote-audio-" + inputFormat.Name.ToLower() + ".mkv");
    }

    protected override AudioSink CreateAudioSink(AudioConfig config)
    {
        return new FM.LiveSwitch.NAudio.Sink(config);
    }

    protected override AudioDecoder CreateOpusDecoder(AudioConfig config)
    {
        return new FM.LiveSwitch.Opus.Decoder(config);
    }

    // Remote Video
    protected override VideoSink CreateVideoRecorder(VideoFormat inputFormat)
    {
        return new FM.LiveSwitch.Matroska.VideoSink(Id + "-remote-video-" + inputFormat.Name.ToLower() + ".mkv");
    }

    protected override ViewSink<System.Windows.Controls.Image> CreateViewSink()
    {
        return new FM.LiveSwitch.Wpf.ImageSink()
        {
            ViewScale = LayoutScale.Contain
        };
    }

    protected override VideoDecoder CreateVp8Decoder()
    {
        return new FM.LiveSwitch.Vp8.Decoder();
    }

    protected override VideoDecoder CreateVp9Decoder()
    {
        return new FM.LiveSwitch.Vp9.Decoder();
    }

    protected override VideoDecoder CreateH264Decoder()
    {
        return null;
    }

    protected override VideoPipe CreateImageConverter(VideoFormat outputFormat)
    {
        return new FM.LiveSwitch.Yuv.ImageConverter(outputFormat);
    }
}

Paste the following code to the RemoteMedia.java file:

public class RemoteMedia extends RtcRemoteMedia<FrameLayout> {

    private final Context context;

    // Enable AEC
    public RemoteMedia(final Context context, boolean disableAudio, boolean disableVideo, AecContext aecContext) {
        super(disableAudio, disableVideo, aecContext);
        this.context = context;
        super.initialize();
    }

    // Remote Audio
    @Override
    protected AudioSink createAudioRecorder(AudioFormat audioFormat) {
        return new fm.liveswitch.matroska.AudioSink(getId() + "-remote-audio-" + audioFormat.getName().toLowerCase() + ".mkv");
    }

    @Override
    protected AudioSink createAudioSink(AudioConfig audioConfig) {
        return new fm.liveswitch.android.AudioTrackSink(audioConfig);
    }

    @Override
    protected AudioDecoder createOpusDecoder(AudioConfig audioConfig) {
        return new fm.liveswitch.opus.Decoder();
    }

    // Remote Video
    @Override
    protected VideoSink createVideoRecorder(VideoFormat videoFormat) {
        return new fm.liveswitch.matroska.VideoSink(getId() + "-remote-video-" + videoFormat.getName().toLowerCase() + ".mkv");
    }

    @Override
    protected ViewSink<FrameLayout> createViewSink() {
        return new OpenGLSink(context);
    }

    @Override
    protected VideoDecoder createVp8Decoder() {
        return new fm.liveswitch.vp8.Decoder();
    }

    @Override
    protected VideoDecoder createVp9Decoder() {
        return new fm.liveswitch.vp9.Decoder();
    }

    @Override
    protected VideoDecoder createH264Decoder() {
        return null;
    }

    @Override
    protected VideoPipe createImageConverter(VideoFormat videoFormat) {
        return new fm.liveswitch.yuv.ImageConverter(videoFormat);
    }
}

Paste the following code to the RemoteMedia.swift file:

class RemoteMedia : FMLiveSwitchRtcRemoteMedia {
    // Enable AEC
    override init!(disableAudio: Bool, disableVideo: Bool, aecContext: FMLiveSwitchAecContext!) {
        super.init(disableAudio: disableAudio, disableVideo: disableVideo, aecContext: aecContext)
        self.initialize()
    }
    
    // Remote Audio
    override func createAudioRecorder(withInputFormat inputFormat: FMLiveSwitchAudioFormat!) -> FMLiveSwitchAudioSink! {
        return FMLiveSwitchMatroskaAudioSink(path: "remote-audio-\(String(describing: inputFormat.name())).mkv")
    }

    override func createAudioSink(with config: FMLiveSwitchAudioConfig!) -> FMLiveSwitchAudioSink! {
        do {
            if #available(iOS 10.0, *) {
                try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .default, options: [.allowBluetooth,  (AVAudioSession.CategoryOptions.defaultToSpeaker)])
            } else {
                AVAudioSession.sharedInstance().perform(NSSelectorFromString("setCategory:withOptions:error:"), with: AVAudioSession.Category.playAndRecord, with: [.allowBluetooth, AVAudioSession.CategoryOptions.defaultToSpeaker])
            }
        } catch {
            FMLiveSwitchLog.warn(withMessage: "Could not set audio session category for remote media")
            return nil
        }
        
        do {
            try AVAudioSession.sharedInstance().setActive(true)
        } catch {
            FMLiveSwitchLog.warn(withMessage: "Could not activate audio session for remote media.")
            return nil
        }
        
        return FMLiveSwitchCocoaAudioUnitSink(config: config)
    }

    override func createOpusDecoder(with config: FMLiveSwitchAudioConfig!) -> FMLiveSwitchAudioDecoder! {
        return FMLiveSwitchOpusDecoder(config: config)
    }
    
    // Remote Video
    override func createVideoRecorder(withInputFormat inputFormat: FMLiveSwitchVideoFormat!) -> FMLiveSwitchVideoSink! {
        return FMLiveSwitchMatroskaVideoSink(path: "remote-video-\(String(describing: inputFormat.name())).mkv")
    }

    override func createViewSink() -> FMLiveSwitchViewSink! {
        return FMLiveSwitchCocoaOpenGLSink(viewScale: FMLiveSwitchLayoutScale.contain)
    } 

    func createImageScaler() -> FMLiveSwitchVideoPipe! {
        return FMLiveSwitchYuvImageScaler(scale: 1.0)
    }

    override func createVp8Decoder() -> FMLiveSwitchVideoDecoder! {
        return FMLiveSwitchVp8Decoder()
    }
    
    override func createVp9Decoder() -> FMLiveSwitchVideoDecoder! {
        return FMLiveSwitchVp9Decoder()
    }
    
    override func createH264Decoder() -> FMLiveSwitchVideoDecoder! {
        return nil
    }
    
    override func createImageConverter(withOutputFormat outputFormat: FMLiveSwitchVideoFormat!) -> FMLiveSwitchVideoPipe! {
        return FMLiveSwitchYuvImageConverter(outputFormat: outputFormat)
    }
}

Congratulations, you've built a LiveSwitch app to handle remote media!