Using aiortc to stream live video feed instead of a video file

Question

I am trying to stream a live feed of the desktop using Mss.

However I cant seem to achieve this and using aortic. Instead I am only able to stream a video file to my client. Here what I'm running.

import asyncio
import json
import logging
import os

from aiohttp import web
from aiortc import RTCPeerConnection, RTCSessionDescription
from aiortc.contrib.media import MediaPlayer

ROOT = os.path.dirname(__file__)

def create_local_tracks(play_from):
    player = MediaPlayer(play_from)
    return player.video 


async def index(request):
    content = open(os.path.join(ROOT, "index.html"), "r").read()
    return web.Response(content_type="text/html", text=content)


async def offer(request):
    params = await request.json()
    offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
    pc = RTCPeerConnection()
    pcs.add(pc)

    # open media source
    video = create_local_tracks("video.mp4")

    pc.addTrack(video)

    await pc.setRemoteDescription(offer)

    answer = await pc.createAnswer()
    await pc.setLocalDescription(answer)

    return web.Response(
        content_type="application/json",
        text=json.dumps(
            {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
        ),
    )

pcs = set()
async def on_shutdown(app):
    coros = [pc.close() for pc in pcs]
    await asyncio.gather(*coros)
    pcs.clear()

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)

    app = web.Application()
    app.on_shutdown.append(on_shutdown)
    app.router.add_get("/", index)
    app.router.add_post("/offer", offer)
    web.run_app(app, host="0.0.0.0", port=8080)

Right now this streams a the video of video.mp4 to the html client when it connects. This works quite well for streaming a static file but I was unable to figure out how to stream a live video of the users screen. I figured the python screen record library called mss would be suitable for this as it delivers high framerate that webRTC is capable of handling. Thanks!

Answer 1

To stream the user's screen, you need to modify your 'create_local_tracks' function and add a new class for capturing the user's screen. Here is an example:

from aiortc import VideoStreamTrack
from av import VideoFrame
import numpy as np
import threading
import asyncio
import queue
import mss

class ScreenCapturing(VideoStreamTrack):
    """
    A custom video stream track for capturing the user's screen.
    """
    def __init__(self) -> None:
        super().__init__()
        self.queue = queue.Queue(10)

    async def recv(self):
        """
        Asynchronously receives frames from the screen capture.
        """
        img = self.queue.get()
        img_rgb = img[:, :, :3] # Convert RGBA to RGB by discarding the alpha channel
        frame = VideoFrame.from_ndarray(img_rgb, format="bgr24")
        pts, time_base = await self.next_timestamp()
        frame.pts = pts
        frame.time_base = time_base
        return frame

    def start(self):
        """
        Starts capturing the screen in a separate thread.
        """
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        with mss.mss() as sct:
            monitor = sct.monitors[1]
            while True:
                im = sct.grab(monitor)
                im_np = np.array(im)
                self.queue.put(im_np)

    def stop(self):
        pass


async def create_local_tracks():
    screencapturing = ScreenCapturing()
    threading.Thread(target=screencapturing.start, daemon=True).start()
    # print(await screencapturing.recv())
    return screencapturing