[Feat] Inbound 네트워크 품질에 따라 Consumer Layer 설정 추가

apps/media/src/mediasoup/mediasoup.service.ts

@@ -113,6 +113,10 @@ export class MediasoupService implements OnModuleInit {
     const peer = room.getPeer(socketId);
     const transport = peer.getTransport(transportId);
 
+    if (appData.mediaTypes !== 'audio') {
+      rtpParameters.encodings = server.PRODUCER_OPTIONS.encodings;
+    }
+
     const producer = await transport.produce({
       kind,
       rtpParameters,
@@ -275,7 +279,7 @@ export class MediasoupService implements OnModuleInit {
     const peer = room.peers.get(socketId);
     const consumer = peer.getConsumer(consumerId);
 
-    if (consumer.producerPaused) {
+    if (consumer?.producerPaused) {
       return { paused: true, consumerId, producerId: consumer.producerId };
     }
 
@@ -292,6 +296,24 @@ export class MediasoupService implements OnModuleInit {
     return consumerIds.map((consumerId) => this.resumeConsumer(socketId, consumerId, roomId));
   }
 
+  changeConsumerPreferredLayers(
+    socketId: string,
+    roomId: string,
+    data: server.NetworkQualityDto[]
+  ) {
+    data.forEach(({ consumerId, networkQuality }) => {
+      const room = this.roomService.getRoom(roomId);
+      const peer = room.peers.get(socketId);
+
+      const consumer = peer.getConsumer(consumerId);
+
+      consumer?.setPreferredLayers({
+        spatialLayer: networkQuality,
+        temporalLayer: networkQuality,
+      });
+    });
+  }
+
   closeRoom(roomId: string) {
     this.roomService.closeRoom(roomId);
   }

apps/media/src/signaling/signaling.gateway.ts

@@ -215,4 +215,13 @@ export class SignalingGateway implements OnGatewayDisconnect {
     const isRecording = this.recordService.getIsRecording(roomId);
     return { isRecording };
   }
+
+  @SubscribeMessage(SOCKET_EVENTS.changeConsumerPreferredLayers)
+  changeConsumerPreferredLayers(
+    @ConnectedSocket() client: Socket,
+    @MessageBody('roomId') roomId: string,
+    @MessageBody('networkQualities') data: server.NetworkQualityDto[]
+  ) {
+    return this.mediasoupService.changeConsumerPreferredLayers(client.id, roomId, data);
+  }
 }

apps/web/src/components/live/StreamView/List/Pinned.tsx

@@ -3,6 +3,7 @@ import { client } from '@repo/mediasoup';
 import PaginationControls from '@/components/live/StreamView/List/PaginationControls';
 import SubVideoGrid from '@/components/live/StreamView/List/SubVideoGrid';
 import VideoPlayer from '@/components/live/StreamView/List/VideoPlayer';
+import useNetworkMonitor from '@/hooks/mediasoup/useNetworkMonitor';
 import usePagination from '@/hooks/usePagination';
 
 const ITEMS_PER_SUB_GRID = 4;
@@ -21,11 +22,13 @@ function PinnedGrid({
   addPinnedVideo,
   getAudioMutedState,
 }: PinnedListProps) {
-  const { paginatedItems: subPaginatedStreams, ...subPaginationControlsProps } = usePagination({
+  const { paginatedItems, ...paginationControlsProps } = usePagination({
     itemsPerPage: ITEMS_PER_SUB_GRID,
     pinnedStream: pinnedVideoStreamData,
   });
 
+  useNetworkMonitor({ streams: [...paginatedItems, pinnedVideoStreamData] });
+
   return (
     <div className="relative flex h-full w-full flex-col gap-5">
       <div className="flex h-3/4 w-full justify-center self-center px-8">
@@ -43,10 +46,10 @@ function PinnedGrid({
         </div>
       </div>
       <div className="relative flex h-1/4 items-center justify-between">
-        <PaginationControls {...subPaginationControlsProps}>
+        <PaginationControls {...paginationControlsProps}>
           <SubVideoGrid
             pinnedVideoStreamData={pinnedVideoStreamData}
-            videoStreamData={subPaginatedStreams}
+            videoStreamData={paginatedItems}
             onVideoClick={addPinnedVideo}
             getAudioMutedState={getAudioMutedState}
           />

apps/web/src/components/live/StreamView/List/UnPinned.tsx

@@ -2,6 +2,7 @@ import { client } from '@repo/mediasoup';
 
 import PaginationControls from '@/components/live/StreamView/List/PaginationControls';
 import VideoGrid from '@/components/live/StreamView/List/VideoGrid';
+import useNetworkMonitor from '@/hooks/mediasoup/useNetworkMonitor';
 import usePagination from '@/hooks/usePagination';
 
 const ITEMS_PER_GRID = 9;
@@ -16,6 +17,8 @@ function UnPinnedGrid({ addPinnedVideo, getAudioMutedState }: UnPinnedListProps)
     itemsPerPage: ITEMS_PER_GRID,
   });
 
+  useNetworkMonitor({ streams: paginatedStreams });
+
   return (
     <PaginationControls {...paginationControlsProps}>
       <VideoGrid

apps/web/src/hooks/mediasoup/useNetworkMonitor.ts

@@ -0,0 +1,157 @@
+import { useParams } from '@tanstack/react-router';
+import { useCallback, useEffect } from 'react';
+import { client, SOCKET_EVENTS } from '@repo/mediasoup';
+
+import { useMediasoupState } from '@/contexts/mediasoup/context';
+
+const TICK = 10000;
+
+const QUALITY_LEVEL = {
+  average: {
+    quality: 1,
+    options: { packetLossRate: 2, jitter: 15, frameDropRate: 5, averageRTT: 150, nackCount: 20 },
+  },
+  poor: {
+    quality: 0,
+    options: { packetLossRate: 5, jitter: 30, frameDropRate: 10, averageRTT: 300, nackCount: 50 },
+  },
+} as const;
+
+interface UseNetworkMonitorProps {
+  streams: client.RemoteStream[];
+}
+
+const useNetworkMonitor = ({ streams }: UseNetworkMonitorProps) => {
+  const { ticleId } = useParams({ from: '/_authenticated/live/$ticleId' });
+  const { socketRef } = useMediasoupState();
+
+  const getNotPausedStreams = (streams: client.RemoteStream[]) => {
+    return streams.filter(
+      (data) => !data?.paused && data.consumer?.closed === false && data.kind === 'video'
+    );
+  };
+
+  const calculatePacketLossRate = useCallback((packetsLost: number, packetsReceived: number) => {
+    const totalExpectedPackets = packetsReceived + packetsLost;
+
+    if (totalExpectedPackets === 0) return 0;
+
+    return (packetsLost / totalExpectedPackets) * 100;
+  }, []);
+
+  const calculateFrameRate = useCallback((framesReceived: number, framesDropped: number) => {
+    if (framesReceived === 0) return 0;
+
+    const totalFrames = framesReceived + framesDropped;
+
+    return (framesDropped / totalFrames) * 100;
+  }, []);
+
+  const calculateAverageRTT = useCallback(
+    (totalRoundTripTime: number, roundTripTimeMeasurements: number) => {
+      if (roundTripTimeMeasurements === 0 || !totalRoundTripTime) return 0;
+
+      return (totalRoundTripTime / roundTripTimeMeasurements) * 1000;
+    },
+    []
+  );
+
+  const getNetworkQuality = useCallback(
+    (
+      packetLossRate: number,
+      jitter: number,
+      frameDropRate: number,
+      averageRTT: number,
+      nackCount: number
+    ) => {
+      if (
+        packetLossRate > QUALITY_LEVEL.average.options.packetLossRate ||
+        jitter > QUALITY_LEVEL.average.options.jitter ||
+        frameDropRate > QUALITY_LEVEL.average.options.frameDropRate ||
+        averageRTT > QUALITY_LEVEL.average.options.averageRTT ||
+        nackCount > QUALITY_LEVEL.average.options.nackCount
+      ) {
+        return 1;
+      }
+
+      if (
+        packetLossRate > QUALITY_LEVEL.poor.options.packetLossRate ||
+        jitter > QUALITY_LEVEL.poor.options.jitter ||
+        frameDropRate > QUALITY_LEVEL.poor.options.frameDropRate ||
+        averageRTT > QUALITY_LEVEL.poor.options.averageRTT ||
+        nackCount > QUALITY_LEVEL.poor.options.nackCount
+      ) {
+        return 0;
+      }
+
+      return 2;
+    },
+    []
+  );
+
+  const checkNetworkQuality = async (streams: client.RemoteStream[]) => {
+    const networkQualities = await Promise.all(
+      streams.map(async (data) => {
+        const { consumer } = data;
+
+        if (!consumer) return;
+
+        let networkQuality = 2; // 0: poor, 1: average, 2: good
+
+        const stats = await consumer.getStats();
+
+        stats.forEach((report) => {
+          if (report.type !== 'inbound-rtp') return;
+
+          const {
+            packetsLost,
+            jitter,
+            packetsReceived,
+            framesDropped,
+            framesReceived,
+            nackCount,
+            totalRoundTripTime,
+            roundTripTimeMeasurements,
+          } = report;
+
+          const packetLossRate = calculatePacketLossRate(packetsLost, packetsReceived);
+          const frameDropRate = calculateFrameRate(framesReceived, framesDropped);
+          const averageRTT = calculateAverageRTT(totalRoundTripTime, roundTripTimeMeasurements);
+
+          networkQuality = getNetworkQuality(
+            packetLossRate,
+            jitter,
+            frameDropRate,
+            averageRTT,
+            nackCount
+          );
+        });
+
+        return { consumerId: consumer.id, networkQuality };
+      })
+    );
+
+    return networkQualities;
+  };
+
+  useEffect(() => {
+    const socket = socketRef.current;
+
+    if (!socket) return;
+
+    const interval = setInterval(async () => {
+      const notPausedStreams = getNotPausedStreams(streams);
+
+      const networkQualities = await checkNetworkQuality(notPausedStreams);
+
+      socket.emit(SOCKET_EVENTS.changeConsumerPreferredLayers, {
+        roomId: ticleId,
+        networkQualities,
+      });
+    }, TICK);
+
+    return () => clearInterval(interval);
+  }, [streams, socketRef, ticleId]);
+};
+
+export default useNetworkMonitor;

apps/web/src/hooks/mediasoup/useRemoteStream.ts

@@ -147,11 +147,7 @@ const useRemoteStream = () => {
         return stream;
       });
 
-      return newStreams.sort((a, b) => {
-        if (a.paused === b.paused) return 0;
-
-        return a.paused ? 1 : -1;
-      });
+      return newStreams;
     };
   };
 
@@ -166,11 +162,7 @@ const useRemoteStream = () => {
         return stream;
       });
 
-      return newStreams.sort((a, b) => {
-        if (a.paused === b.paused) return 0;
-
-        return a.paused ? 1 : -1;
-      });
+      return newStreams;
     };
   };
 
@@ -269,11 +261,7 @@ const useRemoteStream = () => {
       stream.consumer?.pause();
       stream.paused = true;
 
-      return newStreams.sort((a, b) => {
-        if (a.paused === b.paused) return 0;
-
-        return a.paused ? 1 : -1;
-      });
+      return newStreams;
     };
 
     setVideoStreams(getNewStreams);
@@ -304,11 +292,7 @@ const useRemoteStream = () => {
         stream.consumer?.resume();
         stream.paused = false;
 
-        return newStreams.sort((a, b) => {
-          if (a.paused === b.paused) return 0;
-
-          return a.paused ? 1 : -1;
-        });
+        return newStreams;
       };
 
       setVideoStreams(getNewStreams);

apps/web/src/hooks/usePagination.ts

@@ -54,7 +54,6 @@ const usePagination = ({ itemsPerPage, pinnedStream }: PaginationParams) => {
 
     const startIdx = currentPage * itemsPerPage;
     const endIdx = startIdx + itemsPerPage;
-
     return totalItems.slice(startIdx, endIdx);
   }, [videoStreams, currentPage, itemsPerPage, video, screen, nickname]);
 
@@ -70,16 +69,24 @@ const usePagination = ({ itemsPerPage, pinnedStream }: PaginationParams) => {
   const totalPages = Math.ceil(streamLength / itemsPerPage);
 
   const resumeGridStreams = useDebouncedCallback(() => {
-    const gridItems = paginatedItems.filter(
-      (item) => item.socketId !== 'local' && item.consumer?.closed === false
-    ) as client.RemoteStream[];
-    const isExistPinned = gridItems.some((item) => item.socketId === pinnedStream?.socketId);
+    const prevGridItems = prevGridItemsRef.current;
+
+    const target = paginatedItems
+      .filter(
+        (item) =>
+          item.socketId !== 'local' && item.consumer?.closed === false && item.paused === true
+      )
+      .filter((item) => prevGridItems.some((prevItem) => prevItem.socketId === item.socketId));
+
+    const isExistPinned = target.some(
+      (item) => item.socketId === pinnedStream?.socketId && item.paused === true
+    );
 
     if (pinnedStream?.consumer && !isExistPinned) {
-      gridItems.push(pinnedStream as client.RemoteStream);
+      target.push(pinnedStream as client.RemoteStream);
     }
 
-    resumeVideoConsumers(gridItems);
+    resumeVideoConsumers(target);
   }, 300);
 
   const pauseGridStreams = () => {
@@ -89,10 +96,15 @@ const usePagination = ({ itemsPerPage, pinnedStream }: PaginationParams) => {
     if (!socket || prevGridItems.length === 0) return;
 
     const target = prevGridItems
-      .filter((item) => item.consumer?.closed === false && pinnedStream?.socketId !== item.socketId)
       .filter(
-        (item) => !paginatedItems.some((paginatedItem) => paginatedItem.socketId === item.socketId)
-      ) as client.RemoteStream[];
+        (item) =>
+          item.consumer?.closed === false &&
+          pinnedStream?.socketId !== item.socketId &&
+          item.paused === false
+      )
+      .filter((item) => {
+        return !paginatedItems.some((paginatedItem) => paginatedItem.socketId === item.socketId);
+      }) as client.RemoteStream[];
 
     pauseVideoConsumers(target);
   };