Add Video Super Resolution for Xbox

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "third_party/imgui"]
 	path = third_party/imgui
 	url = https://github.com/ocornut/imgui.git
+[submodule "third_party/NVIDIAImageScaling"]
+	path = third_party/NVIDIAImageScaling
+	url = https://github.com/NVIDIAGameWorks/NVIDIAImageScaling.git

Assets/Shader/build_nis.bat

@@ -0,0 +1,17 @@
+@echo off
+setlocal
+pushd "%~dp0" || exit /b 1
+set "HLSL=..\..\third_party\NVIDIAImageScaling\NIS\NIS_Main.hlsl"
+set "EXIT_CODE=0"
+fxc /T cs_5_0 /E main /O3 /D NIS_SCALER=1 /D NIS_HDR_MODE=0 /D NIS_BLOCK_WIDTH=32 /D NIS_BLOCK_HEIGHT=32 /D NIS_THREAD_GROUP_SIZE=128 /D NIS_USE_HALF_PRECISION=1 /Fo "nis_sdr_fp16.cso" "%HLSL%" || goto :fail
+fxc /T cs_5_0 /E main /O3 /D NIS_SCALER=1 /D NIS_HDR_MODE=2 /D NIS_BLOCK_WIDTH=32 /D NIS_BLOCK_HEIGHT=32 /D NIS_THREAD_GROUP_SIZE=128 /D NIS_USE_HALF_PRECISION=1 /Fo "nis_hdr_fp16.cso" "%HLSL%" || goto :fail
+fxc /T cs_5_0 /E main /O3 /D NIS_SCALER=1 /D NIS_HDR_MODE=0 /D NIS_BLOCK_WIDTH=32 /D NIS_BLOCK_HEIGHT=24 /D NIS_THREAD_GROUP_SIZE=128 /D NIS_USE_HALF_PRECISION=0 /Fo "nis_sdr_fp32.cso" "%HLSL%" || goto :fail
+fxc /T cs_5_0 /E main /O3 /D NIS_SCALER=1 /D NIS_HDR_MODE=2 /D NIS_BLOCK_WIDTH=32 /D NIS_BLOCK_HEIGHT=24 /D NIS_THREAD_GROUP_SIZE=128 /D NIS_USE_HALF_PRECISION=0 /Fo "nis_hdr_fp32.cso" "%HLSL%" || goto :fail
+goto :cleanup
+
+:fail
+set "EXIT_CODE=%ERRORLEVEL%"
+
+:cleanup
+popd
+endlocal & exit /b %EXIT_CODE%

Pages/AppPage.xaml.cpp

@@ -166,6 +166,7 @@ void AppPage::Connect(int appId) {
 	config->audioConfig = host->AudioConfig;
 	config->videoCodec = host->VideoCodec;
 	config->playAudioOnPC = host->PlayAudioOnPC;
+	config->videoSuperResolution = host->VideoSuperResolution;
 	config->enableHDR = host->EnableHDR;
 	config->enableSOPS = host->EnableSOPS;
 	config->framePacing = host->FramePacing;

Pages/HostSettingsPage.xaml

@@ -40,6 +40,7 @@
                 <RowDefinition Height="auto"></RowDefinition>
                 <RowDefinition Height="auto"></RowDefinition>
                 <RowDefinition Height="auto"></RowDefinition>
+                <RowDefinition Height="auto"></RowDefinition>
             </Grid.RowDefinitions>
             <TextBlock Grid.Row="0" Grid.Column="0">Resolution</TextBlock>
             <ComboBox x:Name="ResolutionSelector" SelectionChanged="ResolutionSelector_SelectionChanged" SelectedIndex="{x:Bind CurrentResolutionIndex,Mode=TwoWay}" Grid.Row="0" Grid.Column="1" ItemsSource="{x:Bind AvailableResolutions}">
@@ -57,58 +58,60 @@
             <ComboBox x:Name="FPSSelector" SelectionChanged="FPSSelector_SelectionChanged" SelectedItem="{x:Bind Host.FPS,Mode=OneWay}" Grid.Row="1"  Grid.Column="1" ItemsSource="{x:Bind AvailableFPS}"></ComboBox>
             <TextBlock Grid.Row="2" Grid.Column="0">Bitrate (Kbps)</TextBlock>
             <Slider Grid.Row="2" Grid.Column="1" Grid.ColumnSpan="2" Value="{x:Bind Host.Bitrate}" x:DefaultBindMode="TwoWay" Minimum="5000" Maximum="500000" SmallChange="10000" TickFrequency="50000" StepFrequency="50000" />
-            <TextBlock Grid.Row="3" Grid.Column="0">Audio Configuration</TextBlock>
-            <ComboBox ItemsSource="{x:Bind AvailableAudioConfigs}" SelectedItem="{x:Bind Host.AudioConfig,Mode=TwoWay}"  Grid.Row="3"  Grid.Column="1"></ComboBox>
-            <TextBlock Grid.Row="4" Grid.Column="0">Play Audio on Host PC</TextBlock>
-            <CheckBox Grid.Row="4" Grid.Column="1" IsChecked="{x:Bind Host.PlayAudioOnPC, Mode=TwoWay}"></CheckBox>
-            <TextBlock Grid.Row="5" Grid.Column="0">Autostart</TextBlock>
-            <ComboBox  Grid.Row="5"  Grid.Column="1" SelectedIndex="{x:Bind CurrentAppIndex,Mode=TwoWay}" Name="AutoStartSelector" SelectionChanged="AutoStartSelector_SelectionChanged">
+            <TextBlock Grid.Row="3" Grid.Column="0">Video Super Resolution</TextBlock>
+            <CheckBox Grid.Row="3" Grid.Column="1" IsChecked="{x:Bind Host.VideoSuperResolution, Mode=TwoWay}"></CheckBox>
+            <TextBlock Grid.Row="4" Grid.Column="0">Audio Configuration</TextBlock>
+            <ComboBox ItemsSource="{x:Bind AvailableAudioConfigs}" SelectedItem="{x:Bind Host.AudioConfig,Mode=TwoWay}"  Grid.Row="4"  Grid.Column="1"></ComboBox>
+            <TextBlock Grid.Row="5" Grid.Column="0">Play Audio on Host PC</TextBlock>
+            <CheckBox Grid.Row="5" Grid.Column="1" IsChecked="{x:Bind Host.PlayAudioOnPC, Mode=TwoWay}"></CheckBox>
+            <TextBlock Grid.Row="6" Grid.Column="0">Autostart</TextBlock>
+            <ComboBox  Grid.Row="6"  Grid.Column="1" SelectedIndex="{x:Bind CurrentAppIndex,Mode=TwoWay}" Name="AutoStartSelector" SelectionChanged="AutoStartSelector_SelectionChanged">
             </ComboBox>
-            <TextBlock Grid.Row="6" Grid.Column="0">Video Codecs</TextBlock>
-            <ComboBox Name="CodecComboBox" ItemsSource="{x:Bind AvailableVideoCodecs}" SelectedItem="{x:Bind Host.VideoCodec,Mode=TwoWay}"  Grid.Row="6"  Grid.Column="1"></ComboBox>
+            <TextBlock Grid.Row="7" Grid.Column="0">Video Codecs</TextBlock>
+            <ComboBox Name="CodecComboBox" ItemsSource="{x:Bind AvailableVideoCodecs}" SelectedItem="{x:Bind Host.VideoCodec,Mode=TwoWay}"  Grid.Row="7"  Grid.Column="1"></ComboBox>
 
-            <TextBlock Grid.Row="7" Grid.Column="0" >Enable HDR:</TextBlock>
+            <TextBlock Grid.Row="8" Grid.Column="0" >Enable HDR:</TextBlock>
             <CheckBox
-                Name="EnableHDRCheckbox" Grid.Row="7" Grid.Column="1"
+                Name="EnableHDRCheckbox" Grid.Row="8" Grid.Column="1"
                 IsChecked="{x:Bind Host.EnableHDR,Mode=TwoWay}" />
             <TextBlock
-                Name="HDR4KNote" Grid.Row="7" Grid.Column="1" Grid.ColumnSpan="2" Visibility="Collapsed">
+                Name="HDR4KNote" Grid.Row="8" Grid.Column="1" Grid.ColumnSpan="2" Visibility="Collapsed">
                 HDR requires Xbox system resolution set to 4K.
             </TextBlock>
 
-            <TextBlock Grid.Row="8" Grid.Column="0" >Optimize host resolution:</TextBlock>
-            <CheckBox Name="EnableSOPSCheckbox" Grid.Row="8" Grid.Column="1" IsChecked="{x:Bind Host.EnableSOPS,Mode=TwoWay}"></CheckBox>
+            <TextBlock Grid.Row="9" Grid.Column="0" >Optimize host resolution:</TextBlock>
+            <CheckBox Name="EnableSOPSCheckbox" Grid.Row="9" Grid.Column="1" IsChecked="{x:Bind Host.EnableSOPS,Mode=TwoWay}"></CheckBox>
 
-            <TextBlock Grid.Row="9" Grid.Column="0">Frame Pacing:</TextBlock>
-            <ComboBox Name="FramePacingComboBox" ItemsSource="{x:Bind AvailableFramePacing}" SelectedItem="{x:Bind Host.FramePacing,Mode=TwoWay}" SelectionChanged="FramePacing_SelectionChanged" Grid.Row="9" Grid.Column="1"></ComboBox>
+            <TextBlock Grid.Row="10" Grid.Column="0">Frame Pacing:</TextBlock>
+            <ComboBox Name="FramePacingComboBox" ItemsSource="{x:Bind AvailableFramePacing}" SelectedItem="{x:Bind Host.FramePacing,Mode=TwoWay}" SelectionChanged="FramePacing_SelectionChanged" Grid.Row="10" Grid.Column="1"></ComboBox>
             <TextBlock
-                Name="FramePacingImmediateDesc" Grid.Row="9" Grid.Column="2" Visibility="Collapsed">
+                Name="FramePacingImmediateDesc" Grid.Row="10" Grid.Column="2" Visibility="Collapsed">
                 Best for Xbox Series. Lowest latency. Renders frames at the incoming frame rate.
             </TextBlock>
             <TextBlock
-                Name="FramePacingDisplayLockedDesc" Grid.Row="9" Grid.Column="2" Visibility="Collapsed">
+                Name="FramePacingDisplayLockedDesc" Grid.Row="10" Grid.Column="2" Visibility="Collapsed">
                 Best for Xbox One. Locks rendering frame rate to refresh rate and evenly spaces frames.
             </TextBlock>
 
-            <TextBlock Grid.Row="10" Grid.Column="0">Show performance stats:</TextBlock>
+            <TextBlock Grid.Row="11" Grid.Column="0">Show performance stats:</TextBlock>
             <CheckBox
-                Grid.Row="10" Grid.Column="1"
+                Grid.Row="11" Grid.Column="1"
                 x:Name="EnableStatsCheckbox"
                 IsChecked="{x:Bind Host.EnableStats, Mode=TwoWay}" />
 
-            <TextBlock Grid.Row="11" Grid.Column="0" >Show performance graphs:</TextBlock>
+            <TextBlock Grid.Row="12" Grid.Column="0" >Show performance graphs:</TextBlock>
             <CheckBox
-                Grid.Row="11" Grid.Column="1"
+                Grid.Row="12" Grid.Column="1"
                 x:Name="EnableGraphsCheckbox"
                 IsEnabled="{x:Bind Host.EnableStats, Mode=OneWay}"
                 IsChecked="{x:Bind Host.EnableGraphs, Mode=TwoWay}" />
             <TextBlock
-                Name="XboxOneGraphsNote" Grid.Row="11" Grid.Column="1" Grid.ColumnSpan="2" Visibility="Collapsed">
+                Name="XboxOneGraphsNote" Grid.Row="12" Grid.Column="1" Grid.ColumnSpan="2" Visibility="Collapsed">
                 Graphs are unavailable on Xbox One when system resolution is set to 4K.
             </TextBlock>
 
-            <TextBlock Grid.Row="12" Grid.Column="0">Other:</TextBlock>
-            <Button Grid.Row="12" Grid.Column="1" x:Name="GlobalSettingsOption" Click="GlobalSettingsOption_Click">Open Global Settings</Button>
+            <TextBlock Grid.Row="13" Grid.Column="0">Other:</TextBlock>
+            <Button Grid.Row="13" Grid.Column="1" x:Name="GlobalSettingsOption" Click="GlobalSettingsOption_Click">Open Global Settings</Button>
         </Grid>
     </StackPanel>
     </ScrollViewer>

State/ApplicationState.cpp

@@ -44,6 +44,7 @@ Concurrency::task<void> moonlight_xbox_dx::ApplicationState::Init()
 					if (a.contains("autoStartID"))h->AutostartID = a["autoStartID"];
 					if (a.contains("computername")) h->ComputerName = Utils::StringFromStdString(a["computername"].get<std::string>());
 					if (a.contains("playaudioonpc")) h->PlayAudioOnPC = a["playaudioonpc"].get<bool>();
+					if (a.contains("video_super_resolution")) h->VideoSuperResolution = a["video_super_resolution"].get<bool>();
 					if (a.contains("enable_hdr")) h->EnableHDR = a["enable_hdr"].get<bool>();
 					if (a.contains("enable_sops")) h->EnableSOPS = a["enable_sops"].get<bool>();
 					if (a.contains("enable_stats")) h->EnableStats = a["enable_stats"].get<bool>();
@@ -104,6 +105,7 @@ Concurrency::task<void> moonlight_xbox_dx::ApplicationState::UpdateFile()
 			hostJson["framePacing"] = Utils::PlatformStringToStdString(host->FramePacing);
 			hostJson["autoStartID"] = host->AutostartID;
 			hostJson["playaudioonpc"] = host->PlayAudioOnPC;
+			hostJson["video_super_resolution"] = host->VideoSuperResolution;
 			hostJson["enable_hdr"] = host->EnableHDR;
 			hostJson["enable_sops"] = host->EnableSOPS;
 			hostJson["enable_stats"] = host->EnableStats;

State/MoonlightHost.h

@@ -18,6 +18,7 @@ namespace moonlight_xbox_dx {
         bool loading = true;
         bool wolPolling = false;
         bool playAudioOnPC = false;
+        bool videoSuperResolution = false;
         MoonlightClient* client;
         int currentlyRunningAppId;
         int bitrate = 20000;
@@ -249,6 +250,15 @@ namespace moonlight_xbox_dx {
             }
         }
 
+        property bool VideoSuperResolution
+        {
+            bool get() { return this->videoSuperResolution; }
+            void set(bool value) {
+                this->videoSuperResolution = value;
+                OnPropertyChanged("VideoSuperResolution");
+            }
+        }
+
         property bool EnableHDR
         {
             bool get() { return this->enableHDR; }

State/Stats.cpp

@@ -3,6 +3,7 @@
 #include "Utils.hpp"
 #include "../Plot/ImGuiPlots.h"
 #include "../Streaming/FFMpegDecoder.h"
+#include "../State/StreamConfiguration.h"
 
 using namespace moonlight_xbox_dx;
 
@@ -45,7 +46,12 @@ bool Stats::ShouldUpdateDisplay(DX::StepTimer const& timer, bool isVisible, char
 
 		// Move this window into the last window slot and clear it for next window
 		memcpy(&m_LastWndVideoStats, &m_ActiveWndVideoStats, sizeof(VIDEO_STATS));
+
+		// Preserve the scaleRatio which is set directly by the renderer and doesn't accumulate
+		float currentScaleRatio = m_ActiveWndVideoStats.scaleRatio;
 		ZeroMemory(&m_ActiveWndVideoStats, sizeof(VIDEO_STATS));
+		m_ActiveWndVideoStats.scaleRatio = currentScaleRatio;
+
 		m_ActiveWndVideoStats.measurementStartTimestamp = timer.GetTotalSeconds();
 	}
 
@@ -149,6 +155,11 @@ void Stats::SubmitRenderStats(double preWaitTimeMs, double renderTimeMs, double
 	m_ActiveWndVideoStats.totalPresentTimeUs += static_cast<uint64_t>(presentTimeMs * 1000);
 }
 
+void Stats::SubmitScaleRatio(float scaleRatio) {
+	std::lock_guard<std::mutex> lock(m_mutex);
+	m_ActiveWndVideoStats.scaleRatio = scaleRatio;
+}
+
 /// private methods
 
 void Stats::addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_STATS& dst) {
@@ -167,6 +178,7 @@ void Stats::addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_ST
 	dst.totalPreWaitTimeUs += src.totalPreWaitTimeUs;
 	dst.totalPresentTimeUs += src.totalPresentTimeUs;
 	dst.totalPresentDisplayMs += src.totalPresentDisplayMs;
+	dst.scaleRatio = src.scaleRatio;
 
 	if (dst.minHostProcessingLatency == 0) {
 		dst.minHostProcessingLatency = src.minHostProcessingLatency;
@@ -299,19 +311,39 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha
 		double avgVideoMbps = m_bwTracker.GetAverageMbps();
 		double peakVideoMbps = m_bwTracker.GetPeakMbps();
 
-		ret = snprintf(&output[offset],
-					   length - offset,
-					   "Bitrate: %.1f Mbps, Peak (%us): %.1f\n"
-					   "Incoming frame rate from network: %.2f FPS\n"
-					   "Decoding frame rate: %.2f FPS\n"
-					   "Rendering frame rate: %.2f FPS (%s)\n",
-					   avgVideoMbps,
-					   m_bwTracker.GetWindowSeconds(),
-					   peakVideoMbps,
-					   stats.receivedFps,
-					   stats.decodedFps,
-					   stats.renderedFps,
-					   Pacer::instance().getPacingImmediate() ? "immediate" : "display-locked");
+		bool isVSREnabled = GetStreamConfig() != nullptr && GetStreamConfig()->videoSuperResolution;
+
+		if (isVSREnabled) {
+			ret = snprintf(&output[offset],
+						   length - offset,
+						   "Bitrate: %.1f Mbps, Peak (%us): %.1f\n"
+						   "Video Enhancement: (x%.2f) NIS Shader\n"
+						   "Incoming frame rate from network: %.2f FPS\n"
+						   "Decoding frame rate: %.2f FPS\n"
+						   "Rendering frame rate: %.2f FPS (%s)\n",
+						   avgVideoMbps,
+						   m_bwTracker.GetWindowSeconds(),
+						   peakVideoMbps,
+						   stats.scaleRatio,
+						   stats.receivedFps,
+						   stats.decodedFps,
+						   stats.renderedFps,
+						   Pacer::instance().getPacingImmediate() ? "immediate" : "display-locked");
+		} else {
+			ret = snprintf(&output[offset],
+						   length - offset,
+						   "Bitrate: %.1f Mbps, Peak (%us): %.1f\n"
+						   "Incoming frame rate from network: %.2f FPS\n"
+						   "Decoding frame rate: %.2f FPS\n"
+						   "Rendering frame rate: %.2f FPS (%s)\n",
+						   avgVideoMbps,
+						   m_bwTracker.GetWindowSeconds(),
+						   peakVideoMbps,
+						   stats.receivedFps,
+						   stats.decodedFps,
+						   stats.renderedFps,
+						   Pacer::instance().getPacingImmediate() ? "immediate" : "display-locked");
+		}
 		if (ret < 0 || (size_t)ret >= (length - offset)) {
 			Utils::Log("Error: stringifyVideoStats length overflow\n");
 			return;

State/Stats.h

@@ -46,6 +46,7 @@ typedef struct _VIDEO_STATS {
 	double decodedFps;
 	double renderedFps;
 	double measurementStartTimestamp;
+	float scaleRatio;
 } VIDEO_STATS, *PVIDEO_STATS;
 
 namespace moonlight_xbox_dx
@@ -64,6 +65,7 @@ namespace moonlight_xbox_dx
 		void SubmitPacerTime(int64_t pacerTimeQpc);
 		void SubmitPresentPacing(double presentDisplayMs);
 		void SubmitRenderStats(double preWaitTimeMs, double renderTimeMs, double presentTimeMs, bool hitDeadline);
+		void SubmitScaleRatio(float scaleRatio);
 
 	private:
 		void addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_STATS& dst);

State/StreamConfiguration.h

@@ -17,6 +17,7 @@ namespace moonlight_xbox_dx
 		property Platform::String^ framePacing;
 		property bool enableHDR;
 		property bool playAudioOnPC;
+		property bool videoSuperResolution;
 		property bool enableVsync;
 		property bool enableSOPS;
 		property bool enableStats;

Streaming/StatsRenderer.cpp

@@ -6,6 +6,7 @@
 #include "../Plot/PlotDesc.h"
 #include "FFMpegDecoder.h"
 #include "Utils.hpp"
+#include "../State/StreamConfiguration.h"
 
 using namespace DirectX;
 using namespace moonlight_xbox_dx;
@@ -187,17 +188,19 @@ void StatsRenderer::CreateWindowSizeDependentResources() {
 	int right = m_displayWidth / 3;
 	int bottom = 0;
 
-	// 13 lines of text
+	// Check if Video Super Resolution is enabled to allocate space for 14 lines instead of 13
+	bool isVSREnabled = GetStreamConfig() != nullptr && GetStreamConfig()->videoSuperResolution;
+
 	if (m_displayHeight >= 2160) { // 24pt font
 		left = 20;
 		right = m_displayWidth / 2;
-		bottom = 448;
+		bottom = isVSREnabled ? 483 : 448; // 14 lines (483) vs 13 lines (448)
 	} else if (m_displayHeight >= 1440) { // 12pt font
 		left = 14;
-		bottom = 224;
+		bottom = isVSREnabled ? 242 : 224; // 14 lines (242) vs 13 lines (224)
 	} else {
 		left = 10;
-		bottom = 224;
+		bottom = isVSREnabled ? 242 : 224; // 14 lines (242) vs 13 lines (224)
 	}
 
 #if defined(_DEBUG)