Skip to content

Add low latency drop down #565

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
gareth-morgan-nv wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Add low latency drop down #565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
da30a91
Add low latency mode (drop down and CXR.js branch https://gitlab-mast…
gareth-morgan-nv May 29, 2026
3fcc77a
Add uplink throttle UI
gareth-morgan-nv Jun 1, 2026
7e4f3f4
Use new tarball
gareth-morgan-nv Jun 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion deps/cloudxr/.env.default
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# CloudXR Docker Images and Configs
###########################################################
CXR_RUNTIME_SDK_VERSION=6.1.0
CXR_WEB_SDK_VERSION=6.2.0
CXR_WEB_SDK_VERSION=mr.fce66ff6
CXR_HOST_VOLUME_PATH=$HOME/.cloudxr

###########################################################
Expand Down
16 changes: 15 additions & 1 deletion deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,8 @@ export default function CloudXRComponent({
referenceSpace = referenceSpace.getOffsetReferenceSpace(offsetTransform);
}

const lowLatency = config.networkLatencyMode === 'LOW';

// Fill in CloudXR session options.
const cloudXROptions: CloudXR.SessionOptions = {
serverAddress: connectionConfig.serverIP,
Expand All @@ -244,7 +246,13 @@ export default function CloudXRComponent({
gl: gl,
referenceSpace: referenceSpace,
deviceFrameRate: config.deviceFrameRate,
maxStreamingBitrateKbps: config.maxStreamingBitrateMbps * 1000, // Convert Mbps to Kbps
networkLatencyMode: lowLatency
? CloudXR.NetworkLatencyMode.Low
: CloudXR.NetworkLatencyMode.Default,
// LOW preset sets bitrate and DRC in SessionImpl; do not override.
...(lowLatency
? {}
: { maxStreamingBitrateKbps: config.maxStreamingBitrateMbps * 1000 }),
enablePoseSmoothing: config.enablePoseSmoothing,
posePredictionFactor: config.posePredictionFactor,
enableTexSubImage2D: config.enableTexSubImage2D,
Expand All @@ -262,6 +270,12 @@ export default function CloudXRComponent({
},
};

// Requires cloudxr-js with UplinkThrottle support; harmless on older SDK tarballs.
Object.assign(cloudXROptions, {
enableTrackingUplinkThrottle: config.enableTrackingUplinkThrottle ?? false,
trackingUplinkMaxHz: config.trackingUplinkMaxHz ?? 30,
});

// Store the render target and key GL bindings to restore after CloudXR rendering
const cloudXRDelegates: CloudXR.SessionDelegates = {
onWebGLStateChangeBegin: () => {
Expand Down
9 changes: 9 additions & 0 deletions deps/cloudxr/webxr_client/helpers/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,15 @@ export interface CloudXRConfig {
/** Maximum streaming bitrate in Megabits per second (Mbps) */
maxStreamingBitrateMbps: number;

/** CloudXR network latency preset (`LOW` caps downstream bitrate for congested links). */
networkLatencyMode?: 'DEFAULT' | 'LOW';

/** Cap tracking bulk uplink rate (pose, controllers, hands, body) */
enableTrackingUplinkThrottle?: boolean;

/** Max tracking uplink rate in Hz when throttle is enabled */
trackingUplinkMaxHz?: number;

/** Preferred video codec used for streaming */
codec?: 'h264' | 'h265' | 'av1';

Expand Down
2 changes: 1 addition & 1 deletion deps/cloudxr/webxr_client/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"clean": "rimraf dist"
},
"dependencies": {
"@nvidia/cloudxr": "file:../nvidia-cloudxr-6.2.0.tgz",
"@nvidia/cloudxr": "file:../nvidia-cloudxr-mr.fce66ff6.tgz",
"@preact/signals-react": "^3.10.0",
"@react-three/drei": "^10.7.7",
"@react-three/fiber": "^9.6.0",
Expand Down
75 changes: 75 additions & 0 deletions deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ export class CloudXR2DUI {
private immersiveSelect!: HTMLSelectElement;
/** Dropdown to select device frame rate (FPS) */
private deviceFrameRateSelect!: HTMLSelectElement;
/** Dropdown for CloudXR networkLatencyMode preset */
private networkLatencyModeSelect!: HTMLSelectElement;
/** Dropdown to select max streaming bitrate (Mbps) */
private maxStreamingBitrateMbpsSelect!: HTMLSelectElement;
/** Dropdown to select preferred streaming codec */
Expand Down Expand Up @@ -154,6 +156,10 @@ export class CloudXR2DUI {
private controllerModelVisibilitySelect!: HTMLSelectElement;
/** Skip client CloudXR `render` (headless: client blit off; tracking on) */
private headlessInput!: HTMLInputElement;
/** Cap tracking bulk uplink rate (pose, controllers, hands, body) */
private trackingUplinkThrottleInput!: HTMLInputElement;
/** Max tracking uplink rate when throttle is enabled */
private trackingUplinkMaxHzSelect!: HTMLSelectElement;
/** Breadcrumb subtitle in header (e.g. "for Real Robot › GEAR › Dexmate"). */
private teleopModeSubtitle!: HTMLElement;
/** Hierarchical project selector in header */
Expand Down Expand Up @@ -213,6 +219,8 @@ export class CloudXR2DUI {
// Set initial display value
this.posePredictionFactorValue.textContent = this.posePredictionFactorInput.value;
this.updateConfiguration();
this.updateNetworkLatencyModeUi();
this.updateTrackingUplinkThrottleUi();
this.updateDeviceProfileWarning(resolveDeviceProfileId(this.deviceProfileSelect.value));
this.updateConnectButtonState();
this.initialized = true;
Expand Down Expand Up @@ -361,6 +369,7 @@ export class CloudXR2DUI {
this.proxyUrlInput = this.getElement<HTMLInputElement>('proxyUrl');
this.immersiveSelect = this.getElement<HTMLSelectElement>('immersive');
this.deviceFrameRateSelect = this.getElement<HTMLSelectElement>('deviceFrameRate');
this.networkLatencyModeSelect = this.getElement<HTMLSelectElement>('networkLatencyMode');
this.maxStreamingBitrateMbpsSelect =
this.getElement<HTMLSelectElement>('maxStreamingBitrateMbps');
this.codecSelect = this.getElement<HTMLSelectElement>('codec');
Expand Down Expand Up @@ -408,6 +417,10 @@ export class CloudXR2DUI {
'controllerModelVisibility'
);
this.headlessInput = this.getElement<HTMLInputElement>('cloudxrHeadless');
this.trackingUplinkThrottleInput = this.getElement<HTMLInputElement>(
'cloudxrTrackingUplinkThrottle'
);
this.trackingUplinkMaxHzSelect = this.getElement<HTMLSelectElement>('trackingUplinkMaxHz');
this.teleopModeSubtitle = this.getElement<HTMLElement>('teleopModeSubtitle');
this.teleopProjectSelect = this.getElement<HTMLSelectElement>('teleopProjectSelect');
}
Expand Down Expand Up @@ -444,6 +457,7 @@ export class CloudXR2DUI {
reprojectionGridRows: 0,
deviceFrameRate: 90,
maxStreamingBitrateMbps: 150,
networkLatencyMode: 'DEFAULT',
codec: 'av1',
immersiveMode: 'ar',
deviceProfileId: 'custom',
Expand All @@ -458,6 +472,8 @@ export class CloudXR2DUI {
useQuestColorWorkaround: false,
hideControllerModel: false,
headless: false,
enableTrackingUplinkThrottle: false,
trackingUplinkMaxHz: 30,
teleopPath: DEFAULT_TELEOP_PATH,
};
}
Expand All @@ -477,6 +493,7 @@ export class CloudXR2DUI {
enableLocalStorage(this.reprojectionGridRowsInput, 'reprojectionGridRows');
enableLocalStorage(this.proxyUrlInput, 'proxyUrl');
enableLocalStorage(this.deviceFrameRateSelect, 'deviceFrameRate');
enableLocalStorage(this.networkLatencyModeSelect, 'networkLatencyMode');
enableLocalStorage(this.maxStreamingBitrateMbpsSelect, 'maxStreamingBitrateMbps');
enableLocalStorage(this.codecSelect, 'codec');
enableLocalStorage(this.enablePoseSmoothingSelect, 'enablePoseSmoothing');
Expand All @@ -493,6 +510,17 @@ export class CloudXR2DUI {
enableLocalStorage(this.mediaAddressInput, 'mediaAddress');
enableLocalStorage(this.mediaPortInput, 'mediaPort');
enableLocalStorage(this.controllerModelVisibilitySelect, 'controllerModelVisibility');
enableLocalStorage(this.trackingUplinkMaxHzSelect, 'trackingUplinkMaxHz');
this.loadTrackingUplinkThrottleFromLocalStorage();
}

private loadTrackingUplinkThrottleFromLocalStorage(): void {
try {
this.trackingUplinkThrottleInput.checked =
localStorage.getItem('cloudxrTrackingUplinkThrottle') === 'true';
} catch {
this.trackingUplinkThrottleInput.checked = false;
}
}

/**
Expand Down Expand Up @@ -563,6 +591,10 @@ export class CloudXR2DUI {
addListener(this.reprojectionGridRowsInput, 'keyup', updateGridValidation);
this.updateGridValidationMessage();
addListener(this.deviceFrameRateSelect, 'change', onProfileLinkedChange);
addListener(this.networkLatencyModeSelect, 'change', () => {
this.updateNetworkLatencyModeUi();
onProfileLinkedChange();
});
addListener(this.maxStreamingBitrateMbpsSelect, 'change', onProfileLinkedChange);
addListener(this.codecSelect, 'change', onProfileLinkedChange);
addListener(this.enablePoseSmoothingSelect, 'change', onProfileLinkedChange);
Expand Down Expand Up @@ -609,6 +641,21 @@ export class CloudXR2DUI {
this.applyHeadlessImmersiveDropdown();
this.updateConfiguration();
});
addListener(this.trackingUplinkThrottleInput, 'change', () => {
try {
localStorage.setItem(
'cloudxrTrackingUplinkThrottle',
this.trackingUplinkThrottleInput.checked ? 'true' : 'false'
);
} catch {
// ignore
}
this.updateTrackingUplinkThrottleUi();
this.updateConfiguration();
});
addListener(this.trackingUplinkMaxHzSelect, 'change', () => {
this.updateConfiguration();
});

addListener(this.deviceProfileSelect, 'change', () => {
this.applyDeviceProfileToForm(resolveDeviceProfileId(this.deviceProfileSelect.value));
Expand Down Expand Up @@ -745,6 +792,7 @@ export class CloudXR2DUI {
maxStreamingBitrateMbps:
parseInt(this.maxStreamingBitrateMbpsSelect.value) ||
this.getDefaultConfiguration().maxStreamingBitrateMbps,
networkLatencyMode: this.networkLatencyModeSelect.value === 'LOW' ? 'LOW' : 'DEFAULT',
codec:
(this.codecSelect.value as 'h264' | 'h265' | 'av1') || this.getDefaultConfiguration().codec,
// Headless mode turns off the client's CloudXR frame blit but keeps tracking; the WebXR
Expand Down Expand Up @@ -791,6 +839,10 @@ export class CloudXR2DUI {
// See immersiveMode above: when true, callers must start an immersive-vr WebXR session.
headless: this.headlessInput.checked,
panelHiddenAtStart: this.panelHiddenAtStartSelect.value === 'true',
enableTrackingUplinkThrottle: this.trackingUplinkThrottleInput.checked,
trackingUplinkMaxHz:
parseInt(this.trackingUplinkMaxHzSelect.value, 10) ||
this.getDefaultConfiguration().trackingUplinkMaxHz,
teleopPath: this.teleopPath,
};

Expand Down Expand Up @@ -870,6 +922,7 @@ export class CloudXR2DUI {
localStorage.setItem('reprojectionGridCols', this.reprojectionGridColsInput.value);
localStorage.setItem('reprojectionGridRows', this.reprojectionGridRowsInput.value);
localStorage.setItem('deviceFrameRate', this.deviceFrameRateSelect.value);
localStorage.setItem('networkLatencyMode', this.networkLatencyModeSelect.value);
localStorage.setItem('maxStreamingBitrateMbps', this.maxStreamingBitrateMbpsSelect.value);
localStorage.setItem('codec', this.codecSelect.value);
localStorage.setItem('enablePoseSmoothing', this.enablePoseSmoothingSelect.value);
Expand All @@ -881,6 +934,28 @@ export class CloudXR2DUI {
}
}

private updateTrackingUplinkThrottleUi(): void {
const enabled = this.trackingUplinkThrottleInput.checked;
this.trackingUplinkMaxHzSelect.disabled = !enabled;
const help = document.getElementById('trackingUplinkMaxHzHelp');
if (help) {
help.textContent = enabled
? 'Maximum rate for all tracking bulk messages (pose, controllers, hands, body).'
: 'Enable tracking uplink throttle above to set a max rate.';
}
}

private updateNetworkLatencyModeUi(): void {
const low = this.networkLatencyModeSelect.value === 'LOW';
this.maxStreamingBitrateMbpsSelect.disabled = low;
const help = document.getElementById('networkLatencyModeHelp');
if (help) {
help.textContent = low
? 'Low mode sets ~15 Mbit/s max bitrate and Ragnarok DRC. Settings below do not apply.'
: 'Default: Ragnarok ALL adaptive streaming. Set max Mbps below.';
}
}

private updateDeviceProfileWarning(profileId: DeviceProfileId): void {
if (!this.deviceProfileWarning) return;
const profile = getDeviceProfile(profileId);
Expand Down
30 changes: 30 additions & 0 deletions deps/cloudxr/webxr_client/src/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -723,6 +723,36 @@ <h3 class="debug-title">Debug Settings</h3>
</div>
</div>

<div class="config-section">
<label for="networkLatencyMode" class="input-label">Network Latency Mode</label>
<select id="networkLatencyMode" class="ui-input config-input">
<option value="DEFAULT" selected>Default (high quality)</option>
<option value="LOW">Low (congested Wi‑Fi)</option>
</select>
<div id="networkLatencyModeHelp" class="config-text">
Default: set max Mbps below with Ragnarok ALL adaptive streaming. Low: applies ~15 Mbit/s and Ragnarok DRC (ignores Mbps).
</div>
</div>

<div class="config-section">
<label for="cloudxrTrackingUplinkThrottle" class="input-label">Tracking Uplink Throttle</label>
<div class="config-text">
<input type="checkbox" id="cloudxrTrackingUplinkThrottle" class="config-input" style="width: auto; margin-right: 0.5em" aria-label="Cap tracking uplink rate">
<span>Cap tracking uplink (pose, controllers, hands, body) to reduce bandwidth.</span>
</div>
<label for="trackingUplinkMaxHz" class="input-label">Max Uplink Rate (Hz)</label>
<select id="trackingUplinkMaxHz" class="ui-input config-input">
<option value="15">15 Hz</option>
<option value="30" selected>30 Hz</option>
<option value="45">45 Hz</option>
<option value="60">60 Hz</option>
<option value="90">90 Hz</option>
</select>
<div id="trackingUplinkMaxHzHelp" class="config-text">
Maximum rate for all tracking bulk messages when throttle is enabled.
</div>
</div>

<div class="config-section">
<label class="input-label">Maximum Streaming Bitrate</label>
<label for="maxStreamingBitrateMbps" class="input-label">Megabits per second</label>
Expand Down