// Copyright Epic Games, Inc. All Rights Reserved. /** * Class definitions * TODO: Move these to seperate files once we introduce a bundler */ class TwoWayMap { constructor(map = {}) { this.map = map; this.reverseMap = new Map(); for(const key in map) { const value = map[key]; this.reverseMap[value] = key; } } getFromKey(key) { return this.map[key]; } getFromValue(value) { return this.reverseMap[value]; } add(key, value) { this.map[key] = value; this.reverseMap[value] = key; } remove(key, value) { delete this.map[key]; delete this.reverseMap[value]; } } /** * Frontend logic */ // Window events for a gamepad connecting let haveEvents = 'GamepadEvent' in window; let haveWebkitEvents = 'WebKitGamepadEvent' in window; let controllers = {}; let rAF = window.mozRequestAnimationFrame || window.webkitRequestAnimationFrame || window.requestAnimationFrame; let webRtcPlayerObj = null; let print_stats = false; let print_inputs = false; let connect_on_load = false; let ws; const WS_OPEN_STATE = 1; let inputController = null; let autoPlayAudio = true; let qualityController = false; let qualityControlOwnershipCheckBox; let matchViewportResolution; let VideoEncoderQP = "N/A"; // TODO: Remove this - workaround because of bug causing UE to crash when switching resolutions too quickly let lastTimeResized = new Date().getTime(); let resizeTimeout; let responseEventListeners = new Map(); let freezeFrameOverlay = null; let shouldShowPlayOverlay = true; let isFullscreen = false; let isMuted = false; // A freeze frame is a still JPEG image shown instead of the video. let freezeFrame = { receiving: false, size: 0, jpeg: undefined, height: 0, width: 0, valid: false }; let file = { mimetype: "", extension: "", receiving: false, size: 0, data: [], valid: false, timestampStart: undefined }; // Optionally detect if the user is not interacting (AFK) and disconnect them. let afk = { enabled: false, // Set to true to enable the AFK system. warnTimeout: 120, // The time to elapse before warning the user they are inactive. closeTimeout: 10, // The time after the warning when we disconnect the user. active: false, // Whether the AFK system is currently looking for inactivity. overlay: undefined, // The UI overlay warning the user that they are inactive. warnTimer: undefined, // The timer which waits to show the inactivity warning overlay. countdown: 0, // The inactivity warning overlay has a countdown to show time until disconnect. countdownTimer: undefined, // The timer used to tick the seconds shown on the inactivity warning overlay. } // If the user focuses on a UE input widget then we show them a button to open // the on-screen keyboard. JavaScript security means we can only show the // on-screen keyboard in response to a user interaction. let editTextButton = undefined; // A hidden input text box which is used only for focusing and opening the // on-screen keyboard. let hiddenInput = undefined; let MaxByteValue = 255; // The delay between the showing/unshowing of a freeze frame and when the stream will stop/start // eg showing freeze frame -> delay -> stop stream OR show stream -> delay -> unshow freeze frame freezeFrameDelay = 50; // ms let activeKeys = []; let toStreamerMessages = new TwoWayMap(); let fromStreamerMessages = new TwoWayMap(); const MessageDirection = { // A message sent to the streamer. eg Key presses // ie player -> streamer ToStreamer: 0, // A message recevied from the streamer. eg Freeze frames // ie streamer -> player FromStreamer: 1 }; let toStreamerHandlers = new Map(); // toStreamerHandlers[message](args..) let fromStreamerHandlers = new Map(); // fromStreamerHandlers[message](args..) function populateDefaultProtocol() { /* * Control Messages. Range = 0..49. */ toStreamerMessages.add("IFrameRequest", { "id": 0, "byteLength": 0, "structure": [] }); toStreamerMessages.add("RequestQualityControl", { "id": 1, "byteLength": 0, "structure": [] }); toStreamerMessages.add("FpsRequest", { "id": 2, "byteLength": 0, "structure": [] }); toStreamerMessages.add("AverageBitrateRequest", { "id": 3, "byteLength": 0, "structure": [] }); toStreamerMessages.add("StartStreaming", { "id": 4, "byteLength": 0, "structure": [] }); toStreamerMessages.add("StopStreaming", { "id": 5, "byteLength": 0, "structure": [] }); toStreamerMessages.add("LatencyTest", { "id": 6, "byteLength": 0, "structure": [] }); toStreamerMessages.add("RequestInitialSettings", { "id": 7, "byteLength": 0, "structure": [] }); toStreamerMessages.add("TestEcho", { "id": 8, "byteLength": 0, "structure": [] }); /* * Input Messages. Range = 50..89. */ // Generic Input Messages. Range = 50..59. toStreamerMessages.add("UIInteraction", { "id": 50, "byteLength": 0, "structure": [] }); toStreamerMessages.add("Command", { "id": 51, "byteLength": 0, "structure": [] }); // Keyboard Input Message. Range = 60..69. toStreamerMessages.add("KeyDown", { "id": 60, "byteLength": 2, // keyCode isRepeat "structure": ["uint8", "uint8"] }); toStreamerMessages.add("KeyUp", { "id": 61, "byteLength": 1, // keyCode "structure": ["uint8"] }); toStreamerMessages.add("KeyPress", { "id": 62, "byteLength": 2, // charcode "structure": ["uint16"] }); // Mouse Input Messages. Range = 70..79. toStreamerMessages.add("MouseEnter", { "id": 70, "byteLength": 0, "structure": [] }); toStreamerMessages.add("MouseLeave", { "id": 71, "byteLength": 0, "structure": [] }); toStreamerMessages.add("MouseDown", { "id": 72, "byteLength": 5, // button x y "structure": ["uint8", "uint16", "uint16"] }); toStreamerMessages.add("MouseUp", { "id": 73, "byteLength": 5, // button x y "structure": ["uint8", "uint16", "uint16"] }); toStreamerMessages.add("MouseMove", { "id": 74, "byteLength": 8, // x y deltaX deltaY "structure": ["uint16", "uint16", "int16", "int16"] }); toStreamerMessages.add("MouseWheel", { "id": 75, "byteLength": 6, // delta x y "structure": ["int16", "uint16", "uint16"] }); toStreamerMessages.add("MouseDouble", { "id": 76, "byteLength": 5, // button x y "structure": ["uint8", "uint16", "uint16"] }); // Touch Input Messages. Range = 80..89. toStreamerMessages.add("TouchStart", { "id": 80, "byteLength": 8, // numtouches(1) x y idx force valid "structure": ["uint8", "uint16", "uint16", "uint8", "uint8", "uint8"] }); toStreamerMessages.add("TouchEnd", { "id": 81, "byteLength": 8, // numtouches(1) x y idx force valid "structure": ["uint8", "uint16", "uint16", "uint8", "uint8", "uint8"] }); toStreamerMessages.add("TouchMove", { "id": 82, "byteLength": 8, // numtouches(1) x y idx force valid "structure": ["uint8", "uint16", "uint16", "uint8", "uint8", "uint8"] }); // Gamepad Input Messages. Range = 90..99 toStreamerMessages.add("GamepadButtonPressed", { "id": 90, "byteLength": 3, // ctrlerId button isRepeat "structure": ["uint8", "uint8", "uint8"] }); toStreamerMessages.add("GamepadButtonReleased", { "id": 91, "byteLength": 3, // ctrlerId button isRepeat(0) "structure": ["uint8", "uint8", "uint8"] }); toStreamerMessages.add("GamepadAnalog", { "id": 92, "byteLength": 10, // ctrlerId button analogValue "structure": ["uint8", "uint8", "double"] }); fromStreamerMessages.add("QualityControlOwnership", 0); fromStreamerMessages.add("Response", 1); fromStreamerMessages.add("Command", 2); fromStreamerMessages.add("FreezeFrame", 3); fromStreamerMessages.add("UnfreezeFrame", 4); fromStreamerMessages.add("VideoEncoderAvgQP", 5); fromStreamerMessages.add("LatencyTest", 6); fromStreamerMessages.add("InitialSettings", 7); fromStreamerMessages.add("FileExtension", 8); fromStreamerMessages.add("FileMimeType", 9); fromStreamerMessages.add("FileContents", 10); fromStreamerMessages.add("TestEcho", 11); fromStreamerMessages.add("InputControlOwnership", 12); fromStreamerMessages.add("Protocol", 255); } function registerMessageHandlers() { registerMessageHandler(MessageDirection.FromStreamer, "QualityControlOwnership", onQualityControlOwnership); registerMessageHandler(MessageDirection.FromStreamer, "Response", onResponse); registerMessageHandler(MessageDirection.FromStreamer, "Command", onCommand); registerMessageHandler(MessageDirection.FromStreamer, "FreezeFrame", onFreezeFrameMessage); registerMessageHandler(MessageDirection.FromStreamer, "UnfreezeFrame", invalidateFreezeFrameOverlay); registerMessageHandler(MessageDirection.FromStreamer, "VideoEncoderAvgQP", onVideoEncoderAvgQP); registerMessageHandler(MessageDirection.FromStreamer, "LatencyTest", onLatencyTestMessage); registerMessageHandler(MessageDirection.FromStreamer, "InitialSettings", onInitialSettings); registerMessageHandler(MessageDirection.FromStreamer, "FileExtension", onFileExtension); registerMessageHandler(MessageDirection.FromStreamer, "FileMimeType", onFileMimeType); registerMessageHandler(MessageDirection.FromStreamer, "FileContents", onFileContents); registerMessageHandler(MessageDirection.FromStreamer, "TestEcho", () => {/* Do nothing */ }); registerMessageHandler(MessageDirection.FromStreamer, "InputControlOwnership", onInputControlOwnership); registerMessageHandler(MessageDirection.FromStreamer, "Protocol", onProtocolMessage); registerMessageHandler(MessageDirection.ToStreamer, "IFrameRequest", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "RequestQualityControl", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "FpsRequest", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "AverageBitrateRequest", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "StartStreaming", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "StopStreaming", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "LatencyTest", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "RequestInitialSettings", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "TestEcho", () => { /* Do nothing */}); registerMessageHandler(MessageDirection.ToStreamer, "UIInteraction", emitUIInteraction); registerMessageHandler(MessageDirection.ToStreamer, "Command", emitCommand); registerMessageHandler(MessageDirection.ToStreamer, "KeyDown", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "KeyUp", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "KeyPress", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "MouseEnter", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "MouseLeave", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "MouseDown", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "MouseUp", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "MouseMove", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "MouseWheel", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "MouseDouble", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "TouchStart", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "TouchEnd", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "TouchMove", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "GamepadButtonPressed", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "GamepadButtonReleased", sendMessageToStreamer); registerMessageHandler(MessageDirection.ToStreamer, "GamepadAnalog", sendMessageToStreamer); } function registerMessageHandler(messageDirection, messageType, messageHandler) { switch (messageDirection) { case MessageDirection.ToStreamer: toStreamerHandlers[messageType] = messageHandler; break; case MessageDirection.FromStreamer: fromStreamerHandlers[messageType] = messageHandler; break; default: console.log(`Unknown message direction ${messageDirection}`); } } function onQualityControlOwnership(data) { let view = new Uint8Array(data); let ownership = view[1] === 0 ? false : true; console.log("Received quality controller message, will control quality: " + ownership); qualityController = ownership; // If we own the quality control, we can't relinquish it. We only lose // quality control when another peer asks for it if (qualityControlOwnershipCheckBox !== null) { qualityControlOwnershipCheckBox.disabled = ownership; qualityControlOwnershipCheckBox.checked = ownership; } } function onResponse(data) { let response = new TextDecoder("utf-16").decode(data.slice(1)); for (let listener of responseEventListeners.values()) { listener(response); } } function onCommand(data) { let commandAsString = new TextDecoder("utf-16").decode(data.slice(1)); console.log(commandAsString); let command = JSON.parse(commandAsString); if (command.command === 'onScreenKeyboard') { showOnScreenKeyboard(command); } } function onFreezeFrameMessage(data) { let view = new Uint8Array(data); processFreezeFrameMessage(view); } function onVideoEncoderAvgQP(data) { VideoEncoderQP = new TextDecoder("utf-16").decode(data.slice(1)); } function onLatencyTestMessage(data) { let latencyTimingsAsString = new TextDecoder("utf-16").decode(data.slice(1)); console.log("Got latency timings from UE."); console.log(latencyTimingsAsString); let latencyTimingsFromUE = JSON.parse(latencyTimingsAsString); if (webRtcPlayerObj) { webRtcPlayerObj.latencyTestTimings.SetUETimings(latencyTimingsFromUE); } } function onInitialSettings(data) { let settingsString = new TextDecoder("utf-16").decode(data.slice(1)); let settingsJSON = JSON.parse(settingsString); if (settingsJSON.PixelStreaming) { let allowConsoleCommands = settingsJSON.PixelStreaming.AllowPixelStreamingCommands; if (allowConsoleCommands === false) { console.warn("-AllowPixelStreamingCommands=false, sending arbitray console commands from browser to UE is disabled."); } let disableLatencyTest = settingsJSON.PixelStreaming.DisableLatencyTest; if (disableLatencyTest) { document.getElementById("test-latency-button").disabled = true; document.getElementById("test-latency-button").title = "Disabled by -PixelStreamingDisableLatencyTester=true"; console.warn("-PixelStreamingDisableLatencyTester=true, requesting latency report from the the browser to UE is disabled."); } } if (settingsJSON.Encoder) { document.getElementById('encoder-min-qp-text').value = settingsJSON.Encoder.MinQP; document.getElementById('encoder-max-qp-text').value = settingsJSON.Encoder.MaxQP; } if (settingsJSON.WebRTC) { document.getElementById("webrtc-fps-text").value = settingsJSON.WebRTC.FPS; // reminder bitrates are sent in bps but displayed in kbps document.getElementById("webrtc-min-bitrate-text").value = settingsJSON.WebRTC.MinBitrate / 1000; document.getElementById("webrtc-max-bitrate-text").value = settingsJSON.WebRTC.MaxBitrate / 1000; } } function onFileExtension(data) { let view = new Uint8Array(data); processFileExtension(view); } function onFileMimeType(data) { let view = new Uint8Array(data); processFileMimeType(view); } function onFileContents(data) { let view = new Uint8Array(data); processFileContents(view); } function onInputControlOwnership(data) { let view = new Uint8Array(data); let ownership = view[1] === 0 ? false : true; console.log("Received input controller message - will your input control the stream: " + ownership); inputController = ownership; } function onProtocolMessage(data) { try { let protocolString = new TextDecoder("utf-16").decode(data.slice(1)); let protocolJSON = JSON.parse(protocolString); if (!protocolJSON.hasOwnProperty("Direction")) { throw new Error('Malformed protocol received. Ensure the protocol message contains a direction'); } let direction = protocolJSON.Direction; delete protocolJSON.Direction; console.log(`Received new ${ direction == MessageDirection.FromStreamer ? "FromStreamer" : "ToStreamer" } protocol. Updating existing protocol...`); Object.keys(protocolJSON).forEach((messageType) => { let message = protocolJSON[messageType]; switch (direction) { case MessageDirection.ToStreamer: // Check that the message contains all the relevant params if (!message.hasOwnProperty("id") || !message.hasOwnProperty("byteLength")) { console.error(`ToStreamer->${messageType} protocol definition was malformed as it didn't contain at least an id and a byteLength\n Definition was: ${JSON.stringify(message, null, 2)}`); // return in a forEach is equivalent to a continue in a normal for loop return; } if(message.byteLength > 0 && !message.hasOwnProperty("structure")) { // If we specify a bytelength, will must have a corresponding structure console.error(`ToStreamer->${messageType} protocol definition was malformed as it specified a byteLength but no accompanying structure`); // return in a forEach is equivalent to a continue in a normal for loop return; } if(messageType === "GamepadAnalog") { // We don't want to update the GamepadAnalog message type as UE sends it with an incorrect bytelength return; } if (toStreamerHandlers[messageType]) { // If we've registered a handler for this message type we can add it to our supported messages. ie registerMessageHandler(...) toStreamerMessages.add(messageType, message); } else { console.error(`There was no registered handler for "${messageType}" - try adding one using registerMessageHandler(MessageDirection.ToStreamer, "${messageType}", myHandler)`); } break; case MessageDirection.FromStreamer: // Check that the message contains all the relevant params if (!message.hasOwnProperty("id")) { console.error(`FromStreamer->${messageType} protocol definition was malformed as it didn't contain at least an id\n Definition was: ${JSON.stringify(message, null, 2)}`); // return in a forEach is equivalent to a continue in a normal for loop return; } if (fromStreamerHandlers[messageType]) { // If we've registered a handler for this message type. ie registerMessageHandler(...) fromStreamerMessages.add(messageType, message.id); } else { console.error(`There was no registered handler for "${message}" - try adding one using registerMessageHandler(MessageDirection.FromStreamer, "${messageType}", myHandler)`); } break; default: throw new Error(`Unknown direction: ${direction}`); } }); // Once the protocol has been received, we can send our control messages requestInitialSettings(); requestQualityControl(); } catch (e) { console.log(e); } } // https://w3c.github.io/gamepad/#remapping const gamepadLayout = { // Buttons RightClusterBottomButton: 0, RightClusterRightButton: 1, RightClusterLeftButton: 2, RightClusterTopButton: 3, LeftShoulder: 4, RightShoulder: 5, LeftTrigger: 6, RightTrigger: 7, SelectOrBack: 8, StartOrForward: 9, LeftAnalogPress: 10, RightAnalogPress: 11, LeftClusterTopButton: 12, LeftClusterBottomButton: 13, LeftClusterLeftButton: 14, LeftClusterRightButton: 15, CentreButton: 16, // Axes LeftStickHorizontal: 0, LeftStickVertical: 1, RightStickHorizontal: 2, RightStickVertical: 3 }; function scanGamepads() { let gamepads = navigator.getGamepads ? navigator.getGamepads() : (navigator.webkitGetGamepads ? navigator.webkitGetGamepads() : []); for (let i = 0; i < gamepads.length; i++) { if (gamepads[i] && (gamepads[i].index in controllers)) { controllers[gamepads[i].index].currentState = gamepads[i]; } } } function updateStatus() { scanGamepads(); // Iterate over multiple controllers in the case the mutiple gamepads are connected for (let j in controllers) { let controller = controllers[j]; let currentState = controller.currentState; let prevState = controller.prevState; // Iterate over buttons for (let i = 0; i < currentState.buttons.length; i++) { let currButton = currentState.buttons[i]; let prevButton = prevState.buttons[i]; if (currButton.pressed) { // press if (i == gamepadLayout.LeftTrigger) { // UEs left analog has a button index of 5 toStreamerHandlers.GamepadAnalog("GamepadAnalog", [j, 5, currButton.value]); } else if (i == gamepadLayout.RightTrigger) { // UEs right analog has a button index of 6 toStreamerHandlers.GamepadAnalog("GamepadAnalog", [j, 6, currButton.value]); } else { toStreamerHandlers.GamepadButtonPressed("GamepadButtonPressed", [j, i, prevButton.pressed]); } } else if (!currButton.pressed && prevButton.pressed) { // release if (i == gamepadLayout.LeftTrigger) { // UEs left analog has a button index of 5 toStreamerHandlers.GamepadAnalog("GamepadAnalog", [j, 5, 0]); } else if (i == gamepadLayout.RightTrigger) { // UEs right analog has a button index of 6 toStreamerHandlers.GamepadAnalog("GamepadAnalog", [j, 6, 0]); } else { toStreamerHandlers.GamepadButtonReleased("GamepadButtonReleased", [j, i]); } } } // Iterate over gamepad axes (we will increment in lots of 2 as there is 2 axes per stick) for (let i = 0; i < currentState.axes.length; i += 2) { // Horizontal axes are even numbered let x = parseFloat(currentState.axes[i].toFixed(4)); // Vertical axes are odd numbered // https://w3c.github.io/gamepad/#remapping Gamepad browser side standard mapping has positive down, negative up. This is downright disgusting. So we fix it. let y = -parseFloat(currentState.axes[i + 1].toFixed(4)); // UE's analog axes follow the same order as the browsers, but start at index 1 so we will offset as such toStreamerHandlers.GamepadAnalog("GamepadAnalog", [j, i + 1, x]); // Horizontal axes, only offset by 1 toStreamerHandlers.GamepadAnalog("GamepadAnalog", [j, i + 2, y]); // Vertical axes, offset by two (1 to match UEs axes convention and then another 1 for the vertical axes) } controllers[j].prevState = currentState; } rAF(updateStatus); } function gamepadConnectHandler(e) { console.log("Gamepad connect handler"); gamepad = e.gamepad; controllers[gamepad.index] = {}; controllers[gamepad.index].currentState = gamepad; controllers[gamepad.index].prevState = gamepad; console.log("Gamepad: " + gamepad.id + " connected"); rAF(updateStatus); } function gamepadDisconnectHandler(e) { console.log("Gamepad disconnect handler"); console.log("Gamepad: " + e.gamepad.id + " disconnected"); delete controllers[e.gamepad.index]; } function fullscreen() { // if already full screen; exit // else go fullscreen if ( document.fullscreenElement || document.webkitFullscreenElement || document.mozFullScreenElement || document.msFullscreenElement ) { if (document.exitFullscreen) { document.exitFullscreen(); } else if (document.mozCancelFullScreen) { document.mozCancelFullScreen(); } else if (document.webkitExitFullscreen) { document.webkitExitFullscreen(); } else if (document.msExitFullscreen) { document.msExitFullscreen(); } } else { let element; //HTML elements controls if(!(document.fullscreenEnabled || document.webkitFullscreenEnabled)) { // Chrome and FireFox on iOS can only fullscreen a