servo/support/hololens/ServoApp/ImmersiveMain.cpp

557 lines
23 KiB
C++

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "pch.h"
#include "ImmersiveMain.h"
#include "Common/DirectXHelper.h"
#include <windows.graphics.directx.direct3d11.interop.h>
using namespace Immersive;
using namespace concurrency;
using namespace Microsoft::WRL;
using namespace std::placeholders;
using namespace winrt::Windows::Foundation::Numerics;
using namespace winrt::Windows::Gaming::Input;
using namespace winrt::Windows::Graphics::Holographic;
using namespace winrt::Windows::Graphics::DirectX::Direct3D11;
using namespace winrt::Windows::Perception::Spatial;
using namespace winrt::Windows::UI::Input::Spatial;
// Loads and initializes application assets when the application is loaded.
ImmersiveMain::ImmersiveMain(
std::shared_ptr<DX::DeviceResources> const &deviceResources)
: m_deviceResources(deviceResources) {
// Register to be notified if the device is lost or recreated.
m_deviceResources->RegisterDeviceNotify(this);
// If connected, a game controller can also be used for input.
m_gamepadAddedEventToken =
Gamepad::GamepadAdded(bind(&ImmersiveMain::OnGamepadAdded, this, _1, _2));
m_gamepadRemovedEventToken = Gamepad::GamepadRemoved(
bind(&ImmersiveMain::OnGamepadRemoved, this, _1, _2));
for (Gamepad const &gamepad : Gamepad::Gamepads()) {
OnGamepadAdded(nullptr, gamepad);
}
m_canGetHolographicDisplayForCamera =
winrt::Windows::Foundation::Metadata::ApiInformation::IsPropertyPresent(
L"Windows.Graphics.Holographic.HolographicCamera", L"Display");
m_canGetDefaultHolographicDisplay =
winrt::Windows::Foundation::Metadata::ApiInformation::IsMethodPresent(
L"Windows.Graphics.Holographic.HolographicDisplay", L"GetDefault");
m_canCommitDirect3D11DepthBuffer =
winrt::Windows::Foundation::Metadata::ApiInformation::IsMethodPresent(
L"Windows.Graphics.Holographic.HolographicCameraRenderingParameters",
L"CommitDirect3D11DepthBuffer");
if (m_canGetDefaultHolographicDisplay) {
// Subscribe for notifications about changes to the state of the default
// HolographicDisplay and its SpatialLocator.
m_holographicDisplayIsAvailableChangedEventToken =
HolographicSpace::IsAvailableChanged(
bind(&ImmersiveMain::OnHolographicDisplayIsAvailableChanged, this,
_1, _2));
}
// Acquire the current state of the default HolographicDisplay and its
// SpatialLocator.
OnHolographicDisplayIsAvailableChanged(nullptr, nullptr);
}
void ImmersiveMain::SetHolographicSpace(
HolographicSpace const &holographicSpace) {
UnregisterHolographicEventHandlers();
m_holographicSpace = holographicSpace;
//
// TODO: Add code here to initialize your holographic content.
//
#ifdef DRAW_SAMPLE_CONTENT
// Initialize the sample hologram.
m_spinningCubeRenderer =
std::make_unique<SpinningCubeRenderer>(m_deviceResources);
m_spatialInputHandler = std::make_unique<SpatialInputHandler>();
#endif
// Respond to camera added events by creating any resources that are specific
// to that camera, such as the back buffer render target view.
// When we add an event handler for CameraAdded, the API layer will avoid
// putting the new camera in new HolographicFrames until we complete the
// deferral we created for that handler, or return from the handler without
// creating a deferral. This allows the app to take more than one frame to
// finish creating resources and loading assets for the new holographic
// camera. This function should be registered before the app creates any
// HolographicFrames.
m_cameraAddedToken = m_holographicSpace.CameraAdded(
std::bind(&ImmersiveMain::OnCameraAdded, this, _1, _2));
// Respond to camera removed events by releasing resources that were created
// for that camera. When the app receives a CameraRemoved event, it releases
// all references to the back buffer right away. This includes render target
// views, Direct2D target bitmaps, and so on. The app must also ensure that
// the back buffer is not attached as a render target, as shown in
// DeviceResources::ReleaseResourcesForBackBuffer.
m_cameraRemovedToken = m_holographicSpace.CameraRemoved(
std::bind(&ImmersiveMain::OnCameraRemoved, this, _1, _2));
// Notes on spatial tracking APIs:
// * Stationary reference frames are designed to provide a best-fit position
// relative to the
// overall space. Individual positions within that reference frame are
// allowed to drift slightly as the device learns more about the
// environment.
// * When precise placement of individual holograms is required, a
// SpatialAnchor should be used to
// anchor the individual hologram to a position in the real world - for
// example, a point the user indicates to be of special interest. Anchor
// positions do not drift, but can be corrected; the anchor will use the
// corrected position starting in the next frame after the correction has
// occurred.
}
void ImmersiveMain::UnregisterHolographicEventHandlers() {
if (m_holographicSpace != nullptr) {
// Clear previous event registrations.
m_holographicSpace.CameraAdded(m_cameraAddedToken);
m_cameraAddedToken = {};
m_holographicSpace.CameraRemoved(m_cameraRemovedToken);
m_cameraRemovedToken = {};
}
if (m_spatialLocator != nullptr) {
m_spatialLocator.LocatabilityChanged(m_locatabilityChangedToken);
}
}
ImmersiveMain::~ImmersiveMain() {
// Deregister device notification.
m_deviceResources->RegisterDeviceNotify(nullptr);
UnregisterHolographicEventHandlers();
Gamepad::GamepadAdded(m_gamepadAddedEventToken);
Gamepad::GamepadRemoved(m_gamepadRemovedEventToken);
HolographicSpace::IsAvailableChanged(
m_holographicDisplayIsAvailableChangedEventToken);
}
// Updates the application state once per frame.
HolographicFrame ImmersiveMain::Update() {
// Before doing the timer update, there is some work to do per-frame
// to maintain holographic rendering. First, we will get information
// about the current frame.
// The HolographicFrame has information that the app needs in order
// to update and render the current frame. The app begins each new
// frame by calling CreateNextFrame.
HolographicFrame holographicFrame = m_holographicSpace.CreateNextFrame();
// Get a prediction of where holographic cameras will be when this frame
// is presented.
HolographicFramePrediction prediction = holographicFrame.CurrentPrediction();
// Back buffers can change from frame to frame. Validate each buffer, and
// recreate resource views and depth buffers as needed.
m_deviceResources->EnsureCameraResources(holographicFrame, prediction);
#ifdef DRAW_SAMPLE_CONTENT
if (m_stationaryReferenceFrame != nullptr) {
// Check for new input state since the last frame.
for (GamepadWithButtonState &gamepadWithButtonState : m_gamepads) {
bool buttonDownThisUpdate =
((gamepadWithButtonState.gamepad.GetCurrentReading().Buttons &
GamepadButtons::A) == GamepadButtons::A);
if (buttonDownThisUpdate &&
!gamepadWithButtonState.buttonAWasPressedLastFrame) {
m_pointerPressed = true;
}
gamepadWithButtonState.buttonAWasPressedLastFrame = buttonDownThisUpdate;
}
SpatialInteractionSourceState pointerState =
m_spatialInputHandler->CheckForInput();
SpatialPointerPose pose = nullptr;
if (pointerState != nullptr) {
pose = pointerState.TryGetPointerPose(
m_stationaryReferenceFrame.CoordinateSystem());
} else if (m_pointerPressed) {
pose = SpatialPointerPose::TryGetAtTimestamp(
m_stationaryReferenceFrame.CoordinateSystem(),
prediction.Timestamp());
}
m_pointerPressed = false;
// When a Pressed gesture is detected, the sample hologram will be
// repositioned two meters in front of the user.
m_spinningCubeRenderer->PositionHologram(pose);
}
#endif
m_timer.Tick([this]() {
//
// TODO: Update scene objects.
//
// Put time-based updates here. By default this code will run once per frame,
// but if you change the StepTimer to use a fixed time step this code will
// run as many times as needed to get to the current step.
//
#ifdef DRAW_SAMPLE_CONTENT
m_spinningCubeRenderer->Update(m_timer);
#endif
});
if (!m_canCommitDirect3D11DepthBuffer) {
// On versions of the platform that do not support the
// CommitDirect3D11DepthBuffer API, we can control image stabilization by
// setting a focus point with optional plane normal and velocity.
for (HolographicCameraPose const &cameraPose : prediction.CameraPoses()) {
#ifdef DRAW_SAMPLE_CONTENT
// The HolographicCameraRenderingParameters class provides access to set
// the image stabilization parameters.
HolographicCameraRenderingParameters renderingParameters =
holographicFrame.GetRenderingParameters(cameraPose);
// SetFocusPoint informs the system about a specific point in your scene
// to prioritize for image stabilization. The focus point is set
// independently for each holographic camera. When setting the focus
// point, put it on or near content that the user is looking at. In this
// example, we put the focus point at the center of the sample hologram.
// You can also set the relative velocity and facing of the stabilization
// plane using overloads of this method.
if (m_stationaryReferenceFrame != nullptr) {
renderingParameters.SetFocusPoint(
m_stationaryReferenceFrame.CoordinateSystem(),
m_spinningCubeRenderer->GetPosition());
}
#endif
}
}
// The holographic frame will be used to get up-to-date view and projection
// matrices and to present the swap chain.
return holographicFrame;
}
// Renders the current frame to each holographic camera, according to the
// current application and spatial positioning state. Returns true if the
// frame was rendered to at least one camera.
bool ImmersiveMain::Render(HolographicFrame const &holographicFrame) {
// Don't try to render anything before the first Update.
if (m_timer.GetFrameCount() == 0) {
return false;
}
//
// TODO: Add code for pre-pass rendering here.
//
// Take care of any tasks that are not specific to an individual holographic
// camera. This includes anything that doesn't need the final view or
// projection matrix, such as lighting maps.
//
// Lock the set of holographic camera resources, then draw to each camera
// in this frame.
return m_deviceResources->UseHolographicCameraResources<bool>(
[this,
holographicFrame](std::map<UINT32, std::unique_ptr<DX::CameraResources>>
&cameraResourceMap) {
// Up-to-date frame predictions enhance the effectiveness of image
// stablization and allow more accurate positioning of holograms.
holographicFrame.UpdateCurrentPrediction();
HolographicFramePrediction prediction =
holographicFrame.CurrentPrediction();
bool atLeastOneCameraRendered = false;
for (HolographicCameraPose const &cameraPose :
prediction.CameraPoses()) {
// This represents the device-based resources for a HolographicCamera.
DX::CameraResources *pCameraResources =
cameraResourceMap[cameraPose.HolographicCamera().Id()].get();
// Get the device context.
const auto context = m_deviceResources->GetD3DDeviceContext();
const auto depthStencilView = pCameraResources->GetDepthStencilView();
// Set render targets to the current holographic camera.
ID3D11RenderTargetView *const targets[1] = {
pCameraResources->GetBackBufferRenderTargetView()};
context->OMSetRenderTargets(1, targets, depthStencilView);
// Clear the back buffer and depth stencil view.
if (m_canGetHolographicDisplayForCamera &&
cameraPose.HolographicCamera().Display().IsOpaque()) {
context->ClearRenderTargetView(targets[0],
DirectX::Colors::CornflowerBlue);
} else {
context->ClearRenderTargetView(targets[0],
DirectX::Colors::Transparent);
}
context->ClearDepthStencilView(
depthStencilView, D3D11_CLEAR_DEPTH | D3D11_CLEAR_STENCIL, 1.0f,
0);
//
// TODO: Replace the sample content with your own content.
//
// Notes regarding holographic content:
// * For drawing, remember that you have the potential to fill
// twice as many pixels
// in a stereoscopic render target as compared to a
// non-stereoscopic render target of the same resolution. Avoid
// unnecessary or repeated writes to the same pixel, and only
// draw holograms that the user can see.
// * To help occlude hologram geometry, you can create a depth map
// using geometry
// data obtained via the surface mapping APIs. You can use this
// depth map to avoid rendering holograms that are intended to be
// hidden behind tables, walls, monitors, and so on.
// * On HolographicDisplays that are transparent, black pixels will
// appear transparent
// to the user. On such devices, you should clear the screen to
// Transparent as shown above. You should still use alpha
// blending to draw semitransparent holograms.
//
// The view and projection matrices for each holographic camera will
// change every frame. This function refreshes the data in the
// constant buffer for the holographic camera indicated by cameraPose.
if (m_stationaryReferenceFrame) {
pCameraResources->UpdateViewProjectionBuffer(
m_deviceResources, cameraPose,
m_stationaryReferenceFrame.CoordinateSystem());
}
// Attach the view/projection constant buffer for this camera to the
// graphics pipeline.
bool cameraActive =
pCameraResources->AttachViewProjectionBuffer(m_deviceResources);
#ifdef DRAW_SAMPLE_CONTENT
// Only render world-locked content when positional tracking is
// active.
if (cameraActive) {
// Draw the sample hologram.
m_spinningCubeRenderer->Render();
if (m_canCommitDirect3D11DepthBuffer) {
// On versions of the platform that support the
// CommitDirect3D11DepthBuffer API, we can provide the depth
// buffer to the system, and it will use depth information to
// stabilize the image at a per-pixel level.
HolographicCameraRenderingParameters renderingParameters =
holographicFrame.GetRenderingParameters(cameraPose);
IDirect3DSurface interopSurface =
DX::CreateDepthTextureInteropObject(
pCameraResources->GetDepthStencilTexture2D());
// Calling CommitDirect3D11DepthBuffer causes the system to queue
// Direct3D commands to read the depth buffer. It will then use
// that information to stabilize the image as the HolographicFrame
// is presented.
renderingParameters.CommitDirect3D11DepthBuffer(interopSurface);
}
}
#endif
atLeastOneCameraRendered = true;
}
return atLeastOneCameraRendered;
});
}
void ImmersiveMain::SaveAppState() {
//
// TODO: Insert code here to save your app state.
// This method is called when the app is about to suspend.
//
// For example, store information in the SpatialAnchorStore.
//
}
void ImmersiveMain::LoadAppState() {
//
// TODO: Insert code here to load your app state.
// This method is called when the app resumes.
//
// For example, load information from the SpatialAnchorStore.
//
}
void ImmersiveMain::OnPointerPressed() { m_pointerPressed = true; }
// Notifies classes that use Direct3D device resources that the device resources
// need to be released before this method returns.
void ImmersiveMain::OnDeviceLost() {
#ifdef DRAW_SAMPLE_CONTENT
m_spinningCubeRenderer->ReleaseDeviceDependentResources();
#endif
}
// Notifies classes that use Direct3D device resources that the device resources
// may now be recreated.
void ImmersiveMain::OnDeviceRestored() {
#ifdef DRAW_SAMPLE_CONTENT
m_spinningCubeRenderer->CreateDeviceDependentResources();
#endif
}
void ImmersiveMain::OnLocatabilityChanged(
SpatialLocator const &sender,
winrt::Windows::Foundation::IInspectable const &) {
switch (sender.Locatability()) {
case SpatialLocatability::Unavailable:
// Holograms cannot be rendered.
{
winrt::hstring message(L"Warning! Positional tracking is " +
std::to_wstring(int(sender.Locatability())) +
L".\n");
OutputDebugStringW(message.data());
}
break;
// In the following three cases, it is still possible to place holograms using
// a SpatialLocatorAttachedFrameOfReference.
case SpatialLocatability::PositionalTrackingActivating:
// The system is preparing to use positional tracking.
case SpatialLocatability::OrientationOnly:
// Positional tracking has not been activated.
case SpatialLocatability::PositionalTrackingInhibited:
// Positional tracking is temporarily inhibited. User action may be required
// in order to restore positional tracking.
break;
case SpatialLocatability::PositionalTrackingActive:
// Positional tracking is active. World-locked content can be rendered.
break;
}
}
void ImmersiveMain::OnCameraAdded(
HolographicSpace const &,
HolographicSpaceCameraAddedEventArgs const &args) {
winrt::Windows::Foundation::Deferral deferral = args.GetDeferral();
HolographicCamera holographicCamera = args.Camera();
create_task([this, deferral, holographicCamera]() {
//
// TODO: Allocate resources for the new camera and load any content specific
// to
// that camera. Note that the render target size (in pixels) is a
// property of the HolographicCamera object, and can be used to create
// off-screen render targets that match the resolution of the
// HolographicCamera.
//
// Create device-based resources for the holographic camera and add it to
// the list of cameras used for updates and rendering. Notes:
// * Since this function may be called at any time, the
// AddHolographicCamera function
// waits until it can get a lock on the set of holographic camera
// resources before adding the new camera. At 60 frames per second this
// wait should not take long.
// * A subsequent Update will take the back buffer from the
// RenderingParameters of this
// camera's CameraPose and use it to create the ID3D11RenderTargetView
// for this camera. Content can then be rendered for the
// HolographicCamera.
m_deviceResources->AddHolographicCamera(holographicCamera);
// Holographic frame predictions will not include any information about this
// camera until the deferral is completed.
deferral.Complete();
});
}
void ImmersiveMain::OnCameraRemoved(
HolographicSpace const &,
HolographicSpaceCameraRemovedEventArgs const &args) {
create_task([this]() {
//
// TODO: Asynchronously unload or deactivate content resources (not back
// buffer
// resources) that are specific only to the camera that was removed.
//
});
// Before letting this callback return, ensure that all references to the back
// buffer are released. Since this function may be called at any time, the
// RemoveHolographicCamera function waits until it can get a lock on the set
// of holographic camera resources before deallocating resources for this
// camera. At 60 frames per second this wait should not take long.
m_deviceResources->RemoveHolographicCamera(args.Camera());
}
void ImmersiveMain::OnGamepadAdded(winrt::Windows::Foundation::IInspectable,
Gamepad const &args) {
for (GamepadWithButtonState const &gamepadWithButtonState : m_gamepads) {
if (args == gamepadWithButtonState.gamepad) {
// This gamepad is already in the list.
return;
}
}
GamepadWithButtonState newGamepad = {args, false};
m_gamepads.push_back(newGamepad);
}
void ImmersiveMain::OnGamepadRemoved(winrt::Windows::Foundation::IInspectable,
Gamepad const &args) {
m_gamepads.erase(
std::remove_if(m_gamepads.begin(), m_gamepads.end(),
[&](GamepadWithButtonState &gamepadWithState) {
return gamepadWithState.gamepad == args;
}),
m_gamepads.end());
}
void ImmersiveMain::OnHolographicDisplayIsAvailableChanged(
winrt::Windows::Foundation::IInspectable,
winrt::Windows::Foundation::IInspectable) {
// Get the spatial locator for the default HolographicDisplay, if one is
// available.
SpatialLocator spatialLocator = nullptr;
if (m_canGetDefaultHolographicDisplay) {
HolographicDisplay defaultHolographicDisplay =
HolographicDisplay::GetDefault();
if (defaultHolographicDisplay) {
spatialLocator = defaultHolographicDisplay.SpatialLocator();
}
} else {
spatialLocator = SpatialLocator::GetDefault();
}
if (m_spatialLocator != spatialLocator) {
// If the spatial locator is disconnected or replaced, we should discard all
// state that was based on it.
if (m_spatialLocator != nullptr) {
m_spatialLocator.LocatabilityChanged(m_locatabilityChangedToken);
m_spatialLocator = nullptr;
}
m_stationaryReferenceFrame = nullptr;
if (spatialLocator != nullptr) {
// Use the SpatialLocator from the default HolographicDisplay to track the
// motion of the device.
m_spatialLocator = spatialLocator;
// Respond to changes in the positional tracking state.
m_locatabilityChangedToken = m_spatialLocator.LocatabilityChanged(
std::bind(&ImmersiveMain::OnLocatabilityChanged, this, _1, _2));
// The simplest way to render world-locked holograms is to create a
// stationary reference frame based on a SpatialLocator. This is roughly
// analogous to creating a "world" coordinate system with the origin
// placed at the device's position as the app is launched.
m_stationaryReferenceFrame =
m_spatialLocator.CreateStationaryFrameOfReferenceAtCurrentLocation();
}
}
}