GaussianSplatRenderer.h

Go to the documentation of this file.

// ----------------------------------------------------------------------------
// -                        Open3D: www.open3d.org                            -
// ----------------------------------------------------------------------------
// Copyright (c) 2018-2024 www.open3d.org
// SPDX-License-Identifier: MIT
// ----------------------------------------------------------------------------
 
#pragma once
 
#include <Eigen/Core>
#include <Eigen/Geometry>
#include <cstdint>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <vector>
 
#include "open3d/visualization/rendering/Camera.h"
#include "open3d/visualization/rendering/RendererHandle.h"
 
namespace filament {
class Engine;
}  // namespace filament
 
namespace open3d {
namespace visualization {
namespace rendering {
 
class FilamentResourceManager;
class FilamentScene;
class FilamentView;
 
class GaussianSplatRenderer {
public:
    struct RenderConfig {
        Eigen::Vector2i tile_size = Eigen::Vector2i(16, 16);
        int projection_group_size = 64;
        Eigen::Vector2i composite_group_size = Eigen::Vector2i(16, 16);
        int max_sh_degree = 2;
        // Hard cap on tiles per splat; splats exceeding it are culled rather
        // than cropped. Cropping huge splats can create hard-edged opaque
        // blocks. The cap is independent of buffer sizing, so raising it does
        // not increase GPU memory use. Default is 256.
        std::uint32_t max_tiles_per_splat = 256u;
        // Estimates average tiles per splat for buffer sizing; it is not an
        // enforcement limit. Actual writes are bounds-checked, so excess
        // entries are dropped with a warning rather than overflowing.
        std::uint32_t avg_tiles_per_splat = 32u;
        std::uint32_t max_tile_entries_total = 32u * 1024u * 1024u;
        bool antialias = false;
    };
 
    struct ViewRenderData {
        Eigen::Vector2i viewport_origin = Eigen::Vector2i::Zero();
        Eigen::Vector2i viewport_size = Eigen::Vector2i::Zero();
        Eigen::Vector3f camera_position = Eigen::Vector3f::Zero();
        bool screen_y_down = false;
        Camera::Transform model_matrix = Camera::Transform::Identity();
        Camera::Transform view_matrix = Camera::Transform::Identity();
        Camera::ProjectionMatrix projection_matrix =
                Camera::ProjectionMatrix::Identity();
        Camera::Transform culling_projection_matrix =
                Camera::Transform::Identity();
        Camera::ProjectionInfo projection;
        double near_plane = 0.0;
        double far_plane = 0.0;
    };
 
    struct OutputTargets {
        TextureHandle color;
        TextureHandle depth;
        RenderTargetHandle render_target;
        std::uint32_t width = 0;
        std::uint32_t height = 0;
        std::uint32_t scene_depth_gl_handle = 0;
        std::uint32_t color_gl_handle = 0;
 
        bool uses_vulkan_interop = false;
 
        std::uintptr_t color_vk_image = 0;
        std::uintptr_t color_vk_memory = 0;
        std::uint32_t color_gl_mem_obj = 0;  
        std::uintptr_t depth_vk_image = 0;
        std::uintptr_t depth_vk_memory = 0;
        std::uint32_t depth_gl_mem_obj = 0;  
 
        std::uintptr_t scene_depth_mtl_texture = 0;
        std::uintptr_t gs_color_mtl_texture = 0;
        ViewRenderData render_data;
        RenderTargetHandle gs_readback_rt;
        bool has_render_data = false;
        bool has_valid_output = false;
        bool needs_render = true;
        bool wants_depth_readback = false;
        std::uint64_t last_scene_change_id = 0;
        std::uint64_t last_updated_frame = 0;
    };
 
    class Backend {
    public:
        virtual ~Backend() = default;
 
        virtual const char* GetName() const = 0;
        virtual void BeginFrame(std::uint64_t frame_index) = 0;
        virtual void ForgetView(const FilamentView& view) = 0;
        virtual bool RenderGeometryStage(const FilamentView& view,
                                         const FilamentScene& scene,
                                         const ViewRenderData& render_data,
                                         OutputTargets& targets) = 0;
        virtual bool RenderCompositeStage(const FilamentView& view,
                                          const ViewRenderData& render_data,
                                          OutputTargets& targets) = 0;
 
        virtual bool PrepareOutputTextures(
                FilamentView& view,
                FilamentResourceManager& resource_mgr,
                std::uint32_t width,
                std::uint32_t height,
                OutputTargets& targets) = 0;
 
        virtual void ReleaseOutputTextures(
                FilamentResourceManager& resource_mgr,
                OutputTargets& targets) = 0;
 
        virtual bool ReadMergedDepthToUint16Cpu(const FilamentView& view,
                                                std::vector<std::uint16_t>& out,
                                                std::uint32_t width,
                                                std::uint32_t height) {
            (void)view;
            (void)out;
            (void)width;
            (void)height;
            return false;
        }
 
        virtual bool ReadCompositeDepthToFloatCpu(const FilamentView& view,
                                                  std::vector<float>& out,
                                                  std::uint32_t width,
                                                  std::uint32_t height) {
            (void)view;
            (void)out;
            (void)width;
            (void)height;
            return false;
        }
    };
 
    GaussianSplatRenderer(filament::Engine& engine,
                          FilamentResourceManager& resource_mgr);
    ~GaussianSplatRenderer();
 
    void BeginFrame();
 
    void RenderGeometryStage(FilamentView& view, const FilamentScene& scene);
    bool RenderCompositeStage(FilamentView& view);
    void PruneOutputs(
            const std::unordered_set<const FilamentView*>& live_views);
 
    void InvalidateOutputForView(FilamentView& view);
 
    void RequestRedrawForView(const FilamentView& view);
 
    bool IsEnabled() const;
    void SetEnabled(bool enabled);
    bool IsSupported() const;
 
    bool HasOutput(const FilamentView& view) const;
    TextureHandle GetColorTexture(const FilamentView& view) const;
    TextureHandle GetDepthTexture(const FilamentView& view) const;
    RenderTargetHandle GetColorReadbackRT(const FilamentView& view) const;
 
    bool ReadMergedDepthToUint16Cpu(const FilamentView& view,
                                    std::vector<std::uint16_t>& out,
                                    std::uint32_t width,
                                    std::uint32_t height);
 
    bool ReadCompositeDepthToFloatCpu(const FilamentView& view,
                                      std::vector<float>& out,
                                      std::uint32_t width,
                                      std::uint32_t height);
 
    void RequestDepthReadbackForView(const FilamentView& view,
                                     bool wanted = true);
 
    std::uint32_t GetSceneDepthGLHandle(const FilamentView& view) const;
    const ViewRenderData* GetViewRenderData(const FilamentView& view) const;
    const RenderConfig& GetRenderConfig() const;
    void SetRenderConfig(const RenderConfig& config);
    const char* GetBackendName() const;
 
private:
    using ViewKey = const FilamentView*;
 
    OutputTargets& PrepareOutputTargets(FilamentView& view);
    void ResetOutputTargets(OutputTargets& targets);
    ViewRenderData ExtractViewRenderData(const FilamentView& view) const;
    bool UpdateViewRenderData(OutputTargets& targets, const FilamentView& view);
    bool ValidateRenderConfig(const RenderConfig& config) const;
 
    filament::Engine& engine_;
    FilamentResourceManager& resource_mgr_;
    std::unordered_map<ViewKey, OutputTargets> outputs_;
    RenderConfig render_config_;
    std::unique_ptr<Backend> backend_;
    bool enabled_ = false;
    std::uint64_t frame_index_ = 0;
};
 
}  // namespace rendering
}  // namespace visualization
}  // namespace open3d